Handle votes with joint responsible committees (#1122)

Author: tillprochaska

backend/howtheyvote/alembic/versions/8d1995cb0bed_make_responsible_committee_column_json_.py

@@ -0,0 +1,26 @@
+"""Make responsible_committee column JSON in votes table
+
+Revision ID: 8d1995cb0bed
+Revises: 064daf473f9a
+Create Date: 2025-03-16 17:13:08.000602
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "8d1995cb0bed"
+down_revision = "064daf473f9a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_column("votes", "responsible_committee")
+    op.add_column("votes", sa.Column("responsible_committees", sa.JSON))
+
+
+def downgrade() -> None:
+    op.drop_column("votes", "responsible_committees")
+    op.add_column("votes", sa.Column("responsible_committee", sa.Unicode))

backend/howtheyvote/api/serializers.py

@@ -263,8 +263,8 @@ class BaseVoteDict(TypedDict):
     """Concepts from the [EuroVoc](https://eur-lex.europa.eu/browse/eurovoc.html) thesaurus
     that are related to this vote"""
 
-    responsible_committee: CommitteeDict | None
-    """Committee responsible for the legislative procedure"""
+    responsible_committees: list[CommitteeDict] | None
+    """Committees responsible for the legislative procedure"""
 
     result: VoteResult | None
     """Vote result. This field is only available for votes starting in 2024."""
@@ -273,9 +273,9 @@ class BaseVoteDict(TypedDict):
 def serialize_base_vote(vote: Vote) -> BaseVoteDict:
     geo_areas = [serialize_country(geo_area) for geo_area in vote.geo_areas]
     eurovoc_concepts = [serialize_eurovoc_concept(ec) for ec in vote.eurovoc_concepts]
-    responsible_committee = (
-        serialize_committee(vote.responsible_committee) if vote.responsible_committee else None
-    )
+    responsible_committees = [
+        serialize_committee(committee) for committee in vote.responsible_committees
+    ]
 
     return {
         "id": vote.id,
@@ -285,7 +285,7 @@ def serialize_base_vote(vote: Vote) -> BaseVoteDict:
         "reference": vote.reference,
         "geo_areas": geo_areas,
         "eurovoc_concepts": eurovoc_concepts,
-        "responsible_committee": responsible_committee,
+        "responsible_committees": responsible_committees,
         "result": vote.result,
     }
 

backend/howtheyvote/export/__init__.py

@@ -9,7 +9,7 @@
 
 from ..db import Session
 from ..helpers import parse_procedure_reference
-from ..models import Country, EurovocConcept, Member, Vote
+from ..models import Committee, Country, EurovocConcept, Member, Vote
 from ..vote_stats import count_vote_positions
 from .csvw_helpers import Table
 
@@ -149,9 +149,6 @@ class VoteRow(TypedDict):
     `OLP_SECOND_READING`, `OLP_THIRD_READING`.This field is only available for votes starting
     in 2024 and if the vote is part of an Ordinary Legislative Procedure."""
 
-    responsible_committee_code: str | None
-    """Committee responsible for the legislative procedure"""
-
     count_for: int
     """Number of MEPs who voted in favor"""
 
@@ -239,6 +236,29 @@ class GeoAreaVoteRow(TypedDict):
     """Geographic area code"""
 
 
+class CommitteeRow(TypedDict):
+    """Each row represents a committee of the European Parliament."""
+
+    code: str
+    """Unique identifier of the committee"""
+
+    label: str
+    """Label"""
+
+    abbreviation: str
+    """Abbreviation"""
+
+
+class ResponsibleCommitteeVoteRow(TypedDict):
+    """Committee responsible for the legislative procedure a vote is part of."""
+
+    vote_id: int
+    """Vote ID"""
+
+    committee_code: str
+    """Committee code"""
+
+
 class Export:
     def __init__(self, outdir: pathlib.Path):
         self.outdir = outdir
@@ -313,6 +333,20 @@ def __init__(self, outdir: pathlib.Path):
             primary_key=["vote_id", "geo_area_code"],
         )
 
+        self.committees = Table(
+            row_type=CommitteeRow,
+            outdir=self.outdir,
+            name="committees",
+            primary_key=["code"],
+        )
+
+        self.responsible_committee_votes = Table(
+            row_type=ResponsibleCommitteeVoteRow,
+            outdir=self.outdir,
+            name="responsible_committee_votes",
+            primary_key=["vote_id", "committee_code"],
+        )
+
     def run(self) -> None:
         self.fetch_members()
         self.write_export_timestamp()
@@ -328,12 +362,15 @@ def run(self) -> None:
                 self.eurovoc_concept_votes,
                 self.geo_areas,
                 self.geo_area_votes,
+                self.committees,
+                self.responsible_committee_votes,
             ]
         )
         self.export_members()
         self.export_votes()
         self.export_eurovoc_concepts()
         self.export_geo_areas()
+        self.export_committees()
 
     def fetch_members(self) -> None:
         self.members_by_id: dict[int, Member] = {}
@@ -436,6 +473,7 @@ def export_votes(self) -> None:
             self.member_votes.open() as member_votes,
             self.eurovoc_concept_votes.open() as eurovoc_concept_votes,
             self.geo_area_votes.open() as geo_area_votes,
+            self.responsible_committee_votes.open() as responsible_committee_votes,
         ):
             query = select(Vote).order_by(Vote.id).execution_options(yield_per=500)
             result = Session.scalars(query)
@@ -444,10 +482,6 @@ def export_votes(self) -> None:
                 if idx % 1000 == 0:
                     log.info("Writing vote", index=idx)
 
-                responsible_committee_code = (
-                    vote.responsible_committee.code if vote.responsible_committee else None
-                )
-
                 position_counts = count_vote_positions(vote.member_votes)
                 procedure_reference = (
                     parse_procedure_reference(vote.procedure_reference)
@@ -471,7 +505,6 @@ def export_votes(self) -> None:
                         "procedure_stage": (
                             vote.procedure_stage.value if vote.procedure_stage else None
                         ),
-                        "responsible_committee_code": responsible_committee_code,
                         "count_for": position_counts["FOR"],
                         "count_against": position_counts["AGAINST"],
                         "count_abstention": position_counts["ABSTENTION"],
@@ -496,6 +529,14 @@ def export_votes(self) -> None:
                         }
                     )
 
+                for responsible_committee in vote.responsible_committees:
+                    responsible_committee_votes.write_row(
+                        {
+                            "vote_id": vote.id,
+                            "committee_code": responsible_committee.code,
+                        }
+                    )
+
                 for member_vote in sorted(vote.member_votes, key=lambda mv: mv.web_id):
                     member = self.members_by_id[member_vote.web_id]
                     group = member.group_at(vote.timestamp)
@@ -554,6 +595,26 @@ def export_geo_areas(self) -> None:
                     }
                 )
 
+    def export_committees(self) -> None:
+        log.info("Exporting committees")
+
+        with self.committees.open() as committees:
+            exp = func.json_each(Vote.responsible_committees).table_valued("value")
+            query = (
+                select(func.distinct(exp.c.value)).select_from(Vote, exp).order_by(exp.c.value)
+            )
+            committee_codes = Session.execute(query).scalars()
+
+            for committee_code in committee_codes:
+                committee = Committee[committee_code] if True else None
+                committees.write_row(
+                    {
+                        "code": committee.code,
+                        "label": committee.label,
+                        "abbreviation": committee.abbreviation,
+                    }
+                )
+
 
 def generate_export(path: pathlib.Path) -> None:
     with tempfile.TemporaryDirectory() as outdir:

backend/howtheyvote/models/vote.py

@@ -146,7 +146,7 @@ class Vote(BaseWithId):
     eurovoc_concepts: Mapped[list[EurovocConcept]] = mapped_column(
         ListType(EurovocConceptType())
     )
-    responsible_committee: Mapped[Committee] = mapped_column(CommitteeType())
+    responsible_committees: Mapped[list[Committee]] = mapped_column(ListType(CommitteeType()))
     press_release: Mapped[str | None] = mapped_column(sa.Unicode)
     issues: Mapped[list[DataIssue]] = mapped_column(ListType(sa.Enum(DataIssue)))
 

backend/howtheyvote/scrapers/votes.py

@@ -503,7 +503,7 @@ def _url(self) -> str:
     def _extract_data(self, doc: BeautifulSoup) -> Fragment:
         title = self._title(doc)
         geo_areas = self._geo_areas(doc)
-        responsible_committee = self._responsible_committee(doc)
+        responsible_committees = self._responsible_committees(doc)
         self._log.info(
             "Extracted procedure information",
             title=title,
@@ -517,7 +517,7 @@ def _extract_data(self, doc: BeautifulSoup) -> Fragment:
             data={
                 "procedure_title": title,
                 "geo_areas": geo_areas,
-                "responsible_committee": responsible_committee,
+                "responsible_committees": responsible_committees,
             },
         )
 
@@ -562,30 +562,31 @@ def _geo_areas(self, doc: BeautifulSoup) -> list[str]:
 
         return geo_areas
 
-    def _responsible_committee(self, doc: BeautifulSoup) -> str | None:
+    def _responsible_committees(self, doc: BeautifulSoup) -> set[str]:
+        committees: set[str] = set()
+
         table = doc.select_one(
-            '#erpl_accordion-committee table:has(th:-soup-contains("Committee responsible"))'
+            "#erpl_accordion-committee :where("
+            + 'table:has(th:-soup-contains("Committee responsible")),'
+            + 'table:has(th:-soup-contains("Joint committee responsible"))'
+            + ")"
         )
 
         if not table:
-            return None
+            return committees
 
-        if len(table.select("tbody tr")) > 1:
-            # We assume that there is at most one responsible committee
-            log.warning("More than one responsible committee found")
+        badges = table.select("tbody tr .erpl_badge-committee")
 
-        badge = table.select_one("tbody tr .erpl_badge-committee")
-
-        if not badge:
-            return None
+        for badge in badges:
+            text = badge.text.strip()
+            committee = Committee.get(text)
 
-        text = badge.text.strip()
-        committee = Committee.get(text)
+            if not committee:
+                raise ScrapingError(f"Could not find committee {text}")
 
-        if not committee:
-            raise ScrapingError(f"Could not find committee {text}")
+            committees.add(committee.code)
 
-        return committee.code
+        return committees
 
 
 class EurlexProcedureScraper(BeautifulSoupScraper):

backend/howtheyvote/store/mappings.py

@@ -62,7 +62,9 @@ def map_vote(record: CompositeRecord) -> Vote:
     member_votes = [deserialize_member_vote(mv) for mv in record.first("member_votes")]
     geo_areas = {Country[code] for code in record.chain("geo_areas")}
     eurovoc_concepts = {EurovocConcept[id_] for id_ in record.chain("eurovoc_concepts")}
-    responsible_committee = Committee.get(record.first("responsible_committee"))
+    responsible_committees = {
+        Committee[code] for code in record.chain("responsible_committees")
+    }
     result = VoteResult[record.first("result")] if record.first("result") else None
     procedure_stage = (
         ProcedureStage[record.first("procedure_stage")]
@@ -91,7 +93,7 @@ def map_vote(record: CompositeRecord) -> Vote:
         member_votes=member_votes,
         geo_areas=geo_areas,
         eurovoc_concepts=eurovoc_concepts,
-        responsible_committee=responsible_committee,
+        responsible_committees=responsible_committees,
         press_release=press_release,
         issues=record.chain("issues"),
     )

backend/tests/api/test_votes_api.py

@@ -571,7 +571,7 @@ def test_votes_api_show(records, db_session, api):
         "result": "ADOPTED",
         "geo_areas": [],
         "eurovoc_concepts": [],
-        "responsible_committee": None,
+        "responsible_committees": [],
         "related": [],
         "sources": [
             {

backend/tests/export/test_init.py

@@ -5,6 +5,7 @@
 
 from howtheyvote.export import Export
 from howtheyvote.models import (
+    Committee,
     Country,
     EurovocConcept,
     Group,
@@ -140,6 +141,7 @@ def test_export_votes(db_session, tmp_path):
             Country["MDA"],
             Country["RUS"],
         ],
+        responsible_committees=[Committee["AFET"]],
         result=VoteResult.ADOPTED,
     )
 
@@ -153,8 +155,8 @@ def test_export_votes(db_session, tmp_path):
     votes_meta = tmp_path.joinpath("votes.csv-metadata.json")
 
     expected = (
-        "id,timestamp,display_title,reference,description,is_main,procedure_reference,procedure_title,procedure_type,procedure_stage,responsible_committee_code,count_for,count_against,count_abstention,count_did_not_vote,result\n"
-        "123456,2024-01-01 00:00:00,Lorem Ipsum,,,False,2025/1234(COD),Lorem Ipsum,COD,OLP_FIRST_READING,,1,0,0,0,ADOPTED\n"
+        "id,timestamp,display_title,reference,description,is_main,procedure_reference,procedure_title,procedure_type,procedure_stage,count_for,count_against,count_abstention,count_did_not_vote,result\n"
+        "123456,2024-01-01 00:00:00,Lorem Ipsum,,,False,2025/1234(COD),Lorem Ipsum,COD,OLP_FIRST_READING,1,0,0,0,ADOPTED\n"
     )
 
     assert votes_csv.read_text() == expected
@@ -200,6 +202,24 @@ def test_export_votes(db_session, tmp_path):
     assert geo_areas_csv.read_text() == expected
     assert geo_areas_meta.is_file()
 
+    responsible_committee_votes_csv = tmp_path.joinpath("responsible_committee_votes.csv")
+    responsible_committee_votes_meta = tmp_path.joinpath(
+        "responsible_committee_votes.csv-metadata.json"
+    )
+
+    expected = "vote_id,committee_code\n123456,AFET\n"
+
+    assert responsible_committee_votes_csv.read_text() == expected
+    assert responsible_committee_votes_meta.is_file()
+
+    committees_csv = tmp_path.joinpath("committees.csv")
+    committees_meta = tmp_path.joinpath("committees.csv-metadata.json")
+
+    expected = "code,label,abbreviation\nAFET,Committee on Foreign Affairs,AFET\n"
+
+    assert committees_csv.read_text() == expected
+    assert committees_meta.is_file()
+
 
 def test_export_votes_country_group(db_session, tmp_path):
     member = Member(