Merge pull request #134 from openstates/bugfix-scrape-archive-mode

Bugfix: Google Cloud Storage permissions issue with archiving

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 6.20.1 - Aug 2, 2024
+
+* Fix permissions issue caused by slightly wrong usage of GCP storage client code
+
 ## 6.20.0 - Aug 1, 2024
 
 * Adds support for --archive flag on os-update to archive a full scrape to google cloud storage bucket

openstates/cli/update.py

@@ -215,19 +215,25 @@ def archive_to_cloud_storage(
             "Scrape archiving is turned on, but necessary settings are missing. No archive was done."
         )
         return
-    cloud_storage_client = storage.Client()
-    bucket = cloud_storage_client.bucket(BUCKET_NAME, GCP_PROJECT)
+    logger.info("Beginning archive of scraped files to google cloud storage.")
+    logger.info(f"GCP Project is {GCP_PROJECT} and bucket is {BUCKET_NAME}")
+    cloud_storage_client = storage.Client(project=GCP_PROJECT)
+    bucket = cloud_storage_client.bucket(BUCKET_NAME)
     jurisdiction_id = juris.jurisdiction_id.replace("ocd-jurisdiction/", "")
     destination_prefx = (
         f"{SCRAPE_LAKE_PREFIX}/{jurisdiction_id}/{last_scrape_end_datetime.isoformat()}"
     )
 
     # read files in directory and upload
+    files_count = 0
     for file_path in glob.glob(datadir + "/*.json"):
+        files_count += 1
         blob_name = os.path.join(destination_prefx, os.path.basename(file_path))
         blob = bucket.blob(blob_name)
         blob.upload_from_filename(file_path)
 
+    logger.info(f"Completed archive to Google Cloud Storage, {files_count} files were uploaded.")
+
 
 def do_import(juris: State, args: argparse.Namespace) -> dict[str, typing.Any]:
     # import inside here because to avoid loading Django code unnecessarily

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "openstates"
-version = "6.20.0"
+version = "6.20.1"
 description = "core infrastructure for the openstates project"
 authors = ["James Turk <[email protected]>"]
 license = "MIT"