Merge pull request #155 from jump-cellpainting/update-profile-urls-to-assembled-structure

feat: Update profile URLs to use new profiles_assembled structure + move S3 objects to obey new folder structure

Author: shntnu

manifests/profile_index.json

@@ -1,58 +1,58 @@
 [
   {
     "subset": "orf",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_a917fa7/ORF/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/ORF/v1.0a/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/a917fa79342ff92cf0ea05d6d9174d9028a90f8f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/a917fa79342ff92cf0ea05d6d9174d9028a90f8f/inputs/orf.json",
-    "etag": "c05a241135dcedda4e9cc639480b3f8e-44"
+    "etag": "064759b3a850dc351b357116b3e7b32d"
   },
   {
     "subset": "crispr",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_a917fa7/CRISPR/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony_PCA_corrected/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony_PCA_corrected.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/CRISPR/v1.0a/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony_PCA_corrected.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/a917fa79342ff92cf0ea05d6d9174d9028a90f8f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/a917fa79342ff92cf0ea05d6d9174d9028a90f8f/inputs/crispr.json",
-    "etag": "4c59782c0dd5244f67d14323e8325828-10"
+    "etag": "5903af59605b2037190ff64c1f87c530"
   },
   {
     "subset": "compound",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_a917fa7/COMPOUND/profiles_var_mad_int_featselect_harmony/profiles_var_mad_int_featselect_harmony.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/COMPOUND/v1.0/profiles_var_mad_int_featselect_harmony.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/a917fa79342ff92cf0ea05d6d9174d9028a90f8f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/a917fa79342ff92cf0ea05d6d9174d9028a90f8f/inputs/compound.json",
-    "etag": "1368a48ddbd4c44b1bfbc084591aaf10-338"
+    "etag": "c9371af57a36a51e021935c9ca78e506"
   },
   {
     "subset": "orf_interpretable",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_a917fa7/ORF/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony/profiles_wellpos_cc_var_mad_outlier.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/ORF/v1.0a/profiles_wellpos_cc_var_mad_outlier.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/a917fa79342ff92cf0ea05d6d9174d9028a90f8f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/a917fa79342ff92cf0ea05d6d9174d9028a90f8f/inputs/orf.json",
-    "etag": "97b0c31d7d678ca2a5e2353df5799fd8-217"
+    "etag": "a2ca4063bbfcee09ac303cf48eb8bafd"
   },
   {
     "subset": "crispr_interpretable",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_a917fa7/CRISPR/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony_PCA_corrected/profiles_wellpos_cc_var_mad_outlier.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/CRISPR/v1.0a/profiles_wellpos_cc_var_mad_outlier.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/a917fa79342ff92cf0ea05d6d9174d9028a90f8f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/a917fa79342ff92cf0ea05d6d9174d9028a90f8f/inputs/crispr.json",
-    "etag": "90b08b824c06bcf16dfc5e788e74f099-135"
+    "etag": "0009a142e5d132d007696ed8f71f2da8"
   },
   {
     "subset": "compound_interpretable",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_a917fa7/COMPOUND/profiles_var_mad_int_featselect_harmony/profiles_var_mad_int.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/COMPOUND/v1.0/profiles_var_mad_int.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/a917fa79342ff92cf0ea05d6d9174d9028a90f8f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/a917fa79342ff92cf0ea05d6d9174d9028a90f8f/inputs/compound.json",
-    "etag": "b638fa24310db569bc869af92e16f69c-1444"
+    "etag": "67212e3cbdaa25de511f318cdd0503dc-3"
   },
   {
     "subset": "all",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_0224e0f/ALL/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/ALL/v1.0b/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/0224e0fc23a84e7e84b091f320a9e68b3217343f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/0224e0fc23a84e7e84b091f320a9e68b3217343f/inputs/pipeline_2.json",
-    "etag": "71d03c195e41739af0f1ba64b4f6be73-324"
+    "etag": "96eeaeb01ac8eab9845111bd34a6c82d"
   },
   {
     "subset": "all_interpretable",
-    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles/jump-profiling-recipe_2024_0224e0f/ALL/profiles_wellpos_cc_var_mad_outlier_featselect_sphering_harmony/profiles_wellpos_cc_var_mad_outlier_featselect.parquet",
+    "url": "https://cellpainting-gallery.s3.amazonaws.com/cpg0016-jump-assembled/source_all/workspace/profiles_assembled/ALL/v1.0b/profiles_wellpos_cc_var_mad_outlier_featselect.parquet",
     "recipe_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/tree/0224e0fc23a84e7e84b091f320a9e68b3217343f",
     "config_permalink": "https://github.com/broadinstitute/jump-profiling-recipe/blob/0224e0fc23a84e7e84b091f320a9e68b3217343f/inputs/pipeline_2.json",
-    "etag": "023d74cbf007bb6d837724ac8aa78fb4-324"
+    "etag": "29cafe5726408773300eb53619281a1d"
   }
 ]

manifests/src/README.md

@@ -13,7 +13,7 @@ If necessary, update the associated names for new dataset types and (optionally)
 After updating a URL, the ETag (provided by S3) will no longer match. To update the ETags, run the following command from the home folder:
 
 ```bash
-bash manifests/src/update_etags.sh manifests/profile_index.json | sponge manifests/profile_index.json
+bash manifests/src/update_etags.sh manifests/profile_index.json > manifests/profile_index.json.tmp && mv manifests/profile_index.json.tmp manifests/profile_index.json
 ```
 
 Note: If using Nix, all dependencies are already included in the flake at the root folder. Simply run `nix develop --extra-experimental-features nix-command --extra-experimental-features flakes` before the above command.

manifests/src/update_etags.sh

@@ -17,4 +17,4 @@ JSON_LIST=$(printf '%s\n' "${NEW_ETAGS}" | jq -R . | jq -s .)
 jq --argjson etags "${JSON_LIST}" '[
   range(0; length) as $i
   | {subset: .[$i].subset, url: .[$i].url, recipe_permalink: .[$i].recipe_permalink, config_permalink: .[$i].config_permalink, etag: $etags[$i]}
-]' profile_index.json
+]' ${input_file}