Splitting the weekly update by community

.github/CODEOWNERS

@@ -0,0 +1,17 @@
+# These owners will be the default owners for everything in the repo. Unless a later match takes precedence, @global-owner1 and @global-owner2 will be requested for review when someone opens a pull request.
+*       @scorreard @bebatut @paulzierep
+
+# Any change inside each community directory will require approval from the assigned person
+/communities/biodiversity/ @scorreard
+/communities/earth/ @Marie59
+/communities/imaging/ @nagoue
+/communities/micrgalaxy/ @bebatut
+
+
+#Need to identify assigned person : 
+#/communities/assembly/ 
+#/communities/genome/
+#/communities/genome/
+#/communities/machine-learning/
+#/communities/metabolics/
+#/communities/spoc/

.github/workflows/fetch_filter_resources.yaml

@@ -1,4 +1,4 @@
-name: Weekly resource fetching and community filtering
+name: Weekly resource fetching
 
 on:
   workflow_dispatch:
@@ -137,14 +137,14 @@ jobs:
 #        with:
 #          name: workflows
 #          path: communities/all/resources/citations.json
-  merge-fetch-filter:
+  merge-fetch:
     runs-on: ubuntu-latest
     needs: 
       - fetch-tools-stepwise
       - fetch-tutorials
       - fetch-workflows
 #      - fetch-citations
-    name: Merge tools, filter the resources for communities, and populate labs
+    name: Merge tools
     steps:
       - name: Checkout main
         uses: actions/checkout@v6
@@ -185,55 +185,11 @@ jobs:
       - name: Generate tools wordcloud and interactive table
         run: |
           bash sources/bin/format_tools.sh
-      - name: Filter workflows for communities
-        run: | 
-          bash sources/bin/get_community_workflows.sh
-      - name: Filter tutorials for communities
-        run: | 
-          bash sources/bin/get_community_tutorials.sh
-      - name: Filter citations for communities
-        run: | 
-          bash sources/bin/get_community_citations.sh no-scholarly
-      #- name: Update tool to keep and exclude for communities
-      #  run: | 
-      #    bash sources/bin/update_tools_to_keep_exclude.sh
-      - name: Filter tools for communities
-        run: | 
-          bash sources/bin/get_community_tools.sh
-      - name: Populate Microbiology (microGalaxy) Lab
-        run: | 
-          bash sources/bin/populate_labs.sh
-        env: 
-          COMMUNITY: microgalaxy
-      - name: Populate biodiversity lab
-        run: |
-          bash sources/bin/populate_labs.sh
-        env: 
-          COMMUNITY: biodiversity
-      - name: Populate earth-system Lab
-        run: |
-          bash sources/bin/populate_labs.sh
-        env: 
-          COMMUNITY: earth
-      - name: Populate Imaging lab
-        run: |
-          bash sources/bin/populate_labs.sh
-        env: 
-          COMMUNITY: imaging
-#To initiate a lab, we recommend adding it below. It will automatically create the folders and structures required.
-#      - name: Populate <community-name> Lab
-#        run: |
-#          bash sources/bin/populate_labs.sh
-#        env: 
-#          COMMUNITY: <community-name>
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v8
+      - name: Commit and push changes
+        uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          commit-message: Update resources
-          title: Automatic resources update
-          body: Automatic resource update done via GitHub Action once a week
-          base: main
-          branch: resource-update
-          delete-branch: true
-          add-paths: |
-            communities/
+          commit_message: "Update resources"
+          branch: main
+          commit_user_name: GitHub Actions
+          commit_user_email: actions@github.com
+          commit_author: GitHub Actions <actions@github.com>

.github/workflows/filter_communities.yaml

@@ -1,46 +1,91 @@
-name: Filter community resources
+name: Community weekly resource filtering
 
 on:
   workflow_dispatch:
-  # the workflow it also triggered when the community definitions are changed
   push:
+    branches:
+      - 'main'
+      - 'scorrear_PR_too_big'
     paths:
-    - 'data/communities/*/metadata/*'
-    branches: ["main"]
+      - 'communities/all/resources/tools.tsv'
 
 # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
 # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
 concurrency:
-  group: "filter"
+  group: "tools"
   cancel-in-progress: false
 
 jobs:
-  filter-resources:
-    name: Filter resources for communities
+  set-matrix:
     runs-on: ubuntu-latest
+    outputs:
+      communities: ${{ steps.prepare.outputs.communities }}
     steps:
-    - name: Checkout main
-      uses: actions/checkout@v6
-    - uses: actions/setup-python@v6
-      with:
-        python-version: '3.11' 
-    - name: Install requirement
-      run: python -m pip install -r requirements.txt
-    - name: Filter tutorials for communities
-      run: | 
-        bash bin/get_community_tutorials.sh
-    - name: Update tool to keep and exclude for communities
-      run: | 
-        bash bin/update_tools_to_keep_exclude.sh
-    - name: Filter tools for communities
-      run: | 
-        bash bin/get_community_tools.sh
-    - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v8
-      with:
-        commit-message: Update resources
-        title: Resource filtering
-        body: Resource filering done via GitHub Action when the community definitions are changed 
-        base: main
-        branch: resource-filtering
-        delete-branch: true
+      - name: Checkout main
+        uses: actions/checkout@v6
+      - name: Prepare communities JSON
+        id: prepare
+        run: |
+          # produce a JSON array string, e.g. from a file or script
+          echo '["assembly","biodiversity","earth","genome","imaging","machine-learning","metabolomics","microgalaxy","proteomics","spoc"]' > communities.json
+          communities=$(cat communities.json)
+          echo "communities=$communities" >> $GITHUB_OUTPUT
+
+  community-filter:
+    needs: set-matrix
+    runs-on: ubuntu-latest
+    name: Filter resources and create labs
+    strategy:
+      matrix:
+        community: ${{ fromJson(needs.set-matrix.outputs.communities) }}
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Filter tools for communities
+        run: | 
+          bash sources/bin/get_community_tools.sh
+        env: 
+          COMMUNITY: ${{ matrix.community }}
+      - name: Filter workflows for communities
+        run: | 
+          bash sources/bin/get_community_workflows.sh
+        env: 
+          COMMUNITY: ${{ matrix.community }}
+      - name: Filter tutorials for communities
+        run: | 
+          bash sources/bin/get_community_tutorials.sh
+        env: 
+          COMMUNITY: ${{ matrix.community }}
+      - name: Filter citations for communities
+        run: | 
+          bash sources/bin/get_community_citations.sh no-scholarly
+        env: 
+          COMMUNITY: ${{ matrix.community }}
+      #- name: Update tool to keep and exclude for communities
+      #  run: | 
+      #    bash sources/bin/update_tools_to_keep_exclude.sh
+      #  env: 
+      #    COMMUNITY: ${{ matrix.community }}
+      - name: Populate lab for communities
+        run: |
+          bash sources/bin/populate_labs.sh
+        env: 
+          COMMUNITY: ${{ matrix.community }}
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v8
+        with:
+          commit-message: ${{ matrix.community }} community resources update
+          title: ${{ matrix.community }} community automatic resources update
+          body: ${{ matrix.community }} community automatic resource update done via GitHub Action once a week
+          base: main
+          branch: ${{ matrix.community }}-resource-update
+          delete-branch: true
+          add-paths: |
+            communities/

sources/bin/get_community_citations.sh

@@ -1,28 +1,28 @@
 #!/usr/bin/env bash
 
-for com_data_fp in communities/* ; do
-        if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
-                community=`basename "$com_data_fp"`
-                if [[ ! -z $1  && $1 == "test"  && "$community" != "microgalaxy" ]]; then
+#for com_data_fp in communities/* ; do
+#        if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
+#                community=`basename "$com_data_fp"`
+                if [[ ! -z $1  && $1 == "test"  && "$COMMUNITY" != "microgalaxy" ]]; then
                         continue
                 fi;
 
-                if [[ "$community" != "all" && -f "communities/$community/metadata/citation_keywords" ]]; then
-                        echo "$community";
-                        mkdir -p "communities/$community/resources"
+                if [[ "$COMMUNITY" != "all" && -f "communities/$COMMUNITY/metadata/citation_keywords" ]]; then
+                        echo "$COMMUNITY";
+                        mkdir -p "communities/$COMMUNITY/resources"
 
                         python sources/bin/citation_manager.py \
                                 filter \
                                 --all-json "communities/all/resources/citations.json" \
-                                --filtered-json "communities/$community/resources/citations.json" \
-                                --filtered-yaml "communities/$community/resources/citations.yml" \
-                                --filtered-tsv "communities/$community/resources/citations.tsv" \
-                                --keywords "communities/$community/metadata/citation_keywords"
+                                --filtered-json "communities/$COMMUNITY/resources/citations.json" \
+                                --filtered-yaml "communities/$COMMUNITY/resources/citations.yml" \
+                                --filtered-tsv "communities/$COMMUNITY/resources/citations.tsv" \
+                                --keywords "communities/$COMMUNITY/metadata/citation_keywords"
 
-                        if [[ -f "communities/$community/metadata/citations.yml" ]]; then
-                                mkdir -p _data/communities/$community/
-                                ln -sf ../../../communities/$community/resources/citations.yml _data/communities/$community/citations.yml
+                        if [[ -f "communities/$COMMUNITY/metadata/citations.yml" ]]; then
+                                mkdir -p _data/communities/$COMMUNITY/
+                                ln -sf ../../../communities/$COMMUNITY/resources/citations.yml _data/communities/$COMMUNITY/citations.yml
                         fi;
-                fi;
-        fi;
-done
+                fi
+#        fi;
+#done

sources/bin/get_community_tools.sh

@@ -3,56 +3,56 @@
 # stop on error
 set -e 
 
-for com_data_fp in communities/* ; do
-        if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
-                community=`basename "$com_data_fp"`
+#for com_data_fp in communities/* ; do
+#        if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
+#                community=`basename "$com_data_fp"`
 
-                if [[ ! -z $1  && $1 == "test" && "$community" != "microgalaxy" ]]; then
+                if [[ ! -z $1  && $1 == "test" && "$COMMUNITY" != "microgalaxy" ]]; then
                         continue
                 fi;
 
-                if [[ "$community" != "all" && -f "communities/$community/metadata/categories" ]]; then
-                        echo "$community";
-                        mkdir -p "communities/$community/resources"
+                if [[ "$COMMUNITY" != "all" && -f "communities/$COMMUNITY/metadata/categories" ]]; then
+                        echo "$COMMUNITY";
+                        mkdir -p "communities/$COMMUNITY/resources"
 
                         python sources/bin/extract_galaxy_tools.py \
                                 filter \
                                 --all "communities/all/resources/tools.json" \
-                                --categories "communities/$community/metadata/categories" \
-                                --filtered "communities/$community/resources/tools_filtered_by_ts_categories.json" \
-                                --status "communities/$community/metadata/tool_status.tsv"
+                                --categories "communities/$COMMUNITY/metadata/categories" \
+                                --filtered "communities/$COMMUNITY/resources/tools_filtered_by_ts_categories.json" \
+                                --status "communities/$COMMUNITY/metadata/tool_status.tsv"
 
-                        if [[ -e "communities/$community/metadata/tool_status.tsv" ]]; then
+                        if [[ -e "communities/$COMMUNITY/metadata/tool_status.tsv" ]]; then
                                 python sources/bin/extract_galaxy_tools.py \
                                         curate \
-                                        --filtered "communities/$community/resources/tools_filtered_by_ts_categories.json" \
-                                        --status "communities/$community/metadata/tool_status.tsv" \
-                                        --curated "communities/$community/resources/curated_tools.tsv" \
-                                        --wo-biotools "communities/$community/resources/curated_tools_wo_biotools.tsv" \
-                                        --w-biotools "communities/$community/resources/curated_tools_w_biotools.tsv"\
-                                        --yml "communities/$community/resources/curated_tools.yml"
-
-                                if [[ -e "communities/$community/resources/curated_tools.yml" ]]; then
-                                        mkdir -p _data/communities/$community/
-                                        ln -sf ../../../communities/$community/resources/curated_tools.yml _data/communities/$community/curated_tools.yml
+                                        --filtered "communities/$COMMUNITY/resources/tools_filtered_by_ts_categories.json" \
+                                        --status "communities/$COMMUNITY/metadata/tool_status.tsv" \
+                                        --curated "communities/$COMMUNITY/resources/curated_tools.tsv" \
+                                        --wo-biotools "communities/$COMMUNITY/resources/curated_tools_wo_biotools.tsv" \
+                                        --w-biotools "communities/$COMMUNITY/resources/curated_tools_w_biotools.tsv"\
+                                        --yml "communities/$COMMUNITY/resources/curated_tools.yml"
+
+                                if [[ -e "communities/$COMMUNITY/resources/curated_tools.yml" ]]; then
+                                        mkdir -p _data/communities/$COMMUNITY/
+                                        ln -sf ../../../communities/$COMMUNITY/resources/curated_tools.yml _data/communities/$COMMUNITY/curated_tools.yml
                                 fi;
 
-                                if [[ -e "communities/$community/resources/curated_tools.tsv" ]]; then
+                                if [[ -e "communities/$COMMUNITY/resources/curated_tools.tsv" ]]; then
                                         python sources/bin/create_wordcloud.py \
-                                                --input "communities/$community/resources/curated_tools.tsv" \
+                                                --input "communities/$COMMUNITY/resources/curated_tools.tsv" \
                                                 --name-col "Suite ID" \
                                                 --stat-col "Suite runs (last 5 years) on main servers" \
                                                 --wordcloud_mask "sources/data/usage_stats/wordcloud_mask.png" \
-                                                --output "communities/$community/resources/tools_wordcloud.png"
+                                                --output "communities/$COMMUNITY/resources/tools_wordcloud.png"
 
                                         python sources/bin/create_interactive_table.py \
-                                                --input "communities/$community/resources/curated_tools.tsv" \
+                                                --input "communities/$COMMUNITY/resources/curated_tools.tsv" \
                                                 --template "sources/data/interactive_table_template.html" \
-                                                --output "communities/$community/resources/tools.html"
+                                                --output "communities/$COMMUNITY/resources/tools.html"
                                 fi;
                         fi;
-                fi;
-        fi;
-done
+                fi
+#        fi;
+#done