Skip to content

Commit 24e38c1

Browse files
scorreardscorreard
authored
Merge pull request #652 from galaxyproject/scorrear_PR_too_big
Splitting the weekly update by community
2 parents eac75ef + b4d4ed7 commit 24e38c1

8 files changed

Lines changed: 195 additions & 177 deletions

‎.github/CODEOWNERS‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# These owners will be the default owners for everything in the repo. Unless a later match takes precedence, @global-owner1 and @global-owner2 will be requested for review when someone opens a pull request.
2+
* @scorreard @bebatut @paulzierep
3+
4+
# Any change inside each community directory will require approval from the assigned person
5+
/communities/biodiversity/ @scorreard
6+
/communities/earth/ @Marie59
7+
/communities/imaging/ @nagoue
8+
/communities/micrgalaxy/ @bebatut
9+
10+
11+
#Need to identify assigned person :
12+
#/communities/assembly/
13+
#/communities/genome/
14+
#/communities/genome/
15+
#/communities/machine-learning/
16+
#/communities/metabolics/
17+
#/communities/spoc/

‎.github/workflows/fetch_filter_resources.yaml‎

Lines changed: 10 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Weekly resource fetching and community filtering
1+
name: Weekly resource fetching
22

33
on:
44
workflow_dispatch:
@@ -137,14 +137,14 @@ jobs:
137137
# with:
138138
# name: workflows
139139
# path: communities/all/resources/citations.json
140-
merge-fetch-filter:
140+
merge-fetch:
141141
runs-on: ubuntu-latest
142142
needs:
143143
- fetch-tools-stepwise
144144
- fetch-tutorials
145145
- fetch-workflows
146146
# - fetch-citations
147-
name: Merge tools, filter the resources for communities, and populate labs
147+
name: Merge tools
148148
steps:
149149
- name: Checkout main
150150
uses: actions/checkout@v6
@@ -185,55 +185,11 @@ jobs:
185185
- name: Generate tools wordcloud and interactive table
186186
run: |
187187
bash sources/bin/format_tools.sh
188-
- name: Filter workflows for communities
189-
run: |
190-
bash sources/bin/get_community_workflows.sh
191-
- name: Filter tutorials for communities
192-
run: |
193-
bash sources/bin/get_community_tutorials.sh
194-
- name: Filter citations for communities
195-
run: |
196-
bash sources/bin/get_community_citations.sh no-scholarly
197-
#- name: Update tool to keep and exclude for communities
198-
# run: |
199-
# bash sources/bin/update_tools_to_keep_exclude.sh
200-
- name: Filter tools for communities
201-
run: |
202-
bash sources/bin/get_community_tools.sh
203-
- name: Populate Microbiology (microGalaxy) Lab
204-
run: |
205-
bash sources/bin/populate_labs.sh
206-
env:
207-
COMMUNITY: microgalaxy
208-
- name: Populate biodiversity lab
209-
run: |
210-
bash sources/bin/populate_labs.sh
211-
env:
212-
COMMUNITY: biodiversity
213-
- name: Populate earth-system Lab
214-
run: |
215-
bash sources/bin/populate_labs.sh
216-
env:
217-
COMMUNITY: earth
218-
- name: Populate Imaging lab
219-
run: |
220-
bash sources/bin/populate_labs.sh
221-
env:
222-
COMMUNITY: imaging
223-
#To initiate a lab, we recommend adding it below. It will automatically create the folders and structures required.
224-
# - name: Populate <community-name> Lab
225-
# run: |
226-
# bash sources/bin/populate_labs.sh
227-
# env:
228-
# COMMUNITY: <community-name>
229-
- name: Create Pull Request
230-
uses: peter-evans/create-pull-request@v8
188+
- name: Commit and push changes
189+
uses: stefanzweifel/git-auto-commit-action@v5
231190
with:
232-
commit-message: Update resources
233-
title: Automatic resources update
234-
body: Automatic resource update done via GitHub Action once a week
235-
base: main
236-
branch: resource-update
237-
delete-branch: true
238-
add-paths: |
239-
communities/
191+
commit_message: "Update resources"
192+
branch: main
193+
commit_user_name: GitHub Actions
194+
commit_user_email: actions@github.com
195+
commit_author: GitHub Actions <actions@github.com>

‎.github/workflows/filter_communities.yaml‎

Lines changed: 77 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,91 @@
1-
name: Filter community resources
1+
name: Community weekly resource filtering
22

33
on:
44
workflow_dispatch:
5-
# the workflow it also triggered when the community definitions are changed
65
push:
6+
branches:
7+
- 'main'
8+
- 'scorrear_PR_too_big'
79
paths:
8-
- 'data/communities/*/metadata/*'
9-
branches: ["main"]
10+
- 'communities/all/resources/tools.tsv'
1011

1112
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
1213
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
1314
concurrency:
14-
group: "filter"
15+
group: "tools"
1516
cancel-in-progress: false
1617

1718
jobs:
18-
filter-resources:
19-
name: Filter resources for communities
19+
set-matrix:
2020
runs-on: ubuntu-latest
21+
outputs:
22+
communities: ${{ steps.prepare.outputs.communities }}
2123
steps:
22-
- name: Checkout main
23-
uses: actions/checkout@v6
24-
- uses: actions/setup-python@v6
25-
with:
26-
python-version: '3.11'
27-
- name: Install requirement
28-
run: python -m pip install -r requirements.txt
29-
- name: Filter tutorials for communities
30-
run: |
31-
bash bin/get_community_tutorials.sh
32-
- name: Update tool to keep and exclude for communities
33-
run: |
34-
bash bin/update_tools_to_keep_exclude.sh
35-
- name: Filter tools for communities
36-
run: |
37-
bash bin/get_community_tools.sh
38-
- name: Create Pull Request
39-
uses: peter-evans/create-pull-request@v8
40-
with:
41-
commit-message: Update resources
42-
title: Resource filtering
43-
body: Resource filering done via GitHub Action when the community definitions are changed
44-
base: main
45-
branch: resource-filtering
46-
delete-branch: true
24+
- name: Checkout main
25+
uses: actions/checkout@v6
26+
- name: Prepare communities JSON
27+
id: prepare
28+
run: |
29+
# produce a JSON array string, e.g. from a file or script
30+
echo '["assembly","biodiversity","earth","genome","imaging","machine-learning","metabolomics","microgalaxy","proteomics","spoc"]' > communities.json
31+
communities=$(cat communities.json)
32+
echo "communities=$communities" >> $GITHUB_OUTPUT
33+
34+
community-filter:
35+
needs: set-matrix
36+
runs-on: ubuntu-latest
37+
name: Filter resources and create labs
38+
strategy:
39+
matrix:
40+
community: ${{ fromJson(needs.set-matrix.outputs.communities) }}
41+
steps:
42+
- name: Checkout main
43+
uses: actions/checkout@v6
44+
- uses: actions/setup-python@v6
45+
with:
46+
python-version: '3.11'
47+
- name: Install requirement
48+
run: |
49+
python -m pip install -r requirements.txt
50+
sudo apt-get install jq
51+
- name: Filter tools for communities
52+
run: |
53+
bash sources/bin/get_community_tools.sh
54+
env:
55+
COMMUNITY: ${{ matrix.community }}
56+
- name: Filter workflows for communities
57+
run: |
58+
bash sources/bin/get_community_workflows.sh
59+
env:
60+
COMMUNITY: ${{ matrix.community }}
61+
- name: Filter tutorials for communities
62+
run: |
63+
bash sources/bin/get_community_tutorials.sh
64+
env:
65+
COMMUNITY: ${{ matrix.community }}
66+
- name: Filter citations for communities
67+
run: |
68+
bash sources/bin/get_community_citations.sh no-scholarly
69+
env:
70+
COMMUNITY: ${{ matrix.community }}
71+
#- name: Update tool to keep and exclude for communities
72+
# run: |
73+
# bash sources/bin/update_tools_to_keep_exclude.sh
74+
# env:
75+
# COMMUNITY: ${{ matrix.community }}
76+
- name: Populate lab for communities
77+
run: |
78+
bash sources/bin/populate_labs.sh
79+
env:
80+
COMMUNITY: ${{ matrix.community }}
81+
- name: Create Pull Request
82+
uses: peter-evans/create-pull-request@v8
83+
with:
84+
commit-message: ${{ matrix.community }} community resources update
85+
title: ${{ matrix.community }} community automatic resources update
86+
body: ${{ matrix.community }} community automatic resource update done via GitHub Action once a week
87+
base: main
88+
branch: ${{ matrix.community }}-resource-update
89+
delete-branch: true
90+
add-paths: |
91+
communities/

‎sources/bin/get_community_citations.sh‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
11
#!/usr/bin/env bash
22

3-
for com_data_fp in communities/* ; do
4-
if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
5-
community=`basename "$com_data_fp"`
6-
if [[ ! -z $1 && $1 == "test" && "$community" != "microgalaxy" ]]; then
3+
#for com_data_fp in communities/* ; do
4+
# if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
5+
# community=`basename "$com_data_fp"`
6+
if [[ ! -z $1 && $1 == "test" && "$COMMUNITY" != "microgalaxy" ]]; then
77
continue
88
fi;
99

10-
if [[ "$community" != "all" && -f "communities/$community/metadata/citation_keywords" ]]; then
11-
echo "$community";
12-
mkdir -p "communities/$community/resources"
10+
if [[ "$COMMUNITY" != "all" && -f "communities/$COMMUNITY/metadata/citation_keywords" ]]; then
11+
echo "$COMMUNITY";
12+
mkdir -p "communities/$COMMUNITY/resources"
1313

1414
python sources/bin/citation_manager.py \
1515
filter \
1616
--all-json "communities/all/resources/citations.json" \
17-
--filtered-json "communities/$community/resources/citations.json" \
18-
--filtered-yaml "communities/$community/resources/citations.yml" \
19-
--filtered-tsv "communities/$community/resources/citations.tsv" \
20-
--keywords "communities/$community/metadata/citation_keywords"
17+
--filtered-json "communities/$COMMUNITY/resources/citations.json" \
18+
--filtered-yaml "communities/$COMMUNITY/resources/citations.yml" \
19+
--filtered-tsv "communities/$COMMUNITY/resources/citations.tsv" \
20+
--keywords "communities/$COMMUNITY/metadata/citation_keywords"
2121

22-
if [[ -f "communities/$community/metadata/citations.yml" ]]; then
23-
mkdir -p _data/communities/$community/
24-
ln -sf ../../../communities/$community/resources/citations.yml _data/communities/$community/citations.yml
22+
if [[ -f "communities/$COMMUNITY/metadata/citations.yml" ]]; then
23+
mkdir -p _data/communities/$COMMUNITY/
24+
ln -sf ../../../communities/$COMMUNITY/resources/citations.yml _data/communities/$COMMUNITY/citations.yml
2525
fi;
26-
fi;
27-
fi;
28-
done
26+
fi
27+
# fi;
28+
#done

‎sources/bin/get_community_tools.sh‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,56 +3,56 @@
33
# stop on error
44
set -e
55

6-
for com_data_fp in communities/* ; do
7-
if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
8-
community=`basename "$com_data_fp"`
6+
#for com_data_fp in communities/* ; do
7+
# if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
8+
# community=`basename "$com_data_fp"`
99

10-
if [[ ! -z $1 && $1 == "test" && "$community" != "microgalaxy" ]]; then
10+
if [[ ! -z $1 && $1 == "test" && "$COMMUNITY" != "microgalaxy" ]]; then
1111
continue
1212
fi;
1313

14-
if [[ "$community" != "all" && -f "communities/$community/metadata/categories" ]]; then
15-
echo "$community";
16-
mkdir -p "communities/$community/resources"
14+
if [[ "$COMMUNITY" != "all" && -f "communities/$COMMUNITY/metadata/categories" ]]; then
15+
echo "$COMMUNITY";
16+
mkdir -p "communities/$COMMUNITY/resources"
1717

1818
python sources/bin/extract_galaxy_tools.py \
1919
filter \
2020
--all "communities/all/resources/tools.json" \
21-
--categories "communities/$community/metadata/categories" \
22-
--filtered "communities/$community/resources/tools_filtered_by_ts_categories.json" \
23-
--status "communities/$community/metadata/tool_status.tsv"
21+
--categories "communities/$COMMUNITY/metadata/categories" \
22+
--filtered "communities/$COMMUNITY/resources/tools_filtered_by_ts_categories.json" \
23+
--status "communities/$COMMUNITY/metadata/tool_status.tsv"
2424

25-
if [[ -e "communities/$community/metadata/tool_status.tsv" ]]; then
25+
if [[ -e "communities/$COMMUNITY/metadata/tool_status.tsv" ]]; then
2626
python sources/bin/extract_galaxy_tools.py \
2727
curate \
28-
--filtered "communities/$community/resources/tools_filtered_by_ts_categories.json" \
29-
--status "communities/$community/metadata/tool_status.tsv" \
30-
--curated "communities/$community/resources/curated_tools.tsv" \
31-
--wo-biotools "communities/$community/resources/curated_tools_wo_biotools.tsv" \
32-
--w-biotools "communities/$community/resources/curated_tools_w_biotools.tsv"\
33-
--yml "communities/$community/resources/curated_tools.yml"
34-
35-
if [[ -e "communities/$community/resources/curated_tools.yml" ]]; then
36-
mkdir -p _data/communities/$community/
37-
ln -sf ../../../communities/$community/resources/curated_tools.yml _data/communities/$community/curated_tools.yml
28+
--filtered "communities/$COMMUNITY/resources/tools_filtered_by_ts_categories.json" \
29+
--status "communities/$COMMUNITY/metadata/tool_status.tsv" \
30+
--curated "communities/$COMMUNITY/resources/curated_tools.tsv" \
31+
--wo-biotools "communities/$COMMUNITY/resources/curated_tools_wo_biotools.tsv" \
32+
--w-biotools "communities/$COMMUNITY/resources/curated_tools_w_biotools.tsv"\
33+
--yml "communities/$COMMUNITY/resources/curated_tools.yml"
34+
35+
if [[ -e "communities/$COMMUNITY/resources/curated_tools.yml" ]]; then
36+
mkdir -p _data/communities/$COMMUNITY/
37+
ln -sf ../../../communities/$COMMUNITY/resources/curated_tools.yml _data/communities/$COMMUNITY/curated_tools.yml
3838
fi;
3939

40-
if [[ -e "communities/$community/resources/curated_tools.tsv" ]]; then
40+
if [[ -e "communities/$COMMUNITY/resources/curated_tools.tsv" ]]; then
4141
python sources/bin/create_wordcloud.py \
42-
--input "communities/$community/resources/curated_tools.tsv" \
42+
--input "communities/$COMMUNITY/resources/curated_tools.tsv" \
4343
--name-col "Suite ID" \
4444
--stat-col "Suite runs (last 5 years) on main servers" \
4545
--wordcloud_mask "sources/data/usage_stats/wordcloud_mask.png" \
46-
--output "communities/$community/resources/tools_wordcloud.png"
46+
--output "communities/$COMMUNITY/resources/tools_wordcloud.png"
4747

4848
python sources/bin/create_interactive_table.py \
49-
--input "communities/$community/resources/curated_tools.tsv" \
49+
--input "communities/$COMMUNITY/resources/curated_tools.tsv" \
5050
--template "sources/data/interactive_table_template.html" \
51-
--output "communities/$community/resources/tools.html"
51+
--output "communities/$COMMUNITY/resources/tools.html"
5252
fi;
5353
fi;
54-
fi;
55-
fi;
56-
done
54+
fi
55+
# fi;
56+
#done
5757

5858

0 commit comments

Comments
 (0)