Skip to content

Commit fdf82cb

Browse files
renan-souzarenan-souza
authored
Merge pull request #184 from ORNL/dev
Main < Dev
2 parents de8d04a + 23b6e63 commit fdf82cb

File tree

76 files changed

+2469
-1170
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+2469
-1170
lines changed

‎.github/workflows/create-release-n-publish.yml

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,9 @@ jobs:
2020
with:
2121
python-version: "3.10"
2222

23-
- name: Get branch names
24-
id: branch-name
25-
uses: tj-actions/branch-names@v6
26-
2723
- name: Update version.py
2824
run: |
2925
export PYTHONPATH=$PYTHONPATH:flowcept
30-
export BRANCH_NAME="${{ steps.branch-name.outputs.current_branch }}"
3126
python .github/workflows/version_bumper.py
3227
3328
- name: Commit new version

‎.github/workflows/run-tests-in-container.yml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Tests inside a Container
1+
name: (With and Without Mongo) Tests inside a Container Tests inside a Container
22
on: [pull_request]
33

44
jobs:
@@ -20,9 +20,18 @@ jobs:
2020
run: make build
2121

2222
- name: Start dependent services (Mongo and Redis)
23-
run: make services
23+
run: make services-mongo
2424

2525
- name: Run tests in container
26+
run: make tests-in-container-mongo
27+
28+
- name: Stop services
29+
run: make services-stop-mongo
30+
31+
- name: Start dependent services (Redis, no Mongo)
32+
run: make services
33+
34+
- name: Run tests in container without mongo
2635
run: make tests-in-container
2736

2837
- name: Stop services

‎.github/workflows/run-tests-kafka.yml

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
1-
name: All tests on Kafka MQ
1+
name: (With Mongo) Tests on Kafka MQ
22
on:
33
pull_request:
44
branches: [ "dev", "main" ]
55
types: [opened, synchronize, reopened]
6+
#on:
7+
# push:
68

79
jobs:
810

911
build:
1012
runs-on: ubuntu-latest
13+
env:
14+
MONGO_ENABLED: true
15+
LMDB_ENABLED: false
1116
timeout-minutes: 40
1217
if: "!contains(github.event.head_commit.message, 'CI Bot')"
1318

@@ -22,17 +27,22 @@ jobs:
2227
python-version: "3.10"
2328
cache: "pip"
2429

25-
- name: Install package and dependencies
26-
run: |
27-
python -m pip install --upgrade pip
28-
python -m pip install .[all]
29-
python -m pip install .[ml_dev]
30-
3130
- name: Run docker compose
3231
run: docker compose -f deployment/compose-kafka.yml up -d
3332

34-
- name: Wait for one minute
35-
run: sleep 60
33+
- name: Upgrade pip
34+
run: python -m pip install --upgrade pip
35+
36+
- name: Show Python version
37+
run: python --version && pip --version
38+
39+
- name: Test examples
40+
run: bash .github/workflows/run_examples.sh examples true # with mongo
41+
42+
- name: Install all dependencies
43+
run: |
44+
python -m pip install .[all]
45+
python -m pip install .[ml_dev]
3646
3747
- name: Check liveness
3848
run: |
@@ -45,7 +55,7 @@ jobs:
4555
run: |
4656
export MQ_TYPE=kafka
4757
export MQ_PORT=9092
48-
pytest --ignore=tests/decorator_tests/ml_tests/llm_tests
58+
make tests
4959
5060
- name: Test notebooks
5161
run: pytest --ignore=notebooks/zambeze.ipynb --nbmake "notebooks/" --nbmake-timeout=600 --ignore=notebooks/dask_from_CLI.ipynb

‎.github/workflows/run-tests-py11-all-dbs.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: (With and Without Mongo) Tests on py11
2+
on: [pull_request]
3+
4+
jobs:
5+
6+
build:
7+
runs-on: ubuntu-latest
8+
timeout-minutes: 40
9+
env:
10+
MONGO_ENABLED: true
11+
LMDB_ENABLED: true
12+
13+
if: "!contains(github.event.head_commit.message, 'CI Bot')"
14+
15+
steps:
16+
- uses: actions/checkout@v4
17+
with:
18+
fetch-depth: 1
19+
20+
- name: Set up Python 3.11
21+
uses: actions/setup-python@v5
22+
with:
23+
python-version: "3.11"
24+
cache: "pip"
25+
26+
- name: Show OS Info
27+
run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
28+
29+
- name: Start docker compose with redis
30+
run: make services-mongo
31+
32+
- name: Upgrade pip
33+
run: python -m pip install --upgrade pip
34+
35+
- name: Show Python version
36+
run: python --version && pip --version
37+
38+
- name: Test examples
39+
run: bash .github/workflows/run_examples.sh examples true # with mongo
40+
41+
- name: Install all dependencies
42+
run: |
43+
python -m pip install --upgrade pip
44+
python -m pip install .[all]
45+
python -m pip install .[ml_dev]
46+
47+
- name: List installed packages
48+
run: pip list
49+
50+
- name: Test with pytest and redis
51+
run: |
52+
export MONGO_ENABLED=true
53+
export LMDB_ENABLED=true
54+
python -c "from flowcept.configs import MONGO_ENABLED, LMDB_ENABLED; print('MONGO?', MONGO_ENABLED); print('LMDB?', LMDB_ENABLED)"
55+
make tests
56+
57+
- name: Test notebooks with pytest and redis
58+
run: pytest --nbmake "notebooks/" --nbmake-timeout=600 --ignore="notebooks/dask_from_CLI.ipynb"
59+
60+
- name: Stop services
61+
run: make services-stop-mongo

‎.github/workflows/run-tests-py11-simple.yml

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: (Without Mongo) Tests on py11
2+
on: [pull_request]
3+
4+
jobs:
5+
6+
build:
7+
runs-on: ubuntu-latest
8+
timeout-minutes: 40
9+
env:
10+
MONGO_ENABLED: false
11+
if: "!contains(github.event.head_commit.message, 'CI Bot')"
12+
13+
steps:
14+
- uses: actions/checkout@v4
15+
with:
16+
fetch-depth: 1
17+
18+
- name: Set up Python 3.11
19+
uses: actions/setup-python@v5
20+
with:
21+
python-version: "3.11"
22+
cache: "pip"
23+
24+
- name: Show OS Info
25+
run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
26+
27+
- name: Start docker compose with redis
28+
run: make services
29+
30+
- name: Upgrade pip
31+
run: python -m pip install --upgrade pip
32+
33+
- name: Show Python version
34+
run: python --version && pip --version
35+
36+
- name: Test examples
37+
run: bash .github/workflows/run_examples.sh examples false # without mongo
38+
39+
- name: Install all dependencies
40+
run: |
41+
python -m pip install --upgrade pip
42+
python -m pip install .[all]
43+
python -m pip install .[ml_dev]
44+
45+
- name: List installed packages
46+
run: pip list
47+
48+
- name: Test with pytest and redis
49+
run: |
50+
make tests
51+
52+
- name: Test notebooks with pytest and redis
53+
run: pytest --nbmake "notebooks/" --nbmake-timeout=600 --ignore="notebooks/dask_from_CLI.ipynb" --ignore="notebooks/analytics.ipynb"
54+
55+
- name: Stop services
56+
run: make services-stop

‎.github/workflows/run-tests-py11.yml

Lines changed: 8 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
1-
name: Tests on py11
1+
name: (With Mongo) Tests on py11
22
on: [pull_request]
33

44
jobs:
55

66
build:
77
runs-on: ubuntu-latest
88
timeout-minutes: 40
9+
env:
10+
MONGO_ENABLED: true
11+
LMDB_ENABLED: false
912
if: "!contains(github.event.head_commit.message, 'CI Bot')"
1013

1114
steps:
@@ -23,40 +26,19 @@ jobs:
2326
run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
2427

2528
- name: Start docker compose with redis
26-
run: make services
29+
run: make services-mongo
2730

2831
- name: Upgrade pip
2932
run: python -m pip install --upgrade pip
3033

3134
- name: Show Python version
3235
run: python --version && pip --version
3336

34-
- name: Install default dependencies and run simple test
35-
run: |
36-
pip install .
37-
python examples/simple_instrumented_script.py
38-
39-
- name: Install Dask dependencies alone and run a simple Dask test
40-
run: |
41-
pip uninstall flowcept -y
42-
pip install .[dask]
43-
python examples/dask_example.py
44-
45-
- name: Install MLFlow dependencies alone and run a simple MLFlow test
46-
run: |
47-
pip uninstall flowcept -y
48-
pip install .[mlflow]
49-
python examples/mlflow_example.py
50-
51-
- name: Install Tensorboard dependencies alone and run a simple Tensorboard test
52-
run: |
53-
pip uninstall flowcept -y
54-
pip install .[tensorboard]
55-
python examples/tensorboard_example.py
37+
- name: Test examples
38+
run: bash .github/workflows/run_examples.sh examples true # with mongo
5639

5740
- name: Install all dependencies
5841
run: |
59-
python -m pip install --upgrade pip
6042
python -m pip install .[all]
6143
python -m pip install .[ml_dev]
6244
@@ -71,7 +53,7 @@ jobs:
7153
run: make tests-notebooks
7254

7355
- name: Stop services
74-
run: make services-stop
56+
run: make services-stop-mongo
7557

7658
- name: Clean up
7759
run: |

‎.github/workflows/run-tests-simple.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: (Without Mongo) Unit, integration, and notebook tests
2+
on:
3+
push:
4+
schedule:
5+
- cron: '0 12 * * *' # Runs every day at 12 PM UTC (7 AM EST)
6+
7+
jobs:
8+
9+
build:
10+
runs-on: ubuntu-latest
11+
env:
12+
MONGO_ENABLED: false
13+
LMDB_ENABLED: true
14+
timeout-minutes: 60
15+
if: "!contains(github.event.head_commit.message, 'CI Bot')"
16+
17+
steps:
18+
- uses: actions/checkout@v4
19+
with:
20+
fetch-depth: 1
21+
22+
- name: Set up Python 3.10
23+
uses: actions/setup-python@v5
24+
with:
25+
python-version: "3.10"
26+
cache: "pip"
27+
28+
- name: Show OS Info
29+
run: '[[ "$OSTYPE" == "linux-gnu"* ]] && { echo "OS Type: Linux"; (command -v lsb_release &> /dev/null && lsb_release -a) || cat /etc/os-release; uname -r; } || [[ "$OSTYPE" == "darwin"* ]] && { echo "OS Type: macOS"; sw_vers; uname -r; } || echo "Unsupported OS type: $OSTYPE"'
30+
31+
- name: Start docker compose with redis
32+
run: make services
33+
34+
- name: Upgrade pip
35+
run: python -m pip install --upgrade pip
36+
37+
- name: Test examples
38+
run: bash .github/workflows/run_examples.sh examples false # with mongo
39+
40+
- name: Install all dependencies
41+
run: |
42+
python -m pip install .[all]
43+
python -m pip install .[ml_dev]
44+
45+
- name: List installed packages
46+
run: pip list
47+
48+
- name: Test with pytest and redis
49+
run: |
50+
make tests
51+
52+
- name: Test notebooks with pytest and redis
53+
run: pytest --nbmake "notebooks/" --nbmake-timeout=600 --ignore="notebooks/dask_from_CLI.ipynb" --ignore="notebooks/analytics.ipynb"
54+
55+
- name: Shut down docker compose
56+
run: make services-stop
57+
58+
- name: Clean up
59+
run: |
60+
make clean
61+
find /home/runner/runners/ -type f -name "*.log" -exec sh -c 'echo {}; >"{}"' \;

0 commit comments

Comments
 (0)