Merge pull request #4 from DARPA-ASKEM/ml4ai_merge_nov_2023

Ml4ai merge nov 2023

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+## Summary of Changes
+
+
+### Related issues
+
+Resolves ???

.github/labeler.yml

@@ -1,3 +1,4 @@
+# see https://github.com/actions/labeler
 documentation:
 - any: ["docs/**"]
 # all: ["!docs/changes.md"]
@@ -6,17 +7,30 @@ integration:
 - any: ["skema/rest/**"]
 
 CI:
-- any: [".github/**"]
+- any: [".github/**", ".github/workflows/**"]
+
+docker:
+- any:
+ - "Dockerfile.skema-py"
+ - "Dockerfile.skema-py.dockerignore"
+ - "Dockerfile.skema-rs"
+ - "Dockerfile.skema-rs.dockerignore"
 
 tests:
 - any:
- - "skema/rest/tests/**"
+ - "skema/gromet/execution_engine/tests/**"
+ - "skema/img2mml/tests/**"
  - "skema/program_analysis/tests/**"
  - "skema/program_analysis/comment_extractor/tests/**"
- - "skema/img2mml/tests/**"
- - "skema/text_reading/scala/src/test/**"
+ - "skema/program_analysis/tree_sitter_parsers/tests/**"
+ - "skema/program_analysis/CAST/matlab/tests/**"
+ - "skema/program_analysis/CAST/fortran/tests/**"
+ - "skema/program_analysis/CAST/python/tests/**"
+ - "skema/rest/tests/**"
+ - "skema/skema_py/tests/**"
  - "skema/skema-rs/mathml/tests/**"
  - "skema/skema-rs/skema/tests/**"
+ - "skema/text_reading/scala/src/test/**"
 
 Code2FN:
 - any: ["skema/gromet/**", "skema/program_analysis/**"]
@@ -27,8 +41,16 @@ Equations:
 lang/Fortran:
 - any: ["skema/program_analysis/CAST/fortran/**"]
 
+lang/MATLAB:
+- any: ["skema/program_analysis/CAST/matlab/**"]
+
 lang/Python:
-- any: ["skema/program_analysis/CAST/pythonAST/**"]
+- any: 
+ - "skema/program_analysis/CAST/pythonAST/**"
+ - "skema/program_analysis/CAST/python/**"
+
+lang/Rust:
+- any: ["skema/skema-rs/**"]
 
 MORAE:
 - any: ["skema/skema-rs/**"]

.github/workflows/docker.yml.disabled

@@ -0,0 +1,223 @@
+# name: SKEMA docker
+
+# on:
+# push:
+# branches: [main]
+# tags:
+# - "**"
+# pull_request:
+# branches: [main]
+# release:
+# types: [published]
+
+# # builds and publishes docker images for the default branch.
+# # images are tagged with short commit hash, latest, and any tags.
+
+# jobs:
+# # Determine value for app version
+# app_version: 
+# name: "Determine app version"
+# runs-on: ubuntu-latest
+# outputs:
+# app_version: ${{ steps.app_version.outputs.app_version }}
+# commit: ${{ steps.app_version.outputs.commit }}
+# steps:
+# - name: Set APP_VERSION output
+# id: app_version
+# run: |
+# echo "app_version=${{github.ref_name}}"
+# if [ -z "${app_version}" ]; then
+# app_version=${{github.sha}}
+# echo "app_version=${app_version}"
+# fi
+# echo "app_version=$app_version" >> "$GITHUB_OUTPUT"
+# echo "commit=${{github.sha}}" >> "$GITHUB_OUTPUT"
+# echo "app_version=$app_version"
+# echo "commit=${{github.sha}}"
+# python:
+# name: "docker image for python components"
+# needs: [app_version]
+# runs-on: ubuntu-latest
+# steps:
+# # Setup docker
+# - name: Set up QEMU
+# uses: docker/setup-qemu-action@v2
+# # for multi-arch builds (ex. ARM 64)
+# - name: Set up Docker Buildx
+# id: buildx
+# uses: docker/setup-buildx-action@v2
+# # - name: Prepare buildx builder
+# # run: |
+# # docker buildx create --use --name "multiarch-builder" --platform linux/amd64,linux/arm64 --driver "docker-container"
+# - name: Inspect builder
+# run: |
+# echo "Name: ${{ steps.buildx.outputs.name }}"
+# echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
+# echo "Status: ${{ steps.buildx.outputs.status }}"
+# echo "Flags: ${{ steps.buildx.outputs.flags }}"
+# echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+# # Checkout code
+# - name: Checkout code
+# uses: actions/checkout@v3
+# - name: Login to DockerHub
+# uses: docker/login-action@v2 
+# with:
+# username: ${{ secrets.LUM_ASKEM_DOCKERHUB_USERNAME }}
+# password: ${{ secrets.LUM_ASKEM_DOCKERHUB_TOKEN }}
+# - name: "Free disk space"
+# run: |
+# sudo apt-get update
+# sudo apt clean
+# sudo rm -rf /usr/share/dotnet
+# sudo rm -rf /opt/ghc
+# sudo rm -rf "/usr/local/share/boost"
+# sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+# ########################################
+# # lumai/askem-skema-py (img2mml, isa, etc)
+# ########################################
+# - name: Tags for image
+# id: tags
+# # see https://github.com/docker/metadata-action
+# uses: docker/metadata-action@v4
+
+# with:
+# images: |
+# lumai/askem-skema-py
+# lumai/askem-skema-img2mml
+# tags: |
+# # latest
+# type=raw,value=latest
+# # version
+# type=semver,pattern={{version}}
+# # other tags
+# type=ref,event=tag
+# # short commit hash
+# type=sha
+
+# - name: Build and push image
+# # see https://github.com/docker/build-push-action
+# uses: docker/build-push-action@v4
+# with:
+# context: .
+# file: "Dockerfile.skema-py"
+# platforms: linux/amd64
+# #platforms: linux/amd64,linux/arm64
+# push: ${{ github.event_name != 'pull_request' }}
+# # references `tags` step in steps for current job
+# tags: ${{ steps.tags.outputs.tags }}
+# build-args: |
+# #APP_VERSION=${{needs.app_version.outputs.app_version}}
+# APP_VERSION=${{needs.app_version.outputs.commit}}
+
+# rust:
+# name: "docker image for rust components"
+# needs: [app_version]
+# runs-on: ubuntu-latest
+# steps:
+# # Setup docker
+# - name: Set up QEMU
+# uses: docker/setup-qemu-action@v2
+# # for multi-arch builds (ex. ARM 64)
+# - name: Set up Docker Buildx
+# id: buildx
+# uses: docker/setup-buildx-action@v2
+# # - name: Prepare buildx builder
+# # run: |
+# # docker buildx create --use --name "multiarch-builder" --platform linux/amd64,linux/arm64 --driver "docker-container"
+# - name: Inspect builder
+# run: |
+# echo "Name: ${{ steps.buildx.outputs.name }}"
+# echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
+# echo "Status: ${{ steps.buildx.outputs.status }}"
+# echo "Flags: ${{ steps.buildx.outputs.flags }}"
+# echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+# # Checkout code
+# - name: Checkout code
+# uses: actions/checkout@v3
+# - name: Login to DockerHub
+# uses: docker/login-action@v2 
+# with:
+# username: ${{ secrets.LUM_ASKEM_DOCKERHUB_USERNAME }}
+# password: ${{ secrets.LUM_ASKEM_DOCKERHUB_TOKEN }}
+# - name: "Free disk space"
+# run: |
+# sudo apt clean
+# sudo rm -rf /usr/share/dotnet
+# sudo rm -rf /opt/ghc
+# sudo rm -rf "/usr/local/share/boost"
+# sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+# ########################################
+# # lumai/askem-skema-rs
+# ########################################
+# - name: Tags for image
+# id: tags
+# # see https://github.com/docker/metadata-action
+# uses: docker/metadata-action@v4
+# with:
+# images: lumai/askem-skema-rs
+# tags: |
+# # latest
+# type=raw,value=latest
+# # version
+# type=semver,pattern={{version}}
+# # other tags
+# type=ref,event=tag
+# # short commit hash
+# type=sha
+
+# - name: Build and push image
+# # see https://github.com/docker/build-push-action
+# uses: docker/build-push-action@v4
+# with:
+# context: .
+# file: "Dockerfile.skema-rs"
+# platforms: linux/amd64
+# #platforms: linux/amd64,linux/arm64
+# push: ${{ github.event_name != 'pull_request' }}
+# # references `tags` step in steps for current job
+# tags: ${{ steps.tags.outputs.tags }}
+# build-args: |
+# #APP_VERSION=${{needs.app_version.outputs.app_version}}
+# APP_VERSION=${{needs.app_version.outputs.commit}}
+
+# text_reading:
+# name: "docker image for text reading component"
+# needs: [app_version]
+# runs-on: ubuntu-latest
+# steps:
+# # Setup docker
+# - name: Set up QEMU
+# uses: docker/setup-qemu-action@v2
+# # for multi-arch builds (ex. ARM 64)
+# - name: Set up Docker Buildx
+# id: buildx
+# uses: docker/setup-buildx-action@v2
+# with:
+# buildkitd-flags: --debug
+# # - name: Prepare buildx builder
+# # run: |
+# # docker buildx create --use --name "multiarch-builder" --platform linux/amd64,linux/arm64 --driver "docker-container"
+# - name: Inspect builder
+# run: |
+# echo "Name: ${{ steps.buildx.outputs.name }}"
+# echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}"
+# echo "Status: ${{ steps.buildx.outputs.status }}"
+# echo "Flags: ${{ steps.buildx.outputs.flags }}"
+# echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+# # Checkout code
+# - name: Checkout code
+# uses: actions/checkout@v3
+# - name: Login to DockerHub
+# uses: docker/login-action@v2 
+# with:
+# username: ${{ secrets.LUM_ASKEM_DOCKERHUB_USERNAME }}
+# password: ${{ secrets.LUM_ASKEM_DOCKERHUB_TOKEN }}
+# # - name: "Free disk space"
+# # run: |
+# # sudo apt clean
+# # sudo rm -rf /usr/share/dotnet
+# # sudo rm -rf /opt/ghc
+# # sudo rm -rf "/usr/local/share/boost"
+# # sudo rm -rf "$AGENT_TOOLSDIRECTORY"

.github/workflows/tests-and-docs.yml

@@ -6,9 +6,22 @@ on:
  pull_request:
  branches: [main]
 
+env:
+ # used by both Python and Rust services
+ # to connect to Memgraph.
+ # used by skema-rs services
+ SKEMA_GRAPH_DB_HOST: "127.0.0.1"
+ SKEMA_GRAPH_DB_PORT: "7687"
+ SKEMA_RS_HOST: "127.0.0.1"
+ SKEMA_RS_PORT: "8001"
+ # used by Python services.
+ # unfortunately, there doesn't seem to be a way
+ # to reference SKEMA_RS_PORT in SKEMA_RS_ADDESS ...
+ SKEMA_RS_ADDESS: "http://127.0.0.1:8001"
+
 jobs:
  docs:
- name: "Run tests and generate project documenation"
+ name: "Run tests and generate project documentation"
  runs-on: ${{ matrix.os }}
  strategy:
  matrix:
@@ -20,6 +33,18 @@ jobs:
  # python-version: '3.8'
  # - os: windows-latest
  # python-version: '3.6'
+ services:
+ graphdb:
+ image: "memgraph/memgraph-platform"
+ ports:
+ - "7687:7687"
+ - "3000:3000"
+ - "7444:7444"
+ volumes:
+ - mg_lib:/var/lib/memgraph
+ - mg_log:/var/log/memgraph
+ - mg_etc:/etc/memgraph
+
  steps:
  # Checkout code
  - name: Checkout code
@@ -100,16 +125,43 @@ jobs:
  # sbt clean coverage test
  # sbt coverageReport
  # sbt coverageAggregate
- # code coverage (Python)
- - name: "Code coverage reports for Python components"
+ - name: "Launch skema-rs in background for testing"
+ working-directory: ./skema/skema-rs
+ run: |
+ echo "skema-rs REST service: $SKEMA_RS_HOST:$SKEMA_RS_PORT"
+ cargo build --release --bin skema_service
+ ./target/release/skema_service &
+ echo "results for ${SKEMA_RS_HOST}:${SKEMA_RS_PORT}/version: `curl -s ${SKEMA_RS_HOST}:${SKEMA_RS_PORT}/version`"
+ # test & code coverage (Python)
+ - name: "Unit tests and code coverage reports for Python components"
+ # env:
+ # SKEMA_GRAPH_DB_HOST: "bolt://127.0.0.1"
  run: |
- pytest
+ echo "SKEMA_RS_ADDESS: $SKEMA_RS_ADDESS"
+ python -c "import requests; print(requests.get('$SKEMA_RS_ADDESS/version').text)"
+ SKEMA_RS_ADDRESS=$SKEMA_RS_ADDESS pytest -vvv
+
+ - name: "Code2fn model coverage reports"
+ run: python skema/program_analysis/model_coverage_report/model_coverage_report.py $GITHUB_WORKSPACE/docs/coverage/code2fn_coverage/ all
 
  - name: "Unit tests for Rust components"
  working-directory: ./skema/skema-rs
  run: |
  cargo test --verbose --all
-
+ 
+ - name: Free disk space (aggressively)
+ # https://github.com/jlumbroso/free-disk-space
+ uses: jlumbroso/[email protected]
+ with:
+ android: true
+ dotnet: true
+ haskell: true
+ large-packages: true
+ docker-images: true
+ swap-storage: true
+ # NOTE: this might remove things we actually need.
+ tool-cache: true
+
  # docs (other)
  - name: "Create documentation (other)"
  run: |