CI pipeline #948
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI pipeline | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "*" | |
| schedule: | |
| - cron: "0 0 * * *" | |
| env: | |
| telemetry_local_debug_mode: true | |
| jobs: | |
| pre-commit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| with: | |
| python-version: '3.11.x' | |
| - uses: pre-commit/[email protected] | |
| with: | |
| extra_args: --all-files | |
| run-tests: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| data-source: | |
| - "postgres" | |
| - "snowflake" | |
| - "bigquery" | |
| - "redshift" | |
| - "athena" | |
| # - "db2" | |
| # - "spark" | |
| - "spark_df" | |
| - "sqlserver" | |
| - "mysql" | |
| - "duckdb" | |
| - "dask" | |
| # - "vertica" | |
| env: | |
| DATA_SOURCE: ${{ matrix.data-source }} | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_CI_ACCOUNT }} | |
| SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_CI_USERNAME }} | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} | |
| SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_CI_DATABASE }} | |
| SNOWFLAKE_SCHEMA: "public" | |
| BIGQUERY_ACCOUNT_INFO_JSON: ${{ secrets.BIGQUERY_ACCOUNT_INFO_JSON }} | |
| BIGQUERY_DATASET: "test" | |
| REDSHIFT_HOST: ${{ secrets.REDSHIFT_HOST }} | |
| REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }} | |
| REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} | |
| REDSHIFT_DATABASE: "soda_test" | |
| REDSHIFT_PORT: "5439" | |
| ATHENA_ACCESS_KEY_ID: ${{ secrets.ATHENA_ACCESS_KEY_ID }} | |
| ATHENA_SECRET_ACCESS_KEY: ${{ secrets.ATHENA_SECRET_ACCESS_KEY }} | |
| ATHENA_S3_TEST_DIR: ${{ secrets.ATHENA_STAGING_DIR }} | |
| ATHENA_SCHEMA: ${{ secrets.ATHENA_SCHEMA }} | |
| SPARK_HOST: ${{ secrets.SPARK_HOST }} | |
| SPARK_USERNAME: ${{ secrets.SPARK_USERNAME }} | |
| SPARK_PASSWORD: ${{ secrets.SPARK_PASSWORD }} | |
| SPARK_DATABASE: ${{ secrets.SPARK_DATABASE }} | |
| SQLSERVER_USERNAME: SA | |
| SQLSERVER_PASSWORD: Password1! | |
| SQLSERVER_DATABASE: master | |
| SQLSERVER_SCHEMA: dbo | |
| MYSQL_USER: sodacore | |
| MYSQL_DATABASE: sodacore | |
| MYSQL_PASSWORD: sodacore | |
| MYSQL_ROOT_PASSWORD: sodacore | |
| # TODO: add appropriate vars | |
| SPARK_DF_HOST: ${{ secrets.SPARK_DF_HOST }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - | |
| curl https://packages.microsoft.com/config/ubuntu/21.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null | |
| sudo apt-get update | |
| ACCEPT_EULA=Y sudo apt-get install -y libsasl2-dev msodbcsql18 | |
| python -m pip install --upgrade pip | |
| cat dev-requirements.in | grep tox | xargs pip install | |
| - name: Test with tox | |
| run: | | |
| tox -- soda -k soda/core | |
| tox -- soda -k soda/${{ matrix.data-source }} | |
| env: | |
| test_data_source: ${{ matrix.data-source }} | |
| test-cloud-integration: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libsasl2-dev | |
| python -m pip install --upgrade pip | |
| cat dev-requirements.in | grep tox | xargs pip install | |
| - name: Test with tox | |
| run: | | |
| tox -- soda -k soda/core | |
| env: | |
| test_data_source: postgres | |
| WESTMALLE: BETTER_THAN_LA_TRAPPE | |
| test-scientific: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libsasl2-dev | |
| python -m pip install --upgrade pip | |
| cat dev-requirements.in | grep tox | xargs pip install | |
| - name: Test with tox | |
| run: | | |
| tox -- soda -k soda/scientific | |
| test-contracts: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.9" | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python-version }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libsasl2-dev | |
| python -m pip install --upgrade pip | |
| cat dev-requirements.in | grep tox | xargs pip install | |
| - name: Test with tox | |
| run: | | |
| tox -- soda -k soda/contracts | |
| publish-pypi: | |
| name: Build & Publish Package | |
| if: contains(github.ref, 'refs/tags/') | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-20.04 | |
| needs: [run-tests] | |
| env: | |
| REPORTS_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Python 3.8 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Setup dependencies | |
| run: pip install --upgrade setuptools wheel twine | |
| - name: Get version | |
| id: get_version | |
| run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} | |
| - name: Build and publish packages | |
| # Hacky version for now :) | |
| run: | | |
| FAILED=false | |
| cd soda | |
| for pack in * | |
| do | |
| pushd . | |
| cd $pack | |
| echo "| Building $pack" | |
| python3 setup.py sdist bdist_wheel | |
| echo "| Uploading $pack to pypi" | |
| UPLOAD_FAILED=false | |
| twine upload dist/* || UPLOAD_FAILED=true | |
| if [ $UPLOAD_FAILED = true ]; then | |
| FAILED=true | |
| echo "Failed to upload $pack" | |
| fi | |
| popd | |
| done | |
| if [ $FAILED = true ]; then | |
| echo "There was an error, check the logs please." | |
| exit 1 | |
| fi | |
| trigger-docker-build: | |
| name: Trigger Docker build workflow | |
| needs: [publish-pypi] | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-20.04 | |
| if: contains(github.ref, 'refs/tags/') | |
| steps: | |
| - name: check if a version tag | |
| id: check-version-tag | |
| run: | | |
| if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo ::set-output name=match::true | |
| echo ::set-output name=versiontag::$(echo $GITHUB_REF | cut -d / -f 3) | |
| fi | |
| - name: Repository Dispatch | |
| if: steps.check-version-tag.outputs.match == 'true' | |
| uses: peter-evans/repository-dispatch@v2 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| repository: sodadata/soda-core | |
| event-type: started-from-version-tag | |
| client-payload: '{"tag": "${{ steps.check-version-tag.outputs.versiontag }}", "ref": "${{ github.ref }}"}' |