Chore/replace gitlab ci #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Run Data Pipeline Tests | |
on: | |
push: | |
branches: | |
- main | |
pull_request: | |
branches: | |
- main | |
jobs: | |
run-tests: | |
runs-on: ubuntu-latest | |
steps: | |
# Checkout the code | |
- name: Checkout code | |
uses: actions/checkout@v3 | |
- name: Set up Poetry | |
run: | | |
pip install poetry | |
# Set up Python environment | |
- name: Set up Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: '3.10' | |
cache: 'poetry' | |
# Install dependencies and set up environment | |
- name: Install dependencies | |
run: | | |
poetry install | |
# Install DuckDB CLI | |
- name: Install DuckDB CLI | |
run: | | |
wget https://github.com/duckdb/duckdb/releases/download/v1.1.3/duckdb_cli-linux-amd64.zip | |
unzip duckdb_cli-linux-amd64.zip -d /usr/local/bin/ | |
chmod +x /usr/local/bin/duckdb | |
# Verify DuckDB installation and create database | |
- name: Verify DuckDB and create database | |
run: | | |
duckdb --version | |
duckdb ./dbt_mimesis_example/dev.duckdb "SELECT 'Database created successfully';" | |
# Generate test data, run dbt commands, and execute tests | |
- name: Run tests | |
run: | | |
poetry run python data_generator/main.py --dbt-model-path "./dbt_mimesis_example/seeds/schema.yml" --output-path "./dbt_mimesis_example/seeds" --min-rows 100 --max-rows 1000 | |
cd dbt_mimesis_example | |
poetry run dbt deps | |
poetry run dbt seed | |
poetry run dbt run | |
poetry run dbt test |