Skip to content

fix pipeline

fix pipeline #2

name: Run Data Pipeline Tests
on:
push:
branches:
- main
- 'chore/replace-gitlab-ci'
pull_request:
branches:
- main
jobs:
run-tests:
runs-on: ubuntu-latest
steps:
# Checkout the code
- name: Checkout code
uses: actions/checkout@v3
# Set up Python environment
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.10'
cache: 'poetry'
# Install dependencies and set up environment
- name: Install dependencies
run: |
pip install poetry
poetry install
source `poetry env info --path`/bin/activate
# Install DuckDB CLI
- name: Install DuckDB CLI
run: |
wget https://github.com/duckdb/duckdb/releases/download/v1.1.3/duckdb_cli-linux-amd64.zip
unzip duckdb_cli-linux-amd64.zip -d /usr/local/bin/
chmod +x /usr/local/bin/duckdb
# Verify DuckDB installation and create database
- name: Verify DuckDB and create database
run: |
duckdb --version
duckdb ./dbt_mimesis_example/dev.duckdb "SELECT 'Database created successfully';"
# Generate test data, run dbt commands, and execute tests
- name: Run tests
run: |
python data_generator/main.py --dbt-model-path "./dbt_mimesis_example/seeds/schema.yml" --output-path "./dbt_mimesis_example/seeds" --min-rows 100 --max-rows 1000
cd dbt_mimesis_example
dbt deps
dbt seed
dbt run
dbt test