add github action workflow, remove gitlab-ci.yml #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Run Data Pipeline Tests | |
on: | |
push: | |
branches: | |
- main | |
- 'chore/replace-gitlab-ci' | |
pull_request: | |
branches: | |
- main | |
jobs: | |
run-tests: | |
runs-on: ubuntu-latest | |
steps: | |
# Checkout the code | |
- name: Checkout code | |
uses: actions/checkout@v3 | |
# Set up Python environment | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: 3.10 | |
# Install dependencies and set up environment | |
- name: Install dependencies | |
run: | | |
pip install poetry | |
poetry install | |
source `poetry env info --path`/bin/activate | |
# Install DuckDB CLI | |
- name: Install DuckDB CLI | |
run: | | |
wget https://github.com/duckdb/duckdb/releases/download/v1.1.3/duckdb_cli-linux-amd64.zip | |
unzip duckdb_cli-linux-amd64.zip -d /usr/local/bin/ | |
chmod +x /usr/local/bin/duckdb | |
# Verify DuckDB installation and create database | |
- name: Verify DuckDB and create database | |
run: | | |
duckdb --version | |
duckdb ./dbt_mimesis_example/dev.duckdb "SELECT 'Database created successfully';" | |
# Generate test data, run dbt commands, and execute tests | |
- name: Run tests | |
run: | | |
python data_generator/main.py --dbt-model-path "./dbt_mimesis_example/seeds/schema.yml" --output-path "./dbt_mimesis_example/seeds" --min-rows 100 --max-rows 1000 | |
cd dbt_mimesis_example | |
dbt deps | |
dbt seed | |
dbt run | |
dbt test |