nightly-scrape-done #51
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Import scraped data to configured database | |
| on: | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: [nightly-scrape-done] | |
| jobs: | |
| import_filings: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: "deploy" | |
| - name: Import candidate, PAC, and filing data | |
| run: | | |
| touch .env | |
| docker-compose -f docker-compose.etl.yml run --rm \ | |
| -e AWS_STORAGE_BUCKET_NAME=${{ secrets.AWS_STORAGE_BUCKET_NAME }} \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e DATABASE_URL=${{ secrets.DATABASE_URL }} \ | |
| app make import/candidates import/pacs import/candidate_filings import/pac_filings | |
| import_2023: | |
| runs-on: ubuntu-latest | |
| needs: import_filings | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: "deploy" | |
| - name: Import data for 2023 | |
| run: | | |
| touch .env | |
| docker-compose -f docker-compose.etl.yml run --rm \ | |
| -e AWS_STORAGE_BUCKET_NAME=${{ secrets.AWS_STORAGE_BUCKET_NAME }} \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e DATABASE_URL=${{ secrets.DATABASE_URL }} \ | |
| app make import/CON_2023 import/EXP_2023 | |
| import_2024: | |
| runs-on: ubuntu-latest | |
| needs: import_filings | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: "deploy" | |
| - name: Import data for 2024 | |
| run: | | |
| touch .env | |
| docker-compose -f docker-compose.etl.yml run --rm \ | |
| -e AWS_STORAGE_BUCKET_NAME=${{ secrets.AWS_STORAGE_BUCKET_NAME }} \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e DATABASE_URL=${{ secrets.DATABASE_URL }} \ | |
| app make import/CON_2024 import/EXP_2024 |