schedule_spider_yearly.yaml

name: Schedule Spider Crawl Split Per Year

on:
  workflow_dispatch:
    inputs:
      spider_name:
        description: 'Spider to be scheduled'
        required: true
      start_date:
        description: 'Start date (YYYY-MM-DD)'
        required: true
      end_date:
        description: 'End date (YYYY-MM-DD)'
        required: false

jobs:
  full-crawl-yearly:
    runs-on: ubuntu-latest
    env:
      SHUB_APIKEY: ${{ secrets.SHUB_APIKEY }}
      SCRAPY_CLOUD_PROJECT_ID: ${{ secrets.SCRAPY_CLOUD_PROJECT_ID }}
      FILES_STORE: ${{ secrets.FILES_STORE }}
      QUERIDODIARIO_DATABASE_URL: ${{ secrets.QUERIDODIARIO_DATABASE_URL }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
      AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
      SPIDERMON_DISCORD_FAKE: ${{ secrets.SPIDERMON_DISCORD_FAKE }}
      SPIDERMON_DISCORD_WEBHOOK_URL: ${{ secrets.SPIDERMON_DISCORD_WEBHOOK_URL }}
      ZYTE_SMARTPROXY_APIKEY: ${{ secrets.ZYTE_SMARTPROXY_APIKEY }}
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@v2
      with:
        python-version: '3.10'
    - name: Prepare environment
      run: |
        python -m pip install --upgrade pip
        pip install click python-decouple scrapinghub SQLAlchemy psycopg2     
    - name: Schedule full crawl per year
      if: ${{ !github.event.inputs.end_date }}
      run: |
        cd data_collection/
        spider=${{ github.event.inputs.spider_name }}
        start_date=${{ github.event.inputs.start_date }}
        int_start_date=$(date -d $start_date +"%Y%m%d")
        int_end_date=$(date --date="today" +"%Y%m%d")
        while [[ $int_start_date -lt $int_end_date ]]; do
            int_date_to=$(date -d"$int_start_date + 1 year" +"%Y%m%d")
            if [[ $int_date_to -ge $int_end_date ]]; then
                int_date_to="$int_end_date"
            fi
            date_from=$(date -d"$int_start_date" +"%Y-%m-%d")
            date_to=$(date -d"$int_date_to" +"%Y-%m-%d")
            python scheduler.py schedule-spider --spider_name="$spider" --start_date="$date_from" --end_date="$date_to"
            int_start_date="$int_date_to"
        done
    - name: Schedule partial crawl per year
      if: ${{ github.event.inputs.end_date }}
      run: |
        cd data_collection/
        spider=${{ github.event.inputs.spider_name }}
        start_date=${{ github.event.inputs.start_date }}
        end_date=${{ github.event.inputs.end_date }}
        int_start_date=$(date -d $start_date +"%Y%m%d")
        int_end_date=$(date -d $end_date +"%Y%m%d")
        while [[ $int_start_date -lt $int_end_date ]]; do
            int_date_to=$(date -d"$int_start_date + 1 year" +"%Y%m%d")
            if [[ $int_date_to -ge $int_end_date ]]; then
                int_date_to="$int_end_date"
            fi
            date_from=$(date -d"$int_start_date" +"%Y-%m-%d")
            date_to=$(date -d"$int_date_to" +"%Y-%m-%d")
            python scheduler.py schedule-spider --spider_name="$spider" --start_date="$date_from" --end_date="$date_to"
            int_start_date="$int_date_to"
        done