acquire-transfermarkt-api.yml

name: acquire-transfermarkt-api

on:
  workflow_run:
    workflows: ["acquire-transfermarkt-scraper"]
    branches: [master]
    types:
      - completed
  workflow_dispatch:
    inputs:
      season:
        required: false
        default: "2024"
        description: Season to acquire data for

env:
  SEASON: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.season || '2024' }}
  DATA_DIR: data/raw/transfermarkt-api/${{ github.event_name == 'workflow_dispatch' && github.event.inputs.season || '2024' }}

jobs:
  acquire:
    runs-on: ubuntu-latest
    container:
      image: dcaribou/transfermarkt-datasets:linux-amd64-master
    defaults:
      run:
        shell: bash -l {0}
    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-a-workflow-based-on-the-conclusion-of-another-workflow
    if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
    steps:
      - uses: actions/checkout@v4
        # the players asset is requeried for the transfermarkt-api script to run
      - name: pull data
        run: |
          dvc pull data/raw/transfermarkt-scraper/${{ env.SEASON }}/players.json.gz
      - name: run acquire
        run: |
          make \
            acquire_local \
              ACQUIRER=transfermarkt-api \
              ARGS="--seasons $SEASON"
      - uses: actions/upload-artifact@v4
        with:
          name: market_values
          path: ${{ env.DATA_DIR }}/market_values.json
      - uses: actions/upload-artifact@v4
        with:
          name: transfers
          path: ${{ env.DATA_DIR }}/transfers.json

  dvc-push:
    runs-on: ubuntu-latest
    container:
      image: dcaribou/transfermarkt-datasets:linux-amd64-master
    defaults:
      run:
        shell: bash -l {0}
    needs:
      - acquire
    # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#running-a-workflow-based-on-the-conclusion-of-another-workflow
    if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.PA_GITHUB_TOKEN }}
      - name: pull data
        run: |
          dvc pull data/raw/transfermarkt-api
      - uses: actions/download-artifact@v4
        with:
          name: market_values
          path: ${{ env.DATA_DIR }}
      - uses: actions/download-artifact@v4
        with:
          name: transfers
          path: ${{ env.DATA_DIR }}
      - name: dvc commit and push
        run: |
          dvc commit -f data/raw/transfermarkt-api && dvc push --remote s3
          git config --global --add safe.directory '*'
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      - uses: EndBug/add-and-commit@v9
        with:
          add: 'data/raw/transfermarkt-api.dvc'
          message: '🤖 updated `transfermarkt-api` raw data'
          default_author: github_actions
          pull: '--no-rebase'