Feat (CI): Post SQL diff on PRs #18292
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rotki CI | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - develop | |
| - bugfixes | |
| push: | |
| branches: | |
| - master | |
| - develop | |
| - bugfixes | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-changes: | |
| name: 'Required job check' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend_tasks: ${{ steps.checker.outputs.backend_tasks }} | |
| frontend_tasks: ${{ steps.checker.outputs.frontend_tasks }} | |
| e2e_tasks: ${{ steps.checker.outputs.e2e_tasks }} | |
| documentation_tasks: ${{ steps.checker.outputs.documentation_tasks }} | |
| test_environment: ${{ steps.checker.outputs.test_environment }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run check action | |
| uses: rotki/action-job-checker@v3 | |
| id: checker | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| documentation_paths: | | |
| docs | |
| backend_paths: | | |
| rotkehlchen | |
| requirements.txt | |
| requirements_dev.txt | |
| requirements_lint.txt | |
| frontend_paths: | | |
| frontend | |
| code-analyze-frontend: | |
| name: 'Code analyze frontend' | |
| needs: ['check-changes'] | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| if: ${{ needs.check-changes.outputs.frontend_tasks }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v3 | |
| with: | |
| languages: 'javascript' | |
| - name: Autobuild | |
| uses: github/codeql-action/autobuild@v3 | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v3 | |
| with: | |
| category: "/language:javascript" | |
| lint-frontend: | |
| name: 'Frontend lint' | |
| needs: ['check-changes'] | |
| if: ${{ needs.check-changes.outputs.frontend_tasks }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - name: Load env | |
| uses: rotki/action-env@v2 | |
| with: | |
| env_file: .github/.env.ci | |
| - name: Setup node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v3 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| - name: Get pnpm store directory | |
| id: pnpm-cache | |
| shell: bash | |
| run: | | |
| echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT | |
| - name: Setup pnpm cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cache/Cypress | |
| ${{ steps.pnpm-cache.outputs.STORE_PATH }} | |
| frontend/app/components.d.ts | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Store eslint cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| frontend/.eslintcache | |
| key: ${{ runner.os }}-eslint-${{ github.ref_name }} | |
| restore-keys: | | |
| ${{ runner.os }}-eslint-${{ github.ref_name }} | |
| ${{ runner.os }}-eslint-${{ github.base_ref || github.ref_name }} | |
| - name: Install dependencies | |
| working-directory: ./frontend | |
| run: pnpm install --frozen-lockfile | |
| - name: Build | |
| working-directory: ./frontend | |
| run: | | |
| if [ ${{ github.event_name }} != 'push' ]; then | |
| pnpm run build | |
| fi | |
| - name: Lint code | |
| working-directory: ./frontend | |
| run: pnpm run lint | |
| unittest-frontend: | |
| name: 'Frontend unit tests' | |
| needs: [ 'check-changes'] | |
| if: ${{ github.event_name != 'push' && needs.check-changes.outputs.frontend_tasks }} | |
| uses: ./.github/workflows/task_fe_unit_tests.yml | |
| docs: | |
| name: 'Documentation build' | |
| needs: [ 'check-changes'] | |
| if: ${{ github.event_name != 'push' && needs.check-changes.outputs.documentation_tasks }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Load env | |
| uses: rotki/action-env@v2 | |
| with: | |
| env_file: .github/.env.ci | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade uv | |
| uv pip install --system -r requirements_docs.txt | |
| uv pip install --system -e . | |
| git rev-parse HEAD | |
| - name: Build html docs | |
| run: cd docs && make html | |
| code-analyze-backend: | |
| name: 'Code analyze backend' | |
| needs: ['check-changes'] | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| if: ${{ needs.check-changes.outputs.backend_tasks }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # - name: Initialize CodeQL | |
| # uses: github/codeql-action/init@v3 | |
| # with: | |
| # languages: 'python' | |
| # - name: Autobuild | |
| # uses: github/codeql-action/autobuild@v3 | |
| # - name: Perform CodeQL Analysis | |
| # uses: github/codeql-action/analyze@v3 | |
| # with: | |
| # category: "/language:python" | |
| lint-backend: | |
| name: 'Backend lint' | |
| needs: ['check-changes'] | |
| if: ${{ github.event_name != 'push' && needs.check-changes.outputs.backend_tasks }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # - name: Load env | |
| # uses: rotki/action-env@v2 | |
| # with: | |
| # env_file: .github/.env.ci | |
| # - name: Setup python | |
| # uses: actions/setup-python@v5 | |
| # with: | |
| # python-version: ${{ env.PYTHON_VERSION }} | |
| # - name: Set up uv | |
| # run: | | |
| # echo "VIRTUAL_ENV=${Python_ROOT_DIR}" >> $GITHUB_ENV | |
| # pip install --upgrade pip | |
| # pip install --no-cache uv | |
| # - name: Install dependencies | |
| # run: | | |
| # uv pip install --system -r requirements_lint.txt | |
| # uv pip install --system -e . | |
| # git rev-parse HEAD | |
| # - name: Lint | |
| # run: make lint | |
| sql-diff: | |
| name: 'SQL diff' | |
| # needs: ['lint-backend', 'check-changes'] | |
| needs: ['check-changes'] | |
| if: ${{ needs.check-changes.outputs.backend_tasks }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check if any db is modified | |
| id: changed-db-files | |
| uses: tj-actions/changed-files@v44 | |
| with: | |
| files: | | |
| *.db | |
| **/*.db | |
| - name: Checkout target branch | |
| if: steps.changed-db-files.outputs.any_changed == 'true' | |
| uses: actions/checkout@v4 | |
| - name: Save target DBs | |
| if: steps.changed-db-files.outputs.any_changed == 'true' | |
| env: | |
| ALL_CHANGED_FILES: ${{ steps.changed-db-files.outputs.all_changed_files }} | |
| run: | | |
| echo "All changed files: $ALL_CHANGED_FILES" | |
| mkdir target | |
| for file in $ALL_CHANGED_FILES; do | |
| cp $file target | |
| done | |
| echo "Files in target: `ls -lart target`" | |
| - name: Checkout source branch | |
| if: steps.changed-db-files.outputs.any_changed == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| clean: false | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Save source DBs | |
| if: steps.changed-db-files.outputs.any_changed == 'true' | |
| env: | |
| ALL_CHANGED_FILES: ${{ steps.changed-db-files.outputs.all_changed_files }} | |
| run: | | |
| echo "All changed files: $ALL_CHANGED_FILES" | |
| mkdir source | |
| for file in $ALL_CHANGED_FILES; do | |
| cp $file source | |
| done | |
| echo "Files in source: `ls -lart source`" | |
| - name: Install SQLCipher | |
| if: steps.changed-db-files.outputs.any_changed == 'true' | |
| run: | | |
| git clone https://github.com/sqlcipher/sqlcipher.git | |
| cd sqlcipher | |
| export SQLITE_HAS_CODEC=1 | |
| export SQLITE_TEMP_STORE=2 | |
| ./configure --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC" LDFLAGS="-lcrypto" | |
| make | |
| sudo make install | |
| cd .. | |
| sudo apt-get install -y sqlite3-tools | |
| - name: Get diff of all changed DBs | |
| id: sql-diff | |
| if: steps.changed-db-files.outputs.any_changed == 'true' | |
| env: | |
| ALL_CHANGED_FILES: ${{ steps.changed-db-files.outputs.all_changed_files }} | |
| run: | | |
| sqlcipher --version | |
| output_file="sql_diff.md" | |
| echo "Files in `pwd`: `ls -lart`" | |
| echo "Files in target: `ls -lart target`" | |
| echo "Files in source: `ls -lart source`" | |
| for file in `ls source`; do | |
| if head -c 16 source/$file | grep -q "SQLite format 3"; then | |
| decryption_query="" | |
| else | |
| decryption_query="PRAGMA key = '123';" | |
| fi | |
| echo " | |
| $decryption_query | |
| ATTACH DATABASE 'tmp-from.db' AS plaintext KEY ''; | |
| SELECT sqlcipher_export('plaintext'); | |
| DETACH DATABASE plaintext; | |
| " | sqlcipher target/$file | |
| echo " | |
| $decryption_query | |
| ATTACH DATABASE 'tmp-to.db' AS plaintext KEY ''; | |
| SELECT sqlcipher_export('plaintext'); | |
| DETACH DATABASE plaintext; | |
| " | sqlcipher source/$file | |
| echo "SQL diff for \`$file\`:" >> $output_file | |
| echo "\`\`\`sql" >> $output_file | |
| sqldiff --primarykey tmp-from.db tmp-to.db >> $output_file | |
| echo "\`\`\`" >> $output_file | |
| rm tmp-from.db tmp-to.db | |
| done | |
| rm -r source target | |
| - name: Comment the SQL diff on the PR | |
| if: steps.changed-db-files.outputs.any_changed == 'true' | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| filePath: sql_diff.md | |
| comment_tag: sql-diff | |
| - name: Delete an existing SQL diff comment, if not DB changed | |
| if: steps.changed-db-files.outputs.any_changed == 'false' | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| message: 'No DB changed' | |
| comment_tag: sql-diff | |
| mode: delete | |
| test-backend: | |
| if: ${{ github.event_name != 'push' && needs.check-changes.outputs.backend_tasks }} | |
| needs: ['lint-backend', 'check-changes'] | |
| uses: ./.github/workflows/task_backend_tests.yml | |
| with: | |
| os: ubuntu-20.04 | |
| test_environment: ${{needs.check-changes.outputs.test_environment}} | |
| test-e2e: | |
| name: 'Frontend e2e tests' | |
| needs: check-changes | |
| if: ${{ github.event_name != 'push' && needs.check-changes.outputs.e2e_tasks }} | |
| uses: ./.github/workflows/task_e2e_tests.yml | |
| done: | |
| name: 'Success check' | |
| if: ${{ always() }} | |
| needs: [ 'check-changes', 'lint-frontend', 'unittest-frontend', 'docs', 'lint-backend', 'test-backend', 'test-e2e' ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check if any task failed | |
| run: | | |
| data=($(echo "${{ toJSON(needs.*.result) }}" | sed 's/[][,]//g')) | |
| for i in "${data[@]}" | |
| do | |
| if [[ $i == "failure" ]]; then | |
| echo "::error::At least one required task failed" | |
| exit 1; | |
| fi | |
| done |