PrincetonCourses Imports #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PrincetonCourses Imports | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| run_import_courses: | |
| description: "Run course import (OIT + registrar details)" | |
| type: boolean | |
| default: true | |
| run_import_departments: | |
| description: "Run departments import (OIT)" | |
| type: boolean | |
| default: true | |
| run_import_evals: | |
| description: "Run evaluations scraper (requires PHPSESSID)" | |
| type: boolean | |
| default: true | |
| run_backfill: | |
| description: "Run backfill for missing Quality of Course" | |
| type: boolean | |
| default: false | |
| run_setnew: | |
| description: "Run setNewCourseFlag" | |
| type: boolean | |
| default: false | |
| restart_heroku: | |
| description: "Restart Heroku dynos at end (refresh dept cache)" | |
| type: boolean | |
| default: false | |
| term: | |
| description: "Target term code (e.g., 1252)" | |
| type: string | |
| required: false | |
| subject: | |
| description: "Target subject code (e.g., COS). Leave blank for all" | |
| type: string | |
| required: false | |
| php_sessid: | |
| description: "Registrar PHPSESSID cookie (required for evals)" | |
| type: string | |
| required: false | |
| registrar_fe_api_token: | |
| description: "Registrar FE API token (paste per run; overrides scrape)" | |
| type: string | |
| required: false | |
| concurrency: | |
| group: imports-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| import_courses: | |
| if: ${{ inputs.run_import_courses }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Python deps | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r requirements.txt | |
| - name: Install Node deps | |
| run: npm ci | |
| - name: Load Heroku config vars | |
| env: | |
| HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} | |
| HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }} | |
| run: | | |
| echo "::add-mask::${HEROKU_API_KEY}" | |
| cfg=$(curl -sS \ | |
| -H "Authorization: Bearer ${HEROKU_API_KEY}" \ | |
| -H "Accept: application/vnd.heroku+json; version=3" \ | |
| https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars) | |
| echo "$cfg" > heroku_config.json | |
| for key in MONGODB_URI CONSUMER_KEY CONSUMER_SECRET CHATBOT_API_KEY HOST PORT; do | |
| val=$(node -e "const o=require('./heroku_config.json'); const k='${key}'; if (o[k]) console.log(o[k]);") | |
| if [ -n "$val" ]; then | |
| echo "::add-mask::${val}" | |
| echo "$key=$val" >> $GITHUB_ENV | |
| fi | |
| done | |
| - name: Build query arg | |
| id: q | |
| run: | | |
| q="" | |
| if [ -n "${{ inputs.term }}" ] && [ -n "${{ inputs.subject }}" ]; then | |
| q="term=${{ inputs.term }}&subject=${{ inputs.subject }}" | |
| elif [ -n "${{ inputs.term }}" ]; then | |
| q="term=${{ inputs.term }}&subject=all" | |
| fi | |
| echo "query=${q}" >> $GITHUB_OUTPUT | |
| - name: Run course importer | |
| env: | |
| REGISTRAR_FE_API_TOKEN: ${{ inputs.registrar_fe_api_token }} | |
| run: | | |
| if [ -n "${{ inputs.registrar_fe_api_token }}" ]; then echo "::add-mask::${{ inputs.registrar_fe_api_token }}"; fi | |
| if [ -n "${{ steps.q.outputs.query }}" ]; then | |
| node importers/importBasicCourseDetails.js "${{ steps.q.outputs.query }}" | |
| else | |
| node importers/importBasicCourseDetails.js | |
| fi | |
| import_departments: | |
| if: ${{ inputs.run_import_departments }} | |
| needs: [import_courses] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Python deps | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r requirements.txt | |
| - name: Install Node deps | |
| run: npm ci | |
| - name: Load Heroku config vars | |
| env: | |
| HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} | |
| HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }} | |
| run: | | |
| echo "::add-mask::${HEROKU_API_KEY}" | |
| cfg=$(curl -sS \ | |
| -H "Authorization: Bearer ${HEROKU_API_KEY}" \ | |
| -H "Accept: application/vnd.heroku+json; version=3" \ | |
| https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars) | |
| echo "$cfg" > heroku_config.json | |
| for key in MONGODB_URI CONSUMER_KEY CONSUMER_SECRET; do | |
| val=$(node -e "const o=require('./heroku_config.json'); const k='${key}'; if (o[k]) console.log(o[k]);") | |
| if [ -n "$val" ]; then | |
| echo "::add-mask::${val}" | |
| echo "$key=$val" >> $GITHUB_ENV | |
| fi | |
| done | |
| - name: Run departments importer | |
| run: node importers/importDepartments.js | |
| scrape_evals: | |
| if: ${{ inputs.run_import_evals }} | |
| needs: [import_courses] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Python deps | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r requirements.txt | |
| - name: Install Node deps | |
| run: npm ci | |
| - name: Load Heroku config vars | |
| env: | |
| HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} | |
| HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }} | |
| run: | | |
| echo "::add-mask::${HEROKU_API_KEY}" | |
| cfg=$(curl -sS \ | |
| -H "Authorization: Bearer ${HEROKU_API_KEY}" \ | |
| -H "Accept: application/vnd.heroku+json; version=3" \ | |
| https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars) | |
| echo "$cfg" > heroku_config.json | |
| for key in MONGODB_URI; do | |
| val=$(node -e "const o=require('./heroku_config.json'); const k='${key}'; if (o[k]) console.log(o[k]);") | |
| if [ -n "$val" ]; then | |
| echo "::add-mask::${val}" | |
| echo "$key=$val" >> $GITHUB_ENV | |
| fi | |
| done | |
| - name: Run evaluation scraper (non-interactive) | |
| env: | |
| EVAL_SCRAPE_DELAY_MS: "150" | |
| EVAL_SCRAPE_MAX_RETRIES: "3" | |
| EVAL_SCRAPE_RETRY_BACKOFF_MS: "1000" | |
| EVAL_RANDOMIZE_ORDER: "true" | |
| INPUT_TERM: ${{ inputs.term }} | |
| INPUT_SUBJECT: ${{ inputs.subject }} | |
| PHPSESSID: ${{ inputs.php_sessid }} | |
| run: | | |
| if [ -z "${PHPSESSID}" ]; then | |
| echo "php_sessid input is required to run evaluations" >&2 | |
| exit 1 | |
| fi | |
| echo "::add-mask::${PHPSESSID}" | |
| # Build a query that includes courses missing scores OR ones backfilled from a previous semester. | |
| # This ensures we replace placeholder scores with real evaluations when they are published. | |
| export EVAL_QUERY=$(node -e ' | |
| const term = process.env.INPUT_TERM; | |
| const subj = process.env.INPUT_SUBJECT; | |
| const andConds = []; | |
| if (term) andConds.push({ semester: Number(term) }); | |
| if (subj) { | |
| const S = String(subj).toUpperCase(); | |
| andConds.push({ $or: [ { department: S }, { "crosslistings.department": S } ] }); | |
| } | |
| andConds.push({ $or: [ { "scores.Quality of Course": { $exists: false } }, { scoresFromPreviousSemester: true } ] }); | |
| const q = andConds.length > 1 ? { $and: andConds } : (andConds[0] || {}); | |
| console.log(JSON.stringify(q)); | |
| ') | |
| node importers/scrapeEvaluations.js --skip | |
| backfill_scores: | |
| if: ${{ inputs.run_backfill }} | |
| needs: [scrape_evals] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Install Node deps | |
| run: npm ci | |
| - name: Load Heroku config vars | |
| env: | |
| HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} | |
| HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }} | |
| run: | | |
| echo "::add-mask::${HEROKU_API_KEY}" | |
| cfg=$(curl -sS \ | |
| -H "Authorization: Bearer ${HEROKU_API_KEY}" \ | |
| -H "Accept: application/vnd.heroku+json; version=3" \ | |
| https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars) | |
| echo "$cfg" > heroku_config.json | |
| val=$(node -e "const o=require('./heroku_config.json'); const k='MONGODB_URI'; if (o[k]) console.log(o[k]);") | |
| if [ -n "$val" ]; then | |
| echo "::add-mask::${val}" | |
| echo "MONGODB_URI=$val" >> $GITHUB_ENV | |
| fi | |
| - name: Run backfill | |
| run: | | |
| if [ -n "${{ inputs.term }}" ]; then | |
| node importers/insertMostRecentScoreIntoUnevaluatedSemesters.js "${{ inputs.term }}" | |
| else | |
| node importers/insertMostRecentScoreIntoUnevaluatedSemesters.js | |
| fi | |
| set_new_flag: | |
| if: ${{ inputs.run_setnew }} | |
| needs: [scrape_evals] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18 | |
| - name: Install Node deps | |
| run: npm ci | |
| - name: Load Heroku config vars | |
| env: | |
| HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} | |
| HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }} | |
| run: | | |
| echo "::add-mask::${HEROKU_API_KEY}" | |
| cfg=$(curl -sS \ | |
| -H "Authorization: Bearer ${HEROKU_API_KEY}" \ | |
| -H "Accept: application/vnd.heroku+json; version=3" \ | |
| https://api.heroku.com/apps/${HEROKU_APP_NAME}/config-vars) | |
| echo "$cfg" > heroku_config.json | |
| val=$(node -e "const o=require('./heroku_config.json'); const k='MONGODB_URI'; if (o[k]) console.log(o[k]);") | |
| if [ -n "$val" ]; then | |
| echo "::add-mask::${val}" | |
| echo "MONGODB_URI=$val" >> $GITHUB_ENV | |
| fi | |
| - name: Run setNewCourseFlag | |
| run: node importers/setNewCourseFlag.js | |
| restart_heroku: | |
| if: ${{ inputs.restart_heroku && always() }} | |
| needs: [import_departments] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Restart Heroku dynos and wait | |
| env: | |
| HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} | |
| HEROKU_APP_NAME: ${{ secrets.HEROKU_APP_NAME }} | |
| run: | | |
| echo "::add-mask::${HEROKU_API_KEY}" | |
| set -euo pipefail | |
| base="https://api.heroku.com/apps/${HEROKU_APP_NAME}" | |
| authHeader="Authorization: Bearer ${HEROKU_API_KEY}" | |
| acceptHeader="Accept: application/vnd.heroku+json; version=3" | |
| echo "Fetching current dyno state..." | |
| curl -sS -H "$acceptHeader" -H "$authHeader" "$base/dynos" | tee before.json >/dev/null | |
| echo "Issuing restart (DELETE /dynos)..." | |
| http_code=$(curl -sS -o /dev/null -w "%{http_code}" -X DELETE -H "$acceptHeader" -H "$authHeader" "$base/dynos") | |
| echo "Heroku API status: ${http_code}" | |
| if [ "$http_code" != "202" ] && [ "$http_code" != "200" ]; then | |
| echo "Unexpected response from Heroku API when restarting dynos" >&2 | |
| # Fetch response body for debugging | |
| curl -sS -X DELETE -H "$acceptHeader" -H "$authHeader" "$base/dynos" || true | |
| exit 1 | |
| fi | |
| echo "Waiting for dynos to cycle..." | |
| attempts=0 | |
| max_attempts=30 | |
| sleep_seconds=2 | |
| changed=0 | |
| while [ $attempts -lt $max_attempts ]; do | |
| sleep "$sleep_seconds" | |
| attempts=$((attempts+1)) | |
| now=$(curl -sS -H "$acceptHeader" -H "$authHeader" "$base/dynos") | |
| echo "$now" > after.json | |
| # If jq is available, compare updated_at or created_at; else compare raw payloads | |
| if command -v jq >/dev/null 2>&1; then | |
| before_hash=$(jq -r 'map({name,updated_at})|tostring' before.json 2>/dev/null || echo "") | |
| after_hash=$(jq -r 'map({name,updated_at})|tostring' after.json 2>/dev/null || echo "") | |
| if [ "$before_hash" != "$after_hash" ]; then | |
| changed=1 | |
| break | |
| fi | |
| else | |
| if ! diff -q before.json after.json >/dev/null 2>&1; then | |
| changed=1 | |
| break | |
| fi | |
| fi | |
| done | |
| if [ $changed -eq 1 ]; then | |
| echo "Dynos changed state; restart confirmed." | |
| else | |
| echo "Dynos did not show a state change within timeout; restart may still have occurred. Showing current dynos:" | |
| cat after.json || true | |
| fi |