fix: validate config after dereferencing (#2129) #1

Workflow file for this run

	name: CI
	on:
	pull_request:
	push:
	branches:
	- main
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	env:
	OPENAI_API_KEY: xxx
	ANTHROPIC_API_KEY: xxx
	AZURE_OPENAI_API_HOST: xxx
	VITE_TELEMETRY_DISABLED: 1
	jobs:
	build:
	name: Build and test on Node ${{ matrix.node }} and ${{ matrix.os }}
	timeout-minutes: 12
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	# Temporarily disabled 22.x due to github runner compatibility issues
	# TODO(Michael): Re-enable once issues are resolved, 2024-08-23
	node: ['18.x', '20.x']
	os: [ubuntu-latest, windows-latest, macOS-latest]
	exclude:
	# Remove when https://github.com/nodejs/node/issues/51766 is resolved
	- node: '22.x'
	os: windows-latest

	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Use Node ${{ matrix.node }}
	uses: actions/setup-node@v4
	with:
	node-version: ${{ matrix.node }}
	cache: 'npm'

	- name: Use Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: 3.12

	- name: Install Dependencies
	run: \|
	npm ci
	- name: Test
	run: npm run test -- --ci --coverage --maxWorkers=2

	- name: Build
	run: npm run build

	style-check:
	name: Style Check
	timeout-minutes: 5
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Use Node
	uses: actions/setup-node@v4
	with:
	node-version-file: '.nvmrc'
	cache: 'npm'

	- name: Install Dependencies
	run: \|
	npm ci

	- name: Lint Project
	run: \|
	npm run lint

	- name: Lint UI
	working-directory: src/app
	run: \|
	npm run lint

	- name: Run Style Check
	run: \|
	npm run format:check

	- name: Check Dependency Versions
	run: \|
	npm exec check-dependency-version-consistency

	- name: Check for circular dependencies
	run: npx madge $(git ls-files '*.ts') --circular

	assets:
	name: Generate Assets
	timeout-minutes: 5
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Use Node
	uses: actions/setup-node@v4
	with:
	node-version-file: '.nvmrc'
	cache: 'npm'

	- name: Install Dependencies
	run: npm install

	- name: Generate JSON Schema
	run: npm run jsonSchema:generate

	- name: Check for changes
	run: \|
	if [[ -n $(git status --porcelain) ]]; then
	echo "Changes detected after generating assets:"
	git status --porcelain
	exit 1
	else
	echo "No changes detected."
	fi

	python:
	name: Check Python
	timeout-minutes: 5
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: [3.9, 3.12]
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Use Python ${{ matrix.python-version }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install Dependencies
	run: \|
	pip install ruff

	- name: Check Formatting
	run: \|
	ruff check --select I --fix
	ruff format
	git diff --exit-code \|\| (echo "Files were modified by ruff. Please commit these changes." && exit 1)

	- name: Run Tests
	run: \|
	python -m unittest discover -s src/python -p '*_test.py'

	docs:
	name: Build Docs
	timeout-minutes: 5
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Use Node
	uses: actions/setup-node@v4
	with:
	node-version-file: '.nvmrc'
	cache: 'npm'

	- name: Install Dependencies
	working-directory: site
	run: npm install

	- name: Type Check
	working-directory: site
	run: npm run typecheck

	- name: Build Documentation
	working-directory: site
	run: npm run build

	webui:
	name: webui tests
	timeout-minutes: 5
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Use Node
	uses: actions/setup-node@v4
	with:
	node-version-file: '.nvmrc'
	cache: 'npm'

	- name: Install Dependencies
	run: npm ci

	- name: Run App Tests
	run: npm run test:app

	integration-tests:
	name: Run Integration Tests
	runs-on: ubuntu-latest
	timeout-minutes: 5
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Use Node
	uses: actions/setup-node@v4
	with:
	node-version-file: '.nvmrc'
	cache: 'npm'

	- name: Use Python 3.12
	uses: actions/setup-python@v5
	with:
	python-version: 3.12

	- name: Install Dependencies
	run: \|
	npm ci

	- name: Run Integration Tests
	run: npm run test:integration -- --ci --coverage --maxWorkers=2

	share-test:
	name: Share Test
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Install Dependencies
	run: npm ci

	- name: Run local server
	run: \|
	npm run build
	PROMPTFOO_CONFIG_DIR=~/tmp LOG_LEVEL=DEBUG API_PORT=8500 node dist/src/server/index.js &

	- name: run promptfoo eval
	id: eval
	run: \|
	PROMPTFOO_REMOTE_API_BASE_URL=http://localhost:8500 PROMPTFOO_SHARING_APP_BASE_URL=http://localhost:8500 node dist/src/main.js eval -c .github/workflows/files/promptfooconfig.yaml --share

	- name: Test that the eval results are uploaded
	run: \|
	response=$(curl -s http://localhost:8500/api/results)
	echo "Response: $response"

	# Use jq to extract the array length
	count=$(echo "$response" \| jq '.data \| length')
	echo "Array Length: $count"

	# Check if the count is exactly 1
	if [ "$count" -ne 1 ]; then
	echo "Error: Expected 1 entry, but got $count"
	exit 1
	fi

	- name: Share to cloud
	run: \|
	if [ -n "${{ secrets.PROMPTFOO_STAGING_API_KEY }}" ]; then
	node dist/src/main.js auth login -k ${{ secrets.PROMPTFOO_STAGING_API_KEY }} -h https://api.promptfoo-staging.app
	node dist/src/main.js eval -c .github/workflows/files/promptfooconfig.yaml --share
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix: validate config after dereferencing (#2129) #1

Workflow file

fix: validate config after dereferencing (#2129) #1

Jobs

Run details

Workflow file for this run