diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml new file mode 100644 index 00000000..22056a0b --- /dev/null +++ b/.github/workflows/evals.yml @@ -0,0 +1,63 @@ +name: evals + +on: + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + +# Serializes runs, which share one e2e Prismic account. +concurrency: + group: evals + +permissions: + contents: write + pull-requests: write + +jobs: + evals: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 40 + steps: + # The agent under test runs in this workspace with Bash; keep the + # checkout token out of .git/config so it cannot push with it. + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + - run: npm ci + + - name: Run evals + env: + PRISMIC_ALLOW_EVALS: "true" + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + E2E_PRISMIC_EMAIL: ${{ secrets.E2E_PRISMIC_EMAIL }} + E2E_PRISMIC_PASSWORD: ${{ secrets.E2E_PRISMIC_PASSWORD }} + run: node --run evals || true + + # Results are recorded only from main runs, via a PR the job merges itself. + - name: Record results + if: github.ref_name == 'main' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh auth setup-git + { + echo '```' + node --run evals:report + echo '```' + echo + echo "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + } > "$RUNNER_TEMP/report.md" + title="test(eval): record eval run at $(git rev-parse --short HEAD)" + branch="evals/run-${{ github.run_id }}-${{ github.run_attempt }}" + git switch -c "$branch" + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -m "$title" -- evals/results.json + git push origin "$branch" + gh pr create --base main --head "$branch" --title "$title" \ + --body-file "$RUNNER_TEMP/report.md" + gh pr merge "$branch" --squash --delete-branch