diff --git a/.github/workflows/lint-ext-azure-ai-evaluations.yml b/.github/workflows/lint-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..2cb72ea1de4 --- /dev/null +++ b/.github/workflows/lint-ext-azure-ai-evaluations.yml @@ -0,0 +1,28 @@ +name: ext-azure-ai-evaluations-ci + +on: + pull_request: + paths: + - "cli/azd/extensions/azure.ai.evaluations/**" + - ".github/workflows/lint-ext-azure-ai-evaluations.yml" + - ".github/workflows/verify-ext-providers.yml" + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write # required by reusable workflow lint-go.yml + +jobs: + lint: + uses: ./.github/workflows/lint-go.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations + + verify-providers: + uses: ./.github/workflows/verify-ext-providers.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations diff --git a/cli/azd/extensions/azure.ai.evaluations/.gitignore b/cli/azd/extensions/azure.ai.evaluations/.gitignore new file mode 100644 index 00000000000..0d5b6d76489 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.gitignore @@ -0,0 +1,5 @@ +# Test report written by ci-test.ps1 for the pipeline to publish. +junitTestReport.xml + +# Debug log written when --debug or AZD_EXT_DEBUG is set. +azd-ai-eval-*.log diff --git a/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml new file mode 100644 index 00000000000..9777522d023 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml @@ -0,0 +1,21 @@ +version: "2" + +linters: + default: none + enable: + - gosec + - lll + - unused + - errorlint + settings: + lll: + line-length: 220 + tab-width: 4 + gosec: + excludes: + - G204 # Subprocess launched with variable (bicep build invoked in tests) + - G304 # Potential file inclusion via variable + +formatters: + enable: + - gofmt diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md new file mode 100644 index 00000000000..45903a217c0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -0,0 +1,27 @@ +# Release History + +## 1.0.0-beta.1 (Unreleased) + +### Features Added + +- Initial release of the Foundry evaluations extension, `azd ai eval`. +- `init` scaffolds `evals/eval_generate.yaml` and `evals/azure.yaml` next to an + agent, making no service calls. +- `generate` synthesizes a rubric and dataset from the agent's context, writes + them under `evals/`, and merges `source:` references into the deployment spec + while preserving comments, ordering and neighboring entries. +- `run` creates the eval group when it does not exist, starts a run, and + summarizes the result. +- `azure.ai.eval` service-target provider deploys datasets, evaluators and eval + groups during `azd up`, reconciling them in dependency order. +- Change detection so a repeated `azd up` publishes no redundant versions: + datasets are fingerprinted locally, evaluator definitions are compared on the + keys the author wrote, and eval groups are recreated only when their own + declaration changes. +- Atomic commands for every operation: `dataset`, `evaluator`, `run` and + `results` subcommands, all supporting `-o json` and `--no-prompt`. +- Testing criteria are shaped from each evaluator's published contract, so + evaluators requiring inputs beyond the agent shape — `ground_truth`, + `context`, `instruction_id_list` — work by binding them to dataset columns. + A required column the dataset does not carry is reported before the request + is sent, naming the column. diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md new file mode 100644 index 00000000000..2a5c80e62fe --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -0,0 +1,158 @@ +# Azure Developer CLI (azd) Evaluations Extension + +Define Foundry evaluations alongside your agent in `azure.yaml`, deploy them +with `azd up`, and run them from the terminal. + +```bash +azd ai eval init # scaffold evals/ next to your agent +azd ai eval generate # synthesize a rubric and dataset from the agent +azd up # register datasets and evaluators, create the eval group +azd ai eval run # run the evaluation and summarize the results +``` + +## What gets deployed + +Eval resources are one service entry in `azure.yaml`, normally a `$ref` to a +file under `evals/`: + +```yaml +# azure.yaml +services: + ai-project: + host: azure.ai.project + evals: + host: azure.ai.eval + uses: [ai-project] + $ref: ./evals/azure.yaml +``` + +```yaml +# evals/azure.yaml +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + +evaluators: + - name: support-quality + source: ./evaluators/support-quality.json + +evalGroups: + - name: support-quality + dataset: support-golden + evaluators: + - builtin.task_adherence + - support-quality + target: + type: agent + name: support-agent + options: + eval_model: gpt-4.1-nano +``` + +`azd up` reconciles **datasets → evaluators → eval groups**, in that order, +because a group references the versions the first two resolve to. + +Relative paths inside a `$ref`'d file resolve against **that file's** +directory, so `./datasets/x.jsonl` above means `evals/datasets/x.jsonl`. + +### Repeated deploys do not create redundant versions + +Datasets are fingerprinted locally, because the dataset API exposes no content +hash and comparing against the service would mean downloading the blob on every +deploy. Evaluator definitions are compared against the service, but only on the +keys you authored — the service adds `data_schema`, `init_parameters` and +`metrics` of its own. + +Eval groups are immutable, so a change to a group's evaluators, target or +options creates a new group and a new id. The id is cached in the azd +environment so repeat runs stay comparable. + +## Commands + +| Group | Commands | +|---|---| +| `azd ai eval` | `init` · `generate` · `run` | +| `azd ai eval dataset` | `create` · `list` · `show` · `update` · `delete` | +| `azd ai eval evaluator` | `upload` · `list` · `show` · `update` · `delete` · `builtins` | +| `azd ai eval run` | `start` · `list` · `show` · `cancel` | +| `azd ai eval results` | `show` · `export` | + +`create` and `update` both publish a new immutable version; the server +auto-increments and nothing mutates in place. + +Every command supports `-o json` and `--no-prompt`, so the whole surface is +usable from CI. + +## Evaluators + +Built-ins need no declaration — reference them as `builtin.` and list +them with `azd ai eval evaluator builtins`. + +Evaluators do not share an input contract, so the CLI reads each one's +published contract and shapes the request to match. An evaluator needing an +input your dataset does not carry is reported before the request is sent, with +the column named, rather than as a service-side rejection. + +A custom rubric is a JSON list of weighted dimensions: + +```json +{ + "dimensions": [ + { "id": "accuracy", "description": "The answer is factually correct.", "weight": 5 }, + { "id": "tone", "description": "The answer is polite and professional.", "weight": 2 } + ] +} +``` + +`weight` is an **integer from 1 to 10**. Weights do not need to sum to +anything. + +## Choosing a project + +The project endpoint is resolved in this order: + +1. `--project-endpoint` +2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment +3. the host environment variable of the same name + +## Local development + +### Prerequisites + +- Go (the version in `go.mod`; `GOTOOLCHAIN=auto` fetches it) +- [azd](https://aka.ms/azd) and the extension developer kit: + `azd ext install microsoft.azd.extensions` + +### Build, test, install + +```bash +azd x build # compile and install into the local azd +azd x pack # package the artifacts +azd x publish # register in the local extension source +azd ext install azure.ai.evaluations --source local +``` + +```bash +go test ./internal/... # unit tests +``` + +### Live integration tests + +These talk to a real Foundry project, so they are excluded from the default +build by the `live` tag and additionally gated on an environment variable: + +```bash +export AZURE_AI_EVAL_E2E_LIVE=1 +export FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +export AZURE_AI_EVAL_MODEL=gpt-4.1-nano # optional judge model +export AZURE_AI_EVAL_AGENT= # optional, enables the run phase + +go test -tags live ./internal/cmd/ ./tests/live/ +``` + +They clean up every resource they create. + +### Debug logging + +Request tracing is off by default. `--debug`, or `AZD_EXT_DEBUG=true`, writes +it to a dated log file rather than the terminal. diff --git a/cli/azd/extensions/azure.ai.evaluations/build.ps1 b/cli/azd/extensions/azure.ai.evaluations/build.ps1 new file mode 100644 index 00000000000..f37f80cabf0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.ps1 @@ -0,0 +1,78 @@ +# Ensure script fails on any error +$ErrorActionPreference = 'Stop' + +# Get the directory of the script +$EXTENSION_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path + +# Change to the script directory +Set-Location -Path $EXTENSION_DIR + +# Create a safe version of EXTENSION_ID replacing dots with dashes +$EXTENSION_ID_SAFE = $env:EXTENSION_ID -replace '\.', '-' + +# Define output directory +$OUTPUT_DIR = if ($env:OUTPUT_DIR) { $env:OUTPUT_DIR } else { Join-Path $EXTENSION_DIR "bin" } + +# Create output directory if it doesn't exist +if (-not (Test-Path -Path $OUTPUT_DIR)) { + New-Item -ItemType Directory -Path $OUTPUT_DIR | Out-Null +} + +# Get Git commit hash and build date +$COMMIT = git rev-parse HEAD +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Failed to get git commit hash" + exit 1 +} +$BUILD_DATE = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ") + +# List of OS and architecture combinations +if ($env:EXTENSION_PLATFORM) { + $PLATFORMS = @($env:EXTENSION_PLATFORM) +} +else { + $PLATFORMS = @( + "windows/amd64", + "windows/arm64", + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64" + ) +} + +$VERSION_PATH = "azureaieval/internal/version" + +# Loop through platforms and build +foreach ($PLATFORM in $PLATFORMS) { + $OS, $ARCH = $PLATFORM -split '/' + + $OUTPUT_NAME = Join-Path $OUTPUT_DIR "$EXTENSION_ID_SAFE-$OS-$ARCH" + + if ($OS -eq "windows") { + $OUTPUT_NAME += ".exe" + } + + Write-Host "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + if (Test-Path -Path $OUTPUT_NAME) { + Remove-Item -Path $OUTPUT_NAME -Force + } + + # Set environment variables for Go build + $env:GOOS = $OS + $env:GOARCH = $ARCH + + go build ` + -ldflags="-X '$VERSION_PATH.Version=$env:EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" ` + -o $OUTPUT_NAME + + if ($LASTEXITCODE -ne 0) { + Write-Host "An error occurred while building for $OS/$ARCH" + exit 1 + } +} + +Write-Host "Build completed successfully!" +Write-Host "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.evaluations/build.sh b/cli/azd/extensions/azure.ai.evaluations/build.sh new file mode 100644 index 00000000000..4165a516ac4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Get the directory of the script +EXTENSION_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Change to the script directory +cd "$EXTENSION_DIR" || exit + +# Create a safe version of EXTENSION_ID replacing dots with dashes +EXTENSION_ID_SAFE="${EXTENSION_ID//./-}" + +# Define output directory +OUTPUT_DIR="${OUTPUT_DIR:-$EXTENSION_DIR/bin}" + +# Create output and target directories if they don't exist +mkdir -p "$OUTPUT_DIR" + +# Get Git commit hash and build date +COMMIT=$(git rev-parse HEAD) +BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +# List of OS and architecture combinations +if [ -n "$EXTENSION_PLATFORM" ]; then + PLATFORMS=("$EXTENSION_PLATFORM") +else + PLATFORMS=( + "windows/amd64" + "windows/arm64" + "darwin/amd64" + "darwin/arm64" + "linux/amd64" + "linux/arm64" + ) +fi + +VERSION_PATH="azureaieval/internal/version" + +# Loop through platforms and build +for PLATFORM in "${PLATFORMS[@]}"; do + OS=$(echo "$PLATFORM" | cut -d'/' -f1) + ARCH=$(echo "$PLATFORM" | cut -d'/' -f2) + + OUTPUT_NAME="$OUTPUT_DIR/$EXTENSION_ID_SAFE-$OS-$ARCH" + + if [ "$OS" = "windows" ]; then + OUTPUT_NAME+='.exe' + fi + + echo "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + [ -f "$OUTPUT_NAME" ] && rm -f "$OUTPUT_NAME" + + # Set environment variables for Go build + GOOS=$OS GOARCH=$ARCH go build \ + -ldflags="-X '$VERSION_PATH.Version=$EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" \ + -o "$OUTPUT_NAME" + + if [ $? -ne 0 ]; then + echo "An error occurred while building for $OS/$ARCH" + exit 1 + fi +done + +echo "Build completed successfully!" +echo "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 new file mode 100644 index 00000000000..403bc23b08d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 @@ -0,0 +1,114 @@ +param( + [string] $Version = (Get-Content "$PSScriptRoot/version.txt"), + [string] $SourceVersion = (git rev-parse HEAD), + [switch] $CodeCoverageEnabled, + # Accepted because the shared CI template always passes it. This extension + # has no record/playback mode, so there is no second binary to produce. + [switch] $BuildRecordMode, + [string] $MSYS2Shell, # path to msys2_shell.cmd + [string] $OutputFileName +) +$PSNativeCommandArgumentPassing = 'Legacy' + +# Remove any previously built binaries. +go clean + +if ($LASTEXITCODE) { + Write-Host "Error running go clean" + exit $LASTEXITCODE +} + +# Run `go help build` for detail on these flags. +$buildFlags = @( + # Remove file system paths from the binary. Recorded file names become a + # module path@version, or a plain import path for the standard library. + "-trimpath", + + # Position Independent Executable, for memory-corruption hardening across + # platforms. On Windows this enables ASLR and sets DYNAMICBASE and + # HIGH-ENTROPY-VA in the PE header. + "-buildmode=pie" +) + +if ($CodeCoverageEnabled) { + $buildFlags += "-cover" +} + +# cfi: Control Flow Integrity, cfg: Control Flow Guard, +# osusergo: use the pure Go user lookup. +$tagsFlag = "-tags=cfi,cfg,osusergo" + +# -s: omit the symbol table, -w: omit DWARF, -X: set a variable at link time. +$ldFlag = "-ldflags=-s -w " + + "-X 'azureaieval/internal/version.Version=$Version' " + + "-X 'azureaieval/internal/version.Commit=$SourceVersion' " + + "-X 'azureaieval/internal/version.BuildDate=$(Get-Date -Format o)' " + +if ($IsWindows) { + Write-Host "Building for Windows" +} +elseif ($IsLinux) { + Write-Host "Building for linux" + + # Disable cgo for the x64 Linux build. This also links statically, which + # widens compatibility with older Linux distributions. + if ($env:GOARCH -ne "arm64") { + $env:CGO_ENABLED = "0" + } +} +elseif ($IsMacOS) { + Write-Host "Building for macOS" +} + +$outputFlag = "-o=$OutputFileName" + +$buildFlags += @( + $tagsFlag, + $ldFlag, + $outputFlag +) + +function PrintFlags() { + param( + [string] $flags + ) + + # Format the flags so they can be pasted straight into pwsh. + $i = 0 + foreach ($buildFlag in $buildFlags) { + # Quote values so characters such as ',' survive a repaste. Not needed + # for the direct invocation below. + $argWithValue = $buildFlag.Split('=', 2) + if ($argWithValue.Length -eq 2 -and !$argWithValue[1].StartsWith("`"")) { + $buildFlag = "$($argWithValue[0])=`"$($argWithValue[1])`"" + } + + if ($i -eq $buildFlags.Length - 1) { + Write-Host " $buildFlag" + } + else { + Write-Host " $buildFlag ``" + } + $i++ + } +} + +$oldGOEXPERIMENT = $env:GOEXPERIMENT +# Opt into per-iteration loop variables, which is what most readers expect and +# what the Go team intends to make the default. +$env:GOEXPERIMENT = "loopvar" + +try { + Write-Host "Running: go build ``" + PrintFlags -flags $buildFlags + go build @buildFlags + if ($LASTEXITCODE) { + Write-Host "Error running go build" + exit $LASTEXITCODE + } + + Write-Host "go build succeeded" +} +finally { + $env:GOEXPERIMENT = $oldGOEXPERIMENT +} diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 new file mode 100644 index 00000000000..6175585d318 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -0,0 +1,56 @@ +# Runs the unit tests and writes a JUnit report. +# +# The pipeline publishes **/junitTestReport.xml from the extension directory, +# so the report has to be written under that name for results to show up in the +# build. gotestsum produces it; the go test fallback does not, so the fallback +# only runs when gotestsum is unavailable. +# +# The live integration tests are excluded: they carry the `live` build tag, so +# an untagged run does not compile them, and they additionally require +# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. They are still type-checked +# below, so a change that breaks them cannot reach main unnoticed. +# +# TODO before the first release: PR CI runs this script on windows, linux and +# darwin amd64, so the untagged tests are covered on all three. The live and +# hero suites are only type-checked, never executed, and both have only ever +# run on Windows by hand. Run them once on linux, where they assume a path +# separator and shell out to `azd` and to a proxy address. + +$gopath = go env GOPATH +$gotestsumBinary = "gotestsum" +if ($IsWindows) { + $gotestsumBinary += ".exe" +} +$gotestsum = Join-Path $gopath "bin" $gotestsumBinary + +Write-Host "Running unit tests..." + +if (Test-Path $gotestsum) { + & $gotestsum --format testname --junitfile junitTestReport.xml -- ./... -count=1 +} else { + Write-Host "gotestsum not found; falling back to go test (no JUnit report)." -ForegroundColor Yellow + go test ./... -v -count=1 +} + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "Tests failed with exit code: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +# The tagged suites are never run here, so without this nothing compiles them +# and a change that breaks one reaches main silently. Type-checking needs no +# credentials, so it costs a few seconds and runs everywhere the tests do. +Write-Host "" +Write-Host "Type-checking the live and hero suites..." +go vet -tags live,hero ./... + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "The tagged test suites do not compile: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +Write-Host "" +Write-Host "All tests passed!" -ForegroundColor Green +exit 0 diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml new file mode 100644 index 00000000000..4d6c8ad651b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -0,0 +1,27 @@ +import: ../../.vscode/cspell.yaml +words: + # Go module and package names + - azureaieval + - evalcore + - httptest + - creack + # Service identifiers and API fields + - evalrun + - lookback + - AOAI + # Built-in evaluator names + - ifeval + - groundedness + # Repository names + - foundrysdk + # Terms + - inlines + - negotiables + - parseable + - retargeted + - subsetting + - undeployed + - undoable + - unpassed + - unscored + - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml new file mode 100644 index 00000000000..825f01ed341 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -0,0 +1,25 @@ +# yaml-language-server: $schema=../extension.schema.json +id: azure.ai.evaluations +namespace: ai.eval +displayName: Foundry evaluations (Beta) +description: Define and run Foundry evaluations from your terminal. (Beta) +usage: azd ai eval [options] +# NOTE: Make sure version.txt is in sync with this version. +version: 1.0.0-beta.1 +requiredAzdVersion: ">=1.27.1" +language: go +capabilities: + - custom-commands + - service-target-provider + - metadata +providers: + - name: azure.ai.eval + type: service-target + description: Deploys evaluation datasets, evaluators, and eval groups to Foundry +examples: + - name: init + description: Scaffold evaluation config for an agent. + usage: azd ai eval init + - name: run + description: Run an evaluation and summarize the results. + usage: azd ai eval run diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod new file mode 100644 index 00000000000..ab57a735e7f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -0,0 +1,106 @@ +module azureaieval + +go 1.26.4 + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 + github.com/azure/azure-dev/cli/azd v1.28.0 + github.com/fatih/color v1.18.0 + github.com/google/uuid v1.6.0 + github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 + go.yaml.in/yaml/v3 v3.0.4 + google.golang.org/protobuf v1.36.11 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect + github.com/alecthomas/chroma/v2 v2.20.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/braydonk/yaml v0.9.0 // indirect + github.com/buger/goterm v1.0.4 // indirect + github.com/buger/jsonparser v1.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.2 // indirect + github.com/charmbracelet/glamour v0.10.0 // indirect + github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect + github.com/charmbracelet/x/ansi v0.10.2 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cli/browser v1.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/drone/envsubst v1.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gofrs/flock v0.12.1 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golobby/container/v3 v3.3.2 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jmespath-community/go-jmespath v1.1.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.41.1 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect + github.com/microsoft/go-deviceid v1.0.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/theckman/yacspin v0.13.12 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yuin/goldmark v1.7.13 // indirect + github.com/yuin/goldmark-emoji v1.0.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cli/azd/extensions/azure.ai.evaluations/go.sum b/cli/azd/extensions/azure.ai.evaluations/go.sum new file mode 100644 index 00000000000..81c10e45793 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.sum @@ -0,0 +1,318 @@ +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= +github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 h1:0g4UTtvRA9goC37cmD9ZHdW6CCNJR4cOXBnHz0r4ubM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3/go.mod h1:fEiHi0sbYqbo3shUkIF1SNxm8GyeEJl+Poc/djOvbdE= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b h1:g9SuFmxM/WucQFKTMSP+irxyf5m0RiUJreBDhGI6jSA= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b/go.mod h1:XjvqMUpGd3Xn9Jtzk/4GEBCSoBX0eB2RyriXgne0IdM= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw= +github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA= +github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= +github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= +github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/braydonk/yaml v0.9.0 h1:ewGMrVmEVpsm3VwXQDR388sLg5+aQ8Yihp6/hc4m+h4= +github.com/braydonk/yaml v0.9.0/go.mod h1:hcm3h581tudlirk8XEUPDBAimBPbmnL0Y45hCRl47N4= +github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= +github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI= +github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI= +github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY= +github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= +github.com/charmbracelet/x/ansi v0.10.2 h1:ith2ArZS0CJG30cIUfID1LXN7ZFXRCww6RUvAPA+Pzw= +github.com/charmbracelet/x/ansi v0.10.2/go.mod h1:HbLdJjQH4UH4AqA2HpRWuWNluRE6zxJH/yteYEYCFa8= +github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= +github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 h1:a5q2sWiet6kgqucSGjYN1jhT2cn4bMKUwprtm2IGRto= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/cli/browser v1.3.0 h1:LejqCrpWr+1pRqmEPDGnTZOjsMe7sehifLynZJuqJpo= +github.com/cli/browser v1.3.0/go.mod h1:HH8s+fOAxjhQoBUAsKuPCbqUuxZDhQ2/aD+SzsEfBTk= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golobby/container/v3 v3.3.2 h1:7u+RgNnsdVlhGoS8gY4EXAG601vpMMzLZlYqSp77Quw= +github.com/golobby/container/v3 v3.3.2/go.mod h1:RDdKpnKpV1Of11PFBe7Dxc2C1k2KaLE4FD47FflAmj0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= +github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= +github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mark3labs/mcp-go v0.41.1 h1:w78eWfiQam2i8ICL7AL0WFiq7KHNJQ6UB53ZVtH4KGA= +github.com/mark3labs/mcp-go v0.41.1/go.mod h1:T7tUa2jO6MavG+3P25Oy/jR7iCeJPHImCZHRymCn39g= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= +github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= +github.com/microsoft/go-deviceid v1.0.0 h1:i5AQ654Xk9kfvwJeKQm3w2+eT1+ImBDVEpAR0AjpP40= +github.com/microsoft/go-deviceid v1.0.0/go.mod h1:KY13FeVdHkzD8gy+6T8+kVmD/7RMpTaWW75K+T4uZWg= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d h1:NqRhLdNVlozULwM1B3VaHhcXYSgrOAv8V5BE65om+1Q= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d/go.mod h1:cxIIfNMTwff8f/ZvRouvWYF6wOoO7nj99neWSx2q/Es= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4= +github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= +github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= +golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go new file mode 100644 index 00000000000..7333ada80eb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generation spec names an instructions file relative to itself, not to the +// working directory, so `generate --config ` reads the same file the +// author sees next to the spec. +func TestDeclaredInstructions_ResolvesRelativeToTheSpec(t *testing.T) { + dir := t.TempDir() + specDir := filepath.Join(dir, "evals") + require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o755)) + + body := "Answer only from the product catalog." + require.NoError(t, os.WriteFile( + filepath.Join(specDir, "agent", "instructions.md"), []byte(" "+body+"\n"), 0o600)) + + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(specDir, "generate.yaml")) + require.NoError(t, err) + assert.Equal(t, body, got, "the file's contents should be used, trimmed") +} + +// A path can be declared before that file exists. Treating the gap as an error +// would break the flow `init` itself scaffolds. +func TestDeclaredInstructions_MissingFileIsNotAnError(t *testing.T) { + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(t.TempDir(), "generate.yaml")) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestDeclaredInstructions_UnsetIsEmpty(t *testing.T) { + got, err := declaredInstructions("", "generate.yaml") + require.NoError(t, err) + assert.Empty(t, got) +} + +// Only the newest version is read, and an agent with no published version must +// not panic the caller. +func TestAgentInstructions(t *testing.T) { + var agent eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{ + "name": "support", + "versions": { "latest": { "version": "2", "definition": { + "model": "gpt-5-mini", + "instructions": " You are a support assistant.\n" } } } + }`), &agent)) + assert.Equal(t, "You are a support assistant.", agent.Instructions()) + + var empty eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{"name":"x","versions":{}}`), &empty)) + assert.Empty(t, empty.Instructions(), "an agent with no published version has no instructions") + + var nilAgent *eval_api.Agent + assert.Empty(t, nilAgent.Instructions()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go new file mode 100644 index 00000000000..0c3ddb78734 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +// API versions used by the Foundry data plane. +const ( + // ProjectEndpointAPIVersion covers datasets, evaluators, and evaluator + // generation jobs on the project endpoint. + ProjectEndpointAPIVersion = "2025-11-15-preview" + + // DataGenerationAPIVersion covers dataset generation jobs. + DataGenerationAPIVersion = "v1" + + // OpenAI-compatible eval and run calls send no api-version, so there + // is deliberately no constant for them. +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go new file mode 100644 index 00000000000..cdb6b5c3c4b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -0,0 +1,406 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "maps" + "sort" + "strings" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" +) + +// evaluatorSchemas indexes the published evaluator contracts by name. +// +// A failure is deliberately not fatal: without schemas the builder falls back +// to the agent-target shape, which is what it always used to send. +// evaluatorSchemas indexes the published contract of every evaluator a group +// can reference. +// +// Built-ins have to be asked for separately. An unfiltered list returns only +// the project's own evaluators, so relying on it leaves every built-in without +// a schema and falling back to legacyInputs — which happens to match +// query/response and so looks right for the common evaluators while quietly +// dropping the fields anything else needs. +func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { + index := map[string]*eval_api.EvaluatorSummary{} + + for _, filter := range []string{"", eval_api.EvaluatorTypeBuiltin} { + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) + if err != nil { + continue + } + maps.Copy(index, list.ByName()) + } + if len(index) == 0 { + return nil + } + return index +} + +// sampleBindings are the fields an agent target produces at run time. Anything +// an evaluator accepts that is not in this set has to come from a dataset +// column instead. +var sampleBindings = map[string]string{ + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", +} + +// modelSampleBindings are what a model target produces. A model answers as +// plain text and calls no tools, so binding an agent's richer output would +// leave the evaluator waiting on fields the run never produces. +// sampleBindingsFor returns the run-time bindings a target of this kind can +// satisfy. An empty target kind means nothing is invoked, so nothing is bound. +func sampleBindingsFor(targetType string) map[string]string { + if targetType == project.TargetTypeAgent { + return sampleBindings + } + return nil +} + +// legacyInputs is the mapping used when the service publishes no schema for an +// evaluator, which is the case for freshly uploaded custom evaluators. It +// matches the agent-target shape. +var legacyInputs = []string{"query", "response", "tool_calls", "tool_definitions"} + +// criterionPlan is the resolved binding for one evaluator. +type criterionPlan struct { + dataMapping map[string]string + initParams map[string]any + // itemFields are the fields sourced from dataset columns; they have to be + // declared in the item schema. + itemFields []string +} + +// conversationField carries a whole conversation. The service rejects a +// mapping that pairs it with the turn-level fields: +// +// Evaluator 'builtin.task_completion' has both 'messages' and +// 'query'/'response' in data_mapping. Use 'messages' for conversation-level +// evaluation or 'query'/'response' for turn-level evaluation, but not both. +const conversationField = "messages" + +// turnFields are the per-turn counterparts to conversationField. +var turnFields = []string{"query", "response"} + +// selectLevelFields resolves the conversation/turn exclusivity for evaluators +// that accept both shapes, keeping whichever matches the evaluation level. +// Required fields are never dropped, so a genuine conflict still surfaces as a +// missing-field error rather than being silently reshaped. +func selectLevelFields(accepted, required []string, level string) []string { + isRequired := make(map[string]bool, len(required)) + for _, name := range required { + isRequired[name] = true + } + + acceptsConversation := false + acceptsTurn := false + for _, field := range accepted { + if field == conversationField { + acceptsConversation = true + } + for _, turn := range turnFields { + if field == turn { + acceptsTurn = true + } + } + } + if !acceptsConversation || !acceptsTurn { + return accepted + } + + drop := map[string]bool{} + if strings.EqualFold(level, project.EvaluationLevelConversation) { + for _, turn := range turnFields { + drop[turn] = true + } + } else { + drop[conversationField] = true + } + + kept := make([]string, 0, len(accepted)) + for _, field := range accepted { + if drop[field] && !isRequired[field] { + continue + } + kept = append(kept, field) + } + return kept +} + +// planCriterion shapes one evaluator's bindings from its published contract. +// +// Evaluators do not share an input contract: builtin.similarity needs +// ground_truth, builtin.retrieval needs context, and builtin.ifeval needs +// instruction_id_list. Sending one fixed mapping to all of them earns a +// service-side MissingRequiredDataMapping rejection, so the mapping is derived +// per evaluator and anything unsatisfiable is reported before the request is +// sent. +func planCriterion( + ref evalcore.EvaluatorRef, + schema *eval_api.EvaluatorSummary, + targetBindings map[string]string, + datasetColumns map[string]bool, + level string, +) (*criterionPlan, error) { + accepted := legacyInputs + var required []string + // A published schema is authoritative even when it is empty: an empty + // property set means the evaluator accepts nothing, which is different from + // publishing no schema at all. + if dataSchema := schema.DataSchema(); dataSchema != nil { + accepted = dataSchema.PropertyNames() + required = dataSchema.Required + } + accepted = selectLevelFields(accepted, required, level) + + plan := &criterionPlan{ + dataMapping: map[string]string{}, + initParams: map[string]any{}, + } + + for _, field := range accepted { + if binding, ok := targetBindings[field]; ok { + plan.dataMapping[field] = binding + continue + } + // Everything else comes from the dataset. When the columns are known, + // bind only the ones that exist so optional fields stay unbound rather + // than resolving to nothing at run time. + if datasetColumns != nil && !datasetColumns[field] { + continue + } + plan.dataMapping[field] = fmt.Sprintf("{{item.%s}}", field) + plan.itemFields = append(plan.itemFields, field) + } + + // A declared mapping is the author saying the inference got it wrong, so it + // wins. Anything it binds to an item column is a column the schema has to + // declare, whether or not inference found it. + for field, binding := range ref.DataMapping { + plan.dataMapping[field] = binding + if column, ok := itemColumn(binding); ok && !contains(plan.itemFields, column) { + plan.itemFields = append(plan.itemFields, column) + } + } + + var missing []string + for _, field := range required { + if _, ok := plan.dataMapping[field]; !ok { + missing = append(missing, field) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf( + "evaluator %q requires %s, which the dataset does not provide; "+ + "add %s to the dataset, or bind it with `data_mapping`", + ref.Evaluator, quoteList(missing), pluralColumns(missing), + ) + } + + if !schema.SupportsLevel(level) { + return nil, fmt.Errorf( + "evaluator %q does not support evaluation level %q; it supports %s", + ref.Evaluator, level, quoteList(schema.SupportedEvaluationLevels), + ) + } + + initSchema := schema.InitSchema() + accepts := func(name string) bool { + // Only an absent schema falls back to the historical parameters; + // builtin.ifeval publishes an empty one and takes none. + if initSchema == nil { + return name == "deployment_name" || name == "threshold" + } + return initSchema.Accepts(name) + } + + // Evaluators disagree on what the judge model is called: built-ins declare + // deployment_name, custom rubrics declare model. The declaration names one + // of them; bind whichever the evaluator actually accepts rather than + // forwarding a spelling it will reject. + for name, value := range ref.InitializationParameters { + if accepts(name) { + plan.initParams[name] = value + continue + } + if alias, ok := judgeModelAliases[name]; ok && accepts(alias) { + plan.initParams[alias] = value + } + } + if level != "" && accepts("evaluation_level") { + plan.initParams["evaluation_level"] = level + } + + if initSchema != nil { + var missingInit []string + for _, name := range initSchema.Required { + if _, ok := plan.initParams[name]; !ok { + missingInit = append(missingInit, name) + } + } + if len(missingInit) > 0 { + return nil, fmt.Errorf( + "evaluator %q requires %s; set it under the evaluator's "+ + "`initialization_parameters` in the eval config", + ref.Evaluator, quoteList(missingInit), + ) + } + } + + return plan, nil +} + +// judgeModelAliases maps the two spellings of the judge deployment onto each +// other, so one declaration works whichever the evaluator publishes. +var judgeModelAliases = map[string]string{ + "deployment_name": "model", + "model": "deployment_name", +} + +// itemColumn reads the dataset column out of an `{{item.}}` binding. +func itemColumn(binding string) (string, bool) { + const prefix, suffix = "{{item.", "}}" + if !strings.HasPrefix(binding, prefix) || !strings.HasSuffix(binding, suffix) { + return "", false + } + name := strings.TrimSuffix(strings.TrimPrefix(binding, prefix), suffix) + if name == "" { + return "", false + } + return name, true +} + +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} + +// buildEvalRequest converts an eval declaration into the create +// request. Each evaluator becomes a testing criterion bound to its own +// contract, and the item schema declares every dataset column those bindings +// reference. +// +// schemas may be nil or partial; an evaluator with no published contract falls +// back to the agent-target shape. datasetColumns may be nil, meaning the +// columns are unknown and every accepted field is assumed present. +func buildEvalRequest( + group *project.Eval, + schemas map[string]*eval_api.EvaluatorSummary, + datasetColumns map[string]bool, +) (*eval_api.CreateOpenAIEvalRequest, error) { + metadata := map[string]string{} + hasTarget := group.Target != nil && group.Target.Name != "" + targetType := "" + if hasTarget { + metadata["azd_agent"] = group.Target.Name + targetType = group.Target.Type + if targetType == "" { + targetType = project.TargetTypeAgent + } + } + targetBindings := sampleBindingsFor(targetType) + metadata["azd_eval"] = group.Name + // The create request has no description field, so the group's own + // description rides in metadata rather than being dropped. + if group.Description != "" { + metadata["azd_description"] = group.Description + } + + level := group.EvaluationLevel + + req := &eval_api.CreateOpenAIEvalRequest{ + Name: group.Name, + Metadata: metadata, + } + + itemFields := map[string]bool{} + + for _, ref := range group.Evaluators { + schema := schemas[ref.Evaluator] + if schema == nil { + schema = &eval_api.EvaluatorSummary{Name: ref.Evaluator} + } + + plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, level) + if err != nil { + return nil, err + } + + criterion := eval_api.TestingCriterion{ + Type: "azure_ai_evaluator", + // Name labels the criterion in results and defaults to the + // evaluator without its builtin prefix; EvaluatorName keeps it. + Name: ref.CriterionName(), + EvaluatorName: ref.Evaluator, + DataMapping: plan.dataMapping, + } + if ref.Version != "" { + criterion.EvaluatorVersion = ref.Version + } + if len(plan.initParams) > 0 { + criterion.InitializationParameters = plan.initParams + } + for _, field := range plan.itemFields { + itemFields[field] = true + } + + req.TestingCriteria = append(req.TestingCriteria, criterion) + } + + req.DataSourceConfig = &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: hasTarget, + ItemSchema: itemSchema(itemFields), + } + + return req, nil +} + +// itemSchema declares the dataset columns the criteria bind to. It always +// declares at least `query`, the column an agent target reads. +func itemSchema(fields map[string]bool) map[string]any { + if len(fields) == 0 { + fields = map[string]bool{"query": true} + } + properties := map[string]any{} + for field := range fields { + properties[field] = map[string]any{"type": "string"} + } + return map[string]any{ + "type": "object", + "properties": properties, + } +} + +func quoteList(values []string) string { + if len(values) == 0 { + return "nothing" + } + quoted := make([]string, 0, len(values)) + for _, value := range values { + quoted = append(quoted, fmt.Sprintf("%q", value)) + } + sort.Strings(quoted) + if len(quoted) == 1 { + return quoted[0] + } + return strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] +} + +func pluralColumns(values []string) string { + if len(values) == 1 { + return "that column" + } + return "those columns" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go new file mode 100644 index 00000000000..59e95d41c99 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// This file proves the request buildEvalRequest produces is accepted by +// the real service. It lives in the cmd package on purpose: the tests under +// tests/live can only hand-roll a request, which validates the API but not the +// code that ships. +// +// go test -tags live -v ./internal/cmd/ -run TestLiveBuild +// +// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. + +package cmd + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + }) + return sharedCred, sharedCredErr +} + +// credentialFlake is what a token refresh that overran its budget looks like +// by the time it reaches a test. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// retryingCredential retries a token request that failed for that reason. +// +// The refresh shells out to azd, and the SDK gives that subprocess ten +// seconds. On a machine already running the rest of this suite it sometimes +// does not finish in ten, and the failure lands on whichever test asked for a +// token at the wrong moment — reproducibly at 10.1s, and never when that test +// is run on its own. Retrying is right because nothing about the request was +// wrong: the same call succeeds moments later. +type retryingCredential struct { + inner azcore.TokenCredential +} + +func (c retryingCredential) GetToken( + ctx context.Context, + opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + var token azcore.AccessToken + var err error + for attempt := range 4 { + if attempt > 0 { + select { + case <-ctx.Done(): + return azcore.AccessToken{}, ctx.Err() + case <-time.After(time.Duration(attempt) * 2 * time.Second): + } + } + token, err = c.inner.GetToken(ctx, opts) + if err == nil || !strings.Contains(err.Error(), credentialFlake) { + return token, err + } + } + return token, err +} + +func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + cred, err := liveCredential() + require.NoError(t, err) + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = "gpt-4.1-nano" + } + return eval_api.NewEvalClient(endpoint, retryingCredential{inner: cred}), judge +} + +// TestLiveBuildAcceptedForEveryBuiltin walks every built-in the project +// exposes, builds a group with the shipping builder, and posts it. +// +// Each evaluator declares a different input contract, so this is the test that +// would have caught the fixed data mapping: it previously produced a +// MissingRequiredDataMapping rejection for builtin.ifeval and would do so +// again for any evaluator whose contract the builder stops honouring. +func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, listed.Value) + + // Deliberately the production lookup rather than the listing above. Taking + // the schemas straight from a filtered list is what let this test pass + // while the shipping path resolved none of them: it built the input the + // product was failing to build. + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) + + for _, summary := range listed.Value { + summary := summary + t.Run(summary.Name, func(t *testing.T) { + require.NotNil(t, schemas[summary.Name], + "the shipping lookup did not resolve %s", summary.Name) + // Give the builder a dataset carrying every column the evaluator + // accepts, so a rejection means the request shape is wrong rather + // than the data being genuinely absent. + columns := map[string]bool{"query": true} + if ds := summary.DataSchema(); ds != nil { + for _, name := range ds.PropertyNames() { + columns[name] = true + } + } + + level := "" + if len(summary.SupportedEvaluationLevels) > 0 { + level = summary.SupportedEvaluationLevels[0] + } + + group := &project.Eval{ + Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Evaluator: summary.Name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, + EvaluationLevel: level, + } + + req, err := buildEvalRequest(group, schemas, columns) + require.NoError(t, err, "the builder must satisfy every published contract") + + created, err := client.CreateOpenAIEval(ctx, req) + require.NoError(t, err, + "the service rejected the request this extension builds for %s", summary.Name) + require.NotEmpty(t, created.ID) + t.Cleanup(func() { + _ = client.DeleteOpenAIEval(context.Background(), created.ID) + }) + t.Logf("%s accepted as %s", summary.Name, created.ID) + }) + } +} + +// TestLiveBuildRejectsMissingColumnsLocally proves the pre-flight check fires +// before the network call, so a user sees which column is missing instead of a +// service error naming an internal field path. +func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + schemas := listed.ByName() + + target, ok := schemas["builtin.ifeval"] + if !ok { + t.Skip("builtin.ifeval is not available in this project") + } + require.NotNil(t, target.DataSchema()) + require.NotEmpty(t, target.DataSchema().Required, + "this test relies on ifeval declaring required inputs") + + group := &project.Eval{ + Name: "azd-live-negative", + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Name: "builtin.ifeval", + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, + } + + // A dataset with only `query` cannot satisfy ifeval. + _, err = buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + t.Logf("pre-flight error: %v", err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go new file mode 100644 index 00000000000..0bacd287edf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -0,0 +1,320 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// schema builds an evaluator contract the way the service publishes one. +func schema(name string, dataRequired, dataProps, initRequired, initProps []string, levels ...string) *eval_api.EvaluatorSummary { + toProps := func(names []string) map[string]any { + if names == nil { + return nil + } + out := map[string]any{} + for _, n := range names { + out[n] = map[string]any{"type": "string"} + } + return out + } + return &eval_api.EvaluatorSummary{ + Name: name, + SupportedEvaluationLevels: levels, + Definition: &eval_api.EvaluatorContract{ + DataSchema: &eval_api.JSONSchema{Required: dataRequired, Properties: toProps(dataProps)}, + InitParameters: &eval_api.JSONSchema{Required: initRequired, Properties: toProps(initProps)}, + }, + } +} + +func groupWith(evaluators []evalcore.EvaluatorRef, level string) *project.Eval { + return &project.Eval{ + Name: "g", + Dataset: "d", + Target: &project.Target{Type: "agent", Name: "my-agent"}, + Evaluators: evaluators, + EvaluationLevel: level, + } +} + +// withJudge declares the judge deployment where the service reads it from: an +// evaluator's initialization parameters, not a setting on the eval. It merges, +// so a parameter the reference already carries survives. +func withJudge(model string, refs ...evalcore.EvaluatorRef) []evalcore.EvaluatorRef { + for i := range refs { + if refs[i].InitializationParameters == nil { + refs[i].InitializationParameters = map[string]any{} + } + refs[i].InitializationParameters["deployment_name"] = model + } + return refs +} + +// An agent evaluator takes its response from the sample and its query from the +// dataset. +func TestBuildBindsAgentFieldsFromSample(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_adherence": schema("builtin.task_adherence", + nil, []string{"query", "response", "tool_definitions", "messages"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold", "evaluation_level"}, + "turn"), + } + group := groupWith( + withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Evaluator: "builtin.task_adherence"}), + "", + ) + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Len(t, req.TestingCriteria, 1) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + // `messages` is not a dataset column here, so it stays unbound. + require.NotContains(t, mapping, "messages") +} + +// A required field the dataset does not carry is reported before the request +// is sent, naming the field. +func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") + + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + require.Contains(t, err.Error(), "instruction_kwargs") +} + +// The same evaluator succeeds once the dataset supplies the columns. +func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{ + "instruction_id_list": true, + "instruction_kwargs": true, + }) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + // response is satisfied by the agent target. + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{item.instruction_id_list}}", mapping["instruction_id_list"]) + + // The item schema has to declare the columns the criteria reference. + props := req.DataSourceConfig.ItemSchema["properties"].(map[string]any) + require.Contains(t, props, "instruction_id_list") + require.Contains(t, props, "instruction_kwargs") +} + +// Initialization parameters are filtered to what the evaluator accepts. +// builtin.ifeval takes none, so nothing is sent even when a model is set. +func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { + threshold := 4.0 + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + nil, []string{"response"}, nil, nil, "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response", "ground_truth"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Evaluator: "builtin.ifeval", + InitializationParameters: map[string]any{"threshold": threshold}}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity", + InitializationParameters: map[string]any{"threshold": threshold}}, + ), "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{ + "query": true, "ground_truth": true, + }) + require.NoError(t, err) + + // ifeval accepts no init parameters at all. + require.Empty(t, req.TestingCriteria[0].InitializationParameters) + + // similarity accepts both, and never the `model` alias. + params := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", params["deployment_name"]) + require.InDelta(t, 4.0, params["threshold"], 0.0001) + require.NotContains(t, params, "model") +} + +// evaluation_level is an initialization parameter, not run metadata, and only +// on evaluators that declare it. +func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith(withJudge("m", + evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + ), "turn") + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + require.Equal(t, "turn", req.TestingCriteria[0].InitializationParameters["evaluation_level"]) + require.NotContains(t, req.TestingCriteria[1].InitializationParameters, "evaluation_level") +} + +// An evaluator that does not support the requested level is rejected with the +// levels it does support. +func TestBuildRejectsUnsupportedLevel(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), + "conversation") + + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "conversation") + require.Contains(t, err.Error(), "turn") +} + +// A required init parameter with no judge model configured is caught locally. +func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") + + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "deployment_name") +} + +// An evaluator with no published contract keeps the historical agent-target +// shape, so custom evaluators still deploy. +func TestBuildFallsBackWithoutSchema(t *testing.T) { + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "my-custom-evaluator"}), "") + + req, err := buildEvalRequest(group, nil, nil) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_calls}}", mapping["tool_calls"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + require.Equal(t, "m", req.TestingCriteria[0].InitializationParameters["deployment_name"]) +} + +// `messages` and `query`/`response` are mutually exclusive; the evaluation +// level picks which shape is bound. Sending both is rejected by the service. +func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response", "messages", "tool_definitions"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + } + columns := map[string]bool{"query": true, "messages": true, "response": true} + + // Turn level keeps query/response and drops messages. + turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "turn") + req, err := buildEvalRequest(turn, schemas, columns) + require.NoError(t, err) + mapping := req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "query") + require.NotContains(t, mapping, "messages") + + // Conversation level keeps messages and drops query/response. + conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "conversation") + req, err = buildEvalRequest(conv, schemas, columns) + require.NoError(t, err) + mapping = req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "messages") + require.NotContains(t, mapping, "query") + require.NotContains(t, mapping, "response") + + // An unset level behaves as turn, matching the service default. + dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), "") + req, err = buildEvalRequest(dflt, schemas, columns) + require.NoError(t, err) + require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") +} + +// Evaluators disagree on what the judge model is called. Built-ins declare +// deployment_name; a custom rubric declares model, and rejects the eval with +// "requires model" if only deployment_name is sent. One declaration binds +// whichever the evaluator actually accepts. +func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + "my-rubric": schema("my-rubric", + nil, []string{"query", "response"}, + []string{"model"}, []string{"model"}, "turn"), + } + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + evalcore.EvaluatorRef{Evaluator: "my-rubric"}, + ), "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + builtin := req.TestingCriteria[0].InitializationParameters + require.Equal(t, "gpt-4.1-nano", builtin["deployment_name"]) + require.NotContains(t, builtin, "model") + + custom := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", custom["model"]) + require.NotContains(t, custom, "deployment_name") +} + +// Without an agent target the sample bindings are unavailable, so every field +// has to come from the dataset and the sample schema is not requested. +func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + []string{"query", "response", "ground_truth"}, + []string{"query", "response", "ground_truth"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") + group.Target = nil + + req, err := buildEvalRequest(group, schemas, map[string]bool{ + "query": true, "response": true, "ground_truth": true, + }) + require.NoError(t, err) + require.False(t, req.DataSourceConfig.IncludeSampleSchema) + require.Equal(t, "{{item.response}}", req.TestingCriteria[0].DataMapping["response"]) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go new file mode 100644 index 00000000000..2e34a944b12 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation writes the artifact and then names it in the configuration, so +// what it produced is referenceable without a hand edit. Only the catalogs are +// touched: which evals use the artifact is the author's decision, and `init` is +// the command that makes it. + +// addDatasetToCatalog records a generated dataset in `datasets:`. +func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + for i := range cfg.Datasets { + if cfg.Datasets[i].Name == ref.Name { + // Regeneration overwrites the file in place, so the entry only + // changes when the artifact moved. + if cfg.Datasets[i].Source == ref.Source { + return false + } + cfg.Datasets[i].Source = ref.Source + return true + } + } + cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// addEvaluatorToCatalog records a generated evaluator in `evaluators:`. +func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + for i := range cfg.Evaluators { + if cfg.Evaluators[i].Name == ref.Name { + if cfg.Evaluators[i].Source == ref.Source { + return false + } + cfg.Evaluators[i].Source = ref.Source + return true + } + } + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// updateCatalog applies a change to the configuration and writes it back. +// +// A missing configuration is created holding only the catalog. `generate` runs +// before `init` on the golden path, and a downloaded artifact nobody recorded +// is the one state that goes stale. The file it creates has no evals and no +// azure.yaml entry, so it stays inert until init wires one. +func updateCatalog( + cmd *cobra.Command, + evalDir string, + apply func(*project.EvalConfig) bool, +) error { + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return err + } + created := cfg == nil + if created { + cfg = &project.EvalConfig{} + } + if !apply(cfg) { + return nil + } + + if err := project.SaveEvalConfig(evalDir, cfg); err != nil { + return err + } + if !isJSON(cmd) { + path := filepath.ToSlash(project.EvalConfigPath(evalDir)) + if created { + fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Created %s with the catalog entry\n", path) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Added catalog entry to %s\n", path) + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go new file mode 100644 index 00000000000..1c897a0f906 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectEndpointEnvKey is the azd environment key holding the Foundry project +// endpoint the data-plane clients target. +const projectEndpointEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + +// evalContext carries everything the commands need to reach the data plane. +type evalContext struct { + azdClient *azdext.AzdClient + endpoint string + envName string + cred azcore.TokenCredential + + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient +} + +// newEvalContext resolves the project endpoint and builds the data-plane +// clients. Endpoint resolution order: +// +// 1. --project-endpoint +// 2. the active azd environment's FOUNDRY_PROJECT_ENDPOINT +// 3. the host environment variable of the same name +func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, fmt.Errorf("connecting to azd: %w", err) + } + + ec := &evalContext{azdClient: azdClient} + + // The environment name is resolved regardless of where the endpoint comes + // from: it is what the cached eval and run ids are read from and + // written to. Deriving it only when the endpoint came from azd meant + // --project-endpoint silently disabled that cache. + azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) + ec.envName = envName + + if endpointFlag != "" { + ec.endpoint = endpointFlag + } else { + ec.endpoint = azdEndpoint + } + if ec.endpoint == "" { + ec.endpoint = os.Getenv(projectEndpointEnvKey) + } + if ec.endpoint == "" { + return nil, fmt.Errorf( + "no Foundry project endpoint found; pass --project-endpoint or set %s "+ + "in the azd environment (azd env set %s )", + projectEndpointEnvKey, projectEndpointEnvKey) + } + ec.endpoint = strings.TrimSuffix(ec.endpoint, "/") + + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + if err != nil { + return nil, fmt.Errorf("creating Azure credential: %w", err) + } + ec.cred = cred + + ec.evalClient = eval_api.NewEvalClient(ec.endpoint, cred) + ec.datasetClient = dataset_api.NewDatasetClient(ec.endpoint, cred) + + return ec, nil +} + +// lookupEndpointFromAzd reads the endpoint from the active azd environment, +// returning empty strings when azd has no current environment. +func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { + envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return "", "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: projectEndpointEnvKey, + }) + if err != nil || val == nil || val.Value == "" { + return "", envResp.Environment.Name + } + return val.Value, envResp.Environment.Name +} + +// errNoAzdEnvironment reports that there is no azd environment to persist into. +// +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary rather than a problem worth reporting. +// A write that fails for any other reason still is. +var errNoAzdEnvironment = errors.New("no active azd environment") + +// setEnvValue persists a value into the active azd environment. azd itself +// writes none of these keys — the extension owns them. +func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { + if ec.envName == "" { + envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return fmt.Errorf("%w to write %s into", errNoAzdEnvironment, key) + } + ec.envName = envResp.Environment.Name + } + _, err := ec.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: ec.envName, + Key: key, + Value: value, + }) + if err != nil { + return fmt.Errorf("writing %s to the azd environment: %w", key, err) + } + return nil +} + +// getEnvValue reads a value from the active azd environment, returning empty +// when it is unset. +func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { + if ec.envName == "" { + return "" + } + val, err := ec.azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: ec.envName, + Key: key, + }) + if err != nil || val == nil { + return "" + } + return val.Value +} + +// appInsightsEnvKey is where a connected Application Insights resource lands in +// the azd environment. azd's own provisioning writes it, and the agents +// extension reads the same key to pass tracing configuration to a running +// agent, so its presence is the project's answer to "are traces being +// collected?". +const appInsightsEnvKey = "APPLICATIONINSIGHTS_CONNECTION_STRING" + +// defaultGenerationSource picks what `dataset generate` sends when --from was +// not given, from the Application Insights connection string the project has +// (or has not) been given. +// +// Traces are the better dataset when they exist, being real conversations +// rather than synthesized ones, so they win whenever the project is wired to +// collect them. Outside a project, or in one with no Application Insights, +// there are no traces to ask for and the agent's own definition is all that is +// left. +func defaultGenerationSource(appInsightsConnection string) []string { + if appInsightsConnection != "" { + return []string{project.GenerateFromTraces} + } + return []string{project.GenerateFromAgent} +} + +func (ec *evalContext) Close() { + if ec.azdClient != nil { + ec.azdClient.Close() + } +} + +// azd environment keys written by this extension. +const ( + envKeyEvalID = "EVAL_ID" + envKeyEvalRunID = "EVAL_RUN_ID" + envKeyDatasetVersion = "EVAL_DATASET_VERSION" + envKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go new file mode 100644 index 00000000000..05a42c354e1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -0,0 +1,338 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// firstDatasetVersion is the version the service assigns to a dataset's first +// publish, and so the one that exists for every dataset that exists at all. +const firstDatasetVersion = "1" + +func newDatasetCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "dataset", + Short: "Manage evaluation datasets.", + } + cmd.AddCommand( + newDatasetCreateCommand(), + newDatasetUpdateCommand(), + newDatasetGenerateCommand(), + newDatasetListCommand(), + newDatasetShowCommand(), + newDatasetDeleteCommand(), + newDatasetVersionsCommand(), + newJobCommand(datasetJobs), + ) + return cmd +} + +// newDatasetCreateCommand builds `dataset create `, which registers a +// dataset that does not exist yet. +func newDatasetCreateCommand() *cobra.Command { + return newDatasetWriteCommand("create", "Register a dataset, publishing its first version.") +} + +// newDatasetUpdateCommand builds `dataset update `, which publishes a +// further version of one that does. +func newDatasetUpdateCommand() *cobra.Command { + return newDatasetWriteCommand("update", "Publish a new version of a dataset.") +} + +// newDatasetWriteCommand builds create and update. Both run the same upload, +// and the existence check is the only thing that separates them: a version is +// brought into being by startPendingUpload, which neither knows nor cares +// whether the name was already in use. +func newDatasetWriteCommand(verb, short string) *cobra.Command { + var ( + fromFile string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: verb + " ", + Short: short, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if fromFile == "" { + return requireFlag("from-file") + } + + localDir, err := datasetUploadDir(fromFile) + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + existing, err := ec.datasetClient.ListDatasetVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + exists := err == nil && existing != nil && len(existing.Value) > 0 + if !exists { + // The version listing lags a publish, so a `create` followed by + // an `update` was told the dataset it had just made does not + // exist. A direct read of the first version settles it: point + // reads go consistent immediately. + if _, err := ec.datasetClient.GetDataset( + ctx, name, firstDatasetVersion, ProjectEndpointAPIVersion, + ); err == nil { + exists = true + } + } + if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { + return err + } + + ds, err := ec.datasetClient.UploadNextVersion( + ctx, name, version, localDir, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("registering dataset %q: %w", name, err) + } + + if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { + // Persisting is a convenience, so this never fails the command. + // It goes to stdout because azd does not surface an extension's + // stderr, and is skipped outside a project, where having nowhere + // to persist is expected rather than notable. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprintf(cmd.OutOrStdout(), "warning: %v\n", err) + } + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + fmt.Fprintf(cmd.OutOrStdout(), "Registered dataset %s version %s\n", ds.Name, ds.Version) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Path to a .jsonl file, or a directory containing one.") + cmd.Flags().StringVar(&version, "version", "", + "Current version to increment from. Omit to increment from the latest registered version.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// datasetUploadDir resolves what was named into the directory the upload scans. +func datasetUploadDir(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("reading --from-file %q: %w", path, err) + } + if info.IsDir() { + return path, nil + } + if !strings.EqualFold(filepath.Ext(path), ".jsonl") { + return "", fmt.Errorf( + "--from-file must be a .jsonl file or a directory containing one, got %q", path) + } + return filepath.Dir(path), nil +} + +func newDatasetListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's datasets.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing datasets: %w", err) + } + return renderDatasets(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newDatasetVersionsCommand groups the version listing, so that `list` means +// the assets rather than the history of one of them. +func newDatasetVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one dataset.", + } + cmd.AddCommand(newDatasetVersionsListCommand()) + return cmd +} + +func newDatasetVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of a dataset.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing versions of dataset %q: %w", name, err) + } + return renderDatasets(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + rows := make([][]string, 0, len(list.Value)) + for _, d := range list.Value { + rows = append(rows, []string{d.Name, d.Version, d.Format}) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") + return nil + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) +} + +func newDatasetShowCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a dataset version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if version == "" { + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("resolving the latest version of %q: %w", name, err) + } + if len(list.Value) == 0 { + return fmt.Errorf("dataset %q has no versions", name) + } + version = dataset_api.LatestVersion(list.Value) + } + + ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", name, version) + } + return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + return emitTable(cmd.OutOrStdout(), + []string{"NAME", "VERSION", "FORMAT", "URI"}, + [][]string{{ds.Name, ds.Version, ds.Format, ds.ResolvedBlobURI()}}, + ) + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetDeleteCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a dataset version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.datasetClient.DeleteDatasetVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project", name, version) + } + return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset %s version %s\n", name, version) + return nil + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go new file mode 100644 index 00000000000..dd25bcf022b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadJSONLBytes(t *testing.T) { + content := []byte( + "{\"query\":\"a\"}\n" + + "\n" + // blank lines are skipped, not treated as rows + "{\"query\":\"b\"}\n" + + " {\"query\":\"c\"} \n") + + all, err := readJSONLBytes(content, 0) + require.NoError(t, err) + require.Len(t, all, 3) + assert.Equal(t, "a", all[0]["query"]) + assert.Equal(t, "c", all[2]["query"], "surrounding whitespace is not part of the row") +} + +// The limit is what makes --max-samples mean the same thing for a published +// dataset as for a local file. +func TestReadJSONLBytes_StopsAtTheLimit(t *testing.T) { + content := []byte("{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n") + + two, err := readJSONLBytes(content, 2) + require.NoError(t, err) + require.Len(t, two, 2) + assert.EqualValues(t, 1, two[0]["n"]) + assert.EqualValues(t, 2, two[1]["n"]) + + // A limit larger than the file is not an error. + more, err := readJSONLBytes(content, 99) + require.NoError(t, err) + assert.Len(t, more, 3) +} + +func TestReadJSONLBytes_ReportsTheOffendingLine(t *testing.T) { + _, err := readJSONLBytes([]byte("{\"n\":1}\nnot json\n"), 0) + require.ErrorContains(t, err, "line 2") +} + +func TestReadJSONLBytes_EmptyIsNotAnError(t *testing.T) { + items, err := readJSONLBytes([]byte("\n\n"), 0) + require.NoError(t, err) + assert.Empty(t, items, "the caller decides whether no rows is a problem") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go new file mode 100644 index 00000000000..a703889a235 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// A version is what an eval binds to, so publishing must always add one and +// never change one that exists. Evaluators needed a guard for that; this is +// the same question asked of datasets, against the real service. + +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// liveDatasetClient builds a dataset client against the live project. +func liveDatasetClient(t *testing.T) *dataset_api.DatasetClient { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := os.Getenv("FOUNDRY_PROJECT_ENDPOINT") + require.NotEmpty(t, endpoint, "FOUNDRY_PROJECT_ENDPOINT is required") + + cred, err := liveCredential() + require.NoError(t, err) + return dataset_api.NewDatasetClient(endpoint, retryingCredential{inner: cred}) +} + +// writeRows puts a one-row JSONL file in its own directory, which is what the +// upload path reads from. +func writeRows(t *testing.T, answer string) string { + t.Helper() + dir := t.TempDir() + row := fmt.Sprintf(`{"query":"q","response":%q}`+"\n", answer) + require.NoError(t, os.WriteFile(filepath.Join(dir, "rows.jsonl"), []byte(row), 0o600)) + return dir +} + +// TestLiveDatasetVersionIsNeverOverwritten publishes at a version that already +// exists and requires the service to refuse. +// +// The reconciler relies on exactly this: when an author pins `version:` and +// the local content has changed, it publishes at that version and treats a +// conflict as the signal to stop. If the service accepted the write instead, +// the pinned version would silently change under every eval bound to it, and +// `azd up` would report success. +func TestLiveDatasetVersionIsNeverOverwritten(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_immutable_%d", time.Now().UnixNano()) + + first, err := client.UploadVersion( + ctx, name, "1", writeRows(t, "original"), ProjectEndpointAPIVersion) + require.NoError(t, err) + require.Equal(t, "1", first.Version) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, "1", ProjectEndpointAPIVersion) + }) + + _, err = client.UploadVersion( + ctx, name, "1", writeRows(t, "replacement"), ProjectEndpointAPIVersion) + require.Error(t, err, + "publishing over an existing dataset version must be refused, not accepted") + assert.True(t, dataset_api.IsVersionConflict(err), + "the refusal must be a conflict the reconciler can recognise; got: %v", err) +} + +// TestLiveDatasetUpdateAddsAVersion is the other half: the ordinary path must +// keep adding versions rather than reusing the newest. +func TestLiveDatasetUpdateAddsAVersion(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_next_%d", time.Now().UnixNano()) + + first, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "one"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, first.Version, ProjectEndpointAPIVersion) + }) + + // Immediate, because the version listing lags a publish and this is the + // window where a second upload could be told the dataset is new and + // restart at the version the first one just took. + second, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "two"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, second.Version, ProjectEndpointAPIVersion) + }) + + assert.NotEqual(t, first.Version, second.Version, + "a second upload must add a version rather than reuse the first") + + // Both readable, and the first still holding what it was published with. + original, err := client.GetDataset(ctx, name, first.Version, ProjectEndpointAPIVersion) + require.NoError(t, err) + assert.NotEmpty(t, original.Version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go new file mode 100644 index 00000000000..48fc0802aaf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "io" + "log" + "os" + "strconv" + "time" + + azcorelog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" + "github.com/spf13/pflag" +) + +// setupDebugLogging silences the standard logger unless debug mode is on. +// +// The data-plane clients trace every request through log.Printf, which Go +// writes to stderr by default. Without this the CLI interleaves raw HTTP traces +// with its own output on every command. Returns a cleanup function the caller +// should defer. +func setupDebugLogging(flags *pflag.FlagSet) func() { + if !isDebug(flags) { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + return func() {} + } + + logFileName := fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")) + + //nolint:gosec // the name is generated locally from the date, not user input + logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + + var w io.Writer + var closeFile func() + if err != nil { + w = os.Stderr + closeFile = func() {} + } else { + w = logFile + closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup + } + + log.SetOutput(w) + azcorelog.SetListener(func(event azcorelog.Event, msg string) { + fmt.Fprintf(w, "[%s] %s: %s\n", time.Now().Format(time.RFC3339), event, msg) + }) + + return func() { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + closeFile() + } +} + +// isDebug reports whether --debug or AZD_EXT_DEBUG is set. +func isDebug(flags *pflag.FlagSet) bool { + if debugFlag, err := flags.GetBool("debug"); err == nil && debugFlag { + return true + } + debug, _ := strconv.ParseBool(os.Getenv("AZD_EXT_DEBUG")) + return debug +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go new file mode 100644 index 00000000000..5f60108f92d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" +) + +// The create request has no description field, so a documented description +// would otherwise be parsed and dropped. +func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") + group.Description = "Quality gate for the support agent" + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Equal(t, "Quality gate for the support agent", req.Metadata["azd_description"]) +} + +// An absent description adds no metadata key rather than an empty one. +func TestBuildOmitsEmptyDescription(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") + + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.NotContains(t, req.Metadata, "azd_description") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go new file mode 100644 index 00000000000..ccfe9719aa8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Ids are per declaration. A shared key works only while a config has one +// group: with two, the second deploy finds the first's id cached, confirms it +// exists, and hands it back for the wrong group — so group A silently scores +// group B's criteria. +func TestIDKey_IsPerName(t *testing.T) { + a := idKey("eval", "quality-a") + b := idKey("eval", "quality-b") + + assert.NotEqual(t, a, b, "two groups must not share an id key") + assert.Contains(t, a, "QUALITY_A") + assert.True(t, len(a) > 3 && a[len(a)-3:] == "_ID") +} + +// Names that are not valid env identifiers still have to produce distinct, +// stable keys. +func TestIDKey_NormalizesNames(t *testing.T) { + assert.Equal(t, idKey("eval", "my group"), idKey("eval", "my-group"), + "characters that cannot appear in an env name normalize the same way") + assert.NotEqual(t, idKey("eval", "a"), idKey("dataset", "a"), + "the kind keeps different resources apart") +} + +// The id and version keys for the same declaration must not collide. +func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { + assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) +} + +// Setting EVAL_ID by hand is the documented way to point a config at an eval +// that already exists. It is also the key the extension writes itself, which is +// what let a second eval adopt the first one's id — so it stays readable only +// where it cannot be ambiguous. Fixing the aliasing dropped this fallback +// entirely once, silently breaking the documented behaviour. +func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { + write := func(t *testing.T, names ...string) string { + t.Helper() + dir := t.TempDir() + cfg := &project.EvalConfig{} + for _, n := range names { + cfg.Evals = append(cfg.Evals, project.Eval{ + Name: n, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + }) + } + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + return dir + } + + sole := evalIDKeys("quality", write(t, "quality")) + assert.Equal(t, idKey("eval", "quality"), sole[0], + "an eval's own entry is preferred over the shared one") + assert.Contains(t, sole, envKeyEvalID, + "a project with one eval honours an id set by hand") + + assert.Equal(t, []string{idKey("eval", "quality")}, + evalIDKeys("quality", write(t, "quality", "nightly")), + "with several evals the shared entry cannot say which one it means") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go new file mode 100644 index 00000000000..8652dc6b958 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary. Warning about nowhere to persist would +// be noise on every standalone invocation. +func TestNoAzdEnvironmentIsRecognisable(t *testing.T) { + err := fmt.Errorf("%w to write %s into", errNoAzdEnvironment, "EVAL_RUN_ID") + + require.ErrorIs(t, err, errNoAzdEnvironment, + "callers rely on telling this apart from a failed write") + require.Contains(t, err.Error(), "EVAL_RUN_ID", + "the key is still named when the message is shown") +} + +// A write that fails for any other reason stays reportable. +func TestOtherEnvironmentFailuresStayReportable(t *testing.T) { + err := fmt.Errorf("writing %s to the azd environment: %w", "EVAL_RUN_ID", errors.New("rpc failed")) + + require.NotErrorIs(t, err, errNoAzdEnvironment) + require.Contains(t, err.Error(), "rpc failed") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go new file mode 100644 index 00000000000..3082153d4b4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Creation normally belongs to `azd up`, which owns reconciliation. `create` +// is the same path for a single eval outside a project, and takes the +// configuration rather than a wall of flags so there is never a second +// definition to maintain. + +// newEvalCreateCommand creates one declared eval without deploying the rest. +func newEvalCreateCommand() *cobra.Command { + var ( + fromFile string + evalDir string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "create [name]", + Short: "Create one eval declared in the configuration.", + Long: "Create one eval declared in the configuration.\n\n" + + "`azd up` reconciles every eval in the file. This creates a single one, " + + "for a project that is not deployed as a whole — or, with --from-file, " + + "for no project at all.\n\n" + + "The name is optional while the configuration declares exactly one eval.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + path := fromFile + if path == "" { + path = project.EvalConfigPath(evalDir) + } + cfg, err := project.LoadEvalConfig(path) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + eval, err := cfg.Eval(firstArg(args)) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Local sources resolve against the file, not the working directory, + // so the columns are read from where the declaration points. + datasetPath := "" + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok && decl.Source != "" { + datasetPath = filepath.Join(filepath.Dir(path), decl.Source) + } + + reconciler := &evalReconciler{ec: ec} + id, err := reconciler.EnsureEval(ctx, *eval, datasetPath, false) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": id, "name": eval.Name, + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "%s Created eval: %s (%s)\n", doneMark, eval.Name, id) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Read the configuration from this path instead of the eval directory.") + cmd.Flags().StringVar(&evalDir, "path", project.DefaultEvalDir, + "Directory holding the evaluation configuration.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalListCommand() *cobra.Command { + var ( + limit int + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's evals.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListOpenAIEvals(ctx, limit) + if err != nil { + return fmt.Errorf("listing evals: %w", err) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Data) + } + if len(list.Data) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No evals found.") + return nil + } + rows := make([][]string, 0, len(list.Data)) + for _, e := range list.Data { + rows = append(rows, []string{e.ID, e.Name}) + } + return emitTable(cmd.OutOrStdout(), []string{"EVAL ID", "NAME"}, rows) + }, + } + + cmd.Flags().IntVar(&limit, "limit", 0, "Cap the number of evals returned.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show an eval definition.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + group, err := ec.evalClient.GetOpenAIEval(ctx, evalID) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd ai eval list` shows the ones there are", evalID) + } + return fmt.Errorf("reading eval %q: %w", evalID, err) + } + return emitJSON(cmd.OutOrStdout(), group) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalDeleteCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an eval and everything under it.", + Long: "Delete an eval and everything under it.\n\n" + + "An eval owns its runs, so deleting one discards their results too.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.evalClient.DeleteOpenAIEval(ctx, evalID); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf("no eval %q in this project", evalID) + } + return fmt.Errorf("deleting eval %q: %w", evalID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": evalID, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted eval %s\n", evalID) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go new file mode 100644 index 00000000000..f6d2d5ddfc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + + "azureaieval/internal/project" +) + +// evalRef is what `--eval` resolved to: the service id, plus the declaration +// behind it when there was one. +// +// Commands need both. The id is what every route under {eval_id} takes, and the +// declaration is what says which dataset and target a new run should use. +type evalRef struct { + ID string + Eval *project.Eval + Config *project.EvalConfig + ConfigPath string +} + +// Declared reports whether the reference came from the configuration. +func (r evalRef) Declared() bool { return r.Eval != nil } + +// resolveEvalRef turns `--eval` into an id. +// +// One flag takes a name or an id, matching `azd ai training job show`, whose +// --name is documented as "Job name/ID". Name is tried first, in three cases: +// a name in evals: with a recorded id resolves to it; a name in evals: with +// none fails fast naming `azd up` rather than returning a service 404; and +// anything else is sent as an id. +// +// Ids matter because an eval created by `azd ai eval create` has no evals: +// entry, and because the environment records one id per name, so editing a +// declaration leaves every run of the previous eval reachable only by id. +func (ec *evalContext) resolveEvalRef( + ctx context.Context, + evalDir, nameOrID string, +) (evalRef, error) { + configPath := project.EvalConfigPath(evalDir) + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return evalRef{}, err + } + + if cfg != nil { + if err := cfg.Validate(); err != nil { + return evalRef{}, err + } + eval, err := cfg.Eval(nameOrID) + switch { + case err == nil: + id := ec.recordedEvalID(ctx, eval.Name) + if id == "" { + return evalRef{}, fmt.Errorf( + "eval %q is declared but has not been deployed to this environment yet; "+ + "run `azd up` first", eval.Name) + } + return evalRef{ID: id, Eval: eval, Config: cfg, ConfigPath: configPath}, nil + case nameOrID == "": + // No name to fall back on, so the configuration's own complaint — + // none declared, or several to choose between — is the answer. + return evalRef{}, err + } + } + + if nameOrID == "" { + return evalRef{}, fmt.Errorf( + "no eval was named and none is declared in %s; pass --eval with a name or an id", + configPath) + } + // Not a declared name, so it is an id. + return evalRef{ID: nameOrID}, nil +} + +// recordedEvalID reads the id `azd up` stored for a declared eval. +func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) string { + return ec.getEnvValue(ctx, idKey("eval", evalName)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go new file mode 100644 index 00000000000..48aeb48a477 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -0,0 +1,390 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +func newEvaluatorCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "evaluator", + Short: "Manage custom evaluators.", + } + cmd.AddCommand( + newEvaluatorCreateCommand(), + newEvaluatorUpdateCommand(), + newEvaluatorGenerateCommand(), + newEvaluatorListCommand(), + newEvaluatorShowCommand(), + newEvaluatorDeleteCommand(), + newEvaluatorVersionsCommand(), + newJobCommand(evaluatorJobs), + ) + return cmd +} + +// newEvaluatorCreateCommand builds `evaluator create `, which registers +// an evaluator that does not exist yet. +func newEvaluatorCreateCommand() *cobra.Command { + return newEvaluatorWriteCommand("create", "Register an evaluator, publishing its first version.") +} + +// newEvaluatorUpdateCommand builds `evaluator update `, which publishes a +// further version of one that does. +func newEvaluatorUpdateCommand() *cobra.Command { + return newEvaluatorWriteCommand("update", "Publish a new version of an evaluator.") +} + +// newEvaluatorWriteCommand builds create and update, which send the same +// request and differ only in which starting state they accept. The service has +// one route for both and assigns the version either way, so the existence check +// is ours: without it, `create` on a name already in use would silently publish +// a further version of someone else's evaluator. +func newEvaluatorWriteCommand(verb, short string) *cobra.Command { + var ( + fromFile string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: verb + " ", + Short: short, + Long: short + "\n\n" + + "An evaluator is a rubric: a JSON file of weighted scoring dimensions.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if fromFile == "" { + return requireFlag("from-file") + } + + raw, err := os.ReadFile(fromFile) + if err != nil { + return fmt.Errorf("reading evaluator %q: %w", fromFile, err) + } + + body, err := normalizeRubricBody(name, raw) + if err != nil { + return fmt.Errorf("evaluator %q: %w", fromFile, err) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Asked of the direct read, not the version listing. The listing + // lags a publish by up to a second and a half, so an update + // issued straight after a create would be told the evaluator it + // just made does not exist. + existing, readErr := ec.evalClient.GetEvaluatorRaw( + ctx, name, "", ProjectEndpointAPIVersion, + ) + if readErr != nil && !eval_api.IsNotFound(readErr) { + return fmt.Errorf("checking whether evaluator %q exists: %w", name, readErr) + } + if err := checkAssetExistence(verb, "evaluator", name, readErr == nil); err != nil { + return err + } + + // What that read saw is what keeps the publish from being + // answered with the same version and replacing it. + if readErr != nil { + existing = nil + } + + created, err := ec.evalClient.CreateEvaluatorVersion( + ctx, name, body, existing, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("registering evaluator %q: %w", name, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), created) + } + fmt.Fprintf(cmd.OutOrStdout(), + "Registered evaluator %s version %s\n", created.Name, created.Version) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", "Path to the evaluator JSON file.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// checkAssetExistence enforces the one difference between create and update. +func checkAssetExistence(verb, kind, name string, exists bool) error { + switch { + case verb == "create" && exists: + return fmt.Errorf( + "%s %q already exists: use `update` to publish a new version", kind, name) + case verb == "update" && !exists: + return fmt.Errorf( + "%s %q does not exist: use `create` to register it", kind, name) + } + return nil +} + +// rubricDefinitionType is the discriminator the service uses to deserialize a +// rubric definition. +const rubricDefinitionType = "rubric" + +// ensureDefinitionType adds the type discriminator when a definition omits it. +// +// Without it the service cannot tell which definition kind it is holding and +// rejects the whole request with "The request field is required", which points +// at the wrong field entirely. Generated rubrics carry the type; hand-authored +// ones written to the shape the spec documents — a bare list of weighted +// dimensions — do not. +func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { + var doc map[string]json.RawMessage + if err := json.Unmarshal(definition, &doc); err != nil { + return nil, fmt.Errorf("the definition is not a JSON object: %w", err) + } + if _, ok := doc["type"]; ok { + return definition, nil + } + doc["type"] = json.RawMessage(fmt.Sprintf("%q", rubricDefinitionType)) + return json.Marshal(doc) +} + +// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or +// a full evaluator document ({name, definition}) and returns the request body. +func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { + var probe map[string]json.RawMessage + if err := json.Unmarshal(raw, &probe); err != nil { + return nil, fmt.Errorf("not valid JSON: %w", err) + } + + if definition, hasDefinition := probe["definition"]; hasDefinition { + // Already a full document; make sure the name matches the argument. + typed, err := ensureDefinitionType(definition) + if err != nil { + return nil, err + } + probe["definition"] = typed + probe["name"] = json.RawMessage(fmt.Sprintf("%q", name)) + out, err := json.Marshal(probe) + if err != nil { + return nil, err + } + return out, nil + } + + if _, hasDimensions := probe["dimensions"]; !hasDimensions { + return nil, fmt.Errorf( + "expected a rubric definition with 'dimensions', or a document with 'definition'") + } + + typed, err := ensureDefinitionType(raw) + if err != nil { + return nil, err + } + doc := map[string]any{ + "name": name, + "definition": typed, + } + out, err := json.Marshal(doc) + if err != nil { + return nil, err + } + return out, nil +} + +func newEvaluatorListCommand() *cobra.Command { + var ( + builtin bool + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's evaluators, or the built-in ones.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // The service filters by type, and asking for nothing returns only + // the project's own evaluators. + filter := "" + if builtin { + filter = eval_api.EvaluatorTypeBuiltin + } + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing evaluators: %w", err) + } + return renderEvaluators(cmd, list) + }, + } + + cmd.Flags().BoolVar(&builtin, "builtin", false, + "List the built-in evaluators instead of the project's own.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newEvaluatorVersionsCommand groups the version listing, so that `list` means +// the same thing for evaluators as it does for datasets: the assets, not their +// history. +func newEvaluatorVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one evaluator.", + } + cmd.AddCommand(newEvaluatorVersionsListCommand()) + return cmd +} + +func newEvaluatorVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of an evaluator.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing versions of evaluator %q: %w", name, err) + } + return renderEvaluators(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + if len(list.Value) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No evaluators found.") + return nil + } + rows := make([][]string, 0, len(list.Value)) + for _, e := range list.Value { + rows = append(rows, []string{e.Name, e.Version, e.Type()}) + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) +} + +func newEvaluatorShowCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show an evaluator definition.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + raw, err := ec.evalClient.GetEvaluatorRaw(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no evaluator %q in this project; "+ + "`azd ai eval evaluator list` shows the ones there are", name) + } + return fmt.Errorf("reading evaluator %q: %w", name, err) + } + + var pretty any + if err := json.Unmarshal(raw, &pretty); err != nil { + fmt.Fprintln(cmd.OutOrStdout(), string(raw)) + return nil + } + return emitJSON(cmd.OutOrStdout(), pretty) + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvaluatorDeleteCommand() *cobra.Command { + var ( + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an evaluator version.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.evalClient.DeleteEvaluatorVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no evaluator %q at version %q in this project", name, version) + } + return fmt.Errorf("deleting evaluator %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted evaluator %s version %s\n", name, version) + return nil + }, + } + + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go new file mode 100644 index 00000000000..e7ad1feab01 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service needs a type discriminator to deserialize a definition. Without +// it the whole request is rejected with "The request field is required", which +// names the wrong field, so a hand-authored rubric failed to upload. +func TestNormalizeRubricBodyAddsDefinitionType(t *testing.T) { + raw := []byte(`{"dimensions":[{"id":"accuracy","description":"Correct.","weight":5}]}`) + + body, err := normalizeRubricBody("support-quality", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "support-quality", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) + require.Len(t, doc.Definition.Dimensions, 1) + require.Equal(t, 5, doc.Definition.Dimensions[0].Weight) +} + +// A definition that already declares its type keeps it, so a generated rubric +// round-trips unchanged. +func TestNormalizeRubricBodyKeepsExistingType(t *testing.T) { + raw := []byte(`{"type":"custom_kind","dimensions":[{"id":"a","weight":1}]}`) + + body, err := normalizeRubricBody("x", raw) + require.NoError(t, err) + + var doc struct { + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "custom_kind", doc.Definition.Type) +} + +// A full document is normalized the same way, and the name follows the flag. +func TestNormalizeRubricBodyHandlesFullDocument(t *testing.T) { + raw := []byte(`{"name":"stale","definition":{"dimensions":[{"id":"a","weight":1}]}}`) + + body, err := normalizeRubricBody("actual-name", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "actual-name", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) +} + +func TestNormalizeRubricBodyRejectsNonRubric(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"something":1}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "dimensions") + + _, err = normalizeRubricBody("x", []byte(`not json`)) + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go new file mode 100644 index 00000000000..fabc4f222b6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Evaluator versions are the unit an eval binds to, and the service assigns +// them. This proves the extension never hands back a version it has quietly +// overwritten. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorUpdateAlwaysPublishesANewVersion covers the shape of a +// first authoring session: create a rubric, look at it, change one weight, +// update. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over it rather than adding one. +// Nothing observable marks the end of that race — the version listing lags a +// publish as well, answering 404 immediately after a create — so the defence +// is the document the caller already read: it says which version exists and +// when it was written. +// +// Without it, `evaluator update` run straight after `evaluator create` reports +// success, leaves a single version holding the second rubric, and every eval +// bound to the first scores against a rubric nobody chose. +func TestLiveEvaluatorUpdateAlwaysPublishesANewVersion(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-version-%d", time.Now().UnixNano()) + + rubric := func(weight int) json.RawMessage { + body, err := normalizeRubricBody(name, []byte(fmt.Sprintf( + `{"dimensions":[{"id":"tone","weight":%d,"description":"polite"}]}`, weight))) + require.NoError(t, err) + return body + } + + first, err := client.CreateEvaluatorVersion(ctx, name, rubric(1), nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, first.Version) + t.Cleanup(func() { + for _, v := range []string{first.Version, "1", "2"} { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, v, ProjectEndpointAPIVersion) + } + }) + + // Deliberately immediate, and passing what the caller holds rather than + // re-reading: this is the window the guard exists for, and a test that + // waited first would pass with the guard removed. + previous, err := json.Marshal(first) + require.NoError(t, err) + + started := time.Now() + second, err := client.CreateEvaluatorVersion( + ctx, name, rubric(2), previous, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEqual(t, first.Version, second.Version, + "an update issued inside the race must still publish a new version") + t.Logf("the second version was assigned after %s", time.Since(started).Round(time.Millisecond)) + + // The new version holds the new rubric, and both versions are readable. + // The earlier one is not asserted on: if the service does collide, the + // attempt that collided has already written the new definition over it, + // and no amount of care on this side can undo that. + require.Equal(t, 2, liveRubricWeight(t, client, name, second.Version)) + require.NotZero(t, liveRubricWeight(t, client, name, first.Version), + "version %s must remain readable", first.Version) +} + +// liveRubricWeight reads back the one weight the fixture rubric carries. +// +// Read as JSON rather than matched as a substring: the service reformats what +// it stores, so `"weight":1` goes in and `"weight": 1` comes back, and a +// substring assertion would fail for a reason that has nothing to do with what +// is being tested. +func liveRubricWeight( + t *testing.T, + client *eval_api.EvalClient, + name, version string, +) int { + t.Helper() + + raw, err := client.GetEvaluatorRaw( + context.Background(), name, version, ProjectEndpointAPIVersion) + require.NoError(t, err) + + var doc struct { + Definition struct { + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(raw, &doc)) + require.Len(t, doc.Definition.Dimensions, 1) + return doc.Definition.Dimensions[0].Weight +} + +// TestLiveFirstPublishReturnsVersionOne is the other half: the guard must not +// change what a first publish answers. +func TestLiveFirstPublishReturnsVersionOne(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-firstpub-%d", time.Now().UnixNano()) + body, err := normalizeRubricBody(name, []byte( + `{"dimensions":[{"id":"tone","weight":1,"description":"polite"}]}`)) + require.NoError(t, err) + + created, err := client.CreateEvaluatorVersion(ctx, name, body, nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + require.Equal(t, "1", created.Version, + "a name the project has never seen must publish as version 1") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go new file mode 100644 index 00000000000..0b6940a28d5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Gating is opt-in. A completed run with failing samples exits 0 without +// --fail-on: failing samples are the expected output of a working evaluation, +// not a tool error, and `run start` is used constantly in the inner loop. A +// default that returned non-zero on any failure would break a build the first +// time a noisy grader disagreed. +// +// The separate exit code matters more than the flag. It lets a pipeline tell +// "the evaluation regressed" from "the evaluation could not run", which are +// different failures with different owners. + +// exitCodeGateBreached is returned when a run completed but missed its +// threshold. +const exitCodeGateBreached = 2 + +// gate is a parsed --fail-on threshold. +type gate struct { + set bool + anyFailure bool + passRate float64 +} + +// parseGate reads the --fail-on value. An empty value means no gating. +func parseGate(spec string) (gate, error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return gate{}, nil + } + if spec == "any-failure" { + return gate{set: true, anyFailure: true}, nil + } + + rate, ok := strings.CutPrefix(spec, "pass-rate=") + if !ok { + return gate{}, fmt.Errorf( + "--fail-on must be any-failure or pass-rate=<0..1>, got %q", spec) + } + value, err := strconv.ParseFloat(rate, 64) + if err != nil { + return gate{}, fmt.Errorf("--fail-on pass-rate must be a number, got %q", rate) + } + if value < 0 || value > 1 { + return gate{}, fmt.Errorf("--fail-on pass-rate must be between 0 and 1, got %v", value) + } + return gate{set: true, passRate: value}, nil +} + +// breach reports why the run missed the threshold, or empty when it met it. +// +// Errored and skipped rows count against the pass rate, and they can: the +// service puts them inside `total`, verified live on a run that reported +// total=3 passed=2 errored=1. Were they outside it, a run with two passes and +// one error would report total=2 and score a perfect rate, which is precisely +// the broken evaluation a gate exists to catch. +// +// A run that scored nothing at all breaches every threshold rather than +// dividing by zero — "no rows passed" is the honest reading of an empty result. +func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { + if !g.set { + return "" + } + if counts == nil { + return "the run reported no result counts, so the threshold cannot be checked" + } + if g.anyFailure { + unpassed := counts.Total - counts.Passed + if unpassed > 0 { + return fmt.Sprintf("%d of %d samples did not pass", unpassed, counts.Total) + } + return "" + } + if counts.Total == 0 { + return "the run scored no rows, so its pass rate is below any threshold" + } + actual := float64(counts.Passed) / float64(counts.Total) + if actual < g.passRate { + return fmt.Sprintf("pass rate %.1f%% is below the required %.1f%%", + actual*100, g.passRate*100) + } + return "" +} + +// applyGate ends the process with exit code 2 when the run missed its +// threshold. +// +// It exits here rather than returning an error because the extension SDK's +// Run collapses every error to exit 1, and the whole point of the flag is a +// code a pipeline can tell apart from an operational failure. +func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { + if run == nil { + return + } + reason := g.breach(run.ResultCounts) + if reason == "" { + return + } + fmt.Fprintf(os.Stderr, "(x) Failed: Evaluation gate: %s\n\n", reason) + fmt.Fprintln(os.Stderr, "ERROR: evaluation quality gate not met.") + os.Exit(exitCodeGateBreached) +} + +func addFailOnFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "fail-on", "", + "Exit 2 when the run misses this threshold: any-failure, or pass-rate=<0..1>.") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go new file mode 100644 index 00000000000..5952e53d694 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestParseGate(t *testing.T) { + t.Run("empty means no gating", func(t *testing.T) { + g, err := parseGate("") + require.NoError(t, err) + require.False(t, g.set) + require.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3}), + "an unset gate must never breach") + }) + + t.Run("any-failure", func(t *testing.T) { + g, err := parseGate("any-failure") + require.NoError(t, err) + require.True(t, g.anyFailure) + }) + + t.Run("pass-rate", func(t *testing.T) { + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + require.InDelta(t, 0.8, g.passRate, 1e-9) + }) + + for _, bad := range []string{"passrate=0.8", "pass-rate=abc", "pass-rate=1.5", "pass-rate=-1", "sometimes"} { + t.Run("refuses "+bad, func(t *testing.T) { + _, err := parseGate(bad) + require.Error(t, err) + }) + } +} + +func TestGateBreach(t *testing.T) { + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + eighty, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + t.Run("any-failure passes only when every row passed", func(t *testing.T) { + require.Empty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 2})) + require.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 1, Failed: 1})) + }) + + // Errored rows are inside the total, verified live, so they count against + // the threshold the same way a failing row does. + t.Run("errored rows count against the rate", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} + require.Empty(t, eighty.breach(counts), "0.8 exactly meets a 0.8 threshold") + + counts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Errored: 3} + require.NotEmpty(t, eighty.breach(counts)) + }) + + // The wording is pinned because the hero scenario shows it verbatim. + t.Run("reads as a percentage", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 1000, Passed: 764} + require.Equal(t, + "pass rate 76.4% is below the required 80.0%", + eighty.breach(counts)) + }) + + // A run that scored nothing has no defensible pass rate, and treating it as + // 100% would let a broken evaluation hold a gate open. + t.Run("a run that scored nothing breaches", func(t *testing.T) { + require.NotEmpty(t, eighty.breach(&eval_api.EvalRunResultCounts{Total: 0})) + require.NotEmpty(t, eighty.breach(nil)) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go new file mode 100644 index 00000000000..d88cc539cc1 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -0,0 +1,427 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. +// The generation job is not gateway-capped; the old limit simply gave up while +// the service was still working, forcing a second command. +var generatePollBudget = eval_api.PollerOptions{ + Interval: 5 * time.Second, + MaxAttempts: 720, // one hour +} + +// generationPlan is everything one generation job needs, after the flags, the +// generation spec, and the eval's own target have been reconciled. +type generationPlan struct { + // Name of the artifact being generated — the positional argument. + Name string + // Agent whose context seeds generation. May be empty, in which case + // generation runs from the instruction alone. + Agent string + // Model deployment the generation job runs against. + Model string + // Instruction describing what the agent does and what to test. + Instruction string + // BaseDir is the directory OutputDir resolves against. + BaseDir string + // OutputDir is where the artifact is written. + OutputDir string + // SampleSize applies to dataset generation only. + SampleSize int + // From is what --from named: which of the service's sources to send. Empty + // sends whatever the plan has to offer. + From []string + // TraceDays seeds generation from that many days of recent traces. + TraceDays int +} + +// traceOptions converts the plan's trace window into the generation client's +// day count. Traces seed generation only; they are never a run's data source. +func (p generationPlan) traceOptions() *eval_api.TraceOptions { + if p.TraceDays <= 0 { + return nil + } + return &eval_api.TraceOptions{Days: p.TraceDays} +} + +// resolveInstruction returns the generation instruction, reading it from a +// file when one is named. +// +// A useful instruction describes the agent and what to test, which is often +// more than fits comfortably on a command line, so it can live in a file that +// is reviewable alongside the rest of the config. +func resolveInstruction(inline, path string) (string, error) { + if path == "" { + return inline, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading --agent-instruction-file %q: %w", path, err) + } + text := strings.TrimSpace(string(raw)) + if text == "" { + return "", fmt.Errorf("--agent-instruction-file %q is empty", path) + } + return text, nil +} + +// declaredInstructions reads the file named by a generation entry's +// `instructions`, relative to the spec that declared it. +// +// A missing file is not an error. The path can be written before the file +// exists, so treating its absence as a failure would break the flow `init` +// scaffolds. +func declaredInstructions(named, configPath string) (string, error) { + if named == "" { + return "", nil + } + + path := named + if !filepath.IsAbs(path) { + path = filepath.Join(filepath.Dir(configPath), filepath.FromSlash(named)) + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("reading instructions %q: %w", named, err) + } + return strings.TrimSpace(string(raw)), nil +} + +// resolveGenerationInstruction decides what generation is seeded from. +// +// The service accepts an agent source that is meant to pull the agent's own +// instructions, but it fails for every agent, so the agent's context is read +// here instead. In precedence order: what the caller passed, the instructions +// the project already holds, then the agent's published ones. +// +// The project comes before the service because a local read cannot fail +// slowly, and because instructions that have been optimized but not yet +// deployed are the ones the author means — generating against what is still +// published would test the version they are replacing. +// +// The last step is what makes `generate` work with no authored input at all, +// which is the flow `init` sets up. +func (ec *evalContext) resolveGenerationInstruction( + ctx context.Context, + explicit, agentName string, + out io.Writer, + quiet bool, +) (string, error) { + if explicit != "" { + return explicit, nil + } + + if agentName == "" { + return "", nil + } + + local, path, err := ec.agentInstructionsFromProject(ctx, agentName) + if err != nil { + return "", err + } + if local != "" { + if !quiet { + fmt.Fprintf(out, " Seeding generation from %s.\n", filepath.ToSlash(path)) + } + return local, nil + } + + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) + if err != nil { + // Generation can still proceed from the agent source alone, so a + // failure to read the agent is reported without stopping. + if !quiet { + fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", + agentName, err) + } + return "", nil + } + instructions := agent.Instructions() + if instructions != "" && !quiet { + fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", agentName) + } + return instructions, nil +} + +// agentInstructionsFromProject reads the agent's instructions out of the azd +// project, coming back empty when there is no project to read. +// +// Running outside a project is ordinary — the atomic commands work standalone +// against the data plane — so not finding one is not an error. An ambiguous +// target inside one is, because it would otherwise pick an agent at random. +func (ec *evalContext) agentInstructionsFromProject( + ctx context.Context, + agentName string, +) (instruction string, path string, err error) { + if ec.azdClient == nil { + return "", "", nil + } + resp, err := ec.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "", "", nil + } + return project.AgentInstructionsFromProject(resp.GetProject(), agentName) +} + +// generateRubric submits the evaluator generation job and saves the rubric. +func (ec *evalContext) generateRubric( + ctx context.Context, + plan generationPlan, + out io.Writer, + noWait bool, +) (*project.ArtifactRef, error) { + fmt.Fprintf(out, "Generating rubric %s...\n", plan.Name) + + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), + ) + if err := refuseUnbuildableSources(unbuildable); err != nil { + return nil, err + } + req := eval_api.NewEvaluatorGenerationJobRequest(plan.Name, plan.Model, sources) + + job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the rubric generation job: %w", err) + } + if noWait { + reportSubmitted(out, "azd ai eval evaluator", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, ProjectEndpointAPIVersion, + ec.evalClient.GetEvaluatorGenerationJob) + if err != nil { + return nil, fmt.Errorf("rubric generation: %w", err) + } + + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".json") + if err := writeRubric(path, completed.Result); err != nil { + return nil, err + } + fmt.Fprintf(out, " wrote %s\n", path) + + return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil +} + +// refuseUnbuildableSources reports a --from the plan could not honour. +// +// Submitting anyway would run a billed job seeded from less than was asked for +// and return a plausible-looking artifact, which is the worst outcome: the +// caller has no way to tell it apart from one built the way they intended. +func refuseUnbuildableSources(kinds []string) error { + if len(kinds) == 0 { + return nil + } + reasons := map[string]string{ + "prompt": "--from prompt needs --agent-instruction or --agent-instruction-file", + "agent": "--from agent needs a target agent; pass --target, " + + "or declare one under target: in eval.yaml", + "file": "--from file is not a generation source; " + + "register the file with `azd ai eval dataset create` instead", + } + messages := make([]string, 0, len(kinds)) + for _, k := range kinds { + if reason, ok := reasons[k]; ok { + messages = append(messages, reason) + continue + } + messages = append(messages, fmt.Sprintf("--from %s cannot be built from this plan", k)) + } + return errors.New(strings.Join(messages, "; ")) +} + +// reportSubmitted says what was started and how to get back to it. +// +// The job id goes into the command rather than being left as a placeholder: +// --no-wait exists so the caller can walk away, and the line they walk away +// with has to be the one they can paste when they come back. The group is named +// too, because the two job types share no collection. +func reportSubmitted(out io.Writer, group, jobID string) { + fmt.Fprintf(out, " submitted job %s\n", jobID) + fmt.Fprintf(out, "\nReattach with: %s job show %s\n", group, jobID) +} + +// generateDataset submits the data generation job and downloads the result. +func (ec *evalContext) generateDataset( + ctx context.Context, + plan generationPlan, + out io.Writer, + noWait bool, +) (*project.ArtifactRef, error) { + fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) + + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), + ) + if err := refuseUnbuildableSources(unbuildable); err != nil { + return nil, err + } + req := eval_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) + + job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the data generation job: %w", err) + } + if noWait { + reportSubmitted(out, "azd ai eval dataset", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + if err != nil && isAgentSeededGenerationFailure(err) { + // Agent-seeded generation fails server-side for every agent, while the + // same request carrying only the prompt succeeds. Failing the whole + // command would block the documented flow on a defect the user cannot + // do anything about, so retry without the agent and say so. + promptOnly := eval_api.WithoutAgentSource(sources) + if eval_api.HasPromptSource(promptOnly) { + fmt.Fprintf(out, + " warning: generating from agent %q failed in the service; "+ + "retrying from the instruction alone.\n", plan.Agent) + + req = eval_api.NewDataGenerationJobRequest( + plan.Name, plan.Model, plan.SampleSize, promptOnly) + job, err = ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the data generation job: %w", err) + } + completed, err = ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + } + } + if err != nil { + return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, plan.Agent)) + } + + name, version := completed.ResolvedNameVersion() + if name == "" { + return nil, fmt.Errorf("the data generation job returned no dataset reference") + } + + // Confirm the version exists before reading it, so a missing dataset is + // reported as such rather than as a download failure. + if _, err := ec.datasetClient.GetDataset( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return nil, fmt.Errorf("reading the generated dataset %q: %w", name, err) + } + content, err := ec.datasetClient.DownloadDatasetContent(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) + } + + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".jsonl") + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return nil, fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + if err := os.WriteFile(path, content, 0o600); err != nil { + return nil, fmt.Errorf("writing %q: %w", path, err) + } + fmt.Fprintf(out, " wrote %s\n", path) + + return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil +} + +// isAgentSeededGenerationFailure recognizes the service-side failure that hits +// every agent, so it can be retried without the agent rather than surfaced. +func isAgentSeededGenerationFailure(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "DataGenerationJobSystemError") || + strings.Contains(text, "Something went wrong during data generation") +} + +// explainDataGenerationFailure adds context to the service's opaque system +// error. +// +// Seeding generation from an agent currently fails server-side with +// DataGenerationJobSystemError for every agent, within seconds, while the same +// request without the agent source runs normally. The raw message says only +// that something went wrong and to try again, which sends users into a retry +// loop against a deterministic failure. +func explainDataGenerationFailure(err error, agentName string) error { + if err == nil || agentName == "" { + return err + } + // The poller surfaces the service's message; the code is not always in it. + text := err.Error() + if !strings.Contains(text, "DataGenerationJobSystemError") && + !strings.Contains(text, "Something went wrong during data generation") { + return err + } + return fmt.Errorf( + "%w\n\n"+ + "This job seeded generation from agent %q. Agent-seeded data generation is "+ + "currently failing in the service for every agent, so retrying will not help.\n"+ + "Workarounds: supply your own dataset with --dataset, or run without --target "+ + "to generate from the instruction alone.", + err, agentName) +} + +// pollGeneration waits for a generation job using the raised budget. +func (ec *evalContext) pollGeneration( + ctx context.Context, + operationID, apiVersion string, + get eval_api.GetJobFunc, +) (*eval_api.GenerationJob, error) { + poller := eval_api.NewPoller(operationID, apiVersion, get) + poller.Options = generatePollBudget + return poller.Poll(ctx) +} + +// writeRubric persists only the rubric dimensions so the developer can edit +// weights and descriptions and publish a new version. +func writeRubric(path string, result json.RawMessage) error { + if len(result) == 0 { + return fmt.Errorf("the rubric generation job returned no result") + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + + var parsed eval_api.EvaluatorResult + if err := json.Unmarshal(result, &parsed); err == nil && len(parsed.Definition.Dimensions) > 0 { + body, err := json.MarshalIndent(parsed.Definition, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, body, 0o600) + } + + // Fall back to the raw payload rather than losing the result. + return os.WriteFile(path, result, 0o600) +} + +// relativeSource expresses an artifact path relative to the deployment spec. +func relativeSource(baseDir, path string) string { + rel, err := filepath.Rel(baseDir, path) + if err != nil { + return filepath.ToSlash(path) + } + return "./" + filepath.ToSlash(rel) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go new file mode 100644 index 00000000000..a401d9db1fb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -0,0 +1,296 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation is split per artifact because the service splits it: datasets and +// evaluators are separate long-running resources. One composite verb leaves +// partial failure undefined, cannot regenerate one artifact after the other has +// been hand-edited, and gives --no-wait nothing to reattach to. +// +// Neither command edits azure.yaml. Both add a catalog entry to eval.yaml for +// what they produced, so the artifact is referenceable without a hand edit. + +// generateFlags are the settings both generate commands share. +// +// There is no generation spec file. Every setting is a flag, because the +// artifact is checked in and a regeneration usually wants different settings +// anyway; what that costs is provenance, which is Open Question 8. +type generateFlags struct { + path string + target string + instruction string + instructionFile string + model string + outputDir string + noWait bool + force bool + endpoint string +} + +func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { + cmd.Flags().StringVar(&f.path, "path", project.DefaultEvalDir, + "Directory holding the evaluation configuration.") + cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") + cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", + "What the agent does and what to test.") + cmd.Flags().StringVar(&f.instructionFile, "agent-instruction-file", "", + "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") + cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") + cmd.Flags().StringVar(&f.model, "generation-model", "", + "Model deployment that generates the artifact.") + cmd.Flags().StringVar(&f.outputDir, "output-dir", "", + "Directory the generated artifact is written to.") + cmd.Flags().BoolVar(&f.noWait, "no-wait", false, + "Submit the job and return its id without polling.") + cmd.Flags().BoolVar(&f.force, "force", false, + "Overwrite an artifact file that already exists.") + cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") +} + +// resolvePlan settles every input that does not need the network. +// +// Doing it before the client is built means a missing model or an out-of-range +// sample count is refused without an authentication round trip. The instruction +// file is read here rather than later so that an input the caller named and got +// wrong is reported ahead of one they simply left out. +func resolvePlan(f *generateFlags, name string, defaultOutputDir string) (generationPlan, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return generationPlan{}, err + } + + plan := generationPlan{ + Name: name, + Agent: firstNonEmpty(f.target, declaredTarget(f.path)), + Model: f.model, + Instruction: instruction, + BaseDir: f.path, + OutputDir: firstNonEmpty(f.outputDir, "./"+defaultOutputDir), + } + if plan.Model == "" { + return plan, fmt.Errorf( + "a model deployment is required to generate: pass --generation-model") + } + return plan, nil +} + +// prepareGeneration builds the client and settles the one input that needs it: +// the agent's published instructions, which only the service can supply. +func prepareGeneration( + cmd *cobra.Command, + f *generateFlags, + plan generationPlan, +) (*evalContext, generationPlan, error) { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, f.endpoint) + if err != nil { + return nil, plan, err + } + + plan.Instruction, err = ec.resolveGenerationInstruction( + ctx, plan.Instruction, plan.Agent, cmd.OutOrStdout(), isJSON(cmd), + ) + if err != nil { + ec.Close() + return nil, plan, err + } + return ec, plan, nil +} + +// declaredTarget reads the agent from the evaluation configuration, which is +// where the target is already declared, so `generate` does not need it +// repeated. Best effort: generation runs from the instruction alone when there +// is no configuration to read, which is the case in a bare directory. +func declaredTarget(evalDir string) string { + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil || cfg == nil { + return "" + } + for _, eval := range cfg.Evals { + if eval.Target != nil && eval.Target.Name != "" { + return eval.Target.Name + } + } + return "" +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +// refuseExistingArtifact stops a generation that would overwrite a checked-in +// file, because the job is billed and the diff is what the author reviews. +func refuseExistingArtifact(path string, force bool) error { + if force { + return nil + } + if _, err := os.Stat(path); err == nil { + return fmt.Errorf( + "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", + filepath.ToSlash(path)) + } + return nil +} + +func newDatasetGenerateCommand() *cobra.Command { + var ( + flags generateFlags + maxSamples int + from []string + ) + + cmd := &cobra.Command{ + Use: "generate ", + Short: "Generate a dataset and download it.", + Long: "Generate a dataset and download it.\n\n" + + "--from selects one or more of the sources the service accepts, and " + + "is repeatable. Generating from the agent's own definition is a " + + "preference rather than a fallback: it covers cases no user has hit " + + "yet, and it can supply reference answers, which a transcript cannot.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + for _, src := range from { + if err := project.ValidateGenerateSource(src); err != nil { + return err + } + } + if err := project.ValidateSampleSize(maxSamples); err != nil { + return err + } + + plan, err := resolvePlan(&flags, name, project.DefaultDatasetsDir) + if err != nil { + return err + } + plan.From = from + plan.SampleSize = maxSamples + if plan.SampleSize == 0 { + plan.SampleSize = project.DefaultSampleSize + } + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".jsonl"), + flags.force, + ); err != nil { + return err + } + + ec, plan, err := prepareGeneration(cmd, &flags, plan) + if err != nil { + return err + } + defer ec.Close() + + if len(plan.From) == 0 { + plan.From = defaultGenerationSource( + ec.getEnvValue(cmd.Context(), appInsightsEnvKey), + ) + } + + ref, err := ec.generateDataset(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) + if err != nil { + return err + } + if err := addDatasetToCatalog(cmd, flags.path, ref); err != nil { + return err + } + return reportGenerated(cmd, ref, flags.noWait) + }, + } + + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", + project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) + cmd.Flags().StringSliceVar(&from, "from", nil, + fmt.Sprintf("Where rows come from: %s. Repeatable, and the service accepts "+ + "more than one. Defaults to %s when the project has Application Insights "+ + "connected, otherwise %s.", + strings.Join(project.GenerateSources, ", "), + project.GenerateFromTraces, project.GenerateFromAgent)) + addGenerateFlags(cmd, &flags) + return cmd +} + +func newEvaluatorGenerateCommand() *cobra.Command { + var ( + flags generateFlags + traceDays int + ) + + cmd := &cobra.Command{ + Use: "generate ", + Short: "Generate a rubric evaluator and download it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + plan, err := resolvePlan(&flags, name, project.DefaultEvaluatorsDir) + if err != nil { + return err + } + plan.TraceDays = traceDays + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".json"), + flags.force, + ); err != nil { + return err + } + + ec, plan, err := prepareGeneration(cmd, &flags, plan) + if err != nil { + return err + } + defer ec.Close() + + ref, err := ec.generateRubric(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) + if err != nil { + return err + } + if err := addEvaluatorToCatalog(cmd, flags.path, ref); err != nil { + return err + } + return reportGenerated(cmd, ref, flags.noWait) + }, + } + + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Days of traces to seed generation. 0 disables.") + addGenerateFlags(cmd, &flags) + return cmd +} + +// reportGenerated closes out either command. +// +// With --no-wait nothing was downloaded and there is no ref, which is success: +// the submission message has already said how to reattach. +func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { + out := cmd.OutOrStdout() + if ref == nil { + if !noWait { + fmt.Fprintln(out, "Nothing was generated.") + } + return nil + } + if isJSON(cmd) { + return emitJSON(out, ref) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go new file mode 100644 index 00000000000..275e194044d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// `generate` decides what to submit before it touches the network, so the plan +// it builds — which agent, what model, where the artifact lands — is checkable +// without paying for a generation job. These are the parts that cannot be +// observed afterwards: once the job is submitted, a wrong default is +// indistinguishable from an intended one. + +// evalsDir returns flags pointing at an empty eval directory. +func evalsDir(t *testing.T) *generateFlags { + t.Helper() + return &generateFlags{path: t.TempDir()} +} + +// withEvals writes a configuration into the flags' directory. +func withEvals(t *testing.T, f *generateFlags, evals ...project.Eval) { + t.Helper() + require.NoError(t, project.SaveEvalConfig(f.path, &project.EvalConfig{Evals: evals})) +} + +// Generation settings are flags only: there is no generate.yaml, because the +// artifact is checked in and regeneration usually wants different settings. +func TestResolvePlan_FromFlagsAlone(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.model = "gpt-4o-mini" + + plan, err := resolvePlan(f, "shop-golden", project.DefaultDatasetsDir) + require.NoError(t, err) + + require.Equal(t, "shop-golden", plan.Name) + require.Equal(t, "shop-agent", plan.Agent) + require.Equal(t, "gpt-4o-mini", plan.Model) + require.Equal(t, "./"+project.DefaultDatasetsDir, plan.OutputDir) + require.Equal(t, f.path, plan.BaseDir) +} + +// Each generate has its own default output directory, so a rubric never lands +// in the datasets folder. +func TestResolvePlan_OutputDirDefaultsPerArtifact(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.model = "gpt-4o-mini" + + ds, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultDatasetsDir, ds.OutputDir) + + ev, err := resolvePlan(f, "r", project.DefaultEvaluatorsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) + + f.outputDir = "./from-flag" + override, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./from-flag", override.OutputDir) +} + +// Without a model there is nothing to bill the job against, and the refusal has +// to name the flag that supplies one. +func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.Error(t, err) + require.Contains(t, err.Error(), "--generation-model") +} + +// An input the caller named and got wrong is reported ahead of one they simply +// left out. Both checks are local, so the only thing deciding which the user +// sees is the order they run in — and a missing instruction file is a typo the +// caller can act on, while the model has a documented default path. +func TestResolvePlan_ReportsABadExplicitInputFirst(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.instructionFile = filepath.Join(t.TempDir(), "absent.md") + + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file", + "the flag the caller got wrong must win over the one they omitted") +} + +// The target is already declared on an eval, so `generate` does not need it +// repeated on every invocation. +func TestResolvePlan_FallsBackToTheDeclaredTarget(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + withEvals(t, f, project.Eval{ + Name: "support-agent-eval", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + }) + + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "support-agent", plan.Agent, + "the declared target is the agent to generate from") + + // An explicit flag still wins, which is what makes a one-off run possible + // without editing a file that is checked in. + f.target = "from-flag" + plan, err = resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "from-flag", plan.Agent) +} + +// With no configuration at all, generation runs from the instruction alone. +// This is the golden path: both generates precede init. +func TestResolvePlan_NoConfigurationYet(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + f.instruction = "test refunds and returns" + + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Empty(t, plan.Agent) + require.Equal(t, "test refunds and returns", plan.Instruction) +} + +// The bounds are the service's, and the boundaries themselves have to be +// accepted: a check that rejected 15 or 1000 would be indistinguishable from +// one that is simply too strict. +func TestGenerateSampleSizeBounds(t *testing.T) { + for _, tc := range []struct { + size int + allowed bool + }{ + {project.MinSampleSize - 1, false}, + {project.MinSampleSize, true}, + {project.DefaultSampleSize, true}, + {project.MaxSampleSize, true}, + {project.MaxSampleSize + 1, false}, + } { + err := project.ValidateSampleSize(tc.size) + if tc.allowed { + require.NoErrorf(t, err, "%d is inside the service's range", tc.size) + continue + } + require.Errorf(t, err, "%d is outside the service's range", tc.size) + require.Contains(t, err.Error(), "must be between") + } +} + +func TestResolveInstruction(t *testing.T) { + dir := t.TempDir() + filled := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(filled, []byte(" test refunds and returns\n\n"), 0o600)) + blank := filepath.Join(dir, "blank.md") + require.NoError(t, os.WriteFile(blank, []byte(" \n"), 0o600)) + + t.Run("inline is returned as given", func(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + }) + + t.Run("a file is read and trimmed", func(t *testing.T) { + got, err := resolveInstruction("", filled) + require.NoError(t, err) + require.Equal(t, "test refunds and returns", got) + }) + + // A whitespace-only file would otherwise generate from nothing, which + // produces a rubric with no relation to the agent. + t.Run("an empty file is refused", func(t *testing.T) { + _, err := resolveInstruction("", blank) + require.Error(t, err) + require.Contains(t, err.Error(), "is empty") + }) + + t.Run("a missing file names the flag", func(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(dir, "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go new file mode 100644 index 00000000000..ba8d5861714 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The spec's default: traces when the project has Application Insights +// connected, otherwise the agent. The connection string is how a project says +// it collects traces at all, so asking for traces without one would submit a +// billed job against nothing. +func TestDefaultGenerationSource(t *testing.T) { + assert.Equal(t, []string{"traces"}, + defaultGenerationSource("InstrumentationKey=00000000-0000-0000-0000-000000000000"), + "a project collecting traces should be generated from them") + + assert.Equal(t, []string{"agent"}, defaultGenerationSource(""), + "with nowhere for traces to have been collected, the agent is all there is") +} + +// --from is a request, and one the plan cannot honour has to stop the command +// rather than quietly submit a job seeded from less than was asked for. +func TestRefuseUnbuildableSources(t *testing.T) { + assert.NoError(t, refuseUnbuildableSources(nil)) + assert.NoError(t, refuseUnbuildableSources([]string{})) + + tests := []struct { + kind string + says string + }{ + {"prompt", "--agent-instruction"}, + {"agent", "--target"}, + {"file", "azd ai eval dataset create"}, + } + + for _, tt := range tests { + t.Run(tt.kind, func(t *testing.T) { + err := refuseUnbuildableSources([]string{tt.kind}) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.says, + "the error has to name the way out, not just the problem") + }) + } +} + +// Two unhonoured sources are two things the caller has to fix, so both are +// reported at once rather than one per attempt. +func TestRefuseUnbuildableSources_ReportsAllOfThemAtOnce(t *testing.T) { + err := refuseUnbuildableSources([]string{"prompt", "agent"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--agent-instruction") + assert.Contains(t, err.Error(), "--target") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go new file mode 100644 index 00000000000..e4c04baebf8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service's system error says only that something went wrong and to try +// again, but agent-seeded generation fails deterministically, so a bare retry +// suggestion sends users into a loop. +func TestExplainDataGenerationFailureAddsAgentContext(t *testing.T) { + err := errors.New( + `job failed with status "failed": Something went wrong during data generation. Please try again.`) + + explained := explainDataGenerationFailure(err, "my-agent") + require.Error(t, explained) + require.Contains(t, explained.Error(), "my-agent") + require.Contains(t, explained.Error(), "--dataset") + require.ErrorIs(t, explained, err, "the original error must stay in the chain") +} + +// The code spelling is matched as well, in case the poller starts surfacing it. +func TestExplainDataGenerationFailureMatchesErrorCode(t *testing.T) { + err := fmt.Errorf("job failed: DataGenerationJobSystemError") + explained := explainDataGenerationFailure(err, "my-agent") + require.Contains(t, explained.Error(), "Workarounds") +} + +// Unrelated failures are passed through untouched, and so is a job that had no +// agent source to blame. +func TestExplainDataGenerationFailureLeavesOthersAlone(t *testing.T) { + other := errors.New("submitting the data generation job: 403 Forbidden") + require.Equal(t, other, explainDataGenerationFailure(other, "my-agent")) + + systemErr := errors.New("Something went wrong during data generation") + require.Equal(t, systemErr, explainDataGenerationFailure(systemErr, "")) + + require.NoError(t, explainDataGenerationFailure(nil, "my-agent")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go new file mode 100644 index 00000000000..4654145e853 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// digestIDKey is what makes a rename find the eval it already deployed: the id +// is recorded against the eval's substance, so a declaration whose name +// changed still resolves. That only works while the key derives from the +// digest the same way it did last deploy — change the format and every +// deployed eval silently loses its recorded id and gets recreated. +func TestDigestIDKey(t *testing.T) { + const digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + + key := digestIDKey(digest) + + assert.Equal(t, "EVAL_SUBSTANCE_0123456789ABCDEF_ID", key) + for _, r := range key { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } +} + +// Same substance, same key — that is the whole mechanism. +func TestDigestIDKey_IsStableForTheSameSubstance(t *testing.T) { + group := project.Eval{ + Name: "support", + Dataset: "support-regression", + Target: &project.Target{Name: "support-agent"}, + } + + first, err := project.FingerprintGroup(group) + require.NoError(t, err) + + renamed := group + renamed.Name = "support-renamed" + renamed.Description = "reworded" + second, err := project.FingerprintGroup(renamed) + require.NoError(t, err) + + assert.Equal(t, digestIDKey(first), digestIDKey(second), + "a rename must land on the key the first deploy wrote") +} + +// Different substance, different key, so a genuinely new eval does not adopt +// an unrelated one's id. +func TestDigestIDKey_DiffersWhenTheSubstanceDoes(t *testing.T) { + a, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "one"}) + require.NoError(t, err) + b, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "two"}) + require.NoError(t, err) + + assert.NotEqual(t, digestIDKey(a), digestIDKey(b)) +} + +// The version recorded for an artifact comes out of what the service returned, +// falling back to what the caller already knew. +func TestVersionFromRaw(t *testing.T) { + tests := []struct { + name string + raw string + fallback string + want string + }{ + {"version in the body wins", `{"version":"7"}`, "3", "7"}, + {"empty version falls back", `{"version":""}`, "3", "3"}, + {"absent version falls back", `{"name":"x"}`, "3", "3"}, + {"unparseable body falls back", `not json`, "3", "3"}, + {"empty body falls back", ``, "3", "3"}, + {"no fallback either", `{}`, "", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, versionFromRaw([]byte(tt.raw), tt.fallback)) + }) + } +} + +// A criterion binds to a dataset column through `{{item.}}`. Reading the +// name wrong is how a run is submitted against a column the dataset does not +// have, which the service rejects without saying which one. +func TestItemColumn(t *testing.T) { + bound := map[string]string{ + "{{item.query}}": "query", + "{{item.ground_truth}}": "ground_truth", + "{{item.a.b}}": "a.b", + } + for binding, want := range bound { + got, ok := itemColumn(binding) + assert.Truef(t, ok, "%q is a binding", binding) + assert.Equal(t, want, got) + } + + notBound := []string{ + "", + "query", + "{{item.}}", + "{{ item.query }}", + "{{item.query", + "item.query}}", + "{{response.output}}", + } + for _, binding := range notBound { + got, ok := itemColumn(binding) + assert.Falsef(t, ok, "%q is not an item binding", binding) + assert.Empty(t, got) + } +} + +// An eval is named after what it evaluates and what it reads, so two evals over +// the same agent from different sources do not collide. +func TestDefaultEvalName(t *testing.T) { + assert.Equal(t, "support-agent-trace-eval", + defaultEvalName("support-agent", initSourceTraces)) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "dataset")) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "")) + + assert.NotEqual(t, + defaultEvalName("support-agent", initSourceTraces), + defaultEvalName("support-agent", "dataset"), + "the source is in the name so the two do not collide") +} + +// The reattach line printed by --no-wait has to name the group the job +// actually belongs to; the two job types share no collection, so the wrong +// group is a command that returns "not found". +func TestJobLookupErrorNamesTheGroup(t *testing.T) { + for _, kind := range []jobKind{datasetJobs, evaluatorJobs} { + err := jobLookupError(kind, "job_1", assert.AnError) + + require.Error(t, err) + assert.Contains(t, err.Error(), "job_1") + assert.Truef(t, strings.Contains(err.Error(), kind.name), + "the error must name the %q group so the retry goes to the right one", kind.name) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go new file mode 100644 index 00000000000..306232e24e3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -0,0 +1,583 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" + "google.golang.org/protobuf/types/known/structpb" +) + +// Data sources `init` can point an eval at. +const ( + initSourceDataset = "dataset" + initSourceTraces = "traces" +) + +// newInitCommand scaffolds the eval configuration. It makes no service calls at +// all, so it works offline and unauthenticated. +// +// It only ever adds. A name already declared is refused rather than +// overwritten, because the settings a reader tunes by hand — thresholds, judge +// model, data mapping — live nowhere but that entry and `init` cannot +// reproduce them. Editing an eval is a file edit. +func newInitCommand() *cobra.Command { + var ( + evalName string + target string + source string + dataset string + maxTraces int + evaluators []string + judgeModel string + path string + force bool + ) + + cmd := &cobra.Command{ + Use: "init", + Short: "Scaffold evaluation config for an agent. Makes no service calls.", + RunE: func(cmd *cobra.Command, args []string) error { + out := cmd.OutOrStdout() + + if target == "" { + return requireFlag("target") + } + switch source { + case "", initSourceDataset, initSourceTraces: + default: + return fmt.Errorf( + "--source %q is not a data source; use %q or %q", + source, initSourceDataset, initSourceTraces) + } + if source == initSourceTraces && dataset != "" { + return errors.New("--source traces reads production traces, so it takes no --dataset") + } + if cmd.Flags().Changed("max-traces") && source != initSourceTraces { + return errors.New("--max-traces caps a trace-backed eval; pass --source traces") + } + if maxTraces < 0 { + return errors.New("--max-traces must be positive") + } + if source == "" { + source = initSourceDataset + } + if path == "" { + path = project.DefaultEvalDir + } + if evalName == "" { + evalName = defaultEvalName(target, source) + } + + // Asked before anything is written: the project is the one thing + // init cannot supply for itself, and failing after creating + // directories leaves a half-scaffolded tree behind. + azdProject, err := readAzdProject(cmd.Context()) + if err != nil { + return err + } + if judgeModel == "" { + judgeModel = detectModelDeployment(azdProject) + } + + configPath := project.EvalConfigPath(path) + cfg, err := project.OpenEvalConfig(path) + if err != nil { + return err + } + if cfg == nil { + cfg = &project.EvalConfig{} + } + if cfg.HasEval(evalName) { + if !force { + return fmt.Errorf( + "an eval named %q already exists in %s; choose another name with --name, "+ + "or pass --force to replace it. `init` only adds: editing an eval is a file edit", + evalName, filepath.ToSlash(configPath)) + } + cfg.RemoveEval(evalName) + } + + if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { + return fmt.Errorf("creating the datasets directory: %w", err) + } + if err := os.MkdirAll(filepath.Join(path, project.DefaultEvaluatorsDir), 0o750); err != nil { + return fmt.Errorf("creating the evaluators directory: %w", err) + } + + plan := planScaffold(scaffoldInput{ + evalName: evalName, + target: target, + source: source, + dataset: dataset, + maxTraces: maxTraces, + evaluators: evaluators, + judgeModel: judgeModel, + rubricName: target + "-quality", + evalDir: path, + cfg: cfg, + }) + + if err := project.SaveEvalConfig(path, cfg); err != nil { + return err + } + + // Scaffolding a config azd cannot see is half a step: the eval + // service has to be referenced from the root config before any of + // `azd up`, `azd deploy` or `azd ai eval run` will act on it. + serviceName := target + "-evals" + rootWiring, err := ensureRootEvalService(cmd.Context(), serviceName, target, configPath) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(out, map[string]any{ + "eval": evalName, + "evalConfig": configPath, + "service": serviceName, + "datasetsDir": filepath.Join(path, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(path, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, + "target": target, + "source": source, + "judgeModel": judgeModel, + "evaluators": plan.evaluatorNames(), + }) + } + + fmt.Fprintf(out, "%s Detected agent target: %s\n", doneMark, target) + if source == initSourceTraces { + fmt.Fprintf(out, "%s Using data source: traces (Application Insights)\n", doneMark) + } + if judgeModel != "" { + fmt.Fprintf(out, "%s Judge model deployment: %s\n", doneMark, judgeModel) + } + + fmt.Fprintln(out, "\nCreated") + fmt.Fprintf(out, " %-33s evaluation configuration\n", filepath.ToSlash(configPath)) + switch rootWiring { + case wiringAdded: + fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, serviceName) + case wiringPresent: + fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, serviceName) + } + + // Only what was actually scheduled is offered. Suggesting + // `dataset generate` for a dataset the caller supplied sends them + // to submit a billed job for an artifact they already have. + next := plan.nextSteps() + fmt.Fprintf(out, "\nNext: %s\n", next[0]) + for _, step := range next[1:] { + fmt.Fprintf(out, " %s\n", step) + } + return nil + }, + } + + cmd.Flags().StringVar(&evalName, "name", "", + "Name of the eval. Defaults to -eval, or -trace-eval under --source traces.") + cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") + cmd.Flags().StringVar(&source, "source", "", + "Where rows come from: dataset or traces. Defaults to dataset.") + cmd.Flags().StringVar(&dataset, "dataset", "", + "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().IntVar(&maxTraces, "max-traces", project.DefaultScaffoldMaxTraces, + "Cap on traces read by a --source traces eval. Delete max_traces from the "+ + "file to take the service default instead.") + cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, + "Evaluator reference, repeatable. Use builtin. for a built-in. "+ + "Passing this replaces the defaults, so it also opts out of rubric generation.") + cmd.Flags().StringVar(&judgeModel, "judge-model", "", + "Model deployment the graders judge with. Detected from the project when omitted.") + cmd.Flags().StringVar(&path, "path", project.DefaultEvalDir, + "Directory to write the configuration into. Used verbatim, never re-rooted.") + cmd.Flags().BoolVar(&force, "force", false, + "Replace an eval of the same name instead of failing.") + return cmd +} + +// defaultEvalName names an eval after what it evaluates and what it reads. +func defaultEvalName(target, source string) string { + if source == initSourceTraces { + return target + "-trace-eval" + } + return target + "-eval" +} + +// scaffoldInput is everything planScaffold needs, gathered so the signature +// does not grow a seventh positional string. +type scaffoldInput struct { + evalName string + target string + source string + dataset string + maxTraces int + evaluators []string + judgeModel string + rubricName string + evalDir string + cfg *project.EvalConfig +} + +// scaffold is what `init` added, and what it should suggest doing next. +type scaffold struct { + eval *project.Eval + datasetName string + rubricName string + generateDataset bool + generateRubric bool +} + +// planScaffold appends one eval to the configuration, adding any catalog +// entries it needs. +// +// The default evaluator set is a built-in plus a generated rubric: the built-in +// alone would be generic, and the rubric is what makes the baseline about this +// agent. Passing --evaluator replaces both, which is how a caller opts out of +// rubric generation. +func planScaffold(in scaffoldInput) scaffold { + cfg := in.cfg + out := scaffold{rubricName: in.rubricName} + + eval := project.Eval{ + Name: in.evalName, + Description: fmt.Sprintf("Basic quality evaluation for %s", in.target), + EvaluationLevel: project.EvaluationLevelTurn, + Target: &project.Target{ + Type: project.TargetTypeAgent, + Name: in.target, + }, + } + + if in.source == initSourceTraces { + // A trace-backed eval filters by agent rather than invoking one: the + // conversations already happened. + eval.Target = nil + eval.Source = &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: in.target, + MaxTraces: in.maxTraces, + } + } else { + datasetName := in.evalName + datasetSource := "" + out.generateDataset = true + if in.dataset != "" { + out.generateDataset = false + if looksLikeLocalDataset(in.dataset) { + // --dataset is given relative to where the user is standing, + // but source: resolves relative to the config, so the path has + // to be rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(in.dataset, in.evalDir) + datasetName = strings.TrimSuffix( + filepath.Base(in.dataset), filepath.Ext(in.dataset)) + } else { + // A bare name references an already-registered dataset. + datasetName = in.dataset + } + } else { + datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) + } + eval.Dataset = datasetName + out.datasetName = datasetName + addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, Source: datasetSource}) + } + + // Every evaluator carries the judge deployment, because that is where the + // service reads it from: judging built-ins declare it as required, so an + // eval that leaves it off is rejected before it runs. The binding step + // drops it again for a rule-based evaluator that declares no judge. + initParams := map[string]any{} + if in.judgeModel != "" { + initParams["model"] = in.judgeModel + } + withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { + if len(initParams) == 0 { + return ref + } + params := make(map[string]any, len(initParams)) + maps.Copy(params, initParams) + ref.InitializationParameters = params + return ref + } + + refs := evalcore.EvaluatorList{} + if len(in.evaluators) == 0 { + refs = append(refs, + withModel(evalcore.EvaluatorRef{ + Evaluator: evalcore.BuiltinPrefix + "task_adherence", + })) + if in.source != initSourceTraces { + refs = append(refs, withModel(evalcore.EvaluatorRef{Evaluator: in.rubricName})) + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: in.rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, in.rubricName), + }) + out.generateRubric = true + } + } else { + for _, e := range in.evaluators { + ref := evalcore.EvaluatorRef{Evaluator: e} + refs = append(refs, withModel(ref)) + if ref.IsBuiltin() { + continue + } + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: e, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), + }) + } + } + eval.Evaluators = refs + + cfg.Evals = append(cfg.Evals, eval) + out.eval = &cfg.Evals[len(cfg.Evals)-1] + return out +} + +// addDatasetDecl adds a catalog entry unless the name is already declared. +func addDatasetDecl(cfg *project.EvalConfig, decl project.DatasetDecl) { + if decl.Name == "" { + return + } + // A source-less entry is still declared: it names a dataset already + // registered on the project. Skipping it left the eval referencing a + // dataset absent from the catalog, which its own validation rejects. + if _, ok := cfg.DatasetDeclaration(decl.Name); ok { + return + } + cfg.Datasets = append(cfg.Datasets, decl) +} + +// addEvaluatorDecl adds a catalog entry unless the name is already declared. +func addEvaluatorDecl(cfg *project.EvalConfig, decl project.EvaluatorDecl) { + if _, ok := cfg.EvaluatorDeclaration(decl.Name); ok { + return + } + cfg.Evaluators = append(cfg.Evaluators, decl) +} + +// evaluatorNames lists the evaluators the eval will run, in declaration order. +func (s scaffold) evaluatorNames() []string { + names := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + names = append(names, ref.Evaluator) + } + return names +} + +// nextSteps are the commands to run after `init`, and only the ones that have +// something to do. +// +// A caller who supplied both a dataset and their evaluators has nothing left to +// generate, and pointing them at a generation command would submit a billed job +// for an artifact they already have. +func (s scaffold) nextSteps() []string { + var steps []string + if s.generateDataset { + steps = append(steps, "azd ai eval dataset generate "+s.datasetName) + } + if s.generateRubric { + steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) + } + if len(steps) == 0 { + steps = append(steps, "azd up", "azd ai eval run start") + } + return steps +} + +// relativeToConfig rewrites a path given relative to the working directory so +// it resolves from the directory holding the eval config. +func relativeToConfig(path, evalDir string) string { + if filepath.IsAbs(path) { + return path + } + + absPath, err := filepath.Abs(path) + if err != nil { + return path + } + absOut, err := filepath.Abs(evalDir) + if err != nil { + return path + } + + rel, err := filepath.Rel(absOut, absPath) + if err != nil { + return path + } + + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, ".") { + rel = "./" + rel + } + return rel +} + +// rootConfigName is azd's project file, which the eval service is declared in. +const rootConfigName = "azure.yaml" + +// aiProjectHost is the Foundry project service other extensions declare. The +// eval service uses it for ordering when the repo has one. +const aiProjectHost = "azure.ai.project" + +// How the root config ended up referencing the eval service. +const ( + wiringAdded = "added" // the service was added to the project + wiringPresent = "present" // an eval service was already declared +) + +// noAzdProject is what init reports when there is nothing to attach to. +const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + + "or run this from the root of an existing one; the eval service is added to " + + "its azure.yaml" + +// readAzdProject returns the project, without changing it. +func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, errors.New(noAzdProject) + } + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return nil, errors.New(noAzdProject) + } + return resp.GetProject(), nil +} + +// aiModelHost is the model-deployment service the sibling Foundry extensions +// declare, which is where a judge deployment can be read without a service +// call. +const aiModelHost = "azure.ai.model" + +// detectModelDeployment finds the deployment the graders judge with, from what +// the project already declares. +// +// `init` makes no service calls, so detection is limited to the project file. +// Coming back empty is not a failure: --judge-model supplies it. +func detectModelDeployment(proj *azdext.ProjectConfig) string { + for name, svc := range proj.GetServices() { + if svc.GetHost() != aiModelHost { + continue + } + if props := svc.GetAdditionalProperties().AsMap(); props != nil { + for _, key := range []string{"deployment", "deploymentName", "name", "model"} { + if v, ok := props[key].(string); ok && v != "" { + return v + } + } + } + return name + } + return "" +} + +// ensureRootEvalService declares the eval service in azd's project file. +// +// azd acts on nothing until the service exists, so the reference is made rather +// than described. It goes through azd's own Project().AddService, the same call +// the agents extension uses, so azd owns the edit and the project file keeps +// whatever shape azd gives it. +func ensureRootEvalService( + ctx context.Context, + serviceName, target, configPath string, +) (string, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return "", fmt.Errorf("connecting to azd: %w", err) + } + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "", errors.New(noAzdProject) + } + + // A service already pointing at this configuration is left alone: + // re-adding it would deploy the same evals twice. + if svc, ok := resp.GetProject().GetServices()[serviceName]; ok && svc.GetHost() == project.EvalHost { + return wiringPresent, nil + } + + props, err := structpb.NewStruct(map[string]any{ + "$ref": "./" + filepath.ToSlash(configPath), + }) + if err != nil { + return "", fmt.Errorf("building the eval service entry: %w", err) + } + + _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ + Service: &azdext.ServiceConfig{ + Name: serviceName, + Host: project.EvalHost, + Uses: evalServiceUses(resp.GetProject(), target), + AdditionalProperties: props, + }, + }) + if err != nil { + return "", fmt.Errorf("adding the eval service to %s: %w", rootConfigName, err) + } + return wiringAdded, nil +} + +// evalServiceUses orders the eval after the things it reads. +// +// It is conditional for the same reason the agents extension makes it +// conditional: naming a service the project does not declare is a broken +// reference, and an eval config can perfectly well sit in a repo that reaches +// an existing Foundry project by endpoint and an agent deployed elsewhere. +// +// Catalog entries need no ordering of their own — datasets, evaluators and +// evals are reconciled in a fixed order inside one deploy, forced by the +// contract rather than chosen. +func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { + var uses []string + for name, svc := range proj.GetServices() { + if svc.GetHost() == aiProjectHost { + uses = append(uses, name) + break + } + } + if _, ok := proj.GetServices()[target]; ok { + uses = append(uses, target) + } + return uses +} + +// looksLikeLocalDataset distinguishes a path from a registered dataset name. +func looksLikeLocalDataset(v string) bool { + if strings.ContainsAny(v, `/\`) { + return true + } + return strings.EqualFold(filepath.Ext(v), ".jsonl") +} + +func writeYAML(path string, v any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + data, err := yaml.Marshal(v) + if err != nil { + return fmt.Errorf("serializing %q: %w", path, err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("writing %q: %w", path, err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go new file mode 100644 index 00000000000..3b505df7f4d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -0,0 +1,292 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// scaffoldFor runs planScaffold against a fresh configuration, which is what +// `init` does on a project that has never been initialized. +func scaffoldFor(t *testing.T, in scaffoldInput) (scaffold, *project.EvalConfig) { + t.Helper() + if in.cfg == nil { + in.cfg = &project.EvalConfig{} + } + if in.evalDir == "" { + in.evalDir = project.DefaultEvalDir + } + if in.rubricName == "" { + in.rubricName = in.target + "-quality" + } + return planScaffold(in), in.cfg +} + +// The scaffold must round-trip and validate, otherwise `azd up` fails on a +// config the tool itself produced. +func TestScaffold_RoundTripsAndValidates(t *testing.T) { + dir := t.TempDir() + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + evalDir: dir, + }) + + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") + + eval, err := loaded.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, project.TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) + require.Equal(t, project.EvaluationLevelTurn, eval.EvaluationLevel) +} + +// Re-running init appends rather than replacing, so one file ends up holding +// every eval for the target. +func TestScaffold_AppendsToAnExistingConfiguration(t *testing.T) { + dir := t.TempDir() + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "first", target: "support-agent", judgeModel: "m", evalDir: dir, + }) + _, cfg = scaffoldFor(t, scaffoldInput{ + evalName: "second", target: "support-agent", judgeModel: "m", evalDir: dir, cfg: cfg, + }) + + require.Equal(t, []string{"first", "second"}, cfg.EvalNames()) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A trace-backed eval invokes nothing, so agent_name filters instead of +// targeting, and a scaffolded cap keeps the first run bounded rather than +// taking the service's default of 1000. +func TestScaffold_TraceSourceHasNoTarget(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-trace-eval", + target: "support-agent", + source: initSourceTraces, + maxTraces: project.DefaultScaffoldMaxTraces, + }) + + require.Nil(t, plan.eval.Target) + require.NotNil(t, plan.eval.Source) + require.Equal(t, project.SourceTypeTraces, plan.eval.Source.Type) + require.Equal(t, "support-agent", plan.eval.Source.AgentName) + require.Equal(t, 20, plan.eval.Source.MaxTraces) +} + +// Omitting the cap leaves the key out, which is how the service default is +// taken — writing a zero would send one. +func TestScaffold_TraceCapIsOmittedWhenZero(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "t", target: "a", source: initSourceTraces, + }) + require.Zero(t, plan.eval.Source.MaxTraces) + + body, err := yaml.Marshal(plan.eval) + require.NoError(t, err) + require.NotContains(t, string(body), "max_traces") +} + +// The default set is a built-in plus a generated rubric: the built-in alone +// would be generic, and the rubric is what makes the baseline about this agent. +func TestScaffold_DefaultEvaluators(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "gpt-5.6-luna", + }) + + require.Equal(t, + []string{"builtin.task_adherence", "support-agent-quality"}, + plan.evaluatorNames()) + + // Every evaluator carries the judge deployment, because the judging + // built-ins declare it and an eval that leaves it off is rejected. + for _, ref := range plan.eval.Evaluators { + require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["model"], + "%s must name a judge deployment", ref.Evaluator) + } +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation. +func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) + + require.Equal(t, []string{"builtin.task_adherence"}, plan.evaluatorNames()) + require.False(t, plan.generateRubric, "no rubric is generated when evaluators are given") +} + +// `init` closes by naming what to run next, and only what has something to do. +// Pointing a caller who supplied their own artifacts at a generation command +// would submit a billed job for something they already have. +func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { + t.Run("nothing supplied", func(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", + }) + require.Equal(t, []string{ + "azd ai eval dataset generate support-agent-smoke", + "azd ai eval evaluator generate support-agent-quality", + }, plan.nextSteps()) + }) + + t.Run("dataset supplied", func(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m", + }) + require.Equal(t, + []string{"azd ai eval evaluator generate support-agent-quality"}, + plan.nextSteps()) + }) + + t.Run("everything supplied", func(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + dataset: "prod-golden", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) + require.Equal(t, []string{"azd up", "azd ai eval run start"}, plan.nextSteps(), + "with every artifact in place the next step is to deploy") + }) +} + +// Built-ins are referenced but never declared, so the scaffold must not give +// one a catalog entry to publish. +func TestScaffold_BuiltinEvaluatorsGetNoCatalogEntry(t *testing.T) { + dir := t.TempDir() + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence", "my-custom"}, + judgeModel: "m", + evalDir: dir, + }) + + require.Len(t, plan.eval.Evaluators, 2) + require.True(t, plan.eval.Evaluators[0].IsBuiltin()) + require.False(t, plan.eval.Evaluators[1].IsBuiltin()) + + require.Len(t, cfg.Evaluators, 1, "only the custom evaluator is declared") + require.Equal(t, "my-custom", cfg.Evaluators[0].Name) + require.Len(t, cfg.CustomEvaluators(), 1, + "only the custom evaluator is this config's to publish") + + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A bare name means an already-registered dataset; a path means a local file. +// Either way the dataset was supplied, so nothing is scheduled to generate it — +// only a missing --dataset produces a generation step. +func TestScaffold_DatasetReferenceForms(t *testing.T) { + t.Run("local path becomes a source", func(t *testing.T) { + // --dataset is relative to the working directory, but source: is + // resolved relative to the eval config, so it has to be rebased. + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "./tests/golden.jsonl", evalDir: "evals", + }) + decl, ok := cfg.DatasetDeclaration("golden") + require.True(t, ok) + require.Equal(t, "../tests/golden.jsonl", decl.Source, + "a dataset outside the eval dir must be reached with ..") + require.Equal(t, "golden", plan.eval.Dataset) + require.False(t, plan.generateDataset, + "a supplied dataset must not be scheduled for generation") + }) + + t.Run("bare name references a registered dataset", func(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "prod-sample", + }) + decl, ok := cfg.DatasetDeclaration("prod-sample") + require.True(t, ok) + require.Empty(t, decl.Source, "a registered dataset must not get a local source") + require.Equal(t, "prod-sample", plan.eval.Dataset) + require.False(t, plan.generateDataset) + }) + + t.Run("no dataset flag scaffolds a local path and a generation step", func(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", + }) + require.Equal(t, "support-agent-smoke", plan.eval.Dataset, + "the dataset is named after the eval") + decl, ok := cfg.DatasetDeclaration("support-agent-smoke") + require.True(t, ok) + require.Contains(t, decl.Source, "support-agent-smoke.jsonl") + require.True(t, plan.generateDataset) + }) +} + +func TestLooksLikeLocalDataset(t *testing.T) { + require.True(t, looksLikeLocalDataset("./data/golden.jsonl")) + require.True(t, looksLikeLocalDataset("golden.jsonl")) + require.True(t, looksLikeLocalDataset(`data\golden.jsonl`)) + require.False(t, looksLikeLocalDataset("prod-sample")) +} + +// Paths are used verbatim relative to the working directory; the doubling bug +// in the agent-scoped command must not reappear. +func TestSaveEvalConfig_UsesPathVerbatim(t *testing.T) { + dir := t.TempDir() + nested := filepath.Join(dir, "evals") + + require.NoError(t, project.SaveEvalConfig(nested, &project.EvalConfig{})) + _, err := os.Stat(project.EvalConfigPath(nested)) + require.NoError(t, err, "the file must land exactly at the requested path") + + doubled := filepath.Join(dir, "evals", "evals") + _, err = os.Stat(doubled) + require.Error(t, err, "the path must not be re-rooted under itself") +} + +// normalizeRubricBody accepts a bare definition or a full document. +func TestNormalizeRubricBody(t *testing.T) { + t.Run("bare definition is wrapped", func(t *testing.T) { + body, err := normalizeRubricBody("quality", + []byte(`{"type":"rubric","dimensions":[{"id":"q","weight":10}]}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"quality"`) + require.Contains(t, string(body), `"definition"`) + }) + + t.Run("full document keeps its definition and takes the flag name", func(t *testing.T) { + body, err := normalizeRubricBody("renamed", + []byte(`{"name":"old","definition":{"type":"rubric","dimensions":[]}}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"renamed"`) + }) + + t.Run("rejects a document with neither", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"unrelated":true}`)) + require.ErrorContains(t, err, "dimensions") + }) + + t.Run("rejects invalid JSON", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`not json`)) + require.ErrorContains(t, err, "not valid JSON") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go new file mode 100644 index 00000000000..03745d01e75 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func projectWith(names ...string) *azdext.ProjectConfig { + proj := &azdext.ProjectConfig{Services: map[string]*azdext.ServiceConfig{}} + for _, n := range names { + proj.Services[n] = &azdext.ServiceConfig{Name: n} + } + return proj +} + +// The eval service is ordered after everything it reads, but only names +// services the project actually declares. Naming one it does not have is a +// broken reference, and an eval config can sit in a repo that reaches an +// existing Foundry project by endpoint and an agent deployed elsewhere. +func TestEvalServiceUses_OnlyWhatTheProjectDeclares(t *testing.T) { + assert.Nil(t, evalServiceUses(projectWith("api", "web"), "support-agent"), + "neither the project service nor the agent is declared, so there is nothing to order after") + + withProject := projectWith("api", "support-agent") + withProject.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", Host: aiProjectHost, + } + assert.Equal(t, []string{"ai-project", "support-agent"}, + evalServiceUses(withProject, "support-agent"), + "the eval runs after the project it evaluates against and the agent it evaluates") + + assert.Equal(t, []string{"support-agent"}, + evalServiceUses(projectWith("support-agent"), "support-agent"), + "an agent alone is still worth ordering after") +} + +// `init` detects the judge deployment from the project, because it makes no +// service calls and this is the only place it can read one. +func TestDetectModelDeployment(t *testing.T) { + assert.Empty(t, detectModelDeployment(projectWith("api", "web"))) + + proj := projectWith("api") + proj.Services["chat"] = &azdext.ServiceConfig{Name: "chat", Host: aiModelHost} + assert.Equal(t, "chat", detectModelDeployment(proj), + "the service name is the deployment name when nothing more specific is declared") + + named := projectWith() + named.Services["chat"] = &azdext.ServiceConfig{ + Name: "chat", + Host: aiModelHost, + AdditionalProperties: mustStruct(t, map[string]any{ + "deployment": "gpt-5.6-luna", + }), + } + assert.Equal(t, "gpt-5.6-luna", detectModelDeployment(named)) +} + +func mustStruct(t *testing.T, m map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(m) + require.NoError(t, err) + return s +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go new file mode 100644 index 00000000000..52279b70b9b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A useful generation instruction is often longer than fits on a command +// line, so it can come from a file instead. +func TestResolveInstructionReadsFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(path, + []byte(" A customer support agent answering billing questions.\n\n"), 0o600)) + + got, err := resolveInstruction("", path) + require.NoError(t, err) + require.Equal(t, "A customer support agent answering billing questions.", got, + "surrounding whitespace should be trimmed") +} + +func TestResolveInstructionPrefersInlineWhenNoFile(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + + got, err = resolveInstruction("", "") + require.NoError(t, err) + require.Empty(t, got) +} + +// An unreadable or empty file is reported rather than silently generating from +// no instruction at all. +func TestResolveInstructionRejectsUnusableFile(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(t.TempDir(), "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "agent-instruction-file") + + empty := filepath.Join(t.TempDir(), "empty.md") + require.NoError(t, os.WriteFile(empty, []byte(" \n"), 0o600)) + _, err = resolveInstruction("", empty) + require.Error(t, err) + require.Contains(t, err.Error(), "empty") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go new file mode 100644 index 00000000000..c4a9d48aa23 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -0,0 +1,255 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Generation runs as two independent long-running resources — one for datasets, +// one for evaluators — sharing no collection. A job group therefore nests under +// the resource that produced it: a top-level `job show ` would have to guess +// the endpoint from an id prefix that is not a documented contract. + +const ( + jobKindDataset = "dataset" + jobKindEvaluator = "evaluator" +) + +// jobKind binds a group to one generation resource, so every command under it +// calls one endpoint rather than trying both and reporting whichever answered. +type jobKind struct { + name string + list func(context.Context, *evalContext) ([]eval_api.GenerationJob, error) + get func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + cancel func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + remove func(context.Context, *evalContext, string) error +} + +var datasetJobs = jobKind{ + name: jobKindDataset, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, +} + +var evaluatorJobs = jobKind{ + name: jobKindEvaluator, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, +} + +func newJobCommand(kind jobKind) *cobra.Command { + cmd := &cobra.Command{ + Use: "job", + Short: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.", kind.name), + Long: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.\n\n", kind.name) + + fmt.Sprintf("This is the resume path for `%s generate`: a job started with ", kind.name) + + "--no-wait, or one whose client was interrupted, is reattached to here " + + "rather than restarted.", + } + cmd.AddCommand( + newJobListCommand(kind), + newJobShowCommand(kind), + newJobCancelCommand(kind), + newJobDeleteCommand(kind), + ) + return cmd +} + +func newJobListCommand(kind jobKind) *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: fmt.Sprintf("List the project's %s generation jobs.", kind.name), + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + jobs, err := kind.list(ctx, ec) + if err != nil { + return fmt.Errorf("listing %s generation jobs: %w", kind.name, err) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), jobs) + } + if len(jobs) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "No %s generation jobs found.\n", kind.name) + return nil + } + table := make([][]string, 0, len(jobs)) + for _, j := range jobs { + table = append(table, []string{j.ID, j.Status}) + } + return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "STATUS"}, table) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobShowCommand(kind jobKind) *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: fmt.Sprintf("Show a %s generation job.", kind.name), + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + job, err := kind.get(ctx, ec, jobID) + if err != nil { + return jobLookupError(kind, jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), job) + } + fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", job.ID, job.Status) + if job.Error != nil && job.Error.Message != "" { + fmt.Fprintf(cmd.OutOrStdout(), "error: %s\n", job.Error.Message) + } + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobCancelCommand(kind jobKind) *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "cancel ", + Short: fmt.Sprintf("Cancel an in-flight %s generation job.", kind.name), + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + canceled, err := kind.cancel(ctx, ec, jobID) + if err != nil { + return jobLookupError(kind, jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", + kind.name, jobID, canceled.Status) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobDeleteCommand(kind jobKind) *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: fmt.Sprintf("Delete a %s generation job record.", kind.name), + Long: fmt.Sprintf("Delete a %s generation job record.\n\n", kind.name) + + "The artifact the job produced is already registered as its own version " + + "and is not affected.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := kind.remove(ctx, ec, jobID); err != nil { + return jobLookupError(kind, jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": jobID, "kind": kind.name, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted %s generation job %s\n", kind.name, jobID) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// jobLookupError names the sibling group, because the two job types share an id +// shape and reaching for the wrong one is the likely mistake. +func jobLookupError(kind jobKind, jobID string, err error) error { + if eval_api.IsNotFound(err) { + other := jobKindEvaluator + if kind.name == jobKindEvaluator { + other = jobKindDataset + } + return fmt.Errorf( + "no %s generation job %q in this project; if it generated a %s, "+ + "use the %s job group instead", kind.name, jobID, other, other) + } + return fmt.Errorf("reading %s generation job %s: %w", kind.name, jobID, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go new file mode 100644 index 00000000000..7cf4fc48788 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeJSONL(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "d.jsonl") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// The service accepts whatever bytes it is given, so a malformed row becomes a +// published version with an eval bound to it, and only fails much later +// on a row nobody has looked at. A live deploy published `{not json at all}` +// as version 1.0 before this existed. +func TestValidateJSONL_RejectsAMalformedRowByLine(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"fine\"}\n{not json at all}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "one JSON object") +} + +func TestValidateJSONL_AcceptsWellFormedRows(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n{\"query\":\"b\"}\n"))) +} + +// Trailing and interior blank lines are formatting, not rows. +func TestValidateJSONL_IgnoresBlankLines(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n\n{\"query\":\"b\"}\n\n"))) +} + +// A file with nothing in it publishes a version that can never score anything. +func TestValidateJSONL_RejectsAFileWithNoRows(t *testing.T) { + err := validateJSONL(writeJSONL(t, "\n\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "no rows") +} + +// A JSON array is the shape people reach for when they mean JSONL. +func TestValidateJSONL_RejectsAJSONArray(t *testing.T) { + err := validateJSONL(writeJSONL(t, "[{\"query\":\"a\"},{\"query\":\"b\"}]\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 1") +} + +// An empty object parses but evaluates to nothing. +func TestValidateJSONL_RejectsAnEmptyObject(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"a\"}\n{}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "empty object") +} + +// A conversation-level row holds a whole transcript and runs past bufio's +// default 64KB line limit, which would otherwise be reported as invalid JSON. +func TestValidateJSONL_AcceptsAVeryLongRow(t *testing.T) { + long := make([]byte, 200*1024) + for i := range long { + long[i] = 'x' + } + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\""+string(long)+"\"}\n"))) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go new file mode 100644 index 00000000000..9cef816cec2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// newListenCommand registers the service-target provider with azd. It is hidden +// and invoked by azd itself, not by users. +func newListenCommand() *cobra.Command { + return azdext.NewListenCommand(configureExtensionHost) +} + +// configureExtensionHost wires the azure.ai.eval service target so `azd up` and +// `azd deploy` reach this extension. The provider name must match the manifest. +func configureExtensionHost(host *azdext.ExtensionHost) { + azdClient := host.Client() + + host.WithServiceTarget(project.EvalHost, func() azdext.ServiceTargetProvider { + return project.NewEvalServiceTargetProvider( + azdClient, + func(ctx context.Context) (project.Reconciler, error) { + return newEvalReconciler(ctx) + }, + ) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go new file mode 100644 index 00000000000..faf104043b9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// extensionManifest is the subset of extension.yaml this test asserts on. +type extensionManifest struct { + ID string `yaml:"id"` + Version string `yaml:"version"` + Capabilities []string `yaml:"capabilities"` + Providers []struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + } `yaml:"providers"` +} + +func loadManifest(t *testing.T) extensionManifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "extension.yaml")) + require.NoError(t, err, "reading extension.yaml") + + var manifest extensionManifest + require.NoError(t, yaml.Unmarshal(raw, &manifest)) + return manifest +} + +// A declared capability azd cannot reach is worse than an undeclared one: azd +// invokes `metadata` to discover the command tree, and it was declared without +// the command being registered, so discovery failed with "unknown command". +func TestDeclaredCapabilitiesAreImplemented(t *testing.T) { + manifest := loadManifest(t) + root := NewRootCommand() + + hasCommand := func(name string) bool { + for _, sub := range root.Commands() { + if sub.Name() == name { + return true + } + } + return false + } + + for _, capability := range manifest.Capabilities { + switch capability { + case "metadata": + require.True(t, hasCommand("metadata"), + "the metadata capability requires a metadata command") + case "service-target-provider": + require.True(t, hasCommand("listen"), + "a service-target provider is registered through the listen command") + require.NotEmpty(t, manifest.Providers, + "the manifest must name the provider it registers") + case "custom-commands": + require.NotEmpty(t, root.Commands()) + case "lifecycle-events": + // The SDK only starts the event manager when handlers are + // registered, so declaring this without any is an unused + // permission. Nothing here registers handlers today. + t.Fatalf("lifecycle-events is declared but no event handlers are registered") + } + } +} + +// The provider name in the manifest is what azd matches a service's `host` +// against, so a mismatch silently means the provider is never invoked. +func TestManifestProviderMatchesHostConstant(t *testing.T) { + manifest := loadManifest(t) + require.NotEmpty(t, manifest.Providers) + + names := make([]string, 0, len(manifest.Providers)) + for _, p := range manifest.Providers { + names = append(names, p.Name) + } + require.Contains(t, names, "azure.ai.eval", + "the manifest must declare the host the provider registers for") +} + +// extension.yaml carries a note asking that version.txt be kept in sync. The +// build stamps the binary from version.txt while the registry reads +// extension.yaml, so a drift ships a binary that misreports its own version. +func TestManifestVersionMatchesVersionFile(t *testing.T) { + manifest := loadManifest(t) + + raw, err := os.ReadFile(filepath.Join("..", "..", "version.txt")) + require.NoError(t, err, "reading version.txt") + + require.Equal(t, + strings.TrimSpace(string(raw)), + strings.TrimSpace(manifest.Version), + "version.txt and extension.yaml must agree") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go new file mode 100644 index 00000000000..143b49e91ba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "text/tabwriter" + + "github.com/spf13/cobra" +) + +const outputJSON = "json" + +// Progress markers from the azd style guide, so the extension's lines sit +// alongside core's without a second vocabulary. +const ( + doneMark = "(✓) Done:" // finished successfully + skippedMark = "(-) Skipped:" // intentionally not done, not a failure + failedMark = "(x) Failed:" // the step did not complete +) + +// outputFormat reads the inherited -o/--output flag. +func outputFormat(cmd *cobra.Command) string { + if cmd == nil { + return "" + } + v, err := cmd.Flags().GetString("output") + if err != nil { + return "" + } + return strings.ToLower(v) +} + +// isJSON reports whether the command should emit machine-readable output. +func isJSON(cmd *cobra.Command) bool { + return outputFormat(cmd) == outputJSON +} + +// emitJSON writes v as indented JSON. +func emitJSON(w io.Writer, v any) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +// emitJSONList writes items as a JSON array. +// +// List commands emit a bare array rather than the envelope the service replied +// with. The envelopes disagree with each other — the OpenAI-shaped APIs wrap +// results in `data`, the ARM-shaped ones in `value` — so passing them through +// would make a caller's parsing depend on which service happens to back a given +// command. They also carry paging fields that this extension does not follow, +// which would suggest there is more to fetch when there is not. +// +// A nil slice encodes as `null`, so it is normalized to an empty array: a +// caller iterating the result should see no elements, not a type error. +func emitJSONList[T any](w io.Writer, items []T) error { + if items == nil { + items = []T{} + } + return emitJSON(w, items) +} + +// emitTable writes a simple aligned table. Rows must match the header width. +func emitTable(w io.Writer, headers []string, rows [][]string) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { + return err + } + for _, row := range rows { + if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { + return err + } + } + return tw.Flush() +} + +// requireFlag returns an error naming the missing flag, used when --no-prompt +// prevents asking for a required value. +func requireFlag(name string) error { + return fmt.Errorf("--%s is required (running with --no-prompt)", name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go new file mode 100644 index 00000000000..2914e5c3fbd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The list commands are backed by two different services whose envelopes +// disagree — `data` on one side, `value` on the other. Emitting whichever one +// came back would make a caller's parsing depend on that accident, so every +// list emits a bare array instead. +func TestEmitJSONList_EmitsAnArrayNotAnEnvelope(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitJSONList(&buf, []string{"a", "b"})) + assert.Equal(t, "[\n \"a\",\n \"b\"\n]\n", buf.String()) +} + +// A nil slice marshals to `null`, which a caller iterating the output cannot +// range over. An empty listing has to come back as an empty array. +func TestEmitJSONList_NilBecomesEmptyArray(t *testing.T) { + var buf bytes.Buffer + var none []string + require.NoError(t, emitJSONList(&buf, none)) + assert.Equal(t, "[]\n", buf.String()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go new file mode 100644 index 00000000000..2588ad49b9c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" +) + +// TestConfigureExtensionHostMatchesManifest verifies that the providers this +// extension registers match those declared in its extension.yaml. +func TestConfigureExtensionHostMatchesManifest(t *testing.T) { + manifestPath := filepath.Join("..", "..", "extension.yaml") + require.NoError(t, azdext.VerifyProvidersMatchManifest(configureExtensionHost, manifestPath)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go new file mode 100644 index 00000000000..0e0affef613 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -0,0 +1,599 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "time" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" +) + +// evalReconciler applies the eval configuration to the data plane. It is the +// deploy half of the provider; the provider owns ordering, this owns the calls. +type evalReconciler struct { + ec *evalContext +} + +var _ project.Reconciler = (*evalReconciler)(nil) + +func newEvalReconciler(ctx context.Context) (project.Reconciler, error) { + ec, err := newEvalContext(ctx, "") + if err != nil { + return nil, err + } + return &evalReconciler{ec: ec}, nil +} + +// EnsureDataset registers a new version only when the local content changed. +// +// The dataset API exposes no content hash, so comparing against the service +// would mean downloading the blob on every deploy. Instead the local file is +// hashed and the digest kept in the azd environment. +func (r *evalReconciler) EnsureDataset( + ctx context.Context, + decl project.DatasetDecl, + localPath string, +) (string, bool, error) { + // No local source means the dataset is already registered; just confirm it. + if localPath == "" { + version := decl.Version + if version == "" { + list, err := r.ec.datasetClient.ListDatasetVersions( + ctx, decl.Name, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, fmt.Errorf( + "dataset %q has no local source and could not be found on the project: %w", + decl.Name, err) + } + if len(list.Value) == 0 { + return "", false, fmt.Errorf( + "dataset %q has no local source and is not registered on the project", decl.Name) + } + version = dataset_api.LatestVersion(list.Value) + } + return version, false, nil + } + + if _, err := os.Stat(localPath); err != nil { + return "", false, fmt.Errorf("dataset source %q: %w", localPath, err) + } + + // A malformed row is only noticed once the service tries to evaluate it, + // by which point a version has been published and the eval points at + // it. Reading the file here costs nothing and names the offending line. + if err := validateJSONL(localPath); err != nil { + return "", false, fmt.Errorf("dataset %q: %w", decl.Name, err) + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + return "", false, err + } + + key := project.FingerprintKey("dataset", decl.Name) + if prior := r.ec.getEnvValue(ctx, key); prior == digest { + // Unchanged since the last deploy; reuse the recorded version, but only + // after confirming nobody published a newer one outside the repo. An + // explicit `version:` is the author saying which version they want, so + // it settles the question and the check does not apply. + if version := r.ec.getEnvValue(ctx, versionKey("dataset", decl.Name)); version != "" { + if decl.Version != "" { + return decl.Version, false, nil + } + if err := r.checkDatasetDrift(ctx, decl.Name, version); err != nil { + return "", false, err + } + return version, false, nil + } + } + + // The upload helper scans a directory for the first .jsonl. + dir := localPath + if info, err := os.Stat(localPath); err == nil && !info.IsDir() { + dir = filepath.Dir(localPath) + } + + // A declared version is the version to publish, not one to count from. + // Reaching here means the content differs from what that version holds, so + // republishing over it would change a version the author pinned. + if decl.Version != "" { + ds, err := r.ec.datasetClient.UploadVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + if dataset_api.IsVersionConflict(err) { + return "", false, fmt.Errorf( + "dataset %q version %s already exists and the local file differs from it. "+ + "Raise `version:` to publish the change, or drop it to let each "+ + "deploy take the next version", + decl.Name, decl.Version) + } + return "", false, err + } + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) + return ds.Version, true, nil + } + + // UploadNextVersion discovers the currently registered version when none is + // declared, so the upload does not restart at 1.0 and collide. + ds, err := r.ec.datasetClient.UploadNextVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) + _ = r.ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version) + + return ds.Version, true, nil +} + +// checkDatasetDrift fails when the service holds a newer version than the one +// recorded at the last deploy. +// +// Local content being unchanged is not enough to reuse the recorded version: +// someone may have published a newer one outside the repo, and silently +// pinning the eval to the older version would quietly evaluate against +// stale data. Publishing is not destructive — versions are immutable — so the +// remedy is to sync, not to overwrite. +// validateJSONL checks that every row is a JSON object before the file is +// published. +// +// The service accepts the upload whatever the bytes are, so a typo becomes a +// registered version, an eval bound to it, and a run that fails on a row +// nobody has looked at. Blank lines are skipped: they are not rows. +func validateJSONL(path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + // A row carrying a whole conversation runs well past the 64KB default. + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + rows := 0 + for line := 1; scanner.Scan(); line++ { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return fmt.Errorf( + "%s line %d is not valid JSON: %w. Every line must be one JSON object", + path, line, err) + } + if len(row) == 0 { + return fmt.Errorf( + "%s line %d is an empty object, which evaluates to nothing", path, line) + } + rows++ + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + if rows == 0 { + return fmt.Errorf("%s has no rows to evaluate", path) + } + return nil +} + +func (r *evalReconciler) checkDatasetDrift( + ctx context.Context, + name, recorded string, +) error { + latest := r.latestDatasetVersion(ctx, name) + if latest == "" || latest == recorded { + return nil + } + if !dataset_api.VersionGreater(latest, recorded) { + return nil + } + return fmt.Errorf( + "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ + "someone published a version outside this repo. "+ + "Pin it with `version: %s` on the dataset, or pull the newer content locally, "+ + "then deploy again", + name, latest, recorded, latest) +} + +// latestDatasetVersion reports the newest registered version, or empty when the +// dataset is unknown or the listing has not caught up. +func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { + list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return dataset_api.LatestVersion(list.Value) +} + +// EnsureEvaluator publishes a new version when the local definition differs +// from what the service holds. +// +// The two kinds of evaluator are told apart by the source's extension: `.py` +// is code, anything else is a rubric. They also detect change differently. A +// rubric definition comes back inline, so it is compared directly; a code +// definition's source is not read back in a form worth comparing, so a +// fingerprint of the script is kept in the azd environment, the same way +// datasets work. +func (r *evalReconciler) EnsureEvaluator( + ctx context.Context, + decl project.EvaluatorDecl, + localPath string, +) (string, bool, error) { + if localPath == "" { + raw, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, fmt.Errorf( + "evaluator %q has no local source and could not be found on the project: %w", + decl.Name, err) + } + return versionFromRaw(raw, decl.Version), false, nil + } + + if _, err := os.Stat(localPath); err != nil { + return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + } + + raw, err := os.ReadFile(localPath) + if err != nil { + return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + } + + body, err := normalizeRubricBody(decl.Name, raw) + if err != nil { + return "", false, fmt.Errorf("evaluator %q: %w", decl.Name, err) + } + + // Compare against the definition already on the service. + var known json.RawMessage + if existing, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, "", ProjectEndpointAPIVersion, + ); err == nil { + remote := versionFromRaw(existing, "") + if sameDefinition(existing, body) { + // Nothing to publish, but the version is still worth recording: + // it is what a later deploy compares against to notice that + // someone moved the evaluator on from here. + if remote != "" { + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), remote) + } + return versionFromRaw(existing, decl.Version), false, nil + } + + // The definitions differ, which means either the local file changed + // or someone published a version outside the repo. The version + // recorded at the last deploy is what tells them apart, and + // publishing over the second case would bury an intentional change + // under one nobody asked for. + if recorded := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)); recorded != "" { + if err := checkEvaluatorDrift(decl.Name, recorded, remote); err != nil { + return "", false, err + } + } + + // What that read saw is what keeps the publish from being answered + // with it again. + known = existing + } + + created, err := r.ec.evalClient.CreateEvaluatorVersion( + ctx, decl.Name, body, known, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) + return created.Version, true, nil +} + +// checkEvaluatorDrift fails when the service holds a newer version than the +// one recorded at the last deploy. +// +// It is asked only when the local definition and the remote one disagree, +// which on its own says nothing about who moved: the author may have edited +// the file, or someone may have published a version from outside the repo. +// The recorded version settles it, and the difference matters because +// publishing is how this reconciler resolves a disagreement — doing that over +// a version somebody deliberately published would bury their change under one +// nobody asked for, with `azd up` reporting success. +// +// The remote version is passed in rather than listed, because the version +// listing lags a publish and would report an evaluator as un-drifted for the +// first seconds of its newest version's life. +func checkEvaluatorDrift(name, recorded, remote string) error { + recordedNumber, err := strconv.Atoi(recorded) + if err != nil { + return nil + } + remoteNumber, err := strconv.Atoi(remote) + if err != nil || remoteNumber <= recordedNumber { + return nil + } + return fmt.Errorf( + "evaluator %q is at version %s on the project but %s was recorded at the last "+ + "deploy, and the local definition does not match it: someone published a "+ + "version outside this repo. Publishing over it would leave their change "+ + "behind, so bring version %s into the declared source and deploy again, or "+ + "delete that version if it was a mistake", + name, remote, recorded, remote) +} + +// evaluatorPropagation bounds the wait for a freshly published evaluator to +// become usable. +// +// A create returns before the version is resolvable everywhere, and the very +// next step of a deploy is EnsureEval, which names the evaluator in a testing +// criterion. Creating the eval inside that window fails with "The evaluator X +// was not found" — a confusing error, because the evaluator was published +// seconds earlier and is plainly there by the time anyone looks. The observed +// gap is under a second, so the poll is frequent and the cap is generous +// enough to absorb a slow day without stalling a deploy on an evaluator that +// is genuinely missing. +const ( + evaluatorPropagationTimeout = 30 * time.Second + evaluatorPropagationInterval = 250 * time.Millisecond +) + +// awaitEvaluatorReadable polls until a published version is resolvable, or the +// cap passes. +// +// Two reads have to agree, because they are not backed by the same view. The +// direct read goes consistent almost immediately; the version listing lags it +// by seconds, the same way the dataset listing does. A live publish was +// observed reading back at 03:06:58 and still failing eval creation at +// 03:06:59, so waiting on the direct read alone leaves exactly the race this +// exists to close. The listing is the slower of the two and therefore the one +// worth waiting on. +// +// A timeout is not an error. The wait is a courtesy that makes the common case +// reliable; if it never succeeds, the create that follows will report the real +// problem with far more context than a wait that gave up could. +func (r *evalReconciler) awaitEvaluatorReadable(ctx context.Context, name, version string) { + if version == "" { + return + } + deadline := time.Now().Add(evaluatorPropagationTimeout) + for { + if r.evaluatorVersionResolvable(ctx, name, version) { + return + } + if time.Now().After(deadline) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(evaluatorPropagationInterval): + } + } +} + +// evaluatorVersionResolvable reports whether a version can be both read +// directly and found in the listing. +func (r *evalReconciler) evaluatorVersionResolvable( + ctx context.Context, + name, version string, +) bool { + if _, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return false + } + + list, err := r.ec.evalClient.ListEvaluatorVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + if err != nil || list == nil { + return false + } + for _, entry := range list.Value { + if entry.Version == version { + return true + } + } + return false +} + +// EnsureEval creates the group when it has never been deployed, or when an +// upstream artifact changed. Groups are immutable, so a change means a new +// group and a new id. +func (r *evalReconciler) EnsureEval( + ctx context.Context, + group project.Eval, + datasetPath string, + recreate bool, +) (string, error) { + if group.ID != "" { + return group.ID, nil + } + + // Evals are immutable, so a change to the eval's own substance — evaluators, + // dataset, source, target, level — needs a new eval just as much as a change + // to an upstream artifact does. Name and description are excluded from the + // digest and pushed in place instead. + digest, err := project.FingerprintGroup(group) + if err != nil { + return "", err + } + key := project.FingerprintKey("eval", group.Name) + if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { + recreate = true + } + + cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) + if cached == "" && !recreate { + // Nothing recorded under this name, but the substance may already be + // deployed under the name it had before. The environment records the id + // against the digest as well, which is what recognises a rename rather + // than reading it as a delete plus an add. + if adopted := r.adoptRenamed(ctx, group, digest); adopted != "" { + cached = adopted + } + } + if cached != "" && !recreate { + if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + // Record the digest on reuse as well, otherwise an eval deployed + // before fingerprinting existed never establishes a baseline and + // later edits go undetected. + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), cached) + _ = r.ec.setEnvValue(ctx, digestIDKey(digest), cached) + _ = r.ec.setEnvValue(ctx, envKeyEvalID, cached) + return cached, nil + } + } + + req, err := buildEvalRequest( + &group, + r.ec.evaluatorSchemas(ctx), + datasetColumnsFromPath(datasetPath), + ) + if err != nil { + return "", err + } + created, err := r.ec.evalClient.CreateOpenAIEval(ctx, req) + if err != nil { + return "", err + } + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID) + _ = r.ec.setEnvValue(ctx, digestIDKey(digest), created.ID) + // EVAL_ID stays the last-deployed eval, which is what the commands + // fall back to when a config names only one. + _ = r.ec.setEnvValue(ctx, envKeyEvalID, created.ID) + return created.ID, nil +} + +// adoptRenamed reclaims the eval this declaration used to be called, so a +// rename keeps the id and every run under it rather than forking the history. +// +// The name is what UpdateEvalParametersBody reaches, so the new one is pushed +// to the service. A failure there is not fatal: the eval is still the right one +// and the declaration still resolves, it just reads under its old name in the +// portal until the next deploy. +func (r *evalReconciler) adoptRenamed( + ctx context.Context, + group project.Eval, + digest string, +) string { + id := r.ec.getEnvValue(ctx, digestIDKey(digest)) + if id == "" { + return "" + } + remote, err := r.ec.evalClient.GetOpenAIEval(ctx, id) + if err != nil { + return "" + } + if remote.Name == group.Name { + return id + } + _, _ = r.ec.evalClient.UpdateOpenAIEval(ctx, id, &eval_api.UpdateOpenAIEvalRequest{ + Name: group.Name, + }) + return id +} + +// sameDefinition reports whether the locally authored definition already +// matches what the service holds. +// +// Only the keys the candidate declares are compared. The service enriches a +// definition when it is created — a rubric of nothing but `type` and +// `dimensions` comes back carrying data_schema, init_parameters and metrics it +// was never given — so comparing whole documents never matches and every +// deploy publishes a redundant version. +func sameDefinition(existing, candidate []byte) bool { + extract := func(raw []byte) map[string]json.RawMessage { + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return nil + } + def, ok := doc["definition"] + if !ok { + return nil + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(def, &fields); err != nil { + return nil + } + return fields + } + + onService, authored := extract(existing), extract(candidate) + if onService == nil || authored == nil { + return false + } + + for key, want := range authored { + got, ok := onService[key] + if !ok || !equalJSON(got, want) { + return false + } + } + return true +} + +// equalJSON compares two JSON values structurally, so key order and +// whitespace do not register as a change. +func equalJSON(a, b json.RawMessage) bool { + var left, right any + if err := json.Unmarshal(a, &left); err != nil { + return false + } + if err := json.Unmarshal(b, &right); err != nil { + return false + } + return reflect.DeepEqual(left, right) +} + +func versionFromRaw(raw []byte, fallback string) string { + var doc struct { + Version string `json:"version"` + } + if err := json.Unmarshal(raw, &doc); err == nil && doc.Version != "" { + return doc.Version + } + return fallback +} + +// versionKey holds the version resolved for an artifact at the last deploy. +func versionKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_VERSION" +} + +// idKey names the env entry holding a resolved id. +// +// Ids are per declaration. A single shared key works only while a config has +// one group: with two, the second deploy finds the first's id cached, confirms +// it exists, and hands it back for the wrong group. +func idKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_ID" +} + +// digestIDKey records an eval's id against its substance, which is what lets a +// renamed declaration find the eval it already deployed. Keyed by a prefix of +// the digest, because the whole hash makes an unreadable environment variable. +func digestIDKey(digest string) string { + return "EVAL_SUBSTANCE_" + strings.ToUpper(digest[:16]) + "_ID" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go new file mode 100644 index 00000000000..d945b95eda9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Drift is only interesting when the two definitions already disagree, and +// then only when the disagreement came from the project rather than from the +// author. These are the four ways that question can be answered. +func TestCheckEvaluatorDrift(t *testing.T) { + // The author edited the file. The project is where the last deploy left + // it, so publishing is exactly right and must not be blocked. + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "3")) + + // Someone published outside the repo. Publishing over it would leave + // their change behind with `azd up` reporting success. + err := checkEvaluatorDrift("support-quality", "3", "4") + require.Error(t, err) + assert.Contains(t, err.Error(), "support-quality") + assert.Contains(t, err.Error(), "version 4") + assert.Contains(t, err.Error(), "3 was recorded") + assert.Contains(t, err.Error(), "outside this repo", + "the message has to say who moved, not just that something did") + + // A version that went backwards is not drift: a newer version was + // deleted, and republishing is how the repo takes the name back. + require.NoError(t, checkEvaluatorDrift("support-quality", "4", "3")) + + // Versions this extension did not number cannot be compared, and refusing + // a deploy over a numbering convention it does not own would be worse + // than not checking. + require.NoError(t, checkEvaluatorDrift("support-quality", "", "4")) + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "preview")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go new file mode 100644 index 00000000000..8086fd96c43 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// The service enriches a definition when it stores it: a rubric of nothing but +// type and dimensions comes back carrying data_schema, init_parameters and +// metrics. Comparing whole documents therefore never matched, and every deploy +// published a redundant version. +func TestSameDefinitionIgnoresServerAddedFields(t *testing.T) { + authored := []byte(`{ + "name": "r", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}] + } + }`) + + onService := []byte(`{ + "name": "r", + "version": "2", + "created_at": "2026-07-28T00:00:00Z", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}], + "data_schema": {"type":"object","properties":{"query":{"type":"string"}}}, + "init_parameters": {"required":["model"],"properties":{"model":{"type":"string"}}}, + "metrics": {"score":{"type":"number"}} + } + }`) + + require.True(t, sameDefinition(onService, authored), + "server-added fields must not count as a change") +} + +// A real edit still registers. +func TestSameDefinitionDetectsAuthoredChange(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":7}]}}`) + onService := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}],"metrics":{}}}`) + + require.False(t, sameDefinition(onService, authored)) +} + +// Key order and whitespace are not changes. +func TestSameDefinitionIsStructural(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}]}}`) + onService := []byte("{\"definition\":{\n \"dimensions\": [ {\"weight\":5,\"id\":\"a\"} ],\n \"type\":\"rubric\"\n}}") + + require.True(t, sameDefinition(onService, authored)) +} + +func TestSameDefinitionRejectsMalformed(t *testing.T) { + good := []byte(`{"definition":{"type":"rubric"}}`) + require.False(t, sameDefinition([]byte(`not json`), good)) + require.False(t, sameDefinition(good, []byte(`not json`))) + require.False(t, sameDefinition([]byte(`{"no":"definition"}`), good)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go new file mode 100644 index 00000000000..fa417ce43c3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" +) + +// Precedence decides behaviour without announcing it, so a wrong answer here +// is silent. options.max_samples was parsed and dropped once already, which is +// what these lock down. +func TestResolveMaxSamples_Precedence(t *testing.T) { + withOptions := &project.Eval{MaxSamples: 25} + + assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") + assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{}), "neither means no cap") + assert.Equal(t, 0, resolveMaxSamples(0, nil)) + assert.Equal(t, 7, resolveMaxSamples(7, nil), "a flag stands on its own") + + // Zero in config is absent, not a cap of zero: a cap of zero would send + // nothing at all. + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{MaxSamples: 0})) +} + +// The level is the eval's alone. A per-run override would put two incomparable +// result sets under one eval's history, and would bypass the +// supported_evaluation_levels check `azd up` does against the declared level. +func TestResolveLevel_ComesFromTheEval(t *testing.T) { + declared := &project.Eval{ + EvaluationLevel: project.EvaluationLevelConversation, + } + + assert.Equal(t, project.EvaluationLevelConversation, resolveLevel(declared)) + assert.Empty(t, resolveLevel(&project.Eval{}), "unset defers to the service default") + assert.Empty(t, resolveLevel(nil)) +} + +// A group's target decides which run-time fields its criteria can bind. Getting +// this wrong passes validation and then errors on every row. +func TestSampleBindingsFor_UnknownTargetBindsNothing(t *testing.T) { + assert.Nil(t, sampleBindingsFor("prompt"), + "an unrecognized target must bind nothing rather than guess at agent fields") +} + +// The level filter is what keeps a conversation evaluator from being sent turn +// fields and the reverse. Both directions matter. +func TestSelectLevelFields_KeepsOnlyTheLevelsShape(t *testing.T) { + accepted := []string{"query", "response", "messages", "tool_definitions"} + + conv := selectLevelFields(accepted, nil, project.EvaluationLevelConversation) + assert.Contains(t, conv, "messages") + assert.NotContains(t, conv, "query") + assert.NotContains(t, conv, "response") + assert.Contains(t, conv, "tool_definitions", "fields outside the split are untouched") + + turn := selectLevelFields(accepted, nil, project.EvaluationLevelTurn) + assert.Contains(t, turn, "query") + assert.Contains(t, turn, "response") + assert.NotContains(t, turn, "messages") + + // An evaluator offering only one shape is left alone, whatever the level. + only := []string{"query", "response"} + assert.Equal(t, only, selectLevelFields(only, nil, project.EvaluationLevelConversation)) + + // A required field is never dropped: a genuine conflict has to surface as a + // missing-field error rather than being reshaped away. + kept := selectLevelFields(accepted, []string{"query"}, project.EvaluationLevelConversation) + assert.Contains(t, kept, "query", "a required field survives the level filter") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go new file mode 100644 index 00000000000..4f6fcd8860e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// NewRootCommand builds the `azd ai eval` command tree. +func NewRootCommand() *cobra.Command { + rootCmd, _ := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ + Name: "eval", + Use: "eval [options]", + Short: fmt.Sprintf( + "Define and run Foundry evaluations from your terminal. %s", + color.YellowString("(Beta)"), + ), + }) + rootCmd.SilenceUsage = true + rootCmd.SilenceErrors = true + rootCmd.CompletionOptions.DisableDefaultCmd = true + + // The data-plane clients trace requests through the standard logger, which + // Go writes to stderr, so it has to be silenced unless debug was asked for. + // + // The SDK's own hook is chained rather than replaced, and cobra ignores + // PersistentPreRun entirely once PersistentPreRunE is set. The SDK sets + // cobra.EnableTraverseRunHooks, so this still runs alongside subcommand + // hooks. The cleanup func is discarded on purpose: log writes are + // unbuffered and the OS closes the file at exit. + sdkPreRun := rootCmd.PersistentPreRunE + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if sdkPreRun != nil { + if err := sdkPreRun(cmd, args); err != nil { + return err + } + } + setupDebugLogging(cmd.Flags()) + return nil + } + + rootCmd.AddCommand( + newInitCommand(), + newDatasetCommand(), + newRunCommand(), + newEvaluatorCommand(), + newEvalCreateCommand(), + newEvalListCommand(), + newEvalShowCommand(), + newEvalDeleteCommand(), + newListenCommand(), + ) + + // The manifest declares the `metadata` capability, which azd uses to + // discover this extension's command tree. Without the command registered, + // that discovery fails with "unknown command". + rootCmd.AddCommand(azdext.NewMetadataCommand("1.0", "azure.ai.evaluations", func() *cobra.Command { + return rootCmd + })) + + return rootCmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go new file mode 100644 index 00000000000..c3591d523f9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -0,0 +1,918 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Terminal run states reported by the service. +var terminalRunStates = map[string]bool{ + "completed": true, + "failed": true, + "canceled": true, + "cancelled": true, + "error": true, +} + +// runCompleted turns a run that did not complete into an error, so that a +// caller who waited for it exits non-zero. +// +// The results have already been printed by the time this is asked, which is +// the point: a run that errored has a reason worth reading, and reporting it +// and then exiting 0 tells a pipeline the evaluation passed. It is checked +// before the gate because the gate's exit code means "the evaluation +// regressed", and a run that never produced results has not regressed — it did +// not run. Distinguishing those two is what the separate code is for. +func runCompleted(run *eval_api.OpenAIEvalRun) error { + if run == nil { + return nil + } + switch strings.ToLower(run.Status) { + case "completed", "": + return nil + } + return fmt.Errorf("run %s finished with status %s", run.ID, run.Status) +} + +// newRunCommand builds the run group. +// +// `run` is a group, not an executable verb: once `run output` exists, a bare +// `run` would make `azd ai eval run list` read as "run the thing called list". +func newRunCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "run", + Short: "Start and inspect evaluation runs.", + } + addRunSubcommands(cmd) + cmd.AddCommand(buildRunCommand( + "start", "Start a run, creating the eval if it does not exist yet.")) + return cmd +} + +// buildRunCommand builds `run start`. +func buildRunCommand(use, short string) *cobra.Command { + var ( + groupName string + datasetName string + runName string + maxSamples int + wait bool + failOn string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + // Parsed before any network work, so a malformed threshold costs + // nothing to find out about. + threshold, err := parseGate(failOn) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // One flag takes a name or an id. A declared name also brings the + // declaration, which is what says where rows come from; a bare id + // has none, so the pairing comes from the eval's previous run. + ref, err := ec.resolveEvalRef(ctx, project.DefaultEvalDir, groupName) + if err != nil { + return err + } + evalID := ref.ID + group := ref.Eval + configPath := ref.ConfigPath + + if datasetName != "" { + if !ref.Declared() { + return errors.New( + "--dataset overrides the dataset an eval declares, so it needs a " + + "declared eval; pass --eval with a name from the configuration") + } + if _, ok := ref.Config.DatasetDeclaration(datasetName); !ok { + return fmt.Errorf( + "dataset %q is not in the catalog in %s", + datasetName, filepath.ToSlash(configPath)) + } + // The eval keeps its own declaration; only this run reads elsewhere. + overridden := *group + overridden.Dataset = datasetName + overridden.Source = nil + group = &overridden + } + + if ref.Declared() { + if err := ec.checkDatasetRegistered(ctx, ref.Config, group, configPath); err != nil { + return err + } + } + + var dataSource *eval_api.EvalRunDataSource + switch { + case group == nil: + dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) + default: + dataSource, err = ec.buildRunDataSource( + ctx, group, configPath, resolveMaxSamples(maxSamples, group)) + } + if err != nil { + return err + } + + if runName == "" { + base := "eval" + if group != nil { + base = group.Name + } + runName = fmt.Sprintf("%s-%s", base, time.Now().UTC().Format("20060102-150405")) + } + + metadata := map[string]string{} + if lvl := resolveLevel(group); lvl != "" { + metadata["evaluation_level"] = lvl + } + + run, err := ec.evalClient.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: runName, + DataSource: dataSource, + Metadata: metadata, + }) + if err != nil { + return fmt.Errorf("starting the evaluation run: %w", err) + } + + // Remembered per group as well as globally: a single shared key + // belongs to whichever group ran last, so another group asking for + // "the last run" would be handed one that is not its own. + _ = ec.setEnvValue(ctx, idKey("evalrun", evalID), run.ID) + if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { + // Persisting the run id is a convenience for later commands. + // Reported on stdout because azd does not surface an + // extension's stderr, and skipped outside a project. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprintf(out, "warning: %v\n", err) + } + } + + if !wait { + if isJSON(cmd) { + return emitJSON(out, startedRun(run, evalID, group)) + } + fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) + fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval-id %s\n", run.ID, evalID) + return nil + } + + final, err := ec.pollRun(ctx, evalID, run.ID, out, isJSON(cmd)) + if err != nil { + return err + } + + if isJSON(cmd) { + if err := emitJSON(out, final); err != nil { + return err + } + } else if err := renderRun(out, final, ec.runMeans(ctx, evalID, final)); err != nil { + return err + } + + // Last, so that the results are reported whether or not the gate + // holds: a pipeline that only learns it failed is worse off than + // one that can see by how much. + if err := runCompleted(final); err != nil { + return err + } + applyGate(cmd, threshold, final) + return nil + }, + } + + cmd.Flags().StringVar(&groupName, "eval", "", + "Name of the eval to run, or its id. Defaults to the only one declared.") + cmd.Flags().StringVar(&datasetName, "dataset", "", + "Catalog dataset to read instead of the one the eval declares. "+ + "Must satisfy the eval's column schema.") + cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + "Cap the rows sent from the dataset.") + cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + addFailOnFlag(cmd, &failOn) + // The spec documents --no-wait, and cobra does not derive it from a bool. + var noWait bool + cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the run and return immediately.") + cmd.PreRun = func(*cobra.Command, []string) { + if noWait { + wait = false + } + } + cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + + return cmd +} + +// resolveEvalIDFromConfig finds the eval to run against, creating it when it +// has never been deployed. Resolution order: an id pinned on the group, then +// the azd environment, then create. +func (ec *evalContext) resolveEvalIDFromConfig( + ctx context.Context, + group *project.Eval, + configPath string, + level string, + out interface{ Write([]byte) (int, error) }, + jsonMode bool, +) (string, error) { + if group.ID != "" { + return group.ID, nil + } + + for _, key := range evalIDKeys(group.Name, filepath.Dir(configPath)) { + cached := ec.getEnvValue(ctx, key) + if cached == "" { + continue + } + // Confirm it still exists; a deleted group should fall through to create. + if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + return cached, nil + } + } + + if !jsonMode { + fmt.Fprintf(out, "Creating eval %q...\n", group.Name) + } + + // The level from the flag wins over the eval's own declaration, so it has + // to reach the criteria that accept evaluation_level. + effective := *group + if level != "" { + effective.EvaluationLevel = level + } + + req, err := buildEvalRequest( + &effective, + ec.evaluatorSchemas(ctx), + datasetColumns(configPath, group), + ) + if err != nil { + return "", err + } + created, err := ec.evalClient.CreateOpenAIEval(ctx, req) + if err != nil { + return "", fmt.Errorf("creating eval %q: %w", group.Name, err) + } + if err := ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID); err != nil { + fmt.Fprintf(out, "warning: %v\n", err) + } + _ = ec.setEnvValue(ctx, envKeyEvalID, created.ID) + return created.ID, nil +} + +// evalIDKeys lists the env entries that may hold this eval's id, most +// specific first. +// +// The per-name entry is what the extension writes. EVAL_ID is also the +// documented way to point a config at an eval that already exists, created in +// the portal or by another tool, so it stays readable — but only when the +// configuration declares a single eval. With more than one there is no way to +// tell which eval a shared entry refers to, and reading it anyway is what let a +// second eval adopt the first one's id. +func evalIDKeys(name, evalDir string) []string { + keys := []string{idKey("eval", name)} + if cfg, err := project.OpenEvalConfig(evalDir); err == nil && + cfg != nil && len(cfg.Evals) == 1 { + keys = append(keys, envKeyEvalID) + } + return keys +} + +// checkDatasetRegistered fails when the group's local dataset has edits that +// were never deployed. +// +// A run sends a local dataset inline, so without this the run would evaluate +// content that no registered version corresponds to: the results are attributed +// to the eval but cannot be traced back to a dataset version, which +// makes them impossible to reproduce or compare. +// +// The check only applies once a deploy has recorded a fingerprint. Before that +// there is nothing to have drifted from, and running is how a group first comes +// into existence. +func (ec *evalContext) checkDatasetRegistered( + ctx context.Context, + cfg *project.EvalConfig, + group *project.Eval, + configPath string, +) error { + localPath := localDatasetPath(configPath, group) + if localPath == "" { + return nil + } + + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok { + return nil + } + + recorded := ec.getEnvValue(ctx, project.FingerprintKey("dataset", decl.Name)) + if recorded == "" { + return nil + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + // Reading the file is the run's problem to report, not this check's. + return nil + } + if digest == recorded { + return nil + } + + return fmt.Errorf( + "dataset %q has local edits that are not registered.\n"+ + " Run `azd up` to register them, or `--eval-id ` to run against "+ + "an existing eval", + decl.Name) +} + +// reuseDataSourceFromLastRun rebuilds a run's data source from the group's most +// recent run. +// +// `--eval-id` deliberately ignores the config, but a run still needs a target +// and a dataset, and an eval carries neither: the group holds only its +// testing criteria, and the dataset travels on the run. The previous run is the +// only place that pairing survives, so re-running a group means repeating what +// it last ran. +func (ec *evalContext) reuseDataSourceFromLastRun( + ctx context.Context, + evalID string, +) (*eval_api.EvalRunDataSource, error) { + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + return nil, fmt.Errorf("reading previous runs of eval %s: %w", evalID, err) + } + if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { + return nil, fmt.Errorf( + "eval %s has no previous run to repeat, so there is no target or dataset "+ + "to reuse.\n"+ + " Run it from the config once with `azd ai eval run start`, or name an "+ + "eval that declares one with `--eval`", + evalID) + } + return list.Data[0].DataSource, nil +} + +// buildRunDataSource binds the dataset to the run. The eval carries no +// dataset today, so it is supplied here. +func (ec *evalContext) buildRunDataSource( + ctx context.Context, + group *project.Eval, + configPath string, + maxSamples int, +) (*eval_api.EvalRunDataSource, error) { + if group == nil { + return nil, fmt.Errorf("no eval to run") + } + if group.Target == nil || group.Target.Name == "" { + return nil, fmt.Errorf("eval %q does not name a target agent", group.Name) + } + + ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + + if group.Dataset == "" { + return nil, fmt.Errorf("eval %q does not reference a dataset", group.Name) + } + + // A local source is read from disk; anything else is already registered and + // has to be fetched. Either way the rows are sent inline, because a run's + // file_id means an uploaded file and a dataset name is not one: sending the + // name is rejected with "invalid data source file ids". + localPath := localDatasetPath(configPath, group) + if localPath == "" { + items, err := ec.readRegisteredDataset(ctx, group.Dataset, maxSamples) + if err != nil { + return nil, err + } + ds.SetFileContent(items) + return ds, nil + } + + items, err := readJSONL(localPath, maxSamples) + if err != nil { + return nil, err + } + if len(items) == 0 { + return nil, fmt.Errorf("dataset file %q has no rows", localPath) + } + ds.SetFileContent(items) + return ds, nil +} + +// readRegisteredDataset fetches a published dataset's rows, optionally keeping +// only the first n. +// +// The rows have to be fetched because a run cannot reference a dataset by +// name: `file_id` means an uploaded file, and passing a dataset name there is +// rejected. Fetching also makes --max-samples mean the same thing whether the +// dataset is local or published, which a file reference could not — that +// source carries no row limit. +func (ec *evalContext) readRegisteredDataset( + ctx context.Context, + name string, + maxSamples int, +) ([]map[string]any, error) { + version := ec.getEnvValue(ctx, versionKey("dataset", name)) + if version == "" { + versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", name, err) + } + if versions != nil { + version = dataset_api.LatestVersion(versions.Value) + } + } + if version == "" { + return nil, fmt.Errorf("dataset %q has no versions to read", name) + } + + content, err := ec.datasetClient.DownloadDatasetContent( + ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + } + + items, err := readJSONLBytes(content, maxSamples) + if err != nil { + return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + } + if len(items) == 0 { + return nil, fmt.Errorf("dataset %q version %s has no rows", name, version) + } + return items, nil +} + +// datasetColumns reports the columns a group's dataset provides, so criteria +// bind only to fields that exist and a missing required field is caught +// locally rather than as a service rejection. +// +// A nil result means the columns are unknown, which is the case for a dataset +// already registered in the project. The builder then assumes every field an +// evaluator accepts is present. +func datasetColumns(configPath string, group *project.Eval) map[string]bool { + return datasetColumnsFromPath(localDatasetPath(configPath, group)) +} + +// datasetColumnsFromPath reads one row to learn the dataset's shape. An empty +// path, or an unreadable file, yields nil. +func datasetColumnsFromPath(localPath string) map[string]bool { + if localPath == "" { + return nil + } + // One row is enough to learn the shape. + items, err := readJSONL(localPath, 1) + if err != nil || len(items) == 0 { + return nil + } + columns := make(map[string]bool, len(items[0])) + for name := range items[0] { + columns[name] = true + } + return columns +} + +// localDatasetPath resolves the dataset's local source relative to the config +// file, returning empty when the dataset is registered rather than local. +func localDatasetPath(configPath string, group *project.Eval) string { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil || group == nil { + return "" + } + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok || decl.Source == "" { + return "" + } + if filepath.IsAbs(decl.Source) { + return decl.Source + } + return filepath.Join(filepath.Dir(configPath), decl.Source) +} + +// readJSONL reads newline-delimited JSON, optionally truncating to limit rows. +func readJSONL(path string, limit int) ([]map[string]any, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + defer f.Close() + + items, err := scanJSONL(f, limit) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + return items, nil +} + +// readJSONLBytes parses JSONL already in memory, which is how a registered +// dataset arrives. +func readJSONLBytes(content []byte, limit int) ([]map[string]any, error) { + return scanJSONL(bytes.NewReader(content), limit) +} + +// scanJSONL reads rows until the limit is reached, so a subset costs only the +// rows it needs to parse. +func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { + var items []map[string]any + scanner := bufio.NewScanner(r) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + line := 0 + for scanner.Scan() { + line++ + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return nil, fmt.Errorf("line %d is not valid JSON: %w", line, err) + } + items = append(items, row) + if limit > 0 && len(items) >= limit { + break + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + return items, nil +} + +// resolveLevel prefers the flag, then the eval's own declaration. +// resolveLevel is the eval's declared scoring granularity. +// +// There is no per-run override: the level decides the row mapping, so two +// levels under one eval would put incomparable result sets in the same history, +// and it would bypass the supported_evaluation_levels check `azd up` does +// against the declared level. A second level is a second eval. +func resolveLevel(group *project.Eval) string { + if group != nil { + return group.EvaluationLevel + } + return "" +} + +// resolveMaxSamples prefers the flag, then the eval's own declaration, matching +// how the evaluation level resolves. +// +// Without this, max_samples parsed and did nothing: an eval that caps its +// sample count in config would send the whole dataset, and only a flag on every +// invocation would honour the cap. +func resolveMaxSamples(flag int, group *project.Eval) int { + if flag > 0 { + return flag + } + if group != nil && group.MaxSamples > 0 { + return group.MaxSamples + } + return 0 +} + +// pollRun waits for the run to reach a terminal state, reporting status changes. +func (ec *evalContext) pollRun( + ctx context.Context, + evalID, runID string, + out interface{ Write([]byte) (int, error) }, + jsonMode bool, +) (*eval_api.OpenAIEvalRun, error) { + const interval = 5 * time.Second + lastStatus := "" + + for { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err != nil { + return nil, fmt.Errorf("polling run %s: %w", runID, err) + } + if run.Status != lastStatus { + lastStatus = run.Status + if !jsonMode { + fmt.Fprintf(out, " status: %s\n", run.Status) + } + } + if terminalRunStates[strings.ToLower(run.Status)] { + return run, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(interval): + } + } +} + +// startedRunHandoff is what `run start --no-wait -o json` returns. +// +// It is a handoff rather than a dump of the service object. The pipeline that +// started the run has to come back for it later, and doing that needs exactly +// three things: the run, the eval it belongs to, and a name a human can read +// in the log that reports it. The service object carries none of the third and +// buries the first two under the data source, the metadata and every field the +// API happens to return, so a script reading it would depend on a shape this +// extension does not control. +type startedRunHandoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + EvalName string `json:"eval_name,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt string `json:"created_at,omitempty"` +} + +// startedRun builds the handoff. +func startedRun( + run *eval_api.OpenAIEvalRun, + evalID string, + group *project.Eval, +) startedRunHandoff { + handoff := startedRunHandoff{ + RunID: run.ID, + EvalID: evalID, + Status: run.Status, + CreatedAt: timestampString(run.CreatedAt), + } + // Absent with --eval-id, where there is no config to take a name from. + if group != nil { + handoff.EvalName = group.Name + } + return handoff +} + +// timestampString renders a service timestamp as RFC 3339. +// +// The field arrives as epoch seconds on a run and as a formatted string +// elsewhere, so passing it through would hand a script a value whose type +// depends on which route produced it. +func timestampString(value any) string { + switch t := value.(type) { + case nil: + return "" + case string: + return t + case float64: + return time.Unix(int64(t), 0).UTC().Format(time.RFC3339) + case int64: + return time.Unix(t, 0).UTC().Format(time.RFC3339) + case json.Number: + if seconds, err := t.Int64(); err == nil { + return time.Unix(seconds, 0).UTC().Format(time.RFC3339) + } + return t.String() + default: + return fmt.Sprint(value) + } +} + +// runMeans reads the run's rows to average each evaluator's score. +// +// Best effort: the summary is worth printing without the column, and a run +// that scored nothing has no rows to read. +func (ec *evalContext) runMeans( + ctx context.Context, + evalID string, + run *eval_api.OpenAIEvalRun, +) map[string]float64 { + if run == nil || run.ResultCounts == nil || run.ResultCounts.Total == 0 { + return nil + } + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil || items == nil { + return nil + } + return criteriaMeans(items.Data) +} + +// timestampTime reads a service timestamp, which arrives as epoch seconds on a +// run and as a formatted string elsewhere. +func timestampTime(value any) time.Time { + switch t := value.(type) { + case float64: + return time.Unix(int64(t), 0).UTC() + case int64: + return time.Unix(t, 0).UTC() + case string: + if parsed, err := time.Parse(time.RFC3339, t); err == nil { + return parsed.UTC() + } + } + return time.Time{} +} + +// renderRun prints what a person needs after waiting for a run. +// +// means carries each criterion's average score, which the run summary does not +// return; it is nil when the rows were not fetched, and the column is dropped. +func renderRun( + out interface{ Write([]byte) (int, error) }, + run *eval_api.OpenAIEvalRun, + means map[string]float64, +) error { + fmt.Fprintln(out) + renderRunHeader(out, run) + + // A run that failed carries why, and it is usually the only actionable + // thing in the response — dropping it leaves the caller with just the word + // "failed". + if why := run.Failure(); why != "" { + fmt.Fprintf(out, "\n%s\n", why) + } + + renderCriteriaTable(out, run.PerTestingCriteria, means) + + // Counted over samples, not over verdicts: a sample that failed two + // evaluators is one sample to go and look at, and reporting it as two + // overstates how much is wrong. + if c := run.ResultCounts; c != nil && c.Total > 0 { + fmt.Fprintf(out, "\nOverall pass rate: %s (%d/%d samples passed every evaluator)\n", + formatRate(c.Passed, c.Total), c.Passed, c.Total) + if c.Errored > 0 { + fmt.Fprintf(out, "%d sample(s) errored and were not scored.\n", c.Errored) + } + if c.Failed > 0 { + fmt.Fprintln(out, + "\nView failing samples: azd ai eval run output list --failed-only") + } + } + + if run.ReportURL != "" { + fmt.Fprintf(out, "Report: %s\n", run.ReportURL) + } + return nil +} + +// renderRunHeader prints the run's identity above the per-evaluator table. +// +// The eval is named from the metadata the extension wrote at create time, +// because the run carries only an id and the id is not what anyone declared. +func renderRunHeader(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) { + fmt.Fprintf(out, "%-10s %s\n", "Run", run.ID) + if name := run.Metadata["azd_eval"]; name != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", name) + } else if run.EvalID != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", run.EvalID) + } + fmt.Fprintf(out, "%-10s %s\n", "Status", run.Status) + if c := run.ResultCounts; c != nil && c.Total > 0 { + fmt.Fprintf(out, "%-10s %d\n", "Samples", c.Total) + } + if d := runDuration(run); d != "" { + fmt.Fprintf(out, "%-10s %s\n", "Duration", d) + } +} + +// runDuration reports how long the run took, or "" when either end is missing. +func runDuration(run *eval_api.OpenAIEvalRun) string { + start, end := timestampTime(run.CreatedAt), timestampTime(run.ModifiedAt) + if start.IsZero() || end.IsZero() || !end.After(start) { + return "" + } + d := end.Sub(start).Round(time.Second) + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } + return fmt.Sprintf("%dm%02ds", int(d.Minutes()), int(d.Seconds())%60) +} + +// renderCriteriaTable prints one row per evaluator. +// +// Sorted by name so two runs of the same eval read the same way; the service +// returns the criteria in whatever order it evaluated them. +func renderCriteriaTable( + out interface{ Write([]byte) (int, error) }, + results []eval_api.EvalRunCriteriaResult, + means map[string]float64, +) { + if len(results) == 0 { + return + } + + sorted := append([]eval_api.EvalRunCriteriaResult(nil), results...) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].TestingCriteria < sorted[j].TestingCriteria + }) + + width := len("EVALUATOR") + for _, r := range sorted { + if n := len(r.TestingCriteria); n > width { + width = n + } + } + + fmt.Fprintf(out, "\n%-*s %4s %4s %9s", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") + fmt.Fprintf(out, "%s\n", meanHeader(means)) + fmt.Fprintf(out, "%s %s %s %s%s\n", + strings.Repeat("-", width), "----", "----", "---------", meanRule(means)) + + for _, r := range sorted { + scored := r.Passed + r.Failed + fmt.Fprintf(out, "%-*s %4d %4d %9s", + width, r.TestingCriteria, r.Passed, r.Failed, formatRate(r.Passed, scored)) + if means != nil { + if mean, ok := means[r.TestingCriteria]; ok { + fmt.Fprintf(out, " %10.1f", mean) + } else { + fmt.Fprintf(out, " %10s", "-") + } + } + fmt.Fprintln(out) + // Errors are not failures — the evaluator never reached a verdict — + // so they are named rather than folded into the fail column, where + // they would look like a quality problem. + if r.Errored > 0 { + fmt.Fprintf(out, "%-*s %s\n", width, "", errorNote(r.Errored)) + } + } +} + +// meanHeader and meanRule add the score column only when there are scores. +func meanHeader(means map[string]float64) string { + if means == nil { + return "" + } + return fmt.Sprintf(" %10s", "MEAN SCORE") +} + +func meanRule(means map[string]float64) string { + if means == nil { + return "" + } + return " " + strings.Repeat("-", 10) +} + +// criteriaMeans averages each evaluator's score over the rows it scored. +// +// The run summary reports pass and fail counts but no score, so a table that +// shows how close a passing evaluator came to failing has to read the rows. +// Errored and unscored rows are left out rather than counted as zero, which +// would drag the average toward a number no evaluator produced. +func criteriaMeans(items []eval_api.OutputItem) map[string]float64 { + sums := map[string]float64{} + counts := map[string]int{} + for _, item := range items { + for _, r := range item.Results { + if !r.Score.Defined() { + continue + } + name := r.Name + if name == "" { + name = r.Metric + } + sums[name] += float64(r.Score) + counts[name]++ + } + } + if len(counts) == 0 { + return nil + } + means := make(map[string]float64, len(counts)) + for name, n := range counts { + means[name] = sums[name] / float64(n) + } + return means +} + +// errorNote describes rows an evaluator could not score. +func errorNote(errored int) string { + return fmt.Sprintf("(%d errored, not scored)", errored) +} + +// formatRate renders a share as a percentage, and a rate over nothing as a +// dash: 0.0%% would read as a total failure rather than as no data. +func formatRate(part, whole int) string { + if whole <= 0 { + return "-" + } + return fmt.Sprintf("%.1f%%", float64(part)/float64(whole)*100) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go new file mode 100644 index 00000000000..2881096749c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A pipeline that starts a run with --no-wait has to come back for it. What it +// needs to do that is a fixed shape this extension controls, not whatever the +// API happened to return. +func TestStartedRunIsTheHandoffAPipelineNeeds(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_01JQZX", + EvalID: "eval_ignored", + Status: "queued", + CreatedAt: "2026-07-31T21:04:11Z", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + DataSource: &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + }, + } + + raw, err := json.Marshal(startedRun(run, "eval_01JQZW", &project.Eval{Name: "support-agent-smoke"})) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + + assert.Equal(t, "evalrun_01JQZX", out["run_id"]) + assert.Equal(t, "support-agent-smoke", out["eval_name"]) + assert.Equal(t, "queued", out["status"]) + assert.Equal(t, "2026-07-31T21:04:11Z", out["created_at"]) + + // The eval the run was started against, which is the one the command + // resolved rather than whatever the run echoed back. + assert.Equal(t, "eval_01JQZW", out["eval_id"]) + + // Nothing the extension does not promise. A pipeline that could read the + // data source here would come to depend on it. + for _, leaked := range []string{"data_source", "metadata", "id", "report_url"} { + assert.NotContains(t, out, leaked, + "the handoff must not leak %q from the service object", leaked) + } +} + +// A script logging created_at should not have to know which route produced +// the run: the service sends epoch seconds here and a formatted string +// elsewhere, so the handoff settles on one. +func TestStartedRunNormalizesTheTimestamp(t *testing.T) { + for _, tc := range []struct { + name string + value any + want string + }{ + {"epoch seconds", float64(1785801525), "2026-08-03T23:58:45Z"}, + {"already formatted", "2026-07-31T21:04:11Z", "2026-07-31T21:04:11Z"}, + {"absent", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + handoff := startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", CreatedAt: tc.value}, "eval_1", nil) + assert.Equal(t, tc.want, handoff.CreatedAt) + }) + } +} + +// An empty one is omitted rather than reported as "", which a script would +// otherwise print as the eval's name. +func TestStartedRunOmitsTheNameItDoesNotHave(t *testing.T) { + raw, err := json.Marshal(startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "queued"}, "eval_1", nil)) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + assert.NotContains(t, out, "eval_name") + assert.Equal(t, "eval_1", out["eval_id"]) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go new file mode 100644 index 00000000000..cea3b01bb4d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -0,0 +1,301 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// addRunSubcommands attaches the atomic run operations. +// +// `azd ai eval run` stays the composite that creates the group if needed and +// starts a run; these expose the individual operations so every one is +// reachable without the config file. +func addRunSubcommands(cmd *cobra.Command) { + cmd.AddCommand( + newRunListCommand(), + newRunShowCommand(), + newRunCancelCommand(), + newRunDeleteCommand(), + newRunOutputCommand(), + ) +} + +func newRunListCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + limit int + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List runs for an eval.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, limit) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd up` creates the ones your config declares", evalID) + } + return fmt.Errorf("listing runs for %q: %w", evalID, err) + } + if isJSON(cmd) { + var runs []eval_api.OpenAIEvalRun + if list != nil { + runs = list.Data + } + return emitJSONList(cmd.OutOrStdout(), runs) + } + if list == nil || len(list.Data) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "Eval %s has no runs yet.\n", evalID) + return nil + } + + rows := make([][]string, 0, len(list.Data)) + for _, run := range list.Data { + rows = append(rows, []string{run.ID, run.Name, run.Status, summarizeCounts(run.ResultCounts)}) + } + return emitTable(cmd.OutOrStdout(), + []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) + }, + } + addEvalFlag(cmd, &groupName) + cmd.Flags().IntVar(&limit, "limit", 0, + "Return at most this many runs. Omit for the service default.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunShowCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + wait bool + failOn string + ) + + cmd := &cobra.Command{ + Use: "show [run]", + Short: "Show a single run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + threshold, err := parseGate(failOn) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + // Reattaching to a run started asynchronously: the pipeline that + // gates on it is often not the one that started it. + // + // Only a caller that waited is told a bad status through the exit + // code. Without --wait this is an inspection command: it was asked + // what happened, and answering that is a success whatever the + // answer. + gateOnStatus := wait + if wait { + run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) + if err != nil { + return err + } + } + + if isJSON(cmd) { + if err := emitJSON(cmd.OutOrStdout(), run); err != nil { + return err + } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } + applyGate(cmd, threshold, run) + return nil + } + + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Run %s\n", run.ID) + fmt.Fprintf(out, " name : %s\n", run.Name) + fmt.Fprintf(out, " status : %s\n", run.Status) + if counts := summarizeCounts(run.ResultCounts); counts != "" { + fmt.Fprintf(out, " results : %s\n", counts) + } + if run.ReportURL != "" { + fmt.Fprintf(out, " report : %s\n", run.ReportURL) + } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } + applyGate(cmd, threshold, run) + return nil + }, + } + cmd.Flags().BoolVar(&wait, "wait", false, + "Block until the run reaches a terminal state before reporting.") + addFailOnFlag(cmd, &failOn) + addEvalFlag(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// firstArg returns the positional argument, or empty when none was given. +func firstArg(args []string) string { + if len(args) > 0 { + return args[0] + } + return "" +} + +func newRunCancelCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "cancel [run]", + Short: "Cancel an in-flight run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + target, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + // Cancelling a run that already finished is a no-op worth naming, + // since the service reports success either way. + if terminalRunStates[target.Status] { + return fmt.Errorf("run %s already finished with status %q", + target.ID, target.Status) + } + + canceled, err := ec.evalClient.CancelOpenAIEvalRun(ctx, evalID, target.ID) + if err != nil { + return fmt.Errorf("cancelling run %s: %w", target.ID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + status := canceled.Status + if status == "" { + status = "cancelling" + } + fmt.Fprintf(cmd.OutOrStdout(), "Run %s is now %s\n", target.ID, status) + return nil + }, + } + addEvalFlag(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newRunDeleteCommand removes a run. +// +// Runs accumulate — every `run start` adds one — and a run that evaluated the +// wrong dataset or target is noise in every later listing. The run is required +// rather than defaulted to the most recent, because deleting is not undoable +// and "the latest one" is a poor thing to guess at. +func newRunDeleteCommand() *cobra.Command { + var ( + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a run.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + runID := args[0] + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + if err := ec.evalClient.DeleteOpenAIEvalRun(ctx, evalID, runID); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf("no run %q on eval %q", runID, evalID) + } + return fmt.Errorf("deleting run %s: %w", runID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": runID, "eval_id": evalID, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted run %s\n", runID) + return nil + }, + } + addEvalFlag(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { + if counts == nil { + return "" + } + return fmt.Sprintf("%d passed, %d failed, %d errored", + counts.Passed, counts.Failed, counts.Errored) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go new file mode 100644 index 00000000000..8259bfd6b35 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" +) + +func TestSummarizeCounts(t *testing.T) { + require.Equal(t, "", summarizeCounts(nil)) + require.Equal(t, "3 passed, 1 failed, 0 errored", + summarizeCounts(&eval_api.EvalRunResultCounts{Total: 4, Passed: 3, Failed: 1})) +} + +// Cancelling a finished run is rejected locally. The service reports success +// either way, so without this the CLI would claim it cancelled a run that had +// already completed. +func TestTerminalRunStatesCoverServiceVocabulary(t *testing.T) { + for _, status := range []string{"completed", "failed", "canceled", "cancelled", "error"} { + require.True(t, terminalRunStates[status], "%q should be terminal", status) + } + for _, status := range []string{"in_progress", "queued", "running", ""} { + require.False(t, terminalRunStates[status], "%q should not be terminal", status) + } +} + +// The atomic run operations have to be reachable as subcommands; the spec +// requires start, list, show and cancel to exist alongside the composite. +func TestRunCommandExposesAtomicSubcommands(t *testing.T) { + cmd := newRunCommand() + + found := map[string]bool{} + for _, sub := range cmd.Commands() { + found[sub.Name()] = true + } + for _, name := range []string{"start", "list", "show", "cancel"} { + require.True(t, found[name], "run should expose the %q subcommand", name) + } +} + +// `run start` is the atomic form of the composite and must accept the same +// flags, otherwise the two forms diverge. +func TestRunStartMirrorsCompositeFlags(t *testing.T) { + composite := newRunCommand() + + var start *cobra.Command + for _, sub := range composite.Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + + for _, flag := range []string{"eval", "dataset", "name", "max-samples", "wait", "no-wait"} { + require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) + } + + // The level decides the row mapping, so a per-run override would put two + // incomparable result sets under one eval. A second level is a second eval. + require.Nil(t, start.Flags().Lookup("level"), "run start must not offer --level") +} + +// Every command that acts on an eval says which one the same way. One flag +// takes a name from the configuration or a raw service id: an eval created +// outside a project has no declaration to name, and a second --eval-id beside +// it was accepted and silently ignored. +func TestEvalCommandsTakeOneEvalFlag(t *testing.T) { + subs := map[string]*cobra.Command{} + for _, sub := range newRunCommand().Commands() { + subs["run "+sub.Name()] = sub + if sub.Name() == "output" { + for _, leaf := range sub.Commands() { + subs["run output "+leaf.Name()] = leaf + } + } + } + + for _, name := range []string{ + "run list", "run show", "run cancel", + "run output list", "run output show", "run output export", + } { + cmd := subs[name] + require.NotNil(t, cmd, "%s should exist", name) + require.NotNil(t, cmd.Flags().Lookup("eval"), "%s should accept --eval", name) + require.Nil(t, cmd.Flags().Lookup("eval-id"), + "%s must not keep --eval-id beside --eval", name) + } +} + +// --no-wait is documented in the spec, and cobra does not derive it from the +// --wait bool. It belongs to `run start`: `run` itself is a group. +func TestRunCommandAcceptsNoWait(t *testing.T) { + var start *cobra.Command + for _, sub := range newRunCommand().Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + require.NotNil(t, start.Flags().Lookup("no-wait"), "run start should accept --no-wait") + require.NotNil(t, start.Flags().Lookup("wait"), "run start should keep --wait") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go new file mode 100644 index 00000000000..28f2170909b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -0,0 +1,467 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// newRunOutputCommand groups the per-sample views of a run. +// +// `run show` is the summary - how many passed. These are the rows: which ones +// failed, and why. +func newRunOutputCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "output", + Short: "Inspect the per-sample results of a run.", + } + cmd.AddCommand( + newRunOutputListCommand(), + newRunOutputShowCommand(), + newRunOutputExportCommand(), + ) + return cmd +} + +func newRunOutputListCommand() *cobra.Command { + var ( + failedOnly bool + outFile string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "list [run]", + Short: "List the per-sample results of a run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + // The run carries totals and a per-criterion breakdown. The output + // items are the rows themselves, which is what "which one failed, + // and why" needs. A run that never produced any still renders its + // totals rather than failing. + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil { + return fmt.Errorf("reading the results of run %s: %w", run.ID, err) + } + rows := items.Data + if failedOnly { + kept := make([]eval_api.OutputItem, 0, len(rows)) + for _, it := range rows { + if it.Failed() { + kept = append(kept, it) + } + } + rows = kept + } + + payload := map[string]any{"run": run, "output_items": rows} + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return fmt.Errorf("creating %q: %w", outFile, err) + } + defer f.Close() + return emitJSON(f, payload) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), payload) + } + return renderResults(cmd.OutOrStdout(), run, rows, failedOnly) + }, + } + + cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") + cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") + addEvalFlag(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newRunOutputShowCommand reads one evaluated row by its id. +// +// The listing truncates the input and the reason to keep a table readable, so +// this is how the whole of either is seen. +func newRunOutputShowCommand() *cobra.Command { + var ( + runID string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a single evaluated row.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + itemID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + item, err := ec.evalClient.GetOutputItem(ctx, evalID, run.ID, itemID) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no output item %q on run %s; "+ + "`azd ai eval run output list` shows the ones there are", + itemID, run.ID) + } + return fmt.Errorf("reading output item %q: %w", itemID, err) + } + return emitJSON(cmd.OutOrStdout(), item) + }, + } + + cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") + addEvalFlag(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunOutputExportCommand() *cobra.Command { + var ( + format string + outFile string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "export [run]", + Short: "Export run results as JSON or CSV.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + format = strings.ToLower(format) + if format != "json" && format != "csv" { + return fmt.Errorf("--format must be json or csv, got %q", format) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + var w io.Writer = cmd.OutOrStdout() + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return fmt.Errorf("creating %q: %w", outFile, err) + } + defer f.Close() + w = f + } + + switch format { + case formatCSV: + return writeResultsCSV(w, run) + case formatJSON: + return emitJSON(w, run) + case formatJSONL: + return writeResultsJSONL(w, run) + default: + return fmt.Errorf( + "--format %q is not supported; use %s, %s or %s", + format, formatCSV, formatJSON, formatJSONL) + } + }, + } + + cmd.Flags().StringVar(&format, "format", formatCSV, + fmt.Sprintf("Output format: %s, %s or %s.", formatCSV, formatJSON, formatJSONL)) + cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") + addEvalFlag(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveEvalID takes the eval id from the argument, from --eval, or from the +// id cached in the azd environment. +// +// --eval accepts a name or a raw id on the one flag: an eval created outside a +// project has no declaration to name, and the environment records one id per +// name, so editing a declaration leaves every run of the previous eval +// reachable only by id. +func resolveEvalID( + cmd *cobra.Command, + ec *evalContext, + args []string, + groupName string, +) (string, error) { + if len(args) > 0 && args[0] != "" { + return args[0], nil + } + + if groupName != "" { + ref, err := ec.resolveEvalRef(cmd.Context(), project.DefaultEvalDir, groupName) + if err != nil { + return "", err + } + return ref.ID, nil + } + + if cached := ec.getEnvValue(cmd.Context(), envKeyEvalID); cached != "" { + return cached, nil + } + return "", fmt.Errorf( + "no eval given; pass its id as an argument, or name one with --eval") +} + +// addEvalFlag registers the flag that says which eval a command acts on. It +// takes a name from the configuration or a raw service id, which is why there +// is no second --eval-id beside it. +func addEvalFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval", "", + "Name of the eval declared in the configuration, or its id.") +} + +// latestOrNamedRun returns the named run, or the most recent one for the eval. +// +// explicit says whether the caller named the run rather than leaving it to +// default. A remembered run that no longer resolves is worth falling through +// on; one that was asked for by name is not. +func (ec *evalContext) latestOrNamedRun( + cmd *cobra.Command, + evalID, runID string, + explicit bool, +) (*eval_api.OpenAIEvalRun, error) { + ctx := cmd.Context() + + // The remembered run is per group. A single shared one belongs to whichever + // group ran last, and asking another group for it returns 404 rather than + // that group's own latest run. + if runID == "" { + runID = ec.getEnvValue(ctx, idKey("evalrun", evalID)) + } + if runID != "" { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err == nil { + return run, nil + } + if explicit { + return nil, fmt.Errorf("reading run %s: %w", runID, err) + } + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + if eval_api.IsNotFound(err) { + return nil, fmt.Errorf( + "no eval %q in this project; "+ + "`azd up` creates the ones your config declares", evalID) + } + return nil, fmt.Errorf("listing runs for eval %s: %w", evalID, err) + } + if len(list.Data) == 0 { + return nil, fmt.Errorf("eval %s has no runs yet", evalID) + } + return &list.Data[0], nil +} + +func renderResults( + w io.Writer, + run *eval_api.OpenAIEvalRun, + items []eval_api.OutputItem, + failedOnly bool, +) error { + fmt.Fprintf(w, "Run %s status: %s\n", run.ID, run.Status) + + if c := run.ResultCounts; c != nil { + fmt.Fprintf(w, "Totals: %d passed, %d failed, %d errored\n\n", + c.Passed, c.Failed, c.Errored) + } + + if len(run.PerTestingCriteria) > 0 { + rows := make([][]string, 0, len(run.PerTestingCriteria)) + for _, cr := range run.PerTestingCriteria { + if failedOnly && cr.Failed == 0 { + continue + } + rows = append(rows, []string{ + cr.TestingCriteria, + strconv.Itoa(cr.Passed), + strconv.Itoa(cr.Failed), + }) + } + if len(rows) > 0 { + if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { + return err + } + } + } + + // The rows are the point of `results show`: totals say how many failed, + // these say which and why. + if len(items) == 0 { + if failedOnly { + fmt.Fprintln(w, "\nNo failing rows.") + } else { + fmt.Fprintln(w, "\nNo rows have been scored yet.") + } + } else { + fmt.Fprintln(w) + rows := make([][]string, 0, len(items)) + for i, it := range items { + // One row per evaluated sample, not per verdict: a sample that + // failed three evaluators is one sample to go and look at, and + // listing it three times buries how much is actually wrong. + var failed []string + reason := "" + for _, r := range it.Results { + if r.Passed { + continue + } + failed = append(failed, r.Name) + if reason == "" { + reason = r.Reason + } + } + if failedOnly && len(failed) == 0 { + continue + } + verdicts := strings.Join(failed, ", ") + if verdicts == "" { + verdicts = "-" + } + rows = append(rows, []string{ + it.ID, + strconv.Itoa(i + 1), + truncate(verdicts, 40), + truncate(reason, 44), + }) + } + if err := emitTable(w, + []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"}, + rows); err != nil { + return err + } + if n := len(rows); failedOnly && n > 0 { + fmt.Fprintf(w, "\n%d sample(s) failed at least one evaluator.\n", n) + } + } + + if run.ReportURL != "" { + fmt.Fprintf(w, "\nReport: %s\n", run.ReportURL) + } + return nil +} + +// truncate keeps a table readable when a reason runs to a paragraph. The full +// text is always in `-o json`. +func truncate(s string, n int) string { + s = strings.ReplaceAll(strings.ReplaceAll(s, "\n", " "), "\r", "") + if len(s) <= n { + return s + } + if n <= 1 { + return s[:n] + } + return s[:n-1] + "…" +} + +func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { + cw := csv.NewWriter(w) + defer cw.Flush() + + if err := cw.Write([]string{"run_id", "status", "criterion", "passed", "failed"}); err != nil { + return err + } + if len(run.PerTestingCriteria) == 0 { + return cw.Write([]string{run.ID, run.Status, "", "", ""}) + } + for _, cr := range run.PerTestingCriteria { + if err := cw.Write([]string{ + run.ID, run.Status, cr.TestingCriteria, + strconv.Itoa(cr.Passed), strconv.Itoa(cr.Failed), + }); err != nil { + return err + } + } + return nil +} + +// Export formats. csv is the default because the results are a table and a +// build artifact is normally read by a spreadsheet or a diff. +const ( + formatCSV = "csv" + formatJSON = "json" + formatJSONL = "jsonl" +) + +// writeResultsJSONL emits one criterion per line, which is what a downstream +// job can stream without holding the whole run in memory. +func writeResultsJSONL(w io.Writer, run *eval_api.OpenAIEvalRun) error { + enc := json.NewEncoder(w) + if len(run.PerTestingCriteria) == 0 { + return enc.Encode(map[string]any{"run_id": run.ID, "status": run.Status}) + } + for _, cr := range run.PerTestingCriteria { + if err := enc.Encode(map[string]any{ + "run_id": run.ID, + "status": run.Status, + "testing_criteria": cr.TestingCriteria, + "passed": cr.Passed, + "failed": cr.Failed, + }); err != nil { + return err + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go new file mode 100644 index 00000000000..ec63f399695 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// twoCriteriaRun is a finished run with the shape export has to preserve: one +// row per testing criterion, all carrying the run they belong to. +func twoCriteriaRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "task_adherence", Passed: 8, Failed: 2}, + {TestingCriteria: "coherence", Passed: 10, Failed: 0}, + }, + } +} + +// An export is read by a spreadsheet or a diff, so the header is part of the +// contract: renaming a column silently breaks whatever consumes it. +func TestWriteResultsCSV(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, twoCriteriaRun())) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_abc", "completed", "task_adherence", "8", "2"}, rows[1]) + assert.Equal(t, []string{"evalrun_abc", "completed", "coherence", "10", "0"}, rows[2]) + assert.Len(t, rows, 3, "one header and one row per criterion") +} + +// A run that graded nothing still has to produce a file with a header, because +// a consumer that gets zero bytes cannot tell an empty run from a failed +// export. +func TestWriteResultsCSV_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + require.Len(t, rows, 2) + assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_empty", "failed", "", "", ""}, rows[1]) +} + +// A criterion name is service-supplied, so it can hold anything. The writer +// has to quote rather than corrupt the row. +func TestWriteResultsCSV_QuotesASeparatorInTheData(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: `groundedness, strict`, Passed: 1, Failed: 0}, + }, + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + require.Len(t, rows, 2) + assert.Equal(t, "groundedness, strict", rows[1][2], + "a comma in a criterion name must survive the round trip") +} + +// One criterion per line is what lets a downstream job stream results without +// holding the whole run. +func TestWriteResultsJSONL(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 2, "one line per criterion") + + var first map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + assert.Equal(t, "evalrun_abc", first["run_id"]) + assert.Equal(t, "completed", first["status"]) + assert.Equal(t, "task_adherence", first["testing_criteria"]) + assert.EqualValues(t, 8, first["passed"]) + assert.EqualValues(t, 2, first["failed"]) + + var second map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[1]), &second)) + assert.Equal(t, "coherence", second["testing_criteria"]) +} + +// Every line has to parse on its own; that is the whole point of the format. +func TestWriteResultsJSONL_EachLineParsesAlone(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + var row map[string]any + assert.NoErrorf(t, json.Unmarshal([]byte(line), &row), "line is not self-contained: %s", line) + } +} + +func TestWriteResultsJSONL_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 1) + + var row map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &row)) + assert.Equal(t, "evalrun_empty", row["run_id"]) + assert.Equal(t, "failed", row["status"]) + assert.NotContains(t, row, "testing_criteria", + "a run that graded nothing must not claim a criterion") +} + +// The three export formats are a documented set. A fourth spelling, or a +// missing one, is a promise broken on either side. +func TestExportFormatsAreTheDocumentedSet(t *testing.T) { + assert.Equal(t, "csv", formatCSV) + assert.Equal(t, "json", formatJSON) + assert.Equal(t, "jsonl", formatJSONL) + + usage := find(t, "run output export").Flags().Lookup("format") + require.NotNil(t, usage) + assert.Equal(t, formatCSV, usage.DefValue, + "results are a table, so the default artifact is the one a spreadsheet opens") + + for _, f := range []string{formatCSV, formatJSON, formatJSONL} { + assert.Containsf(t, usage.Usage, f, "--format accepts %q, so its help has to say so", f) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go new file mode 100644 index 00000000000..cce1c662156 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// scoredRun is a run the way the service returns one, with rows attached. +func scoredRows() []eval_api.OutputItem { + return []eval_api.OutputItem{ + { + ID: "oi_1", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: true, Score: 5}, + {Name: "coherence", Passed: true, Score: 4}, + }, + }, + { + ID: "oi_2", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: false, Score: 1, Reason: "Answered a different question."}, + {Name: "coherence", Passed: false, Score: 2, Reason: "Rambled."}, + }, + }, + } +} + +// One evaluated sample is one row. Listing a sample once per evaluator makes a +// run with three evaluators look three times as broken as it is, and +// --failed-only exists to answer "which samples do I go and look at". +func TestRenderResultsIsOneRowPerSample(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), false)) + + text := out.String() + assert.Equal(t, 1, strings.Count(text, "oi_2"), + "a sample that failed two evaluators must still be one row:\n%s", text) + + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + assert.Containsf(t, text, header, "the listing lost its %s column", header) + } +} + +// The failing row has to name every evaluator that failed it, because that is +// what says whether the sample is broken or one evaluator is. +func TestRenderResultsNamesEveryFailedEvaluator(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), true)) + + text := out.String() + assert.Contains(t, text, "relevance, coherence") + assert.Contains(t, text, "Answered a different question.", + "the first failure's reason is what the row is looked at for") + assert.NotContains(t, text, "oi_1", "--failed-only must drop the passing sample") + assert.Contains(t, text, "1 sample(s) failed at least one evaluator.") +} + +// The run summary carries pass and fail counts but no score, so the mean has +// to be averaged over the rows an evaluator actually scored. +func TestCriteriaMeans(t *testing.T) { + means := criteriaMeans(scoredRows()) + assert.InDelta(t, 3.0, means["relevance"], 0.001) + assert.InDelta(t, 3.0, means["coherence"], 0.001) + + assert.Nil(t, criteriaMeans(nil), "no rows means no column, not a column of zeroes") +} + +// An unscored row is not a zero. Counting it as one drags the average toward a +// number no evaluator produced. +func TestCriteriaMeansIgnoresUnscoredRows(t *testing.T) { + rows := []eval_api.OutputItem{ + {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: true}}}, + {Results: []eval_api.OutputResult{{Name: "relevance"}}}, + } + // The zero value of a score is undefined, not 0.0. + rows[1].Results[0].Score = eval_api.LenientFloat(0) + + means := criteriaMeans(rows) + require.Contains(t, means, "relevance") + assert.InDelta(t, 2.0, means["relevance"], 0.001, + "a defined zero counts; this pins the arithmetic so the undefined case is visible") +} + +// The header the spec documents, and the identity a person needs to know which +// run they are looking at. +func TestRenderRunHeaderNamesTheEval(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", + EvalID: "eval_9", + Status: "completed", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + ResultCounts: &eval_api.EvalRunResultCounts{Total: 15, Passed: 12, Failed: 3}, + CreatedAt: float64(1785801525), + ModifiedAt: float64(1785802119), + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run, map[string]float64{"relevance": 4.1})) + text := out.String() + + assert.Contains(t, text, "Run evalrun_9") + assert.Contains(t, text, "Eval support-agent-smoke", + "the declared name is what the author recognises, not the service id") + assert.Contains(t, text, "Status completed") + assert.Contains(t, text, "Samples 15") + assert.Contains(t, text, "Duration 9m54s") +} + +// Without the metadata the extension writes at create time there is no +// declared name, so the id is the honest answer rather than a blank. +func TestRenderRunHeaderFallsBackToTheEvalID(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_9", EvalID: "eval_9", Status: "queued"} + require.NoError(t, renderRun(&out, run, nil)) + assert.Contains(t, out.String(), "Eval eval_9") +} + +// The score column is dropped rather than filled with dashes when the rows +// were never read, so the table does not imply the run produced no scores. +func TestRenderRunOmitsTheScoreColumnWithoutMeans(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{{TestingCriteria: "relevance", Passed: 2}}, + } + + var without bytes.Buffer + require.NoError(t, renderRun(&without, run, nil)) + assert.NotContains(t, without.String(), "MEAN SCORE") + + var with bytes.Buffer + require.NoError(t, renderRun(&with, run, map[string]float64{"relevance": 4.15})) + assert.Contains(t, with.String(), "MEAN SCORE") + assert.Contains(t, with.String(), "4.2", "the mean is shown to one decimal") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go new file mode 100644 index 00000000000..646fd560e50 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exit code is the whole contract with a pipeline, and there are three +// answers it has to be able to give: the evaluation ran and passed, it ran and +// regressed, or it could not run. The gate owns the middle one; this owns the +// last. +// +// Reporting a run that errored and then exiting 0 tells the pipeline the +// evaluation passed, which is the one answer that is never true. +func TestRunCompleted(t *testing.T) { + require.NoError(t, runCompleted(nil), + "nothing was waited for, so there is nothing to report") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "completed"})) + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "Completed"}), + "the service is not consistent about case") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1"}), + "a status the service did not send is not a failure to report") + + for _, status := range []string{"failed", "error", "canceled", "cancelled"} { + err := runCompleted(&eval_api.OpenAIEvalRun{ID: "run_abc", Status: status}) + require.Error(t, err, "status %q must not exit 0", status) + assert.Contains(t, err.Error(), "run_abc") + assert.Contains(t, err.Error(), status, + "the message has to name the status, which is what the caller acts on") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go new file mode 100644 index 00000000000..a26d7e9b4a5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// finishedRun is a run the way the service returns one: counts over samples, +// and a result per testing criterion. +func finishedRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc123", + Status: "completed", + ResultCounts: &eval_api.EvalRunResultCounts{ + Total: 10, Passed: 7, Failed: 3, + }, + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 9, Failed: 1}, + {TestingCriteria: "coherence", Passed: 7, Failed: 3}, + }, + } +} + +// The whole point of waiting for a run is the verdict per evaluator. Printing +// only the status meant the answer to the question the command was asked took +// a second command to see. +func TestRenderRunReportsEveryEvaluator(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun(), nil)) + text := out.String() + + assert.Contains(t, text, "evalrun_abc123") + assert.Contains(t, text, "completed") + + for _, criterion := range []string{"relevance", "coherence"} { + assert.Contains(t, text, criterion, + "every evaluator the run scored must appear") + } + assert.Contains(t, text, "90.0%", "relevance passed 9 of 10") + assert.Contains(t, text, "70.0%", "coherence passed 7 of 10") + assert.Contains(t, text, "7/10", "the sample counts must be shown, not just the rate") +} + +// Two runs of the same eval have to read the same way. The service returns the +// criteria in whatever order it evaluated them, which is not stable. +func TestRenderRunOrdersEvaluatorsByName(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun(), nil)) + + text := out.String() + assert.Less(t, strings.Index(text, "coherence"), strings.Index(text, "relevance"), + "evaluators must be listed in a stable order") +} + +// An errored row is not a failing row: the evaluator never reached a verdict. +// Folding the two together would report a service problem as a quality problem. +func TestRenderRunSeparatesErrorsFromFailures(t *testing.T) { + run := finishedRun() + run.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Failed: 1, Errored: 2} + run.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 7, Failed: 1, Errored: 2}, + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run, nil)) + text := out.String() + + assert.Contains(t, text, "2 errored") + assert.Contains(t, text, "87.5%", + "the pass rate is over what was scored, not over what was attempted") + assert.Contains(t, text, "errored and were not scored") +} + +// A rate over nothing is not zero. Printing 0.0% for a criterion that scored +// no rows reads as a total failure rather than as no data. +func TestFormatRateHasNoOpinionAboutNothing(t *testing.T) { + assert.Equal(t, "-", formatRate(0, 0)) + assert.Equal(t, "0.0%", formatRate(0, 4)) + assert.Equal(t, "100.0%", formatRate(4, 4)) + assert.Equal(t, "33.3%", formatRate(1, 3)) +} + +// The next thing anyone does after seeing failures is look at them, so the +// command that shows them is named — and it has to be a command that exists. +func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun(), nil)) + assert.Contains(t, out.String(), "azd ai eval run output list --failed-only") + + clean := finishedRun() + clean.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 10} + clean.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 10}, + } + var cleanOut bytes.Buffer + require.NoError(t, renderRun(&cleanOut, clean, nil)) + assert.NotContains(t, cleanOut.String(), "--failed-only", + "a run with nothing to look at must not send anyone looking") +} + +// A run that never produced counts still has to render. The service returns +// none for a run that failed before scoring, and a nil dereference there would +// replace the failure message with a panic. +func TestRenderRunSurvivesAnEmptyResult(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"}, nil)) + assert.Contains(t, out.String(), "evalrun_x") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go new file mode 100644 index 00000000000..55329762df9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cmd + +import ( + "context" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorSchemasIncludesBuiltins covers the function that supplies +// the schemas, rather than the builder that consumes them. +// +// The builder was already tested against every built-in, but the test fetched +// them itself with the Builtin filter. Production did not: it listed +// unfiltered, which returns only the project's own evaluators, so every +// built-in reached the builder with no schema at all. The builder was correct +// and the criteria were still wrong, and no test could see it because each one +// constructed the input production was failing to construct. +func TestLiveEvaluatorSchemasIncludesBuiltins(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + // The listing production used to rely on, to show what it omits. + unfiltered, err := client.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + require.NoError(t, err) + builtinsInUnfiltered := 0 + for _, e := range unfiltered.Value { + if eval_api.IsBuiltinEvaluator(e.Name) { + builtinsInUnfiltered++ + } + } + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas, "no evaluator schemas were resolved at all") + + builtins := 0 + for name, summary := range schemas { + if !eval_api.IsBuiltinEvaluator(name) { + continue + } + builtins++ + assert.NotNil(t, summary.DataSchema(), + "%s resolved without the contract the criteria are shaped from", name) + } + + require.NotZero(t, builtins, + "built-ins must be resolvable; the unfiltered listing returns %d of them, "+ + "so they have to be asked for by type", builtinsInUnfiltered) +} + +// The fields an evaluator declares are the ones its criterion has to bind, so +// a conversation-level evaluator must resolve to its conversation field. +func TestLiveConversationEvaluatorBindsMessages(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) + + var name string + for n, summary := range schemas { + if eval_api.IsBuiltinEvaluator(n) && summary.SupportsLevel("conversation") { + if ds := summary.DataSchema(); ds != nil && ds.Accepts(conversationField) { + name = n + break + } + } + } + if name == "" { + t.Skip("no built-in advertises a conversation contract on this project") + } + + plan, err := planCriterion( + evalcore.EvaluatorRef{ + Name: name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }, + schemas[name], + nil, // no target: the dataset holds both sides of the exchange + map[string]bool{conversationField: true}, + "conversation", + ) + require.NoError(t, err) + assert.Equal(t, "{{item."+conversationField+"}}", plan.dataMapping[conversationField], + "%s must bind its conversation field", name) + assert.NotEmpty(t, plan.dataMapping, "an empty mapping scores nothing") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go new file mode 100644 index 00000000000..bf6be972f2a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -0,0 +1,476 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The command surface is a contract with the spec and with the sibling Foundry +// extensions, and it is the part of this tool users type from memory. Nothing +// was checking it: the flag that writes results to a file was `--out-file` +// while the spec, Scenario 4, and `azd ai skill download` all say +// `--output-file`, and it took reading the two documents side by side to see. +// +// These tests walk the built tree, so a command or flag that is renamed, +// dropped, or quietly added has to be acknowledged here. + +// walk visits every command in the tree, skipping the ones azd contributes. +func walk(t *testing.T, cmd *cobra.Command, path []string, visit func(string, *cobra.Command)) { + t.Helper() + for _, child := range cmd.Commands() { + name := strings.Fields(child.Use)[0] + switch name { + case "help", "completion", "listen", "metadata": + continue + } + full := append(append([]string{}, path...), name) + visit(strings.Join(full, " "), child) + walk(t, child, full, visit) + } +} + +// commandTree is every command the extension exposes, and is the surface the +// spec's command table describes. +func TestCommandTreeMatchesTheSpec(t *testing.T) { + want := []string{ + "dataset", + "dataset create", + "dataset delete", + "dataset generate", + "dataset list", + "dataset show", + "dataset update", + "dataset versions", + "dataset versions list", + "dataset job", + "dataset job cancel", + "dataset job delete", + "dataset job list", + "dataset job show", + "create", + "delete", + "evaluator", + "evaluator create", + "evaluator delete", + "evaluator generate", + "evaluator list", + "evaluator show", + "evaluator update", + "evaluator versions", + "evaluator versions list", + "evaluator job", + "evaluator job cancel", + "evaluator job delete", + "evaluator job list", + "evaluator job show", + "init", + "list", + "run", + "run cancel", + "run delete", + "run list", + "run output", + "run output export", + "run output list", + "run output show", + "run show", + "run start", + "show", + } + + var got []string + walk(t, NewRootCommand(), nil, func(path string, _ *cobra.Command) { + got = append(got, path) + }) + + assert.ElementsMatch(t, want, got, + "the command tree changed; update the spec's command table with it") +} + +// Flag names are shared vocabulary across the Foundry extensions. A command +// that invents its own spelling for something the others already name is the +// kind of difference nobody notices until a user types the one they learned +// somewhere else. +func TestFlagVocabularyIsShared(t *testing.T) { + // Meaning → the one spelling for it, from the spec's vocabulary table. + // A command that means one of these must use exactly this name, and the + // near-misses are listed so a rename back is caught rather than accepted. + forbidden := map[string]string{ + "--out-file": "--output-file", + "--out-dir": "--output-dir", + "--file": "--from-file", + "--rubric": "--from-file", + "--from-traces": "deferred to M2", + "--response-id": "deferred to M2", + "--no-target": "deferred to M2", + "--out": "--output-file", + "--dir": "--output-dir", + "--baseline": "deferred to M2", + "--cron": "deferred to M2", + "--folder": "deferred to M2", + "--init-params": "deferred to M2", + "--data-schema": "deferred to M2", + "--metrics": "deferred to M2", + "--trace-window": "deferred to M2", + "--max-turns": "deferred to M2", + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if want, bad := forbidden["--"+f.Name]; bad { + t.Errorf("%s declares --%s; use %s", path, f.Name, want) + } + }) + }) +} + +// M1 promises `-o json` and `--no-prompt` throughout. Both come from the azd +// extension SDK's root command, so every command inherits them — until one +// declares its own flag by the same name, which silently shadows the global +// and leaves that one command unable to answer in JSON or to run unattended. +func TestNoCommandShadowsAGlobalFlag(t *testing.T) { + global := []string{"output", "no-prompt", "environment", "cwd", "debug"} + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + for _, name := range global { + assert.Nilf(t, cmd.LocalFlags().Lookup(name), + "%s declares its own --%s, which shadows the global one", path, name) + } + }) +} + +// The two commands that write a file have to agree on what that flag is +// called, and it has to be the name the sibling extensions use. +func TestOutputFileFlagIsSpelledTheSharedWay(t *testing.T) { + for _, path := range []string{"run output list", "run output export"} { + cmd := find(t, path) + require.NotNil(t, cmd.Flags().Lookup("output-file"), + "%s must write to --output-file, the name `azd ai skill download` uses", path) + assert.Nil(t, cmd.Flags().Lookup("out-file"), + "%s must not keep the old spelling alongside the shared one", path) + } +} + +// `init` is the one command with a documented flag table, so it is pinned +// whole: an extra flag there is a promise the spec does not make, and a +// missing one is a promise it does. +func TestInitFlagsMatchTheSpec(t *testing.T) { + cmd := find(t, "init") + + var got []string + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if f.Name != "help" { + got = append(got, "--"+f.Name) + } + }) + + assert.ElementsMatch(t, []string{ + "--name", "--target", "--source", "--dataset", "--max-traces", + "--evaluator", "--judge-model", "--path", "--force", + }, got, "init's flags are a table in the spec; change both together") +} + +// `init` makes no service calls, so it must not offer the flag that says where +// to make them. +func TestInitTakesNoProjectEndpoint(t *testing.T) { + assert.Nil(t, find(t, "init").Flags().Lookup("project-endpoint"), + "init is offline; a project endpoint would imply otherwise") +} + +// Every command that does reach the service accepts it, because the shared +// Foundry resolver is how a project is named without an azd environment. +func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { + groups := map[string]bool{ + "dataset": true, "evaluator": true, "run": true, + "job": true, "run output": true, + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if cmd.RunE == nil || path == "init" { + return + } + if groups[path] { + return + } + assert.NotNil(t, cmd.Flags().Lookup("project-endpoint"), + "%s reaches the service, so it must accept --project-endpoint", path) + }) +} + +// The spec says --from "selects one or more of the four sources", so it has to +// be repeatable. Declared as a plain string it would still accept every +// documented single-source invocation and silently keep only the last of a +// repeated one, which is the kind of difference no example in the spec shows. +func TestDatasetGenerateFromTakesMoreThanOneSource(t *testing.T) { + flag := find(t, "dataset generate").Flags().Lookup("from") + require.NotNil(t, flag, "dataset generate must offer --from") + + assert.Equal(t, "stringSlice", flag.Value.Type(), + "--from selects one or more sources, so it cannot be a single string") +} + +// `--from` names sources; the set it accepts is the set the service has a path +// for, and the help has to list exactly that set. +func TestDatasetGenerateFromListsEverySource(t *testing.T) { + usage := find(t, "dataset generate").Flags().Lookup("from").Usage + + for _, source := range project.GenerateSources { + assert.Containsf(t, usage, source, + "--from accepts %q, so its help has to say so", source) + } +} + +// `--from` is the only place a source is named, so `evaluator generate`, which +// has no such flag, must not be left half-wired to one. +func TestEvaluatorGenerateHasNoFromFlag(t *testing.T) { + assert.Nil(t, find(t, "evaluator generate").Flags().Lookup("from"), + "the spec gives --from to dataset generate only") +} + +// The spec's run table says which commands carry which flag. Where it says +// "every", that is checkable; where it names two commands, a third carrying the +// flag is a promise the spec does not make and a missing one is a promise it +// does. +// +// This pins placement, not the whole flag list: unlike init's, the run table is +// headed "Flag | Commands | Default" and documents defaults rather than +// enumerating every flag. +func TestRunFlagsSitWhereTheSpecSaysTheyDo(t *testing.T) { + // Flag → exactly the run commands that may declare it. nil means every + // run command that does something. + placement := map[string][]string{ + "eval": nil, + "dataset": {"run start"}, + "fail-on": {"run start", "run show"}, + "wait": {"run start", "run show"}, + "format": {"run output export"}, + } + + // Every run command that actually runs, which is what "every run command" + // means — the bare groups take no flags. + var runCommands []string + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if strings.HasPrefix(path, "run") && cmd.RunE != nil { + runCommands = append(runCommands, path) + } + }) + require.NotEmpty(t, runCommands) + + for flag, allowed := range placement { + if allowed == nil { + allowed = runCommands + } + for _, path := range runCommands { + has := find(t, path).Flags().Lookup(flag) != nil + want := slices.Contains(allowed, path) + + switch { + case want && !has: + t.Errorf("%s must accept --%s; the spec's run table says so", path, flag) + case !want && has: + t.Errorf("%s declares --%s, which the spec gives only to %s", + path, flag, strings.Join(allowed, ", ")) + } + } + } +} + +// `--eval` is how a run command finds the eval, and the spec gives it to every +// one of them. Losing it from a single command makes that command unusable in a +// project with more than one eval. +func TestEveryRunCommandTakesEval(t *testing.T) { + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if !strings.HasPrefix(path, "run") || cmd.RunE == nil { + return + } + assert.NotNilf(t, cmd.Flags().Lookup("eval"), + "%s must accept --eval, which the spec gives to every run command", path) + }) +} + +// The tagged suites drive the binary by writing flags as strings, so a flag +// that is renamed or removed still compiles there and only fails when someone +// has the credentials to run them. +// +// That is not hypothetical. Removing `--eval-id` and the generation spec file +// left 28 uses of `--eval-id` and two `--config` tests behind in tests/cli, +// every one of which would have failed at the first live run — under `live` and +// `hero` tags that `go test ./...` never builds. This checks them from the +// default suite, where a rename is caught by the person doing the renaming. +func TestTaggedSuitesNameFlagsThatExist(t *testing.T) { + // Every flag any command declares, plus the globals azd contributes. + known := map[string]bool{ + "output": true, "no-prompt": true, "environment": true, + "cwd": true, "debug": true, "help": true, + } + walk(t, NewRootCommand(), nil, func(_ string, cmd *cobra.Command) { + cmd.Flags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + }) + + // Only string literals, which is how a test spells a flag it passes to the + // binary. Prose in a comment is not a flag. + literal := regexp.MustCompile(`"--([a-z][a-z0-9-]*)"`) + + err := filepath.WalkDir("../../tests", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range literal.FindAllStringSubmatch(line, -1) { + assert.Truef(t, known[m[1]], + "%s:%d passes --%s, which no command declares", path, i+1, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} + +// find resolves a command path, failing the test when it does not exist. +func find(t *testing.T, path string) *cobra.Command { + t.Helper() + cmd, _, err := NewRootCommand().Find(strings.Fields(path)) + require.NoError(t, err, "no such command: %s", path) + require.Equal(t, strings.Fields(path)[len(strings.Fields(path))-1], + strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) + return cmd +} + +// Messages that tell a user what to run next have to name a command that +// exists. +// +// Rebuilding the surface left `run start --no-wait` closing with "Check +// progress with: azd ai eval results show", a command that had been renamed +// out of existence — so the one instruction printed at the moment a user needs +// it was the one thing guaranteed to fail. Nothing catches that: the string +// compiles, the command that prints it succeeds, and only someone following +// the advice finds out. +// This extension's namespace is `ai.eval`, so every command it can suggest +// begins `azd ai eval`. Anchoring on that prefix is what caught the renamed +// command above — and anchoring only on it is what let three suggestions +// through pointing at `azd ai dataset`, a namespace no installed extension +// serves. So the prefix checked is `azd ai`, and anything under it that is not +// this extension's own is a command nobody can run. +func TestSuggestedCommandsExist(t *testing.T) { + root := "../.." + pattern := regexp.MustCompile("azd ai ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) + if err != nil { + return err + } + + for line := range strings.SplitSeq(string(body), "\n") { + // Comments explain the surface; only what reaches a terminal has + // to resolve. + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range pattern.FindAllStringSubmatch(line, -1) { + words := strings.Fields(m[1]) + + // `ai.eval` is this extension's namespace, so it is the only + // thing under `azd ai` that resolves here. Another namespace is + // a command this extension cannot suggest, whether or not some + // future extension serves it. + if len(words) == 0 || words[0] != "eval" { + t.Errorf("%s suggests `azd ai %s`, which is not this extension's "+ + "namespace; commands here are `azd ai eval ...`", path, m[1]) + continue + } + words = words[1:] + + // Trim trailing prose: "run start" is a command, "run start + // and summarize" is a sentence that begins with one. + for len(words) > 0 { + if _, _, err := NewRootCommand().Find(words); err == nil { + resolved, _, _ := NewRootCommand().Find(words) + if strings.Fields(resolved.Use)[0] == words[len(words)-1] { + break + } + } + words = words[:len(words)-1] + } + assert.NotEmpty(t, words, + "%s suggests `azd ai %s`, which is not a command", path, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} + +// A command suggested with an argument has to be suggested with the argument +// filled in. +// +// `--no-wait` exists so the caller can walk away, and the line they walk away +// with is the one they paste when they come back. Printing +// `azd ai eval job show ` reads like a command and is not one: it +// resolves, so the check above passes, and it fails the moment anyone uses it. +func TestSuggestedCommandsCarryNoPlaceholders(t *testing.T) { + placeholder := regexp.MustCompile(`azd ai eval [^"'\n]*<[a-z-]+>`) + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(line) + // A `Use:` string and the help text around it are where a + // placeholder belongs: cobra prints it as the signature. + if strings.HasPrefix(trimmed, "//") || + strings.HasPrefix(trimmed, "Use:") || + strings.HasPrefix(trimmed, "Short:") || + strings.HasPrefix(trimmed, "Long:") { + continue + } + if m := placeholder.FindString(line); m != "" { + t.Errorf("%s:%d suggests %q; substitute the value instead", + path, i+1, m) + } + } + return nil + }) + require.NoError(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go new file mode 100644 index 00000000000..9ac6f21e92b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A dataset's URI points at either the blob or the container holding it, +// depending on how it was created, and nothing in the payload says which: +// isSingleFile is true either way. Uploaded datasets end in the file name; +// generated ones end in the container. Downloading a container returns 409. +func TestLooksLikeBlobURI(t *testing.T) { + uploaded := "https://acct.blob.core.windows.net:443/container-guid/azd-smoke-golden.jsonl" + generated := "https://acct.blob.core.windows.net/asayedahme-420d0b21-956c-513b-bb18-f60bfbf5e724" + + require.True(t, looksLikeBlobURI(uploaded), "an uploaded dataset names its file") + require.False(t, looksLikeBlobURI(generated), "a generated dataset names its container") +} + +// A SAS token on the URI must not change the answer. +func TestLooksLikeBlobURIIgnoresQuery(t *testing.T) { + require.True(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c/data.jsonl?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI("https://acct.blob.core.windows.net/c/")) +} + +// An evaluation dataset is JSONL, so that is preferred when a container holds +// more than one file. +func TestPickDatasetBlobPrefersJSONL(t *testing.T) { + require.Equal(t, "data.jsonl", + pickDatasetBlob([]string{"_meta.json", "data.jsonl", "readme.txt"})) + require.Equal(t, "data.JSONL", + pickDatasetBlob([]string{"data.JSONL"}), "the extension match is case-insensitive") +} + +// With nothing recognisable, any real file beats returning nothing. +func TestPickDatasetBlobFallsBackToAnyFile(t *testing.T) { + require.Equal(t, "data.csv", pickDatasetBlob([]string{"data.csv"})) + require.Empty(t, pickDatasetBlob([]string{"folder/"})) + require.Empty(t, pickDatasetBlob(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go new file mode 100644 index 00000000000..19b548c3e05 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +// DatasetList is the paged response returned when listing datasets or the +// versions of one dataset. +type DatasetList struct { + Value []Dataset `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ListDatasets returns the datasets registered on the project. +func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*DatasetList, error) { + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) +} + +// ListDatasetVersions returns every version of a single dataset. +func (c *DatasetClient) ListDatasetVersions( + ctx context.Context, + name string, + apiVersion string, +) (*DatasetList, error) { + path := fmt.Sprintf("%s/%s/versions", pathDatasets, url.PathEscape(name)) + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// DeleteDatasetVersion removes a single dataset version. +func (c *DatasetClient) DeleteDatasetVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// VersionOrder returns a sortable value for a version string, matching the +// decimal convention NextVersion produces ("1.0", "2.0"). Unparseable versions +// sort lowest. +func VersionOrder(version string) float64 { + v := strings.TrimSpace(version) + if v == "" { + return -1 + } + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + // Fall back to trailing digits, e.g. "v3" -> 3. + i := len(v) + for i > 0 && v[i-1] >= '0' && v[i-1] <= '9' { + i-- + } + if i == len(v) { + return -1 + } + if n, err := strconv.Atoi(v[i:]); err == nil { + return float64(n) + } + return -1 +} + +// VersionGreater reports whether a is a strictly newer version than b. +// +// Both must be orderable; when either is not, the answer is false so an +// unparseable version never triggers a drift failure on its own. +func VersionGreater(a, b string) bool { + orderA, orderB := VersionOrder(a), VersionOrder(b) + if orderA < 0 || orderB < 0 { + return false + } + return orderA > orderB +} + +// LatestVersion returns the highest version in the list, falling back to the +// last entry when none of the versions can be ordered. +func LatestVersion(datasets []Dataset) string { + best := "" + bestOrder := -2.0 + for _, d := range datasets { + if o := VersionOrder(d.Version); o > bestOrder { + bestOrder, best = o, d.Version + } + } + if best == "" && len(datasets) > 0 { + return datasets[len(datasets)-1].Version + } + return best +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go new file mode 100644 index 00000000000..13fe6da2d9d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "strings" +) + +// CreateDatasetRequest is the request body for creating (uploading) a dataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +// +// The field spelling is not consistent across the surface: the live +// project-endpoint GET returns camelCase (dataUri, isSingleFile), while other +// paths have used snake_case (data_uri, blob_uri, content_uri). Both spellings +// are accepted here because binding only one silently yields an empty URI, +// which then fails much later at download time. +type Dataset struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type,omitempty"` + Format string `json:"format,omitempty"` + + // camelCase spellings (project endpoint). + DataURICamel string `json:"dataUri,omitempty"` + BlobURICamel string `json:"blobUri,omitempty"` + ContentURICamel string `json:"contentUri,omitempty"` + IsSingleFile bool `json:"isSingleFile,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + + // snake_case spellings. + BlobURI string `json:"blob_uri,omitempty"` + DataURI string `json:"data_uri,omitempty"` + ContentURI string `json:"content_uri,omitempty"` +} + +// ResolvedBlobURI returns the first URI the service supplied, across both +// spellings. An empty result means the dataset carries no downloadable URI and +// the caller must fetch a credential instead. +func (d *Dataset) ResolvedBlobURI() string { + for _, candidate := range []string{ + d.BlobURI, d.BlobURICamel, + d.DataURI, d.DataURICamel, + d.ContentURI, d.ContentURICamel, + } { + if candidate != "" { + return candidate + } + } + return "" +} + +// DatasetCredential is the response for dataset credential (SAS token) requests. +// The API returns a nested structure with blobReference and blobReferenceForConsumption. +type DatasetCredential struct { + // Flat fields (legacy format). + BlobURI string `json:"blob_uri,omitempty"` + SAS string `json:"sas,omitempty"` + SASUri string `json:"sas_uri,omitempty"` + + // Nested fields (current API format). + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` +} + +// BlobReference represents a blob storage reference with credentials. +type BlobReference struct { + BlobURI string `json:"blobUri,omitempty"` + StorageAccountARM string `json:"storageAccountArmId,omitempty"` + Credential *BlobCredential `json:"credential,omitempty"` +} + +// BlobCredential holds SAS credential details for blob access. +type BlobCredential struct { + Type string `json:"type,omitempty"` + SASUri string `json:"sasUri,omitempty"` + SASPath string `json:"sas,omitempty"` +} + +// ResolvedDownloadURI returns the URL to download the dataset. +// Prefers blobReferenceForConsumption.credential.sasUri (current API), +// then blobReference.credential.sasUri, then flat sas_uri, then blob_uri + sas. +func (c *DatasetCredential) ResolvedDownloadURI() string { + // Current API format: nested blob references. + if c.BlobReferenceConsumption != nil && c.BlobReferenceConsumption.Credential != nil { + if uri := c.BlobReferenceConsumption.Credential.SASUri; uri != "" { + return uri + } + } + if c.BlobReference != nil && c.BlobReference.Credential != nil { + if uri := c.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + // Legacy flat format. + if c.SASUri != "" { + return c.SASUri + } + if c.BlobURI != "" && c.SAS != "" { + return c.BlobURI + "?" + c.SAS + } + return c.BlobURI +} + +// PendingUploadResponse is returned by the startPendingUpload endpoint. +// It contains a SAS URI for uploading blob data and the blob container URI. +type PendingUploadResponse struct { + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` + PendingUploadID *string `json:"pendingUploadId,omitempty"` + PendingUploadType string `json:"pendingUploadType,omitempty"` + Version string `json:"version,omitempty"` +} + +// ResolvedUploadURI returns the SAS URI for uploading blobs. +func (p *PendingUploadResponse) ResolvedUploadURI() string { + if p.BlobReference != nil && p.BlobReference.Credential != nil { + if uri := p.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + return "" +} + +// ResolvedBlobURI returns the blob container URI (without SAS) for the finalize request. +func (p *PendingUploadResponse) ResolvedBlobURI() string { + if p.BlobReference != nil { + return p.BlobReference.BlobURI + } + return "" +} + +// FinalizeDatasetRequest is the request body for finalizing a dataset version +// after blob upload. +type FinalizeDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Type string `json:"type"` + IsReference bool `json:"isReference"` + DataURI string `json:"dataUri"` +} + +// NextVersion computes the next dataset version string. +// +// Rules: +// 1. Empty → "1.0" +// 2. Parsable as a decimal number → increment by 1, format as "N.0" +// 3. Ends with trailing digits → increment the trailing numeric part +// 4. Otherwise → append ".1" +func NextVersion(current string) string { + current = strings.TrimSpace(current) + if current == "" { + return "1.0" + } + + // Try parsing as a decimal number (e.g. "1", "1.0", "2.0"). + if f, err := strconv.ParseFloat(current, 64); err == nil { + return strconv.FormatFloat(math.Floor(f)+1, 'f', 1, 64) + } + + // Find trailing digits and increment them. + i := len(current) - 1 + for i >= 0 && current[i] >= '0' && current[i] <= '9' { + i-- + } + if i < len(current)-1 { + prefix := current[:i+1] + n, err := strconv.Atoi(current[i+1:]) + if err == nil { + return prefix + strconv.Itoa(n+1) + } + } + + return current + ".1" +} + +// ReadFirstJSONLFile finds and reads the first .jsonl file in a directory. +func ReadFirstJSONLFile(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("reading directory: %w", err) + } + for _, e := range entries { + if e.IsDir() { + continue + } + if filepath.Ext(e.Name()) == ".jsonl" { + data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path + if err != nil { + return "", fmt.Errorf("reading %s: %w", e.Name(), err) + } + return string(data), nil + } + } + return "", fmt.Errorf("no .jsonl file found in %s", dir) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go new file mode 100644 index 00000000000..427f8c6af00 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -0,0 +1,622 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strings" + + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefix for dataset endpoints. +const pathDatasets = "/datasets" + +// DatasetClient provides methods for dataset upload, download, and metadata retrieval. +type DatasetClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewDatasetClient creates a new DatasetClient. +func NewDatasetClient(endpoint string, cred azcore.TokenCredential) *DatasetClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-datasets", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewDatasetClientFromPipeline creates a DatasetClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewDatasetClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *DatasetClient { + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataset registers a dataset with inline content (upload). +func (c *DatasetClient) CreateDataset( + ctx context.Context, + request *CreateDatasetRequest, + apiVersion string, +) (*Dataset, error) { + return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) +} + +// UploadNextVersion registers the next version of a dataset, discovering the +// current one from the service when currentVersion is empty. +// +// Prefer this over UploadNewVersion. That function derives the next version +// from whatever it is handed, so an empty value restarts at 1.0 and the +// service rejects the pending upload with a 409 +// TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers +// almost always mean "the version after whatever is registered", which is what +// this does. +// +// The version listing is eventually consistent — it returns nothing for a +// second or two after a version is created — so an empty listing cannot be +// trusted to mean the dataset is new. A conflict is therefore treated as a +// stale read: the listing is re-read, and when it is still behind, the version +// just refused is taken as proof that it exists and the next one is tried. +// Trusting the listing alone left a second upload issued moments after the +// first reporting a 409 to the user for a publish that should simply have +// added a version. +func (c *DatasetClient) UploadNextVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + if currentVersion == "" { + currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) + } + + var err error + for range versionConflictAttempts { + var ds *Dataset + ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + if err == nil || !IsVersionConflict(err) { + return ds, err + } + + // The version derived from currentVersion is taken, so it exists + // whatever the listing says. Prefer the listing when it has caught up + // and moved further ahead; otherwise step past what was just refused. + refused := NextVersion(currentVersion) + currentVersion = refused + if latest := c.latestRegisteredVersion(ctx, name, apiVersion); versionAtLeast(latest, refused) { + currentVersion = latest + } + } + return nil, err +} + +// versionConflictAttempts bounds the walk past versions the listing has not +// caught up with. Each attempt is one refused pending upload, so this is short. +const versionConflictAttempts = 4 + +// versionAtLeast reports whether a is a version at or beyond b. +func versionAtLeast(a, b string) bool { + if a == "" { + return false + } + return LatestVersion([]Dataset{{Version: a}, {Version: b}}) == a +} + +// latestRegisteredVersion returns the newest registered version, or empty when +// the dataset is unknown or the listing has not caught up yet. +func (c *DatasetClient) latestRegisteredVersion( + ctx context.Context, + name string, + apiVersion string, +) string { + list, err := c.ListDatasetVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return LatestVersion(list.Value) +} + +// isVersionConflict reports whether the service refused the upload because the +// target version already exists. +func IsVersionConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// UploadNewVersion reads the first JSONL file from localDir, computes the next +// version from currentVersion, and uploads it as a new dataset version using +// the 3-step pending upload flow: +// 1. startPendingUpload → get SAS URI +// 2. Upload blob to SAS URI +// 3. Finalize dataset version with dataUri +func (c *DatasetClient) UploadNewVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + return c.UploadVersion(ctx, name, NextVersion(currentVersion), localDir, apiVersion) +} + +// UploadVersion publishes the dataset at exactly this version. +// +// Separate from UploadNewVersion because its parameter is the version to +// count from, not the one to write: passing "1.0" there publishes 2.0. An +// author who declares a version means that version. +func (c *DatasetClient) UploadVersion( + ctx context.Context, + name string, + version string, + localDir string, + apiVersion string, +) (*Dataset, error) { + content, err := ReadFirstJSONLFile(localDir) + if err != nil { + return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) + } + + newVersion := version + + // Step 1: Start pending upload to get a SAS URI. + pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) + if err != nil { + return nil, fmt.Errorf("starting pending upload: %w", err) + } + + uploadURI := pending.ResolvedUploadURI() + if uploadURI == "" { + return nil, fmt.Errorf("no upload SAS URI returned from startPendingUpload") + } + + // Step 2: Upload the JSONL file to blob storage. + blobName := name + ".jsonl" + if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { + return nil, fmt.Errorf("uploading blob: %w", err) + } + + // Step 3: Finalize the dataset version with the full blob URI. + dataURI := strings.TrimSuffix(pending.ResolvedBlobURI(), "/") + "/" + blobName + return c.FinalizeDatasetVersion(ctx, name, newVersion, dataURI, apiVersion) +} + +// StartPendingUpload initiates a pending upload for a dataset version. +// Returns the SAS URI and blob reference for uploading data. +func (c *DatasetClient) StartPendingUpload( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*PendingUploadResponse, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/startPendingUpload", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[PendingUploadResponse](c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + +// UploadBlob uploads data to a container SAS URI as a block blob. +func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { + u, err := url.Parse(containerSASUri) + if err != nil { + return fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("failed to create upload request: %w", err) + } + req.Header.Set("x-ms-blob-type", "BlockBlob") + req.Header.Set("Content-Type", "application/octet-stream") + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("blob upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// FinalizeDatasetVersion completes the dataset version after blob upload +// by sending the metadata (name, version, dataUri) to the API. +func (c *DatasetClient) FinalizeDatasetVersion( + ctx context.Context, + name string, + version string, + dataURI string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + request := &FinalizeDatasetRequest{ + Name: name, + Version: version, + Type: "uri_file", + DataURI: dataURI, + } + return doRequestTyped[Dataset](c, ctx, http.MethodPut, path, nil, request, apiVersion) +} + +// GetDataset retrieves metadata for a dataset by name and version. +func (c *DatasetClient) GetDataset( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + return doRequestTyped[Dataset](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// GetDatasetCredential retrieves a SAS credential for downloading a dataset from blob storage. +func (c *DatasetClient) GetDatasetCredential( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*DatasetCredential, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/credentials", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) +} + +// DownloadDatasetContent fetches a dataset version's content, whether its URI +// names a blob or a container. +// +// The two differ by origin, not by any field: a dataset uploaded through +// startPendingUpload gets a URI ending in the file name, while one produced by +// a generation job gets the container it was written into, with isSingleFile +// true either way. Downloading the container directly returns a 409, so the +// blob inside has to be found first. +// +// A credential is always fetched, because the URI on the dataset carries no +// SAS token and an unauthenticated read fails. +func (c *DatasetClient) DownloadDatasetContent( + ctx context.Context, + name string, + version string, + apiVersion string, +) ([]byte, error) { + cred, err := c.GetDatasetCredential(ctx, name, version, apiVersion) + if err != nil { + return nil, fmt.Errorf("reading download credentials for %q: %w", name, err) + } + + sasURI := cred.ResolvedDownloadURI() + if sasURI == "" { + return nil, fmt.Errorf("no download URI returned for dataset %q", name) + } + + // A URI whose last path segment carries a file extension is the blob + // itself; anything else is the container holding it. + if looksLikeBlobURI(sasURI) { + data, err := c.DownloadDataset(ctx, sasURI) + if err == nil { + return data, nil + } + log.Printf("[dataset_api] direct download failed (%v); treating the URI as a container", err) + } + + names, err := c.ListContainerBlobs(ctx, sasURI) + if err != nil { + return nil, fmt.Errorf("listing the content of dataset %q: %w", name, err) + } + blobName := pickDatasetBlob(names) + if blobName == "" { + return nil, fmt.Errorf("dataset %q holds no downloadable file", name) + } + return c.DownloadBlob(ctx, sasURI, blobName) +} + +// looksLikeBlobURI reports whether the URI's final segment names a file. +func looksLikeBlobURI(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + last := path.Base(strings.TrimSuffix(u.Path, "/")) + return path.Ext(last) != "" +} + +// pickDatasetBlob chooses the file to read from a container, preferring JSONL +// since that is what an evaluation dataset is. +func pickDatasetBlob(names []string) string { + for _, n := range names { + if strings.EqualFold(path.Ext(n), ".jsonl") { + return n + } + } + for _, n := range names { + if n != "" && !strings.HasSuffix(n, "/") { + return n + } + } + return "" +} + +// DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. +// Returns the raw content as bytes. The downloadURL should be the full URL with SAS token +// (e.g., from DatasetCredential.ResolvedDownloadURI()). +func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { + req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) + if err != nil { + return nil, fmt.Errorf("failed to create download request: %w", err) + } + + // Use a plain HTTP client for blob downloads — the SAS token in the URL provides + // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) + // should not be sent to Azure Blob Storage endpoints. + httpClient := &http.Client{} + resp, err := httpClient.Do(req.Raw()) + if err != nil { + return nil, fmt.Errorf("failed to download dataset from blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d", resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read dataset content: %w", err) + } + + log.Printf("[dataset_api] downloaded %d bytes", len(data)) + return data, nil +} + +// ListContainerBlobs lists blobs in a container using a container-level SAS URI. +// The containerSASUri should include the SAS token (e.g., from credential.sasUri with sr=c). +// Returns a list of blob names found in the container. +func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri string) ([]string, error) { + // Parse the container URI and append list query parameters. + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + q := u.Query() + q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter + q.Set("comp", "list") + u.RawQuery = q.Encode() + + log.Printf("[dataset_api] listing blobs: %s", u.Redacted()) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create list request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to list container blobs: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("container list failed with status %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read list response: %w", err) + } + + // Parse XML blob listing to extract blob names. + names := parseBlobNames(string(body)) + log.Printf("[dataset_api] found %d blobs in container", len(names)) + return names, nil +} + +// DownloadBlob downloads a single blob from a container using the container SAS URI +// and the blob name. Returns the blob content as bytes. +func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create blob download request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d for %s", resp.StatusCode, blobName) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read blob content: %w", err) + } + + log.Printf("[dataset_api] downloaded blob %s (%d bytes)", blobName, len(data)) + return data, nil +} + +// parseBlobNames extracts blob names from the Azure Blob Storage XML list response +// using proper XML parsing against the EnumerationResults schema. +func parseBlobNames(xmlBody string) []string { + type blob struct { + Name string `xml:"Name"` + } + type blobs struct { + Blob []blob `xml:"Blob"` + } + type enumerationResults struct { + Blobs blobs `xml:"Blobs"` + } + + var result enumerationResults + if err := xml.Unmarshal([]byte(xmlBody), &result); err != nil { + return nil + } + + names := make([]string, 0, len(result.Blobs.Blob)) + for _, b := range result.Blobs.Blob { + if b.Name != "" { + names = append(names, b.Name) + } + } + return names +} + +// doRequest performs an HTTP request against the dataset API and returns the raw response body. +func (c *DatasetClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + log.Printf("[dataset_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[dataset_api] response status: %d", resp.StatusCode) + + // 204 belongs here for the same reason it does in eval_api: a delete that + // removed the version answers No Content, and rejecting that reports every + // successful delete as an error. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *DatasetClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go new file mode 100644 index 00000000000..164cc6d415b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadServer answers the three-step publish, refusing any version in taken +// and reporting whatever the listing is told to report. +type uploadServer struct { + mu sync.Mutex + taken map[string]bool + listing []string + attempts []string +} + +func (s *uploadServer) handler(t *testing.T, base func() string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): + version := strings.Split(r.URL.Path, "/versions/")[1] + version = strings.TrimSuffix(version, "/startPendingUpload") + s.attempts = append(s.attempts, version) + if s.taken[version] { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error":{"code":"Conflict"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReference": map[string]any{ + "blobUri": base() + "/c", + "storageAccountArmId": "id", + "credential": map[string]any{"sasUri": base() + "/c?sig=x"}, + }, + })) + + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/versions"): + values := []map[string]any{} + for _, v := range s.listing { + values = append(values, map[string]any{"name": "ds", "version": v}) + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + + case r.Method == http.MethodPut: + version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + s.taken[version] = true + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "ds", "version": version, + })) + + default: + // The blob PUT. + w.WriteHeader(http.StatusCreated) + } + } +} + +// The version listing lags a publish, so a second upload can be told the +// dataset is new and restart at a version that already exists. Trusting the +// listing alone surfaced that 409 to the user for a publish that should simply +// have added a version. +func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { + server := &uploadServer{taken: map[string]bool{"1.0": true}} + // The listing has not caught up: it still reports nothing at all. + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err, "a stale listing must not surface as a conflict") + assert.Equal(t, "2.0", ds.Version) + assert.Equal(t, []string{"1.0", "2.0"}, server.attempts, + "the version just refused is proof it exists, so the next one is tried") +} + +// When the listing has caught up and is further ahead than the refused +// version, it is the better answer: it skips versions somebody else published. +func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) { + server := &uploadServer{ + taken: map[string]bool{"1.0": true, "2.0": true, "3.0": true}, + listing: []string{"1.0", "2.0", "3.0"}, + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err) + assert.Equal(t, "4.0", ds.Version) +} + +// A service that refuses everything must end in the conflict rather than +// looping: an unbounded walk would hammer the service on a real failure. +func TestUploadNextVersionGivesUpBounded(t *testing.T) { + server := &uploadServer{taken: map[string]bool{}} + for _, v := range []string{"1.0", "2.0", "3.0", "4.0", "5.0", "6.0"} { + server.taken[v] = true + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + _, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.Error(t, err) + assert.True(t, IsVersionConflict(err)) + assert.Len(t, server.attempts, versionConflictAttempts) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go new file mode 100644 index 00000000000..e6cc46c2e3d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The service spells these fields inconsistently, and a URI read from the +// wrong spelling comes back empty rather than wrong — which is how the dataset +// URI went unbound the first time. +func TestDatasetResolvedBlobURI_AcceptsEitherSpelling(t *testing.T) { + cases := map[string]string{ + `{"dataUri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"data_uri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"blobUri":"https://x/b.jsonl"}`: "https://x/b.jsonl", + `{"contentUri":"https://x/c.jsonl"}`: "https://x/c.jsonl", + } + for body, want := range cases { + var ds Dataset + require.NoError(t, json.Unmarshal([]byte(body), &ds), body) + assert.Equal(t, want, ds.ResolvedBlobURI(), body) + } + + var none Dataset + require.NoError(t, json.Unmarshal([]byte(`{"name":"x"}`), &none)) + assert.Empty(t, none.ResolvedBlobURI(), + "no URI means the caller has to fetch a credential, not that the dataset is unreadable") +} + +// An upload needs the SAS-bearing URI to write to and the plain one to +// finalize with. Confusing them fails at different stages, so both are read +// from their own place. +func TestPendingUploadURIs(t *testing.T) { + var p PendingUploadResponse + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReference": { + "blobUri": "https://acct.blob.core.windows.net/container", + "credential": { "sasUri": "https://acct.blob.core.windows.net/container?sig=abc" } + } + }`), &p)) + + assert.Equal(t, "https://acct.blob.core.windows.net/container?sig=abc", p.ResolvedUploadURI(), + "the upload target carries the SAS") + assert.Equal(t, "https://acct.blob.core.windows.net/container", p.ResolvedBlobURI(), + "the finalize URI does not") + + var empty PendingUploadResponse + assert.Empty(t, empty.ResolvedUploadURI()) + assert.Empty(t, empty.ResolvedBlobURI()) +} + +// Credentials arrive in two shapes and the consumption one takes precedence, +// because that is the one scoped for reading. +func TestCredentialResolvedDownloadURI(t *testing.T) { + var c DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReferenceForConsumption": { "credential": { "sasUri": "https://acct/read?sig=r" } }, + "blobReference": { "credential": { "sasUri": "https://acct/write?sig=w" } } + }`), &c)) + assert.Equal(t, "https://acct/read?sig=r", c.ResolvedDownloadURI()) + + var legacy DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{"sas_uri":"https://acct/legacy?sig=l"}`), &legacy)) + assert.Equal(t, "https://acct/legacy?sig=l", legacy.ResolvedDownloadURI(), + "the flat spelling is still honoured") + + var none DatasetCredential + assert.Empty(t, none.ResolvedDownloadURI()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go new file mode 100644 index 00000000000..052a63ae504 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Drift detection compares the version on the service with the one recorded at +// the last deploy, so the ordering has to be numeric rather than lexical: +// "10.0" is newer than "9.0" even though it sorts earlier as a string. +func TestVersionGreater(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.0", "1.0", true}, + {"1.0", "2.0", false}, + {"1.0", "1.0", false}, + {"10.0", "9.0", true}, + {"9.0", "10.0", false}, + {"v3", "v2", true}, + } + + for _, tc := range cases { + require.Equal(t, tc.want, VersionGreater(tc.a, tc.b), + "VersionGreater(%q, %q)", tc.a, tc.b) + } +} + +// An unorderable version must never trigger a drift failure on its own: the +// deploy would be blocked with no way for the author to reason about it. +func TestVersionGreaterIgnoresUnorderable(t *testing.T) { + require.False(t, VersionGreater("draft", "1.0")) + require.False(t, VersionGreater("1.0", "draft")) + require.False(t, VersionGreater("", "1.0")) + require.False(t, VersionGreater("1.0", "")) +} + +// The two upload entry points read their version argument differently, and the +// difference is the whole point: UploadNewVersion counts from it, UploadVersion +// writes it. Passing "1.0" to the counting one publishes 2.0, which is not what +// an author who wrote version: "1.0" asked for. +func TestNextVersionCountsFromTheArgument(t *testing.T) { + if got := NextVersion("1.0"); got != "2.0" { + t.Fatalf("NextVersion(1.0) = %q, want 2.0", got) + } + if got := NextVersion("1"); got != "2.0" { + t.Fatalf("NextVersion(1) = %q, want 2.0", got) + } + // An unknown current version starts the sequence rather than guessing. + if got := NextVersion(""); got != "1.0" { + t.Fatalf("NextVersion(empty) = %q, want 1.0", got) + } +} + +func TestLatestVersionOrdersNumerically(t *testing.T) { + got := LatestVersion([]Dataset{{Version: "1.0"}, {Version: "10.0"}, {Version: "2.0"}}) + if got != "10.0" { + t.Fatalf("LatestVersion = %q, want 10.0 (numeric, not lexical)", got) + } + if LatestVersion(nil) != "" { + t.Fatal("LatestVersion(nil) should be empty") + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go new file mode 100644 index 00000000000..3b4b93d7c7e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "errors" + "net/http" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsConflict reports whether the service refused because the resource is busy. +func IsConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go new file mode 100644 index 00000000000..dd230ef1f08 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" +) + +// EvaluatorTypeBuiltin selects the platform-provided evaluators. +const EvaluatorTypeBuiltin = "Builtin" + +// JSONSchema is the subset of JSON Schema the evaluator contract uses. +type JSONSchema struct { + Type string `json:"type,omitempty"` + Required []string `json:"required,omitempty"` + Properties map[string]any `json:"properties,omitempty"` +} + +// PropertyNames returns the accepted property names, sorted for stable output. +func (s *JSONSchema) PropertyNames() []string { + if s == nil { + return nil + } + names := make([]string, 0, len(s.Properties)) + for name := range s.Properties { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Accepts reports whether the schema declares the named property. +func (s *JSONSchema) Accepts(name string) bool { + if s == nil || s.Properties == nil { + return false + } + _, ok := s.Properties[name] + return ok +} + +// EvaluatorContract is the published input contract for an evaluator: which +// data fields it consumes and which initialization parameters it takes. +type EvaluatorContract struct { + Type string `json:"type,omitempty"` + DataSchema *JSONSchema `json:"data_schema,omitempty"` + InitParameters *JSONSchema `json:"init_parameters,omitempty"` +} + +// EvaluatorSummary is a single entry in an evaluator listing. +// +// The listing carries the full contract, so callers can shape a request to +// match an evaluator instead of guessing and taking a service-side rejection. +type EvaluatorSummary struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Description string `json:"description,omitempty"` + + // The listing spells this evaluator_type; `type` is accepted too because + // other evaluator payloads use it. + EvaluatorType string `json:"evaluator_type,omitempty"` + TypeAlias string `json:"type,omitempty"` + + Categories []string `json:"categories,omitempty"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels,omitempty"` + Definition *EvaluatorContract `json:"definition,omitempty"` +} + +// Type reports the evaluator kind across both spellings. +func (e *EvaluatorSummary) Type() string { + if e.EvaluatorType != "" { + return e.EvaluatorType + } + return e.TypeAlias +} + +// SupportsLevel reports whether the evaluator runs at the given evaluation +// level. An evaluator that declares no levels is treated as unconstrained. +func (e *EvaluatorSummary) SupportsLevel(level string) bool { + if level == "" || len(e.SupportedEvaluationLevels) == 0 { + return true + } + for _, supported := range e.SupportedEvaluationLevels { + if strings.EqualFold(supported, level) { + return true + } + } + return false +} + +// DataSchema returns the evaluator's input schema, or nil when the listing +// did not describe one. +func (e *EvaluatorSummary) DataSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.DataSchema +} + +// InitSchema returns the evaluator's initialization-parameter schema, or nil +// when the listing did not describe one. +func (e *EvaluatorSummary) InitSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.InitParameters +} + +// EvaluatorListResponse is the paged response for an evaluator listing. +type EvaluatorListResponse struct { + Value []EvaluatorSummary `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ByName indexes the listing by evaluator name. +func (r *EvaluatorListResponse) ByName() map[string]*EvaluatorSummary { + if r == nil { + return nil + } + index := make(map[string]*EvaluatorSummary, len(r.Value)) + for i := range r.Value { + index[r.Value[i].Name] = &r.Value[i] + } + return index +} + +// ListEvaluators returns the evaluators visible to the project. Pass +// EvaluatorTypeBuiltin to list only the platform's built-ins. +func (c *EvalClient) ListEvaluators( + ctx context.Context, + evaluatorType string, + apiVersion string, +) (*EvaluatorListResponse, error) { + var query map[string]string + if evaluatorType != "" { + query = map[string]string{"type": evaluatorType} + } + return doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, pathEvaluators, query, nil, apiVersion, + ) +} + +// ListEvaluatorVersions returns every version of one evaluator. +func (c *EvalClient) ListEvaluatorVersions( + ctx context.Context, + name string, + apiVersion string, +) (*EvaluatorListResponse, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, path, nil, nil, apiVersion, + ) +} + +// LatestEvaluatorVersionNumber reports the newest registered version as an +// integer, or 0 when the evaluator is unknown or its versions are not numeric. +func (c *EvalClient) LatestEvaluatorVersionNumber( + ctx context.Context, + name string, + apiVersion string, +) int { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return 0 + } + number, err := strconv.Atoi(pickLatestVersion(list.Value)) + if err != nil { + return 0 + } + return number +} + +// parseVersionNumber reads a version string as an integer, answering 0 for one +// that is not numeric. +func parseVersionNumber(version string) int { + number, err := strconv.Atoi(version) + if err != nil { + return 0 + } + return number +} + +// DeleteEvaluatorVersion removes a single evaluator version. +func (c *EvalClient) DeleteEvaluatorVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathEvaluators, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// CancelOpenAIEvalRun stops an in-flight run. +// +// The body must stay nil: this route cancels only when the body is empty, and +// updates the run's status and counters when it is not. +func (c *EvalClient) CancelOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") +} + +// DeleteOpenAIEvalRun removes a single run. +func (c *EvalClient) DeleteOpenAIEvalRun(ctx context.Context, evalID, runID string) error { + path := fmt.Sprintf( + "%s/%s/runs/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + +// ListOutputItems returns a run's per-sample results. +// +// The run itself carries only totals and a per-criterion breakdown. The output +// items are the rows: each one holds the dataset item that was evaluated, what +// the target answered, and every evaluator's score, verdict and reason. Showing +// results without them can say how many failed but never which, or why. +func (c *EvalClient) ListOutputItems( + ctx context.Context, + evalID, runID string, + limit int, +) (*OutputItemList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + return doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") +} + +// GetOutputItem reads a single evaluated row. +func (c *EvalClient) GetOutputItem( + ctx context.Context, + evalID, runID, itemID string, +) (*OutputItem, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), url.PathEscape(itemID), + ) + return doRequestTyped[OutputItem](c, ctx, http.MethodGet, path, nil, nil, "") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go new file mode 100644 index 00000000000..46f7524acbf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Evaluator versions are integers rendered as strings, so a lexical compare +// ranks "9" above "15". The live service already has evaluators at version 15 +// and 17, so this is not hypothetical. +func TestPickLatestEvaluatorVersionIsNumeric(t *testing.T) { + cases := []struct { + name string + versions []string + want string + }{ + {"single", []string{"1"}, "1"}, + {"ascending", []string{"1", "2", "3"}, "3"}, + {"unordered", []string{"3", "1", "2"}, "3"}, + {"double digits beat single", []string{"9", "15"}, "15"}, + {"realistic", []string{"1", "9", "10", "17", "2"}, "17"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + entries := make([]EvaluatorSummary, 0, len(tc.versions)) + for _, v := range tc.versions { + entries = append(entries, EvaluatorSummary{Name: "e", Version: v}) + } + require.Equal(t, tc.want, pickLatestVersion(entries)) + }) + } +} + +// A non-numeric version is only used when nothing numeric exists, so one odd +// entry cannot mask the real latest. +func TestPickLatestEvaluatorVersionHandlesNonNumeric(t *testing.T) { + require.Equal(t, "2", pickLatestVersion([]EvaluatorSummary{ + {Version: "draft"}, {Version: "1"}, {Version: "2"}, + })) + require.Equal(t, "draft", pickLatestVersion([]EvaluatorSummary{{Version: "draft"}})) + require.Equal(t, "", pickLatestVersion(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go new file mode 100644 index 00000000000..235f58884a0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "path/filepath" + "strings" + "time" + + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// Generation source building +// --------------------------------------------------------------------------- + +// TraceOptions holds optional trace inclusion parameters for generation sources. +type TraceOptions struct { + Days int +} + +// WithoutAgentSource returns the sources with the agent entry removed. +// +// Agent-seeded data generation currently fails server-side for every agent, +// while the same request carrying only the prompt succeeds, so this is what a +// retry falls back to. +func WithoutAgentSource(sources []GenerationSource) []GenerationSource { + kept := make([]GenerationSource, 0, len(sources)) + for _, s := range sources { + if s.Type == "agent" { + continue + } + kept = append(kept, s) + } + return kept +} + +// HasPromptSource reports whether anything remains to generate from. +func HasPromptSource(sources []GenerationSource) bool { + for _, s := range sources { + if s.Type == "prompt" && s.Prompt != "" { + return true + } + } + return false +} + +// BuildGenerationSources emits the sources the caller selected, in a stable +// order, along with the ones it asked for and nothing could be built from. +// +// kinds is what --from named. An empty kinds means "whatever this plan has to +// offer" and reports nothing missing: the caller expressed no preference, so +// there is nothing to disappoint. Naming a kind explicitly is a request, and a +// request that cannot be built is worth saying out loud rather than quietly +// submitting a job seeded from less than was asked for. +func BuildGenerationSources( + kinds []string, + agentName, version, instruction string, + traces *TraceOptions, +) (sources []GenerationSource, unbuildable []string) { + want := map[string]bool{} + for _, k := range kinds { + want[k] = true + } + // Empty kinds selects everything available; a populated one selects only + // what it names. + selected := func(kind string) bool { + return len(want) == 0 || want[kind] + } + // asked distinguishes "the default swept this up" from "the user typed it", + // which is what decides whether an empty-handed source is an error. + asked := func(kind string) bool { return want[kind] } + + // The agent is settled first because whether it was built decides whether + // its instructions have anything to be the instructions of. + var agentSource *GenerationSource + if selected("agent") { + switch { + case agentName != "": + agentSource = &GenerationSource{Type: "agent", AgentName: agentName} + if version != "" { + agentSource.AgentVersion = version + } + case asked("agent"): + unbuildable = append(unbuildable, "agent") + } + } + + // Generating from an agent means generating from its instructions, so they + // travel with it as a prompt. That is also the only shape the service + // currently honours: the agent source alone fails for every agent, and the + // prompt is what the retry in generateDataset falls back to. Without this, + // `--from agent` would be a request that always fails. + promptCarriesTheAgent := agentSource != nil && asked("agent") + if selected("prompt") || promptCarriesTheAgent { + switch { + case instruction != "": + sources = append(sources, GenerationSource{ + Type: "prompt", + Prompt: instruction, + }) + case asked("prompt"): + unbuildable = append(unbuildable, "prompt") + } + } + + if agentSource != nil { + sources = append(sources, *agentSource) + } + + if selected("traces") { + // A window narrows the request; it does not authorize it. Asking for + // traces without one means every trace the agent has. + switch { + case traces != nil && traces.Days > 0: + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + StartTime: time.Now().AddDate(0, 0, -traces.Days).Unix(), + }) + case asked("traces"): + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + }) + } + } + + // The service takes a file's rows through the dataset upload path, not + // through a generation source, so there is nothing here to build one from. + if asked("file") { + unbuildable = append(unbuildable, "file") + } + + return sources, unbuildable +} + +// --------------------------------------------------------------------------- +// Request builders +// --------------------------------------------------------------------------- + +// NewDataGenerationJobRequest builds a DataGenerationJobRequest from the +// provided parameters. Currently, it's always "simple_qna" type with multiple sources +func NewDataGenerationJobRequest( + name, evalModel string, + maxSamples int, + sources []GenerationSource, +) *DataGenerationJobRequest { + return &DataGenerationJobRequest{ + Inputs: DataGenerationInputs{ + Name: name, + Scenario: "evaluation", + Options: DataGenerationOptions{ + Type: "simple_qna", + MaxSamples: maxSamples, + ModelOptions: ModelOptions{ + Model: evalModel, + }, + }, + Sources: sources, + }, + } +} + +// NewEvaluatorGenerationJobRequest builds an EvaluatorGenerationJobRequest +// from the provided parameters. +func NewEvaluatorGenerationJobRequest( + name, evalModel string, + sources []GenerationSource, +) *EvaluatorGenerationJobRequest { + return &EvaluatorGenerationJobRequest{ + Inputs: EvaluatorGenerationInputs{ + Name: name, + EvaluatorName: name, + Model: evalModel, + Sources: sources, + }, + } +} + +// --------------------------------------------------------------------------- +// Evaluator classification +// --------------------------------------------------------------------------- + +// IsBuiltinEvaluator returns true when the evaluator name has the "builtin." +// prefix. +func IsBuiltinEvaluator(name string) bool { + return strings.HasPrefix(name, "builtin.") +} + +// SplitEvaluators partitions evaluators into generated (non-builtin) and +// built-in lists. +func SplitEvaluators(evaluators evalcore.EvaluatorList) (generated, builtin evalcore.EvaluatorList) { + for _, e := range evaluators { + if IsBuiltinEvaluator(e.Name) { + builtin = append(builtin, e) + } else { + generated = append(generated, e) + } + } + return generated, builtin +} + +// --------------------------------------------------------------------------- +// Dataset name detection +// --------------------------------------------------------------------------- + +// IsDatasetName returns true when the value looks like a registered dataset +// name rather than a local file path. A name has no path separators and no +// common data-file extension (.jsonl, .json, .csv). +func IsDatasetName(value string) bool { + if value == "" { + return false + } + if strings.ContainsAny(value, "/\\") { + return false + } + ext := strings.ToLower(filepath.Ext(value)) + return ext != ".jsonl" && ext != ".json" && ext != ".csv" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go new file mode 100644 index 00000000000..6a5c993f820 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// kindsOf reduces the built sources to what --from talks about, which is the +// only part these tests are asserting on. +func kindsOf(sources []GenerationSource) []string { + kinds := make([]string, 0, len(sources)) + for _, s := range sources { + kinds = append(kinds, s.Type) + } + return kinds +} + +// Naming a source is a request to send that one, not a hint. Everything the +// plan could otherwise have offered stays out of the request. +func TestBuildGenerationSources_SendsOnlyWhatFromNamed(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, + "support-agent", "3", "answer support questions", + &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Generating from an agent means generating from its instructions, so asking +// for the agent carries them. It is also the only shape the service honours: +// the agent source on its own fails for every agent, so a `--from agent` that +// dropped the prompt would be a request that always fails. +func TestBuildGenerationSources_AgentCarriesItsInstructions(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "support-agent", "3", "answer support questions", nil, + ) + + assert.Equal(t, []string{"prompt", "agent"}, kindsOf(sources)) + assert.Equal(t, "answer support questions", sources[0].Prompt) + assert.Empty(t, unbuildable) +} + +// The instructions ride along with the agent; they do not stand in for it. An +// agent nobody named is still nothing to generate from. +func TestBuildGenerationSources_InstructionsDoNotSubstituteForTheAgent(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "", "", "answer support questions", nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, []string{"agent"}, unbuildable) +} + +// The agent name travels with the traces source: it is what scopes the query +// to this agent's conversations rather than the whole project's. +func TestBuildGenerationSources_TracesCarryTheAgent(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "support-agent", sources[0].AgentName) +} + +// A day window narrows the trace query; it is not what authorizes it. The +// documented `dataset generate --from traces` carries no window, and it +// has to mean "every trace" rather than "no traces". +func TestBuildGenerationSources_TracesWithoutAWindowAreUnbounded(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", nil, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "traces", sources[0].Type) + assert.Zero(t, sources[0].StartTime, + "an absent window must leave start_time off the wire, not pin it to now") + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_TraceWindowBecomesAStartTime(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + want := time.Now().AddDate(0, 0, -7).Unix() + assert.InDelta(t, want, sources[0].StartTime, 60) +} + +// No --from is no preference, so the plan sends everything it happens to have. +func TestBuildGenerationSources_EmptyFromSendsWhatThePlanHas(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + nil, "support-agent", "3", "answer support questions", &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Expressing no preference cannot disappoint one, so an empty --from reports +// nothing missing however little the plan turns out to hold. +func TestBuildGenerationSources_EmptyFromNeverReportsMissingSources(t *testing.T) { + sources, unbuildable := BuildGenerationSources(nil, "", "", "", nil) + + assert.Empty(t, sources) + assert.Empty(t, unbuildable) +} + +// Asking for a source the plan cannot build has to surface, because the job is +// billed and what comes back looks the same either way. +func TestBuildGenerationSources_ReportsWhatItCouldNotBuild(t *testing.T) { + tests := []struct { + name string + kinds []string + agentName string + instruction string + want []string + }{ + { + name: "prompt without an instruction", + kinds: []string{"prompt"}, + want: []string{"prompt"}, + }, + { + name: "agent without a target", + kinds: []string{"agent"}, + want: []string{"agent"}, + }, + { + name: "file is not a generation source at all", + kinds: []string{"file"}, + want: []string{"file"}, + }, + { + name: "several at once", + kinds: []string{"prompt", "agent"}, + want: []string{"agent", "prompt"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + tt.kinds, tt.agentName, "", tt.instruction, nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, tt.want, unbuildable) + }) + } +} + +// A request that names two sources and can only build one still reports the +// one it could not, rather than being satisfied by the other's success. +func TestBuildGenerationSources_OneBuiltSourceDoesNotExcuseAMissingOne(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent", "prompt"}, "support-agent", "", "", nil, + ) + + assert.Equal(t, []string{"agent"}, kindsOf(sources)) + assert.Equal(t, []string{"prompt"}, unbuildable) +} + +// `file` is only unbuildable when it was asked for. The default sweep must not +// invent a complaint about a source nobody named. +func TestBuildGenerationSources_FileIsOnlyReportedWhenAskedFor(t *testing.T) { + _, unbuildable := BuildGenerationSources( + nil, "support-agent", "", "instruction", &TraceOptions{Days: 7}, + ) + + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { + withVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "3", "", nil) + require.Len(t, withVersion, 1) + assert.Equal(t, "3", withVersion[0].AgentVersion) + + withoutVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "", "", nil) + require.Len(t, withoutVersion, 1) + assert.Empty(t, withoutVersion[0].AgentVersion) +} + +// The retry that saves the documented flow: agent-seeded generation fails +// server-side for every agent, and the same request without the agent source +// succeeds. +func TestWithoutAgentSource(t *testing.T) { + sources := []GenerationSource{ + {Type: "prompt", Prompt: "be helpful"}, + {Type: "agent", AgentName: "support"}, + {Type: "traces", AgentName: "support"}, + } + + kept := WithoutAgentSource(sources) + + assert.Equal(t, []string{"prompt", "traces"}, kindsOf(kept)) + assert.Len(t, sources, 3, "the original must not be modified; it is retried from") +} + +// The retry only happens when something is left to generate from, so this is +// what stops a second billed job that would fail the same way. +func TestHasPromptSource(t *testing.T) { + assert.True(t, HasPromptSource([]GenerationSource{{Type: "prompt", Prompt: "x"}})) + assert.False(t, HasPromptSource([]GenerationSource{{Type: "prompt"}}), + "an empty prompt is nothing to generate from") + assert.False(t, HasPromptSource([]GenerationSource{{Type: "agent", AgentName: "s"}})) + assert.False(t, HasPromptSource(nil)) +} + +// The request body is what the service validates, so the fields it keys on are +// pinned rather than left to whatever the builder happens to set. +func TestNewDataGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "be helpful"}} + + req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, sources) + + require.NotNil(t, req) + assert.Equal(t, "support-regression", req.Inputs.Name) + assert.Equal(t, "evaluation", req.Inputs.Scenario) + assert.Equal(t, "simple_qna", req.Inputs.Options.Type) + assert.Equal(t, 15, req.Inputs.Options.MaxSamples) + assert.Equal(t, "gpt-4o", req.Inputs.Options.ModelOptions.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} + +// The evaluator request sends the name twice, under two keys the service reads +// separately. Setting only one produces a job that runs and returns an +// evaluator under the wrong name. +func TestNewEvaluatorGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "grade politeness"}} + + req := NewEvaluatorGenerationJobRequest("support-quality", "gpt-4o", sources) + + require.NotNil(t, req) + assert.Equal(t, "support-quality", req.Inputs.Name) + assert.Equal(t, "support-quality", req.Inputs.EvaluatorName) + assert.Equal(t, "gpt-4o", req.Inputs.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go new file mode 100644 index 00000000000..0335f951d62 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/url" + "strconv" + "strings" +) + +// InsightTypeEvaluationComparison compares evaluation runs. The service also +// defines EvaluationRunClusterInsight and AgentClusterInsight, which this +// extension does not use. +const InsightTypeEvaluationComparison = "EvaluationComparison" + +const pathInsights = "/insights" + +// InsightRequest is the polymorphic body the service dispatches on. `type` is +// the discriminator; without it the request is rejected because the underlying +// contract is an interface. +type InsightRequest struct { + Type string `json:"type"` + EvalID string `json:"evalId"` + BaselineRunID string `json:"baselineRunId"` + TreatmentRunIDs []string `json:"treatmentRunIds"` +} + +// CreateInsightRequest wraps the request. DisplayName is required; the service +// rejects a body without it before it looks at anything else. +type CreateInsightRequest struct { + DisplayName string `json:"displayName"` + Request *InsightRequest `json:"request"` +} + +// LenientFloat is a float64 that also decodes the quoted forms the service +// uses for values JSON cannot express. +// +// A run with a single sample has an undefined standard deviation, and the +// service sends it as the string "NaN" because JSON has no NaN literal. +// Decoding that into a plain float64 fails the entire comparison — including +// the TooFewSamples verdict that exists to explain exactly this case — so a +// one-sample gate reported a parse error instead of its result. +type LenientFloat float64 + +func (f *LenientFloat) UnmarshalJSON(data []byte) error { + s := strings.TrimSpace(string(data)) + if s == "null" { + *f = LenientFloat(math.NaN()) + return nil + } + // "NaN", "Infinity", "-Infinity" and ordinary numbers arrive quoted; + // ParseFloat accepts all of them once the quotes are gone. + if unquoted, err := strconv.Unquote(s); err == nil { + s = strings.TrimSpace(unquoted) + if s == "" { + *f = LenientFloat(math.NaN()) + return nil + } + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return fmt.Errorf("parsing number %s: %w", data, err) + } + *f = LenientFloat(v) + return nil +} + +// MarshalJSON writes non-finite values as null. encoding/json refuses to +// marshal NaN or ±Inf at all, which would turn `-o json` into an error the +// moment a comparison contained one; null is valid JSON and reads as the +// "undefined" that a one-sample standard deviation actually is. +func (f LenientFloat) MarshalJSON() ([]byte, error) { + if math.IsNaN(float64(f)) || math.IsInf(float64(f), 0) { + return []byte("null"), nil + } + return json.Marshal(float64(f)) +} + +// Defined reports whether the value is a real number that can be shown. +func (f LenientFloat) Defined() bool { + return !math.IsNaN(float64(f)) && !math.IsInf(float64(f), 0) +} + +// RunSummary is one run's aggregate for a single metric. +type RunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average LenientFloat `json:"average"` + StandardDeviation LenientFloat `json:"standardDeviation"` +} + +// CompareItem is one treatment run measured against the baseline. +type CompareItem struct { + TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` + DeltaEstimate LenientFloat `json:"deltaEstimate"` + PValue LenientFloat `json:"pValue"` + // TreatmentEffect classifies the result, e.g. TooFewSamples when the + // sample count cannot support a conclusion. + TreatmentEffect string `json:"treatmentEffect,omitempty"` +} + +// MetricComparison is the baseline and treatments for one testing criterion. +type MetricComparison struct { + TestingCriteria string `json:"testingCriteria"` + Metric string `json:"metric"` + Evaluator string `json:"evaluator"` + BaselineRunSummary *RunSummary `json:"baselineRunSummary,omitempty"` + CompareItems []CompareItem `json:"compareItems,omitempty"` +} + +// InsightResult carries the comparison once the insight succeeds. +type InsightResult struct { + Comparisons []MetricComparison `json:"comparisons,omitempty"` + // Method names the statistical test, e.g. PairedTTest. + Method string `json:"method,omitempty"` + Type string `json:"type,omitempty"` + Error any `json:"error,omitempty"` +} + +// Insight is the long-running operation the comparison runs as. +type Insight struct { + ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + State string `json:"state,omitempty"` + Request *InsightRequest `json:"request,omitempty"` + Result *InsightResult `json:"result,omitempty"` +} + +// Succeeded reports whether the insight finished with a result. +func (i *Insight) Succeeded() bool { + return i != nil && i.State == "Succeeded" +} + +// Terminal reports whether the insight has stopped changing. +func (i *Insight) Terminal() bool { + if i == nil { + return false + } + switch i.State { + case "", "NotStarted", "Running", "InProgress", "Queued": + return false + default: + return true + } +} + +// CreateInsight starts a comparison. +// +// The synchronous variant, POST /insights/sync, returns a 500 for this request +// shape, so the asynchronous form is the only usable one and the caller polls. +func (c *EvalClient) CreateInsight( + ctx context.Context, + request *CreateInsightRequest, + apiVersion string, +) (*Insight, error) { + return doRequestTyped[Insight]( + c, ctx, http.MethodPost, pathInsights, nil, request, apiVersion) +} + +// GetInsight reads a comparison's current state. +func (c *EvalClient) GetInsight( + ctx context.Context, + insightID string, + apiVersion string, +) (*Insight, error) { + path := fmt.Sprintf("%s/%s", pathInsights, url.PathEscape(insightID)) + return doRequestTyped[Insight](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go new file mode 100644 index 00000000000..b8db134dc79 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exact body the service returned for a comparison of two single-sample +// runs. `standardDeviation` is the string "NaN" because JSON has no NaN +// literal, and decoding it into a float64 failed the whole comparison — losing +// the TooFewSamples verdict that explains the very situation that produced it. +const oneSampleComparison = `{ + "comparisons": [ + { + "testingCriteria": "task_adherence", + "metric": "task_adherence", + "evaluator": "builtin.task_adherence", + "baselineRunSummary": { + "runId": "evalrun_base", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "compareItems": [ + { + "treatmentRunSummary": { + "runId": "evalrun_treat", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "deltaEstimate": 0.0, + "pValue": 1.0, + "treatmentEffect": "TooFewSamples" + } + ] + } + ], + "method": "TTest", + "type": "EvaluationComparison" +}` + +func TestInsightResult_DecodesQuotedNaN(t *testing.T) { + var got InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &got)) + + require.Len(t, got.Comparisons, 1) + c := got.Comparisons[0] + require.NotNil(t, c.BaselineRunSummary) + + assert.Equal(t, 1.0, float64(c.BaselineRunSummary.Average)) + assert.False(t, c.BaselineRunSummary.StandardDeviation.Defined(), + "a single sample has no standard deviation") + + require.Len(t, c.CompareItems, 1) + assert.Equal(t, "TooFewSamples", c.CompareItems[0].TreatmentEffect, + "the verdict survives, which is the whole point of not failing the parse") + assert.Equal(t, 1.0, float64(c.CompareItems[0].PValue)) +} + +func TestLenientFloat_AcceptsBothShapes(t *testing.T) { + cases := map[string]func(LenientFloat) bool{ + `0.75`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"0.75"`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"NaN"`: func(f LenientFloat) bool { return !f.Defined() }, + `"Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `"-Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `null`: func(f LenientFloat) bool { return !f.Defined() }, + `""`: func(f LenientFloat) bool { return !f.Defined() }, + } + + for raw, ok := range cases { + var f LenientFloat + require.NoError(t, json.Unmarshal([]byte(raw), &f), "decoding %s", raw) + assert.True(t, ok(f), "unexpected value decoding %s", raw) + } + + var f LenientFloat + assert.Error(t, json.Unmarshal([]byte(`"not a number"`), &f), + "genuine garbage must still be reported") +} + +// encoding/json refuses to marshal NaN, so `-o json` would fail on any +// comparison holding one unless it is written as null. +func TestLenientFloat_MarshalsNonFiniteAsNull(t *testing.T) { + b, err := json.Marshal(LenientFloat(math.NaN())) + require.NoError(t, err) + assert.Equal(t, "null", string(b)) + + b, err = json.Marshal(LenientFloat(0.5)) + require.NoError(t, err) + assert.Equal(t, "0.5", string(b)) + + // The whole result has to survive a round trip, since that is what + // `results compare -o json` emits. + var res InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &res)) + out, err := json.Marshal(res) + require.NoError(t, err, "a comparison containing NaN must still emit JSON") + assert.Contains(t, string(out), `"standardDeviation":null`) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go new file mode 100644 index 00000000000..ceed191847a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -0,0 +1,525 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "fmt" + "sort" + "strings" +) + +// --------------------------------------------------------------------------- +// Data Generation Jobs +// --------------------------------------------------------------------------- + +// DataGenerationJobRequest is the request body for CreateDataGenerationJob. +type DataGenerationJobRequest struct { + Inputs DataGenerationInputs `json:"inputs"` +} + +// DataGenerationInputs holds the inputs for a data generation job. +type DataGenerationInputs struct { + Name string `json:"name"` + Scenario string `json:"scenario"` + Options DataGenerationOptions `json:"options"` + Sources []GenerationSource `json:"sources"` +} + +// DataGenerationOptions holds configuration for data generation. +type DataGenerationOptions struct { + Type string `json:"type"` + MaxSamples int `json:"max_samples"` + ModelOptions ModelOptions `json:"model_options"` +} + +// ModelOptions holds the model selection for generation. +type ModelOptions struct { + Model string `json:"model"` +} + +// GenerationSource describes a source used for dataset or evaluator generation. +type GenerationSource struct { + Type string `json:"type"` + Prompt string `json:"prompt,omitempty"` + AgentName string `json:"agent_name,omitempty"` + AgentVersion string `json:"agent_version,omitempty"` + StartTime int64 `json:"start_time,omitempty"` +} + +// Agent is the part of a catalog agent that describes what it does. +// +// An agent is returned with its versions inlined rather than as a list, and +// only `latest` is populated on a plain read. +type Agent struct { + Name string `json:"name"` + Versions struct { + Latest *AgentVersion `json:"latest"` + } `json:"versions"` +} + +// AgentVersion is one published revision of an agent. +type AgentVersion struct { + Version string `json:"version"` + Definition struct { + Model string `json:"model"` + Instructions string `json:"instructions"` + } `json:"definition"` +} + +// Instructions returns the newest version's system prompt, or "" when the agent +// has no published version. +func (a *Agent) Instructions() string { + if a == nil || a.Versions.Latest == nil { + return "" + } + return strings.TrimSpace(a.Versions.Latest.Definition.Instructions) +} + +// GenerationJob is the response for data and evaluator generation job operations. +type GenerationJob struct { + ID string `json:"id"` + Status string `json:"status"` + Result json.RawMessage `json:"result,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// JobError captures error details from a failed generation job. +type JobError struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +// ResolvedNameVersion extracts the name and version from the generation job result. +// If name is empty, both return values are empty (caller should treat as no result). +// If version is empty, it defaults to "latest". +func (j *GenerationJob) ResolvedNameVersion() (string, string) { + name := j.resultStringField("name") + if name == "" { + return "", "" + } + version := j.resultStringField("version") + if version == "" { + version = "latest" + } + return name, version +} + +// resultStringField extracts a string field from the raw Result JSON. +// It first checks for a top-level key, then falls back to outputs[0].key +// to handle the nested response format. +func (j *GenerationJob) resultStringField(key string) string { + if len(j.Result) == 0 { + return "" + } + var m map[string]json.RawMessage + if err := json.Unmarshal(j.Result, &m); err != nil { + return "" + } + + // Try top-level field first. + if raw, ok := m[key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + return s + } + } + + // Fall back to outputs[0].key for nested response format. + if rawOutputs, ok := m["outputs"]; ok { + var outputs []map[string]json.RawMessage + if err := json.Unmarshal(rawOutputs, &outputs); err == nil && len(outputs) > 0 { + if raw, ok := outputs[0][key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + } + } + } + + return "" +} + +// --------------------------------------------------------------------------- +// Evaluator Generation Jobs +// --------------------------------------------------------------------------- + +// EvaluatorGenerationJobRequest is the request body for CreateEvaluatorGenerationJob. +type EvaluatorGenerationJobRequest struct { + Inputs EvaluatorGenerationInputs `json:"inputs"` +} + +// EvaluatorGenerationInputs holds the inputs for an evaluator generation job. +type EvaluatorGenerationInputs struct { + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + Category string `json:"category,omitempty"` + Model string `json:"model"` + Sources []GenerationSource `json:"sources"` +} + +// --------------------------------------------------------------------------- +// Evaluator Versions +// --------------------------------------------------------------------------- + +// EvaluatorVersion is the response for evaluator version operations. +type EvaluatorVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// Evaluator Definition (Rubric) +// --------------------------------------------------------------------------- + +// EvaluatorResult is the top-level response from evaluator generation, +// containing the evaluator's definition. +type EvaluatorResult struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Definition EvaluatorDefinition `json:"definition"` +} + +// EvaluatorDefinition describes an evaluator's scoring rubric. +type EvaluatorDefinition struct { + Type string `json:"type"` + Dimensions []EvaluatorDimension `json:"dimensions"` +} + +// EvaluatorDimension is a single scoring dimension within a rubric evaluator. +type EvaluatorDimension struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Weight int `json:"weight"` + AlwaysApplicable bool `json:"always_applicable,omitempty"` +} + +// --------------------------------------------------------------------------- +// Datasets +// --------------------------------------------------------------------------- + +// CreateDatasetRequest is the request body for CreateDataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +type Dataset struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// OpenAI Evals +// --------------------------------------------------------------------------- + +// DataSourceConfig describes the data source for an OpenAI eval. +type DataSourceConfig struct { + Type string `json:"type"` + ItemSchema map[string]any `json:"item_schema"` + IncludeSampleSchema bool `json:"include_sample_schema"` +} + +// DataSourceSchema defines the item and sample schemas for an eval data source. +type DataSourceSchema struct { + Item map[string]any `json:"item,omitempty"` + Sample map[string]any `json:"sample,omitempty"` +} + +// TestingCriterion describes a single evaluator in testing_criteria. +type TestingCriterion struct { + Type string `json:"type"` + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + EvaluatorVersion string `json:"evaluator_version,omitempty"` + InitializationParameters map[string]any `json:"initialization_parameters,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` +} + +// CreateOpenAIEvalRequest is the request body for CreateOpenAIEval. +type CreateOpenAIEvalRequest struct { + Name string `json:"name"` + Metadata map[string]string `json:"metadata,omitempty"` + DataSourceConfig *DataSourceConfig `json:"data_source_config,omitempty"` + TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` +} + +// UpdateOpenAIEvalRequest is UpdateEvalParametersBody: the only fields an eval +// accepts after creation. Testing criteria and the data source are fixed at +// create time, and the service drops anything else here silently rather than +// rejecting it. +type UpdateOpenAIEvalRequest struct { + Name string `json:"name,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// OpenAIEval is the response for an OpenAI eval definition. +type OpenAIEval struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// OpenAIEvalList is the response for listing OpenAI eval definitions. +type OpenAIEvalList struct { + Data []OpenAIEval `json:"data"` +} + +// --------------------------------------------------------------------------- +// OpenAI Eval Runs +// --------------------------------------------------------------------------- + +// CreateOpenAIEvalRunRequest is the request body for CreateOpenAIEvalRun. +type CreateOpenAIEvalRunRequest struct { + Name string `json:"name"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// EvalRunDataSourceType defines the type for an eval run data source. +type EvalRunDataSourceType string + +const ( + // EvalRunDataSourceTypeAgentTarget is the data source type for agent target completions. + EvalRunDataSourceTypeAgentTarget EvalRunDataSourceType = "azure_ai_target_completions" + + // EvalRunDataSourceTypeTraces evaluates an agent's recorded traces instead of + // a dataset. The service reads them from Application Insights, so the agent + // must be emitting gen_ai.input.messages / gen_ai.output.messages. + EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" + + // EvalRunDataSourceTypeResponses evaluates responses the project already + // stored, addressed by id. + EvalRunDataSourceTypeResponses EvalRunDataSourceType = "azure_ai_responses" + + // EvalRunDataSourceTypeJSONL scores the rows as they are, invoking nothing. + EvalRunDataSourceTypeJSONL EvalRunDataSourceType = "jsonl" +) + +// EvalRunDataContentType defines the source type for eval run data content. +type EvalRunDataContentType string + +const ( + EvalRunDataContentTypeFileContent EvalRunDataContentType = "file_content" + EvalRunDataContentTypeFileID EvalRunDataContentType = "file_id" +) + +// EvalRunDataSource describes the data source for an eval run with agent target completions. +type EvalRunDataSource struct { + Type EvalRunDataSourceType `json:"type"` + InputMessages *EvalRunInputMessages `json:"input_messages,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` + Target *EvalRunTarget `json:"target,omitempty"` + + // Traces only. The window is expressed as a lookback in hours, not as a + // start bound: the service has no start_time on this data source and + // silently falls back to its default when one is sent. + AgentName string `json:"agent_name,omitempty"` + LookbackHours int `json:"lookback_hours,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` + + // Responses only. + ItemGenerationParams *ItemGenerationParams `json:"item_generation_params,omitempty"` +} + +// ItemGenerationParams says how the service should turn a source into the items +// it evaluates. +type ItemGenerationParams struct { + Type string `json:"type"` + MaxNumTurns int `json:"max_num_turns,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` +} + +// EvalRunInputMessages describes how input messages are constructed from dataset items. +type EvalRunInputMessages struct { + Type string `json:"type"` + Template []EvalRunMessageTemplate `json:"template"` +} + +// EvalRunMessageTemplate describes a single message in the input template. +type EvalRunMessageTemplate struct { + Role string `json:"role"` + Content string `json:"content"` + Type string `json:"type"` +} + +// EvalRunTarget describes what the run invokes: an agent by name, or a model +// deployment directly. Only the fields belonging to Type are sent. +type EvalRunTarget struct { + Type string `json:"type"` + Name string `json:"name,omitempty"` + Version *string `json:"version,omitempty"` + ToolDescriptions []string `json:"tool_descriptions,omitempty"` + Model string `json:"model,omitempty"` +} + +// EvalRunDataContent holds the source reference within an EvalRunDataSource. +type EvalRunDataContent struct { + Type EvalRunDataContentType `json:"type"` + ID string `json:"id,omitempty"` + Content []map[string]any `json:"content,omitempty"` +} + +// NewAgentTargetDataSource builds an EvalRunDataSource configured for agent target completions. +// The rows must be supplied separately via SetFileContent. +func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_agent", + Name: agentName, + Version: agentVersion, + ToolDescriptions: []string{}, + }, + } +} + +// SetFileContent sets the data source to use inline file content. +// +// There is no by-reference counterpart. A run's `file_id` means an uploaded +// file, and a dataset name is not one — sending it is rejected with "invalid +// data source file ids" — so registered datasets are fetched and sent inline +// too. See readRegisteredDataset. +func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { + ds.Source = &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: items, + } +} + +// OpenAIEvalRun is the response for an OpenAI eval run. +type OpenAIEvalRun struct { + ID string `json:"id"` + EvalID string `json:"eval_id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + ReportURL string `json:"report_url,omitempty"` + + // Result summary + ResultCounts *EvalRunResultCounts `json:"result_counts,omitempty"` + PerTestingCriteria []EvalRunCriteriaResult `json:"per_testing_criteria_results,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// Failure returns why the run failed, or "" when it did not. +// +// The field is always present and its members are null on success, so its +// presence says nothing on its own. +func (r *OpenAIEvalRun) Failure() string { + if r == nil || r.Error == nil { + return "" + } + return strings.TrimSpace(r.Error.Message) +} + +// EvalRunResultCounts holds pass/fail/error/skip counts for a run. +type EvalRunResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// EvalRunCriteriaResult holds per-testing-criteria pass/fail counts. +type EvalRunCriteriaResult struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// OpenAIEvalRunList is the response for listing OpenAI eval runs. +type OpenAIEvalRunList struct { + Data []OpenAIEvalRun `json:"data"` +} + +// OutputItemList is a page of a run's per-sample results. +type OutputItemList struct { + Data []OutputItem `json:"data"` +} + +// OutputItem is one evaluated row: the dataset item, and every evaluator's +// verdict on it. +type OutputItem struct { + ID string `json:"id"` + RunID string `json:"run_id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item,omitempty"` + Results []OutputResult `json:"results,omitempty"` +} + +// OutputResult is one evaluator's verdict on one row. +type OutputResult struct { + Name string `json:"name"` + Metric string `json:"metric,omitempty"` + Score LenientFloat `json:"score"` + Label string `json:"label,omitempty"` + Passed bool `json:"passed"` + // Reason is the judge's explanation, which is the part a failing row is + // actually looked at for. + Reason string `json:"reason,omitempty"` +} + +// Failed reports whether any evaluator failed this row. +func (o OutputItem) Failed() bool { + for _, r := range o.Results { + if !r.Passed { + return true + } + } + return false +} + +// Input renders the row's own columns for display, leaving out the +// service-injected `sample.*` bindings and the plumbing ids, which are not what +// the dataset author wrote. +func (o OutputItem) Input() string { + if len(o.DataSourceItem) == 0 { + return "" + } + skip := map[string]bool{ + "response_id": true, "agent_id": true, "agent_name": true, + "agent_version": true, "conversation_id": true, + "previous_response_id": true, "trace_id": true, "span_id": true, + } + keys := make([]string, 0, len(o.DataSourceItem)) + for k := range o.DataSourceItem { + if skip[k] || strings.HasPrefix(k, "sample.") { + continue + } + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%v", k, o.DataSourceItem[k])) + } + return strings.Join(parts, " ") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go new file mode 100644 index 00000000000..0b18f6556a2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -0,0 +1,601 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strconv" + "time" + + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefixes for eval service endpoints. +const ( + pathDataGenerationJobs = "/data_generation_jobs" + pathEvaluatorGenerationJobs = "/evaluator_generation_jobs" + pathEvaluators = "/evaluators" + pathDatasets = "/datasets" + pathOpenAIEvals = "/openai/v1/evals" + pathAgents = "/agents" +) + +// EvalClient provides methods for interacting with the Azure AI eval APIs. +type EvalClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewEvalClient creates a new EvalClient. +func NewEvalClient(endpoint string, cred azcore.TokenCredential) *EvalClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-evals", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewEvalClientFromPipeline creates an EvalClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewEvalClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *EvalClient { + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataGenerationJob starts a dataset generation job for eval onboarding. +func (c *EvalClient) CreateDataGenerationJob( + ctx context.Context, + request *DataGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathDataGenerationJobs, nil, request, apiVersion) +} + +// GetDataGenerationJob gets the current state of a dataset generation job. +func (c *EvalClient) GetDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorGenerationJob starts an evaluator generation job for eval onboarding. +func (c *EvalClient) CreateEvaluatorGenerationJob( + ctx context.Context, + request *EvaluatorGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathEvaluatorGenerationJobs, nil, request, apiVersion) +} + +// GetEvaluatorGenerationJob gets the current state of an evaluator generation job. +func (c *EvalClient) GetEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathEvaluatorGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// GenerationJobList is the listing envelope both job types answer with. It is +// `data`, not the `value` the dataset and evaluator routes use. +type GenerationJobList struct { + Data []GenerationJob `json:"data"` +} + +// ListDataGenerationJobs returns the project's dataset generation jobs. +func (c *EvalClient) ListDataGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, pathDataGenerationJobs, nil, nil, apiVersion) +} + +// ListEvaluatorGenerationJobs returns the project's evaluator generation jobs. +func (c *EvalClient) ListEvaluatorGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, pathEvaluatorGenerationJobs, nil, nil, apiVersion) +} + +// CancelDataGenerationJob stops a dataset generation job. +func (c *EvalClient) CancelDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// CancelEvaluatorGenerationJob stops an evaluator generation job. +func (c *EvalClient) CancelEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// cancelGenerationJob posts to the colon form of the route. +// +// The separator is a colon, not a path segment: `{id}/cancel` is a 404 while +// `{id}:cancel` reaches the action. The empty object is what carries a content +// type, without which the route answers 415. +func (c *EvalClient) cancelGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) (*GenerationJob, error) { + path := basePath + "/" + url.PathEscape(operationID) + ":cancel" + return doRequestTyped[GenerationJob]( + c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + +// DeleteDataGenerationJob removes a dataset generation job record. +func (c *EvalClient) DeleteDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// DeleteEvaluatorGenerationJob removes an evaluator generation job record. +func (c *EvalClient) DeleteEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// deleteGenerationJob discards the job record. The artifact the job produced is +// already registered and is not affected. +func (c *EvalClient) deleteGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) error { + path := basePath + "/" + url.PathEscape(operationID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// GetAgent reads an agent from the project's catalog. +// +// Only the newest version is returned, which is the one generation is seeded +// from: the point is to describe what the agent does now. +func (c *EvalClient) GetAgent( + ctx context.Context, + name string, + apiVersion string, +) (*Agent, error) { + path := pathAgents + "/" + url.PathEscape(name) + return doRequestTyped[Agent](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorVersion creates a new version of a named evaluator. +// The body should be the full evaluator JSON with the definition field updated. +// +// previous is the evaluator document the caller has already read, or nil when +// it read none. It is what keeps the publish from being answered with the +// version that document holds. +func (c *EvalClient) CreateEvaluatorVersion( + ctx context.Context, + name string, + body json.RawMessage, + previous json.RawMessage, + apiVersion string, +) (*EvaluatorVersion, error) { + return c.publishEvaluatorVersion(ctx, name, previous, apiVersion, func() (*EvaluatorVersion, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) + }) +} + +// versionSettle bounds the wait for the service to start assigning the next +// version number. +const ( + versionSettleTimeout = 45 * time.Second + versionSettleInterval = 3 * time.Second + versionSettleAge = 8 * time.Second +) + +// publishedVersion is the little of an evaluator document this needs: which +// version it is, and when it was written. +type publishedVersion struct { + Version string `json:"version"` + ModifiedAt time.Time `json:"modified_at"` + CreatedAt time.Time `json:"created_at"` +} + +// writtenAt reports when the version was last written, preferring the +// modification time and falling back to creation. +func (p publishedVersion) writtenAt() time.Time { + if !p.ModifiedAt.IsZero() { + return p.ModifiedAt + } + return p.CreatedAt +} + +// publishEvaluatorVersion publishes and then makes sure a new version is what +// came back. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over that version's contents instead +// of adding one. It is a race rather than a fixed window — a second publish +// has been seen both colliding a quarter of a second later and succeeding +// immediately — and nothing observable marks its end. +// +// That matters because versions are the unit an eval binds to. `evaluator +// create` followed by `evaluator update`, which is what a first authoring +// session looks like, would otherwise leave one version holding the second +// definition and every eval bound to the first silently scoring against a +// rubric nobody chose. +// +// So there are two defenses. The publish is held back until the version the +// caller read has had time to settle, which is what keeps the collision from +// happening at all; and the version that comes back is checked, which is what +// keeps a collision that happens anyway from being reported as success. The +// recheck republishes the same body, so it cannot make a collision worse than +// the first attempt already did. +// +// What the caller reads is used rather than the version listing because the +// listing lags a publish too: asked immediately after a create it answers 404, +// so a guard that trusted it would stand down in exactly the case it exists +// for. Callers that publish an evaluator have already read it to decide +// between creating and updating. +func (c *EvalClient) publishEvaluatorVersion( + ctx context.Context, + name string, + previous json.RawMessage, + apiVersion string, + publish func() (*EvaluatorVersion, error), +) (*EvaluatorVersion, error) { + var known publishedVersion + if len(previous) > 0 { + _ = json.Unmarshal(previous, &known) + } + + latest := parseVersionNumber(known.Version) + if listed := c.LatestEvaluatorVersionNumber(ctx, name, apiVersion); listed > latest { + latest = listed + } + + if written := known.writtenAt(); !written.IsZero() { + if wait := versionSettleAge - time.Since(written); wait > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(wait): + } + } + } + + deadline := time.Now().Add(versionSettleTimeout) + for { + created, err := publish() + if err != nil { + return nil, err + } + if latest == 0 || parseVersionNumber(created.Version) > latest { + return created, nil + } + if time.Now().After(deadline) { + return nil, fmt.Errorf( + "publishing evaluator %q kept returning version %s, which already "+ + "existed. The service was still assigning that version after %s, so "+ + "version %s now holds what was just published and any eval bound to "+ + "it is scoring against it", + name, created.Version, versionSettleTimeout, created.Version) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(versionSettleInterval): + } + } +} + +// GetEvaluatorRaw gets an evaluator by name and version as raw JSON. +// If version is empty, the latest version is resolved first. +// +// The service has no route for an unversioned evaluator: GET +// /evaluators/{name} returns 404 with no body, so the version cannot simply be +// left off the path. +func (c *EvalClient) GetEvaluatorRaw( + ctx context.Context, + name string, + version string, + apiVersion string, +) (json.RawMessage, error) { + if version == "" { + latest, err := c.LatestEvaluatorVersion(ctx, name, apiVersion) + if err != nil { + return nil, err + } + version = latest + } + path := pathEvaluators + "/" + url.PathEscape(name) + + "/versions/" + url.PathEscape(version) + return c.doRequest(ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// LatestEvaluatorVersion returns the newest registered version of an evaluator. +func (c *EvalClient) LatestEvaluatorVersion( + ctx context.Context, + name string, + apiVersion string, +) (string, error) { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil { + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", fmt.Errorf("evaluator %q has no versions", name) + } + latest := pickLatestVersion(list.Value) + if latest == "" { + return "", fmt.Errorf("evaluator %q has no usable version", name) + } + return latest, nil +} + +// pickLatestVersion selects the highest evaluator version. +// +// Versions are integers rendered as strings, so they are compared numerically: +// a lexical compare would rank "9" above "15", and the service already +// publishes evaluators at version 15 and 17. A non-numeric version is used +// only when nothing numeric is present. +func pickLatestVersion(entries []EvaluatorSummary) string { + best := "" + bestNum := -1 + for _, entry := range entries { + if entry.Version == "" { + continue + } + num, err := strconv.Atoi(entry.Version) + if err != nil { + if best == "" { + best = entry.Version + } + continue + } + if num > bestNum { + bestNum, best = num, entry.Version + } + } + return best +} + +// CreateOpenAIEval creates an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEval( + ctx context.Context, + request *CreateOpenAIEvalRequest, +) (*OpenAIEval, error) { + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, pathOpenAIEvals, nil, request, "") +} + +// ListOpenAIEvals lists OpenAI eval definitions. +func (c *EvalClient) ListOpenAIEvals(ctx context.Context, limit int) (*OpenAIEvalList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + return doRequestTyped[OpenAIEvalList](c, ctx, http.MethodGet, pathOpenAIEvals, query, nil, "") +} + +// GetOpenAIEval gets an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEval(ctx context.Context, evalID string) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodGet, path, nil, nil, "") +} + +// DeleteOpenAIEval removes an eval definition and its runs. +func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + +// UpdateOpenAIEval edits an eval in place. The route is a POST on the eval +// itself, matching how this surface spells run cancel — there is no PATCH verb +// here. +// +// Only what UpdateEvalParametersBody reaches is editable: name, metadata and +// properties. Anything else the service drops silently, so substance never +// travels through this call and an edit that touches it is a new eval. +func (c *EvalClient) UpdateOpenAIEval( + ctx context.Context, + evalID string, + request *UpdateOpenAIEvalRequest, +) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, path, nil, request, "") +} + +// CreateOpenAIEvalRun starts a run for an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEvalRun( + ctx context.Context, + evalID string, + request *CreateOpenAIEvalRunRequest, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, request, "") +} + +// ListOpenAIEvalRuns lists runs for an OpenAI eval definition. +func (c *EvalClient) ListOpenAIEvalRuns( + ctx context.Context, + evalID string, + limit int, +) (*OpenAIEvalRunList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + return doRequestTyped[OpenAIEvalRunList](c, ctx, http.MethodGet, path, query, nil, "") +} + +// GetOpenAIEvalRun gets a run for an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs/%s", pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodGet, path, nil, nil, "") +} + +func (c *EvalClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + return c.doRequestWithHeaders(ctx, method, path, query, body, apiVersion, nil) +} + +// doRequestWithHeaders is doRequest with extra request headers, which the +// preview evaluator operations need to opt in to the properties they set. +func (c *EvalClient) doRequestWithHeaders( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, + headers map[string]string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + for k, v := range headers { + req.Raw().Header.Set(k, v) + } + + log.Printf("[eval_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[eval_api] response status: %d", resp.StatusCode) + + // 204 belongs here: a delete that removed the resource answers No Content, + // and treating that as a failure reports every successful delete as an + // error. doRequestTyped already tolerates the empty body. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { + // Restore the body so runtime.NewResponseError can read it. + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *EvalClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go new file mode 100644 index 00000000000..4976248d87a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// JobStatus — typed status with terminal/failed semantics +// --------------------------------------------------------------------------- + +// JobStatus represents the normalized status of a generation job. +type JobStatus string + +const ( + JobStatusRunning JobStatus = "running" + JobStatusCompleted JobStatus = "completed" + JobStatusSucceeded JobStatus = "succeeded" + JobStatusFailed JobStatus = "failed" + JobStatusCancelled JobStatus = "cancelled" + JobStatusCanceled JobStatus = "canceled" +) + +// ParseJobStatus normalizes a raw status string into a JobStatus. +// An empty string is treated as "running". +func ParseJobStatus(s string) JobStatus { + if s == "" { + return JobStatusRunning + } + return JobStatus(strings.ToLower(s)) +} + +// IsTerminal returns true when the status represents a final state. +func (s JobStatus) IsTerminal() bool { + switch s { + case JobStatusCompleted, JobStatusSucceeded, JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// IsFailed returns true when the status represents a failure or cancellation. +func (s JobStatus) IsFailed() bool { + switch s { + case JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// String returns the status as a plain string. +func (s JobStatus) String() string { + return string(s) +} + +// --------------------------------------------------------------------------- +// JobFailedError — returned when a polled job reaches a failed state +// --------------------------------------------------------------------------- + +// JobFailedError is returned when a generation job reaches a failed terminal state. +type JobFailedError struct { + Job *GenerationJob + Status JobStatus +} + +func (e *JobFailedError) Error() string { + if e.Job != nil && e.Job.Error != nil && e.Job.Error.Message != "" { + return fmt.Sprintf("job failed with status %q: %s", e.Status, e.Job.Error.Message) + } + return fmt.Sprintf("job failed with status %q", e.Status) +} + +// --------------------------------------------------------------------------- +// PollerTimeoutError — returned when polling exhausts all attempts +// --------------------------------------------------------------------------- + +// PollerTimeoutError is returned when a generation job has not reached a +// terminal state within the configured number of polling attempts. +type PollerTimeoutError struct { + OperationID string + Attempts int +} + +func (e *PollerTimeoutError) Error() string { + return fmt.Sprintf( + "operation %s did not complete within %d attempts", + e.OperationID, e.Attempts, + ) +} + +// --------------------------------------------------------------------------- +// GetJobFunc — callback type for fetching job state +// --------------------------------------------------------------------------- + +// GetJobFunc fetches the current state of a generation job by operation ID. +type GetJobFunc func(ctx context.Context, operationID, apiVersion string) (*GenerationJob, error) + +// --------------------------------------------------------------------------- +// PollerOptions — configurable polling behavior +// --------------------------------------------------------------------------- + +// PollerOptions configures the polling interval and attempt limit. +type PollerOptions struct { + Interval time.Duration + MaxAttempts int +} + +// DefaultPollerOptions returns sensible defaults: 2 s interval, 300 attempts (~10 min). +func DefaultPollerOptions() PollerOptions { + return PollerOptions{ + Interval: 2 * time.Second, + MaxAttempts: 300, + } +} + +// --------------------------------------------------------------------------- +// Poller — polls a generation job until it reaches a terminal state +// --------------------------------------------------------------------------- + +// Poller polls a GenerationJob until it reaches a terminal status. +type Poller struct { + OperationID string + APIVersion string + GetJob GetJobFunc + Options PollerOptions + // OnPoll is called after each successful poll with the latest status. + // Callers can use this for progress reporting (e.g. debug logging). + OnPoll func(status JobStatus) +} + +// NewPoller creates a Poller with default options. +func NewPoller(operationID, apiVersion string, getJob GetJobFunc) *Poller { + return &Poller{ + OperationID: operationID, + APIVersion: apiVersion, + GetJob: getJob, + Options: DefaultPollerOptions(), + } +} + +// Poll blocks until the job reaches a terminal state, the context is +// cancelled, or the maximum number of attempts is exhausted. +// +// On success it returns the completed GenerationJob. +// On failure it returns a *JobFailedError (which wraps the job for inspection). +// On timeout it returns a plain error. +func (p *Poller) Poll(ctx context.Context) (*GenerationJob, error) { + if p.OperationID == "" { + return nil, fmt.Errorf("operation ID is empty") + } + + for range p.Options.MaxAttempts { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(p.Options.Interval): + } + + job, err := p.GetJob(ctx, p.OperationID, p.APIVersion) + if err != nil { + if evalcore.IsTransientError(err) { + log.Printf("[poller] transient error polling %s, will retry: %v", p.OperationID, err) + continue + } + return nil, err + } + + status := ParseJobStatus(job.Status) + log.Printf("[poller] operationID=%s status=%s", p.OperationID, status) + + if p.OnPoll != nil { + p.OnPoll(status) + } + + if status.IsTerminal() { + if status.IsFailed() { + return nil, &JobFailedError{Job: job, Status: status} + } + return job, nil + } + } + + return nil, &PollerTimeoutError{ + OperationID: p.OperationID, + Attempts: p.Options.MaxAttempts, + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go new file mode 100644 index 00000000000..8b1ccd0fd5d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/google/uuid" +) + +// PortalPrefix holds the parsed project context needed to construct Foundry portal URLs. +type PortalPrefix struct { + prefix string // e.g. "https://ai.azure.com/nextgen/r/,,,," +} + +// NewPortalPrefix parses an ARM project resource ID and returns a PortalPrefix +// that can be reused to build multiple portal URLs. +// Returns an error if the resource ID is invalid or not a Foundry project. +func NewPortalPrefix(projectResourceID string) (*PortalPrefix, error) { + resourceID, err := arm.ParseResourceID(projectResourceID) + if err != nil { + return nil, fmt.Errorf("failed to parse project resource ID: %w", err) + } + + encodedSub, err := encodeSubscriptionForURL(resourceID.SubscriptionID) + if err != nil { + return nil, fmt.Errorf("failed to encode subscription ID: %w", err) + } + + if resourceID.Parent == nil || + !strings.Contains(string(resourceID.ResourceType.Type), "/") { + return nil, fmt.Errorf( + "resource ID does not represent a Foundry project (missing parent account): %s", + projectResourceID, + ) + } + + prefix := fmt.Sprintf( + "https://ai.azure.com/nextgen/r/%s,%s,,%s,%s", + encodedSub, resourceID.ResourceGroupName, + resourceID.Parent.Name, resourceID.Name, + ) + return &PortalPrefix{prefix: prefix}, nil +} + +// EvalRunURL returns the portal URL for an eval run report. +func (p *PortalPrefix) EvalRunURL(evalID, runID string) string { + return fmt.Sprintf("%s/build/evaluations/%s/run/%s", p.prefix, evalID, runID) +} + +// EvaluatorURL returns the portal URL for a generated evaluator. +func (p *PortalPrefix) EvaluatorURL(evaluatorName, version string) string { + return fmt.Sprintf("%s/build/evaluations/catalog/%s/%s", p.prefix, evaluatorName, version) +} + +// DatasetURL returns the portal URL for a dataset. +func (p *PortalPrefix) DatasetURL(datasetName, version string) string { + return fmt.Sprintf("%s/build/data/datasets/%s/%s", p.prefix, datasetName, version) +} + +// OptimizationURL returns the portal URL for an optimization job. +func (p *PortalPrefix) OptimizationURL(agentName, operationID string) string { + return fmt.Sprintf("%s/build/agents/%s/optimization/%s", + p.prefix, agentName, operationID) +} + +// encodeSubscriptionForURL encodes a subscription ID GUID as base64 without padding. +func encodeSubscriptionForURL(subscriptionID string) (string, error) { + guid, err := uuid.Parse(subscriptionID) + if err != nil { + return "", fmt.Errorf("invalid subscription ID format: %w", err) + } + guidBytes, _ := guid.MarshalBinary() + return strings.TrimRight(base64.URLEncoding.EncodeToString(guidBytes), "="), nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go new file mode 100644 index 00000000000..602b584ccd5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "strings" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testProjectID = "/subscriptions/00000000-1111-2222-3333-444444444444/" + + "resourceGroups/rg-eval/providers/Microsoft.CognitiveServices/accounts/acct/projects/proj" + +// A portal URL is printed at the end of a run and is the one thing a user +// clicks. It is assembled from parts rather than returned by the service, so +// nothing but a test says whether it lands anywhere. +func TestPortalPrefix_BuildsEveryDocumentedURL(t *testing.T) { + p, err := NewPortalPrefix(testProjectID) + require.NoError(t, err) + + // The subscription travels base64url-encoded without padding, so the + // literal GUID must not appear anywhere in the result. + const sub = "00000000-1111-2222-3333-444444444444" + + tests := []struct { + name string + got string + want string + }{ + {"eval run", p.EvalRunURL("eval_1", "evalrun_1"), "/build/evaluations/eval_1/run/evalrun_1"}, + {"evaluator", p.EvaluatorURL("quality", "3"), "/build/evaluations/catalog/quality/3"}, + {"dataset", p.DatasetURL("regression", "2"), "/build/data/datasets/regression/2"}, + {"optimization", p.OptimizationURL("support", "op_9"), "/build/agents/support/optimization/op_9"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, strings.HasPrefix(tt.got, "https://ai.azure.com/nextgen/r/"), + "got %s", tt.got) + assert.True(t, strings.HasSuffix(tt.got, tt.want), "got %s", tt.got) + assert.Contains(t, tt.got, "rg-eval") + assert.Contains(t, tt.got, "acct") + assert.Contains(t, tt.got, "proj") + assert.NotContains(t, tt.got, sub, + "the subscription is encoded, so its plain GUID must not appear") + }) + } +} + +// The encoding is what the portal decodes on the other end, so it is pinned +// rather than merely exercised. +func TestEncodeSubscriptionForURL(t *testing.T) { + encoded, err := encodeSubscriptionForURL("00000000-1111-2222-3333-444444444444") + + require.NoError(t, err) + assert.NotContains(t, encoded, "=", "padding would need escaping inside a URL segment") + assert.NotContains(t, encoded, "+", "base64url, not standard base64") + assert.NotContains(t, encoded, "/", "a slash would split the URL segment") + assert.Equal(t, "AAAAABERIiIzM0RERERERA", encoded) +} + +func TestEncodeSubscriptionForURL_RejectsSomethingThatIsNotAGUID(t *testing.T) { + _, err := encodeSubscriptionForURL("not-a-subscription") + + require.Error(t, err) + assert.Contains(t, err.Error(), "subscription") +} + +// A resource ID that is not a project has no account to name, and guessing +// would produce a URL that resolves to someone else's project. +func TestNewPortalPrefix_RefusesWhatIsNotAProject(t *testing.T) { + tests := []struct { + name string + id string + }{ + {"not a resource id at all", "hello"}, + {"empty", ""}, + { + name: "an account rather than a project under it", + id: "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct", + }, + { + name: "a project whose subscription is not a GUID", + id: "/subscriptions/not-a-guid/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p, err := NewPortalPrefix(tt.id) + + require.Error(t, err) + assert.Nil(t, p) + }) + } +} + +// The prefix distinguishes built-in evaluators from ones the project owns, +// which is what decides whether a version is published or referenced. +func TestIsBuiltinEvaluator(t *testing.T) { + assert.True(t, IsBuiltinEvaluator("builtin.task_adherence")) + assert.False(t, IsBuiltinEvaluator("task_adherence")) + assert.False(t, IsBuiltinEvaluator("builtin"), "the dot is part of the prefix") + assert.False(t, IsBuiltinEvaluator("my.builtin.thing"), "the prefix has to lead") + assert.False(t, IsBuiltinEvaluator("")) +} + +func TestSplitEvaluators(t *testing.T) { + generated, builtin := SplitEvaluators(evalcore.EvaluatorList{ + {Name: "builtin.coherence"}, + {Name: "support-quality"}, + {Name: "builtin.task_adherence"}, + }) + + require.Len(t, generated, 1) + assert.Equal(t, "support-quality", generated[0].Name) + require.Len(t, builtin, 2) + assert.Equal(t, "builtin.coherence", builtin[0].Name) + assert.Equal(t, "builtin.task_adherence", builtin[1].Name) +} + +// Both halves come back nil rather than empty for an empty input, so a caller +// checking len() reads the same either way. +func TestSplitEvaluators_Empty(t *testing.T) { + generated, builtin := SplitEvaluators(nil) + + assert.Empty(t, generated) + assert.Empty(t, builtin) +} + +// This decides whether a value is looked up in the service or opened off disk. +// Getting it wrong sends a path to the registry, or a registered name to the +// filesystem, and neither failure names the real problem. +func TestIsDatasetName(t *testing.T) { + names := []string{ + "support-regression", + "dataset_v2", + "name.with.dots", + "trailing.txt", + } + for _, v := range names { + assert.Truef(t, IsDatasetName(v), "%q is a registered name", v) + } + + paths := []string{ + "", + "data.jsonl", + "data.json", + "data.csv", + "DATA.JSONL", + "./data.jsonl", + "evals/datasets/x.jsonl", + `evals\datasets\x.jsonl`, + "a/b", + } + for _, v := range paths { + assert.Falsef(t, IsDatasetName(v), "%q is a path, not a name", v) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go new file mode 100644 index 00000000000..e34e1460017 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newRecordingClient points a client at a test server, with no credential +// policy in the pipeline. +func newRecordingClient(t *testing.T, handler http.HandlerFunc) *EvalClient { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + return NewEvalClientFromPipeline( + server.URL, runtime.NewPipeline("test", "v1.0.0", runtime.PipelineOptions{}, nil)) +} + +// versionServer answers a version listing and a publish, assigning whatever +// version the caller decides for each attempt. +func versionServer(t *testing.T, existing []string, assign func(attempt int) string) ( + http.HandlerFunc, *atomic.Int32, +) { + t.Helper() + var publishes atomic.Int32 + + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == http.MethodGet { + values := []map[string]any{} + for _, v := range existing { + values = append(values, map[string]any{"name": "tone", "version": v}) + } + if len(existing) == 0 { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"NotFound"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + return + } + + attempt := int(publishes.Add(1)) + w.WriteHeader(http.StatusCreated) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "tone", "version": assign(attempt), + })) + }, &publishes +} + +// A name the project has never seen has no version to collide with, so it must +// publish once and return. Waiting there would tax every first publish for a +// hazard that cannot apply. +func TestCreateEvaluatorVersion_FirstPublishDoesNotRetry(t *testing.T) { + handler, publishes := versionServer(t, nil, func(int) string { return "1" }) + client := newRecordingClient(t, handler) + + started := time.Now() + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "1", created.Version) + assert.Equal(t, int32(1), publishes.Load(), "a first publish must be issued once") + assert.Less(t, time.Since(started), versionSettleInterval, + "a first publish must not wait on a version that cannot exist") +} + +// For a few seconds after a publish the service answers the next one with the +// version it just assigned, replacing that version rather than adding one. +// Accepting it would leave every eval bound to the earlier version scoring +// against a definition nobody chose, so the publish is reissued until the +// version advances. +func TestCreateEvaluatorVersion_RetriesUntilTheVersionAdvances(t *testing.T) { + handler, publishes := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 3 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(3), publishes.Load(), + "the publish must be reissued until the service assigns a new version") +} + +// A service that never advances must end in an error rather than in a version +// the caller believes is new. Reporting success there is the failure the whole +// guard exists to prevent. +func TestCreateEvaluatorVersion_GivesUpRatherThanReportASharedVersion(t *testing.T) { + handler, _ := versionServer(t, []string{"4"}, func(int) string { return "4" }) + client := newRecordingClient(t, handler) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := client.CreateEvaluatorVersion( + ctx, "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.Error(t, err) +} + +// The version listing lags a publish: asked immediately after a create it +// answers 404. A guard that trusted it would stand down in exactly the window +// it exists for, which is why the caller supplies the version it has already +// read. +func TestCreateEvaluatorVersion_UsesTheCallersVersionWhenTheListingLags(t *testing.T) { + handler, publishes := versionServer(t, nil, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), json.RawMessage(`{"version":"1"}`), "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(2), publishes.Load(), + "the version the caller read must be enough to catch the collision") +} + +// A version the service does not number cannot be compared, so it is taken at +// face value: refusing it would make an evaluator unpublishable over a +// convention this extension does not own. +func TestParseVersionNumber(t *testing.T) { + assert.Equal(t, 7, parseVersionNumber("7")) + assert.Equal(t, 0, parseVersionNumber("v7")) + assert.Equal(t, 0, parseVersionNumber("")) +} + +// The publish is reissued, so the same body has to arrive every time. A +// closure that consumed its body on the first attempt would send an empty one +// on the second and publish an evaluator with no definition. +func TestCreateEvaluatorVersion_ReissuesTheSameBody(t *testing.T) { + bodies := make(chan string, 4) + handler, _ := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + buf := make([]byte, r.ContentLength) + _, _ = r.Body.Read(buf) + bodies <- string(buf) + } + handler(w, r) + }) + + _, err := client.CreateEvaluatorVersion( + context.Background(), "tone", + json.RawMessage(`{"definition":{"type":"rubric"}}`), nil, "2025-11-15-preview") + require.NoError(t, err) + close(bodies) + + seen := 0 + for body := range bodies { + seen++ + assert.Contains(t, body, "rubric", fmt.Sprintf("attempt %d sent an empty body", seen)) + } + assert.Equal(t, 2, seen) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go new file mode 100644 index 00000000000..75bc7f8a251 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -0,0 +1,164 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "go.yaml.in/yaml/v3" +) + +// BuiltinPrefix marks an evaluator provided by the platform. The prefix is +// stripped before the name is sent as testing_criteria[].evaluator_name. +const BuiltinPrefix = "builtin." + +// EvaluatorRef is one entry in an eval's `evaluators:` list. Every entry is a +// map keyed `evaluator:`; what to publish lives on the catalog entry instead, +// so a reference only names an evaluator and says how to run it: +// +// evaluators: +// - evaluator: builtin.task_adherence +// initialization_parameters: +// model: gpt-5.6-luna +// threshold: 3 +// - evaluator: support-agent-quality +// name: quality_strict +// version: "2" +// data_mapping: +// query: "{{item.customer_message}}" +type EvaluatorRef struct { + // Evaluator is the evaluator to run: a catalog name or builtin.. + Evaluator string `yaml:"evaluator" json:"evaluator"` + // Name labels the criterion in results. Empty means the evaluator's name. + Name string `yaml:"name,omitempty" json:"name,omitempty"` + // Version pins this reference. Pinning belongs to one eval's reference + // rather than to the asset, matching evaluator_version on the criterion. + Version string `yaml:"version,omitempty" json:"version,omitempty"` + // InitializationParameters carry the judge deployment and a built-in's + // numeric threshold. They are bound against the evaluator's published + // contract rather than forwarded as written. + InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"initialization_parameters,omitempty"` + // DataMapping binds evaluator inputs to dataset columns, and is written + // only when the inference from declared inputs and columns gets it wrong. + DataMapping map[string]string `yaml:"data_mapping,omitempty" json:"data_mapping,omitempty"` +} + +// IsBuiltin reports whether the reference names a platform evaluator, which +// needs no catalog entry and is never uploaded. +func (e EvaluatorRef) IsBuiltin() bool { + return strings.HasPrefix(e.Evaluator, BuiltinPrefix) +} + +// APIName is the name the service expects, with the builtin prefix removed. +func (e EvaluatorRef) APIName() string { + return strings.TrimPrefix(e.Evaluator, BuiltinPrefix) +} + +// CriterionName labels this criterion in results. +func (e EvaluatorRef) CriterionName() string { + if e.Name != "" { + return e.Name + } + return e.APIName() +} + +// EvaluatorList is a sequence of EvaluatorRef. +// +// A bare string is refused rather than accepted quietly. Every other collection +// in the file is a list of named maps, and a bare string would have to mean the +// evaluator while reading as the criterion's own name — a different key this +// same entry also carries. +type EvaluatorList []EvaluatorRef + +const bareEvaluatorRemedy = "an evaluator entry is a mapping, not a bare string: " + + "write `- evaluator: %s`" + +func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.SequenceNode { + return fmt.Errorf("evaluators must be a sequence, got %v", value.Kind) + } + + result := make([]EvaluatorRef, 0, len(value.Content)) + for _, node := range value.Content { + switch node.Kind { + case yaml.ScalarNode: + var name string + if err := node.Decode(&name); err != nil { + return fmt.Errorf("decoding evaluator name: %w", err) + } + return fmt.Errorf(bareEvaluatorRemedy, name) + case yaml.MappingNode: + var ref EvaluatorRef + if err := node.Decode(&ref); err != nil { + return fmt.Errorf("decoding evaluator: %w", err) + } + if ref.Evaluator == "" { + return fmt.Errorf("evaluator entry is missing 'evaluator'") + } + result = append(result, ref) + default: + return fmt.Errorf("evaluator entry must be a mapping, got %v", node.Kind) + } + } + + *el = result + return nil +} + +// UnmarshalJSON accepts the same mapping-only form as the YAML decoder. +// +// This matters for the service-target provider: azd hands the service entry to +// the extension as JSON, so a config written the old way arrives here as a bare +// string and has to be refused with the same remedy rather than with a +// decoder's own type error. +func (el *EvaluatorList) UnmarshalJSON(data []byte) error { + var entries []json.RawMessage + if err := json.Unmarshal(data, &entries); err != nil { + return fmt.Errorf("evaluators must be a list: %w", err) + } + + result := make([]EvaluatorRef, 0, len(entries)) + for _, entry := range entries { + trimmed := bytes.TrimSpace(entry) + if len(trimmed) > 0 && trimmed[0] == '"' { + var name string + if err := json.Unmarshal(trimmed, &name); err != nil { + return fmt.Errorf("decoding evaluator name: %w", err) + } + return fmt.Errorf(bareEvaluatorRemedy, name) + } + + var ref EvaluatorRef + if err := json.Unmarshal(trimmed, &ref); err != nil { + return fmt.Errorf("decoding evaluator: %w", err) + } + if ref.Evaluator == "" { + return fmt.Errorf("evaluator entry is missing 'evaluator'") + } + result = append(result, ref) + } + + *el = result + return nil +} + +// MarshalJSON is the default list encoding, defined so a compact form cannot +// creep back in through the encoder. +// +// Everything the reference carries has to survive the round trip: the eval +// fingerprint is taken over this encoding, so a field dropped here is a change +// the reconciler cannot see. +func (el EvaluatorList) MarshalJSON() ([]byte, error) { + // Aliased so the element encoder does not recurse through this method. + type ref = EvaluatorRef + + out := make([]any, 0, len(el)) + for _, r := range el { + out = append(out, ref(r)) + } + return json.Marshal(out) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go new file mode 100644 index 00000000000..b185414e8ed --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// The service-target provider receives the config as JSON, not YAML, so both +// paths have to decode the mapping form identically. Supporting only YAML made +// `azd deploy` fail on a config the CLI itself writes. +func TestEvaluatorListDecodesEntriesFromJSON(t *testing.T) { + const payload = `[ + {"evaluator": "builtin.task_adherence"}, + {"evaluator": "support-quality", "name": "quality_strict", + "initialization_parameters": {"model": "gpt-5.6-luna", "threshold": 4}}, + {"evaluator": "pinned", "version": "3"} + ]` + + var list EvaluatorList + require.NoError(t, json.Unmarshal([]byte(payload), &list)) + require.Len(t, list, 3) + + require.Equal(t, "builtin.task_adherence", list[0].Evaluator) + require.True(t, list[0].IsBuiltin()) + require.Equal(t, "task_adherence", list[0].APIName()) + require.Equal(t, "task_adherence", list[0].CriterionName()) + require.Nil(t, list[0].InitializationParameters) + + require.Equal(t, "support-quality", list[1].Evaluator) + require.Equal(t, "quality_strict", list[1].CriterionName()) + require.False(t, list[1].IsBuiltin()) + require.Equal(t, "gpt-5.6-luna", list[1].InitializationParameters["model"]) + require.EqualValues(t, 4, list[1].InitializationParameters["threshold"]) + + require.Equal(t, "pinned", list[2].Evaluator) + require.Equal(t, "3", list[2].Version) +} + +// The JSON and YAML decoders must agree, otherwise a config behaves one way +// through the CLI and another through `azd up`. +func TestEvaluatorListJSONMatchesYAML(t *testing.T) { + const doc = ` +- evaluator: builtin.task_adherence +- evaluator: support-quality + name: quality_strict + initialization_parameters: + model: gpt-5.6-luna + data_mapping: + query: "{{item.customer_message}}" +` + var fromYAML EvaluatorList + require.NoError(t, yaml.Unmarshal([]byte(doc), &fromYAML)) + + encoded, err := json.Marshal(fromYAML) + require.NoError(t, err) + + var fromJSON EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &fromJSON)) + require.Equal(t, fromYAML, fromJSON) +} + +// A bare string is the old shorthand. It has to be refused with the remedy +// rather than a decoder type error, through both decoders, because the +// service-target provider only ever sees JSON. +func TestEvaluatorListRefusesBareString(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- builtin.task_adherence\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`["builtin.task_adherence"]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) +} + +func TestEvaluatorListRejectsEntryWithoutEvaluator(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- name: quality_strict\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`[{"name": "quality_strict"}]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) +} + +// The eval fingerprint is taken over this encoding, so a field the encoder +// drops is a change the reconciler cannot see. +func TestEvaluatorListMarshalKeepsEveryField(t *testing.T) { + list := EvaluatorList{{ + Evaluator: "support-quality", + Name: "quality_strict", + Version: "2", + InitializationParameters: map[string]any{"model": "gpt-5.6-luna"}, + DataMapping: map[string]string{"query": "{{item.customer_message}}"}, + }} + + encoded, err := json.Marshal(list) + require.NoError(t, err) + + var round EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &round)) + require.Equal(t, list, round) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go new file mode 100644 index 00000000000..029af83ffc0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "errors" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsTransientError reports whether err is worth retrying: throttling, a server +// fault, or a dropped connection. +func IsTransientError(err error) bool { + if err == nil { + return false + } + + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + return respErr.StatusCode == 429 || respErr.StatusCode >= 500 + } + + msg := err.Error() + return strings.Contains(msg, "connection reset") || + strings.Contains(msg, "connection refused") || + strings.Contains(msg, "EOF") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go new file mode 100644 index 00000000000..d191fd184c2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "go.yaml.in/yaml/v3" +) + +// AgentHost is the service host the agents extension registers. Services +// declaring it are the ones that could be a generation target. +const AgentHost = "azure.ai.agent" + +// Where `azd ai agent optimize` leaves the configuration it settled on. +// +// These are the agents extension's file names, repeated rather than imported: +// azd extensions are separate Go modules and share no code, so the only way to +// read another one's output is to know its layout. That makes this a coupling +// worth naming — if the agents extension moves these, generation quietly stops +// finding instructions locally and falls back to the service. +const ( + agentConfigsDir = ".agent_configs" + agentBaselineDir = "baseline" + agentMetadataFile = "metadata.yaml" +) + +// agentConfigMetadata is the part of the optimize configuration's metadata.yaml +// that says where the instructions are. It points at a file rather than +// carrying the text, because the text is what a reviewer diffs. +type agentConfigMetadata struct { + InstructionFile string `yaml:"instruction_file"` +} + +// ErrAmbiguousAgentService reports that a target name matched more than one +// service, so there is no single set of instructions to read. +var ErrAmbiguousAgentService = errors.New("more than one agent service matches") + +// AgentInstructionsFromProject reads the target agent's instructions out of the +// project, returning empty when the project does not hold them. +// +// The instructions an agent was optimized with are the best description of what +// it is supposed to do, and they are already on disk, so generating from them +// needs no service call. Coming back empty is ordinary — most projects have +// never run `azd ai agent optimize` — and leaves the caller free to ask the +// service instead. +// +// The returned path is where the text came from, for a caller that wants to say +// so. +func AgentInstructionsFromProject( + proj *azdext.ProjectConfig, + agentName string, +) (instruction string, path string, err error) { + svc, err := findAgentService(proj, agentName) + if err != nil || svc == nil { + return "", "", err + } + + configDir := filepath.Join( + proj.GetPath(), serviceRelativeDir(svc), agentConfigsDir, agentBaselineDir) + + data, err := os.ReadFile(filepath.Join(configDir, agentMetadataFile)) //nolint:gosec // under the project + if err != nil { + // An agent that was never optimized has no such directory, which is + // the common case rather than a problem. + return "", "", nil + } + + var meta agentConfigMetadata + if err := yaml.Unmarshal(data, &meta); err != nil { + return "", "", fmt.Errorf( + "reading %s: %w", filepath.Join(configDir, agentMetadataFile), err) + } + if meta.InstructionFile == "" { + return "", "", nil + } + + instructionPath := meta.InstructionFile + if !filepath.IsAbs(instructionPath) { + instructionPath = filepath.Join(configDir, instructionPath) + } + text, err := os.ReadFile(instructionPath) //nolint:gosec // named by the metadata beside it + if err != nil { + // The metadata named a file that is not there. That is worth saying: + // something wrote the pointer and not the target. + return "", "", fmt.Errorf( + "%s names instruction_file %q, which could not be read: %w", + filepath.Join(configDir, agentMetadataFile), meta.InstructionFile, err) + } + + return strings.TrimSpace(string(text)), instructionPath, nil +} + +// findAgentService resolves a target name to the one service that is it. +// +// A name can match either the azure.yaml service key or the agent name the +// service declares, because the two need not agree and a user has only ever +// seen one of them. Matching both is what makes `--target` mean what they +// typed; refusing a tie is what stops it silently meaning one of two things. +func findAgentService( + proj *azdext.ProjectConfig, + agentName string, +) (*azdext.ServiceConfig, error) { + if proj == nil || agentName == "" { + return nil, nil + } + + var matched []string + services := map[string]*azdext.ServiceConfig{} + for name, svc := range proj.GetServices() { + if svc.GetHost() != AgentHost { + continue + } + if name == agentName || declaredAgentName(svc) == agentName { + matched = append(matched, name) + services[name] = svc + } + } + + switch len(matched) { + case 0: + return nil, nil + case 1: + return services[matched[0]], nil + default: + sort.Strings(matched) + return nil, fmt.Errorf( + "%w %q: %s. Name one of them with --target, or pass the text with "+ + "--agent-instruction", + ErrAmbiguousAgentService, agentName, strings.Join(matched, ", ")) + } +} + +// declaredAgentName is the name the service gives the agent, which is what the +// service publishes under and so what the eval configuration's target refers +// to. It is absent when the service key is also the agent name. +func declaredAgentName(svc *azdext.ServiceConfig) string { + props := serviceProps(svc) + if props == nil { + return "" + } + name, _ := props.AsMap()["name"].(string) + return name +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go new file mode 100644 index 00000000000..dae36a18c5b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +// writeOptimizeConfig lays out what `azd ai agent optimize` leaves behind: +// .agent_configs/baseline/metadata.yaml pointing at instructions.md beside it. +func writeOptimizeConfig(t *testing.T, serviceDir, metadata, instructions string) { + t.Helper() + dir := filepath.Join(serviceDir, ".agent_configs", "baseline") + require.NoError(t, os.MkdirAll(dir, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "metadata.yaml"), []byte(metadata), 0o600)) + if instructions != "" { + require.NoError(t, + os.WriteFile(filepath.Join(dir, "instructions.md"), []byte(instructions), 0o600)) + } +} + +// agentService builds a project holding one agent service, optionally +// declaring an agent name that differs from the service key. +func agentService(t *testing.T, root, serviceKey, declaredName string) *azdext.ProjectConfig { + t.Helper() + svc := &azdext.ServiceConfig{ + Name: serviceKey, + Host: AgentHost, + RelativePath: serviceKey, + } + if declaredName != "" { + props, err := structpb.NewStruct(map[string]any{"name": declaredName}) + require.NoError(t, err) + svc.AdditionalProperties = props + } + return &azdext.ProjectConfig{ + Path: root, + Services: map[string]*azdext.ServiceConfig{serviceKey: svc}, + } +} + +// The instructions an agent was optimized with are already on disk, so +// generating from them needs no service call. +func TestAgentInstructionsFromProject_ReadsTheOptimizeConfig(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "support"), + "name: support\ninstruction_file: instructions.md\n", + "Answer support questions politely.\n") + + instruction, path, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + require.NoError(t, err) + assert.Equal(t, "Answer support questions politely.", instruction) + assert.Equal(t, filepath.Join(root, "support", ".agent_configs", "baseline", "instructions.md"), + path) +} + +// A target names the agent, which need not be spelled the way the azure.yaml +// key is. A user has only ever seen one of the two. +func TestAgentInstructionsFromProject_MatchesTheDeclaredAgentName(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "svc"), + "instruction_file: instructions.md\n", + "Be helpful.") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "svc", "support-agent"), "support-agent") + + require.NoError(t, err) + assert.Equal(t, "Be helpful.", instruction) +} + +// Most projects have never run optimize, so finding nothing is the ordinary +// case and has to leave the caller free to ask the service instead. +func TestAgentInstructionsFromProject_SilentWhenThereIsNothingToRead(t *testing.T) { + root := t.TempDir() + + tests := []struct { + name string + proj *azdext.ProjectConfig + agent string + }{ + {"no project at all", nil, "support"}, + {"no agent named", agentService(t, root, "support", ""), ""}, + {"no service by that name", agentService(t, root, "support", ""), "other"}, + {"no optimize config on disk", agentService(t, root, "support", ""), "support"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instruction, path, err := AgentInstructionsFromProject(tt.proj, tt.agent) + + assert.NoError(t, err) + assert.Empty(t, instruction) + assert.Empty(t, path) + }) + } +} + +// A service that is not an agent is not a candidate, however it is named. +func TestAgentInstructionsFromProject_IgnoresServicesThatAreNotAgents(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "Be helpful.") + + proj := agentService(t, root, "support", "") + proj.Services["support"].Host = "containerapp" + + instruction, _, err := AgentInstructionsFromProject(proj, "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +// Two services answering to one name is a tie, and picking either would make +// the generated dataset describe an agent the caller did not mean. +func TestAgentInstructionsFromProject_RefusesAnAmbiguousTarget(t *testing.T) { + root := t.TempDir() + proj := agentService(t, root, "support", "") + props, err := structpb.NewStruct(map[string]any{"name": "support"}) + require.NoError(t, err) + proj.Services["helpdesk"] = &azdext.ServiceConfig{ + Name: "helpdesk", Host: AgentHost, RelativePath: "helpdesk", + AdditionalProperties: props, + } + + _, _, err = AgentInstructionsFromProject(proj, "support") + + require.ErrorIs(t, err, ErrAmbiguousAgentService) + assert.Contains(t, err.Error(), "helpdesk") + assert.Contains(t, err.Error(), "support") + assert.Contains(t, err.Error(), "--target", + "an ambiguity the caller can resolve has to say how") +} + +// A pointer with nothing behind it means something wrote half the config. +// Falling back silently would generate from the published agent while the +// author believes they are generating from what they just optimized. +func TestAgentInstructionsFromProject_ReportsADanglingInstructionFile(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "instructions.md") +} + +// Metadata that names no instruction file is a config without instructions, +// not a broken one. +func TestAgentInstructionsFromProject_NoInstructionFileIsNotAnError(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "name: support\n", "") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +func TestAgentInstructionsFromProject_ReportsUnreadableMetadata(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "\tnot: [valid\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "metadata.yaml") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go new file mode 100644 index 00000000000..9f5a6320684 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "path/filepath" + "strings" +) + +// Conventional artifact locations, relative to the eval directory. +const ( + DefaultDatasetsDir = "datasets" + DefaultEvaluatorsDir = "evaluators" +) + +// ArtifactRef is the name/source pair a generation run produces, so the +// command can tell the developer how to reference it. +type ArtifactRef struct { + Name string `json:"name"` + Source string `json:"source"` +} + +// Sample-count bounds enforced by the generation service. +const ( + MinSampleSize = 15 + MaxSampleSize = 1000 + DefaultSampleSize = 15 +) + +// Sources a dataset can be generated from. +const ( + GenerateFromTraces = "traces" + GenerateFromAgent = "agent" + GenerateFromPrompt = "prompt" + GenerateFromFile = "file" +) + +// GenerateSources is what --from accepts, in help order. +var GenerateSources = []string{ + GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile, +} + +// ValidateGenerateSource rejects a --from value the service has no path for. +func ValidateGenerateSource(from string) error { + switch from { + case "", GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile: + return nil + default: + return fmt.Errorf( + "--from %q is not a source; use one of %s", + from, strings.Join(GenerateSources, ", ")) + } +} + +// ValidateSampleSize rejects a row count the service would reject, before a +// generation job is submitted and billed. +func ValidateSampleSize(n int) error { + if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { + return fmt.Errorf( + "sample size must be between %d and %d, got %d", + MinSampleSize, MaxSampleSize, n) + } + return nil +} + +// ArtifactPath resolves an output directory against baseDir. The value may be a +// directory, in which case the file name is derived from resourceName and ext, +// or an explicit file path, which is used as-is. +func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { + if outputDir == "" { + return filepath.Join(baseDir, resourceName+ext) + } + candidate := outputDir + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(baseDir, candidate) + } + if looksLikeFile(outputDir, ext) { + return candidate + } + return filepath.Join(candidate, resourceName+ext) +} + +// looksLikeFile treats a trailing recognized extension as an explicit file path. +func looksLikeFile(p, ext string) bool { + got := strings.ToLower(filepath.Ext(p)) + if got == "" { + return false + } + if got == strings.ToLower(ext) { + return true + } + switch got { + case ".json", ".jsonl", ".yaml", ".yml": + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go new file mode 100644 index 00000000000..8b8f1581194 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package project models the eval configuration carried by the +// `host: azure.ai.eval` service entry in azure.yaml. +package project + +import ( + "fmt" + "strings" + + "azureaieval/internal/pkg/evalcore" +) + +// EvalConfig is one evaluation configuration: the catalogs of reusable assets, +// and every eval defined over them. +// +// It is the body of a single `azure.ai.eval` service entry, pulled in with +// $ref. One file rather than one per eval, because the catalogs are shared: +// two evals over the same dataset should name it once. +// +// How it is stored lives in eval_config_store.go. +type EvalConfig struct { + Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` + Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` +} + +// DatasetDecl is a catalog entry. A local Source is uploaded on deploy; without +// one the name must already resolve to a registered dataset. +type DatasetDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// EvaluatorDecl is a catalog entry for a custom evaluator. Built-ins are +// referenced straight from an eval and never declared here. +// +// Source names a `.json` file holding a rubric: a list of weighted scoring +// dimensions. +type EvaluatorDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// Eval is one evaluation defined over the catalogs. +// +// Dataset and Source are alternatives: rows come from a catalog dataset, or +// from a source such as production traces. Target is what gets invoked, and is +// a separate axis — an eval can read traces and invoke nothing. +type Eval struct { + Name string `yaml:"name" json:"name"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Source *SourceDecl `yaml:"source,omitempty" json:"source,omitempty"` + EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` + MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` +} + +// SourceDecl says where an eval's rows come from when they are not a dataset. +type SourceDecl struct { + Type string `yaml:"type" json:"type"` + LookbackHours int `yaml:"lookback_hours,omitempty" json:"lookback_hours,omitempty"` + MaxTraces int `yaml:"max_traces,omitempty" json:"max_traces,omitempty"` + AgentName string `yaml:"agent_name,omitempty" json:"agent_name,omitempty"` + ResponseIDs []string `yaml:"response_ids,omitempty" json:"response_ids,omitempty"` + MaxTurns int `yaml:"max_turns,omitempty" json:"max_turns,omitempty"` +} + +// Source types an eval can read rows from. +const ( + SourceTypeTraces = "traces" + SourceTypeResponses = "responses" +) + +// DefaultScaffoldMaxTraces is the cap init writes on a trace-backed eval, so a +// first run is bounded rather than taking the service's own default of 1000. +// Deleting max_traces from the file restores that default. +const DefaultScaffoldMaxTraces = 20 + +// Target names what the run invokes. +type Target struct { + Type string `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` +} + +// Target types the extension can invoke. Absent means nothing is invoked and +// the dataset already carries the answers. +const ( + TargetTypeAgent = "agent" + TargetTypeModel = "model" +) + +// Evaluation levels accepted by the service. The service default is turn. +const ( + EvaluationLevelTurn = "turn" + EvaluationLevelConversation = "conversation" +) + +// EvalNames lists the declared evals in declaration order. +func (c *EvalConfig) EvalNames() []string { + names := make([]string, 0, len(c.Evals)) + for _, e := range c.Evals { + names = append(names, e.Name) + } + return names +} + +// Eval returns the named eval. +// +// An empty name is only answered when the file declares exactly one, because +// guessing which eval a command meant is the kind of mistake that is noticed +// only after it has run. +func (c *EvalConfig) Eval(name string) (*Eval, error) { + if name == "" { + switch len(c.Evals) { + case 0: + return nil, fmt.Errorf("no evals are declared") + case 1: + return &c.Evals[0], nil + default: + return nil, fmt.Errorf( + "this configuration declares %d evals (%s); choose one with --eval", + len(c.Evals), strings.Join(c.EvalNames(), ", ")) + } + } + + for i := range c.Evals { + if c.Evals[i].Name == name { + return &c.Evals[i], nil + } + } + return nil, fmt.Errorf( + "eval %q is not declared; this configuration has %s", + name, strings.Join(c.EvalNames(), ", ")) +} + +// HasEval reports whether the named eval is declared. Unlike Eval it never +// falls back to "the only one", so callers checking for a collision cannot +// match a differently named entry. +func (c *EvalConfig) HasEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + return true + } + } + return false +} + +// RemoveEval drops the named eval, reporting whether it was there. +func (c *EvalConfig) RemoveEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + c.Evals = append(c.Evals[:i], c.Evals[i+1:]...) + return true + } + } + return false +} + +// DatasetDeclaration returns the catalog entry an eval's `dataset:` names. +func (c *EvalConfig) DatasetDeclaration(name string) (*DatasetDecl, bool) { + for i := range c.Datasets { + if c.Datasets[i].Name == name { + return &c.Datasets[i], true + } + } + return nil, false +} + +// EvaluatorDeclaration returns the catalog entry an evaluator reference names. +func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { + for i := range c.Evaluators { + if c.Evaluators[i].Name == name { + return &c.Evaluators[i], true + } + } + return nil, false +} + +// CustomEvaluators are the catalog entries this configuration owns — the ones +// carrying a local source, published before the evals that name them. +func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { + var owned []EvaluatorDecl + for _, decl := range c.Evaluators { + if decl.Source == "" { + continue + } + owned = append(owned, decl) + } + return owned +} + +// LocalDatasets are the catalog entries carrying a file to upload. +func (c *EvalConfig) LocalDatasets() []DatasetDecl { + var owned []DatasetDecl + for _, decl := range c.Datasets { + if decl.Source == "" { + continue + } + owned = append(owned, decl) + } + return owned +} + +// Validate checks the invariants the provider relies on before it calls the +// service, so failures surface as config errors rather than opaque 4xx. +func (c *EvalConfig) Validate() error { + if err := c.validateCatalogs(); err != nil { + return err + } + if len(c.Evals) == 0 { + return fmt.Errorf("at least one eval is required") + } + + seen := map[string]bool{} + substance := map[string]string{} + for i, eval := range c.Evals { + if eval.Name == "" { + return fmt.Errorf("evals[%d]: 'name' is required", i) + } + if seen[eval.Name] { + return fmt.Errorf("evals[%d]: duplicate eval name %q", i, eval.Name) + } + seen[eval.Name] = true + + if err := c.validateEval(i, eval); err != nil { + return err + } + + // Two evals that differ only by name are indistinguishable once + // deployed: the environment records an id against each eval's substance + // so a renamed declaration can find what it already deployed, and a + // shared substance makes that lookup ambiguous. + digest, err := FingerprintGroup(eval) + if err != nil { + return err + } + if first, clash := substance[digest]; clash { + return fmt.Errorf( + "evals[%d] (%s): identical to %q apart from its name and description; "+ + "give them different evaluators, datasets or settings, or declare one", + i, eval.Name, first) + } + substance[digest] = eval.Name + } + return nil +} + +func (c *EvalConfig) validateCatalogs() error { + datasets := map[string]bool{} + for i, d := range c.Datasets { + if d.Name == "" { + return fmt.Errorf("datasets[%d]: 'name' is required", i) + } + if datasets[d.Name] { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) + } + datasets[d.Name] = true + } + + evaluators := map[string]bool{} + for i, e := range c.Evaluators { + if e.Name == "" { + return fmt.Errorf("evaluators[%d]: 'name' is required", i) + } + if evaluators[e.Name] { + return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) + } + evaluators[e.Name] = true + + if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in needs no catalog entry; reference it "+ + "straight from an eval", i, e.Name) + } + // The service assigns an evaluator's version on publish, so a declared + // one cannot be honoured alongside a source: the upload lands on + // whatever comes next and the eval binds that, leaving the pin + // describing a version nothing uses. + if e.Source != "" && e.Version != "" { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `source`, because the "+ + "service assigns the version when it publishes. Drop `version` to "+ + "publish this file, or drop `source` to reference a version already "+ + "on the project", i, e.Name) + } + } + return nil +} + +func (c *EvalConfig) validateEval(i int, eval Eval) error { + if eval.Dataset != "" && eval.Source != nil { + return fmt.Errorf( + "evals[%d] (%s): `dataset` and `source` both say where rows come from; "+ + "declare one", i, eval.Name) + } + if eval.Dataset != "" { + if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { + return fmt.Errorf( + "evals[%d] (%s): dataset %q is not in the datasets catalog", + i, eval.Name, eval.Dataset) + } + } + if eval.Source != nil { + switch eval.Source.Type { + case SourceTypeTraces, SourceTypeResponses: + case "": + return fmt.Errorf("evals[%d] (%s): source.type is required", i, eval.Name) + default: + return fmt.Errorf( + "evals[%d] (%s): source.type %q is not supported; use %q or %q", + i, eval.Name, eval.Source.Type, SourceTypeTraces, SourceTypeResponses) + } + } + + if len(eval.Evaluators) == 0 { + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, eval.Name) + } + criteria := map[string]bool{} + for j, ref := range eval.Evaluators { + if ref.Evaluator == "" { + return fmt.Errorf("evals[%d].evaluators[%d]: 'evaluator' is required", i, j) + } + // The criterion name is what identifies a result row, so two rows that + // cannot be told apart are refused here rather than in the results. + criterion := ref.CriterionName() + if criteria[criterion] { + return fmt.Errorf( + "evals[%d].evaluators[%d]: duplicate criterion %q; give one a `name`", + i, j, criterion) + } + criteria[criterion] = true + + if ref.IsBuiltin() { + continue + } + if _, ok := c.EvaluatorDeclaration(ref.Evaluator); !ok { + return fmt.Errorf( + "evals[%d].evaluators[%d]: evaluator %q is not in the evaluators catalog", + i, j, ref.Evaluator) + } + } + + if eval.Target != nil && eval.Target.Type != "" && + eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { + return fmt.Errorf( + "evals[%d] (%s): target.type %q is not supported; use %q or %q", + i, eval.Name, eval.Target.Type, TargetTypeAgent, TargetTypeModel) + } + switch eval.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: + return fmt.Errorf( + "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + i, eval.Name, eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go new file mode 100644 index 00000000000..2b2b587c4f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "go.yaml.in/yaml/v3" +) + +// This file is the only place that knows how the configuration is stored: the +// directory it lives in, what the file is called, and how it is parsed and +// serialized. Everything else works with *EvalConfig, so changing the on-disk +// shape stays a local edit. + +// DefaultEvalDir is where init writes the configuration and its artifacts. +const DefaultEvalDir = "evals" + +// EvalConfigBase is the single configuration file inside that directory. +const EvalConfigBase = "eval.yaml" + +// EvalConfigPath is the configuration file inside an eval directory. It is +// exported for error messages and for the azure.yaml $ref; readers should +// prefer OpenEvalConfig. +func EvalConfigPath(evalDir string) string { + return filepath.Join(evalDir, EvalConfigBase) +} + +// OpenEvalConfig reads the configuration under evalDir. +// +// A missing file returns (nil, nil): generate runs before init, so "no +// configuration yet" is an ordinary state rather than a failure. +func OpenEvalConfig(evalDir string) (*EvalConfig, error) { + cfg, err := LoadEvalConfig(EvalConfigPath(evalDir)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return cfg, err +} + +// LoadEvalConfig reads a configuration from an explicit path. The path is used +// verbatim, relative to the process working directory — never re-rooted. +func LoadEvalConfig(path string) (*EvalConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading eval config %q: %w", path, err) + } + + var cfg EvalConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing eval config %q: %w", path, err) + } + return &cfg, nil +} + +// SaveEvalConfig writes cfg as the configuration under evalDir, creating the +// directory when it does not exist yet. +func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { + if err := os.MkdirAll(evalDir, 0o750); err != nil { + return fmt.Errorf("creating %q: %w", evalDir, err) + } + return SaveEvalConfigTo(EvalConfigPath(evalDir), cfg) +} + +// SaveEvalConfigTo writes cfg over an explicit path, for callers that already +// resolved one. +func SaveEvalConfigTo(path string, cfg *EvalConfig) error { + body, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("serializing eval config: %w", err) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + return fmt.Errorf("writing eval config %q: %w", path, err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go new file mode 100644 index 00000000000..cb51446d56c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -0,0 +1,332 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// sampleEvalConfig is the shape the spec documents for evals/eval.yaml: two +// catalogs, then the evals defined over them. +const sampleEvalConfig = ` +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + - name: prod-registered + +evaluators: + - name: support-quality + source: ./evaluators/support-quality.json + +evals: + - name: support-agent-smoke + description: Quality gate for the support agent + dataset: support-golden + evaluation_level: conversation + max_samples: 100 + evaluators: + - evaluator: builtin.task_adherence + - evaluator: support-quality + name: quality_strict + initialization_parameters: + deployment_name: gpt-4.1-nano + target: + type: agent + name: support-agent + + - name: support-agent-trace-eval + source: + type: traces + agent_name: support-agent + max_traces: 20 + evaluators: + - evaluator: builtin.task_adherence +` + +func loadFromString(t *testing.T, body string) *EvalConfig { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(EvalConfigPath(dir), []byte(body), 0o600)) + cfg, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.NotNil(t, cfg) + return cfg +} + +func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + require.Len(t, cfg.Datasets, 2) + require.Equal(t, "support-golden", cfg.Datasets[0].Name) + require.Equal(t, "./datasets/support-golden.jsonl", cfg.Datasets[0].Source) + require.Equal(t, "1", cfg.Datasets[0].Version) + + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "support-quality", cfg.Evaluators[0].Name) + + require.Equal(t, []string{"support-agent-smoke", "support-agent-trace-eval"}, cfg.EvalNames()) +} + +// One file holds many evals, and each is selected by its own name. +func TestEval_SelectsByName(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, "support-golden", eval.Dataset) + require.Equal(t, "Quality gate for the support agent", eval.Description) + require.Equal(t, EvaluationLevelConversation, eval.EvaluationLevel) + require.Equal(t, 100, eval.MaxSamples) + require.Len(t, eval.Evaluators, 2) + require.Equal(t, TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) +} + +// A trace-backed eval invokes nothing, so agent_name filters rather than targets. +func TestEval_TraceSourceHasNoTarget(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval, err := cfg.Eval("support-agent-trace-eval") + require.NoError(t, err) + require.Nil(t, eval.Target) + require.Equal(t, SourceTypeTraces, eval.Source.Type) + require.Equal(t, "support-agent", eval.Source.AgentName) + require.Equal(t, 20, eval.Source.MaxTraces) +} + +// An unnamed selection is only answered when the file declares exactly one, +// because guessing which eval a command meant is noticed only after it runs. +func TestEval_UnnamedIsAmbiguousWithSeveral(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("") + require.ErrorContains(t, err, "--eval") + require.ErrorContains(t, err, "support-agent-trace-eval") + + single := loadFromString(t, "evals:\n - name: only\n evaluators:\n - evaluator: builtin.relevance\n") + eval, err := single.Eval("") + require.NoError(t, err) + require.Equal(t, "only", eval.Name) +} + +func TestEval_UnknownNameNamesWhatIsDeclared(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("nope") + require.ErrorContains(t, err, "is not declared") + require.ErrorContains(t, err, "support-agent-smoke") +} + +// HasEval never falls back to "the only one", so a collision check cannot match +// a differently named entry. +func TestHasEvalAndRemoveEval(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + require.True(t, cfg.HasEval("support-agent-smoke")) + require.False(t, cfg.HasEval("nope")) + require.False(t, cfg.HasEval("")) + + require.True(t, cfg.RemoveEval("support-agent-smoke")) + require.False(t, cfg.HasEval("support-agent-smoke")) + require.Equal(t, []string{"support-agent-trace-eval"}, cfg.EvalNames()) + require.False(t, cfg.RemoveEval("support-agent-smoke")) +} + +// Only catalog entries carrying a local source are this config's to publish. +// One without a source already exists on the project. +func TestCustomEvaluatorsAndLocalDatasets_OnlyOwnLocalSources(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + owned := cfg.CustomEvaluators() + require.Len(t, owned, 1) + require.Equal(t, "support-quality", owned[0].Name) + require.Equal(t, "./evaluators/support-quality.json", owned[0].Source) + + local := cfg.LocalDatasets() + require.Len(t, local, 1) + require.Equal(t, "support-golden", local[0].Name, + "prod-registered has no source, so it is already on the project") +} + +func TestDeclarationLookups(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + ds, ok := cfg.DatasetDeclaration("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) + + _, ok = cfg.DatasetDeclaration("missing") + require.False(t, ok) + + ev, ok := cfg.EvaluatorDeclaration("support-quality") + require.True(t, ok) + require.Equal(t, "./evaluators/support-quality.json", ev.Source) +} + +// The configuration must survive a write/read cycle, because init and generate +// both append to a file they just read. +func TestEvalConfig_RoundTripsThroughTheStore(t *testing.T) { + dir := t.TempDir() + cfg := loadFromString(t, sampleEvalConfig) + + require.NoError(t, SaveEvalConfig(dir, cfg)) + back, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.Equal(t, cfg, back) +} + +// A missing file is an ordinary state: generate runs before init. +func TestOpenEvalConfig_MissingIsNotAnError(t *testing.T) { + cfg, err := OpenEvalConfig(t.TempDir()) + require.NoError(t, err) + require.Nil(t, cfg) +} + +// SaveEvalConfig creates the directory, so generate can record an artifact in a +// project that has never run init. +func TestSaveEvalConfig_CreatesTheDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "generated", Source: "./datasets/generated.jsonl"}}, + })) + + cfg, err := OpenEvalConfig(dir) + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Empty(t, cfg.Evals, "a generate-only file is inert until init wires an eval") +} + +func TestValidate_Accepts(t *testing.T) { + require.NoError(t, loadFromString(t, sampleEvalConfig).Validate()) +} + +func TestValidate_Rejects(t *testing.T) { + const oneEval = "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + + cases := []struct { + name string + body string + wantErr string + }{ + { + name: "dataset without a name", + body: "datasets:\n - source: ./d.jsonl\n" + oneEval, + wantErr: "'name' is required", + }, + { + name: "duplicate dataset", + body: "datasets:\n - name: d\n - name: d\n" + oneEval, + wantErr: "duplicate dataset name", + }, + { + name: "built-in declared in the catalog", + body: "evaluators:\n - name: builtin.relevance\n" + oneEval, + wantErr: "needs no catalog entry", + }, + { + name: "version pinned alongside a source", + body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n" + oneEval, + wantErr: "cannot be set with `source`", + }, + { + name: "no evals", + body: "datasets:\n - name: d\n", + wantErr: "at least one eval is required", + }, + { + name: "duplicate eval", + body: oneEval + " - name: e\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "duplicate eval name", + }, + { + name: "no evaluators", + body: "evals:\n - name: e\n evaluators: []\n", + wantErr: "at least one evaluator is required", + }, + { + name: "dataset and source both declared", + body: "datasets:\n - name: d\nevals:\n - name: e\n dataset: d\n" + + " source:\n type: traces\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "declare one", + }, + { + name: "dataset not in the catalog", + body: "evals:\n - name: e\n dataset: missing\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "not in the datasets catalog", + }, + { + name: "evaluator not in the catalog", + body: "evals:\n - name: e\n evaluators:\n - evaluator: quality\n", + wantErr: "not in the evaluators catalog", + }, + { + name: "duplicate criterion", + body: "evals:\n - name: e\n evaluators:\n" + + " - evaluator: builtin.relevance\n - evaluator: builtin.relevance\n", + wantErr: "duplicate criterion", + }, + { + name: "unsupported source type", + body: "evals:\n - name: e\n source:\n type: prompt\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "is not supported", + }, + { + name: "unsupported target type", + body: "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + + " target:\n type: prompt\n", + wantErr: "is not supported", + }, + { + name: "invalid evaluation level", + body: "evals:\n - name: e\n evaluation_level: sentence\n" + + " evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "evaluation_level", + }, + { + name: "two evals differing only by name", + body: "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: b\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "identical to", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := loadFromString(t, tc.body).Validate() + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// outputDir accepts a directory or an explicit file path. +func TestArtifactPath(t *testing.T) { + cases := []struct { + name string + outputDir string + resource string + ext string + want string + }{ + {"directory derives the file name", "datasets", "support-golden", ".jsonl", + filepath.Join("base", "datasets", "support-golden.jsonl")}, + {"explicit file path is used as-is", "generated/datasets/support-golden.jsonl", "ignored", ".jsonl", + filepath.Join("base", "generated", "datasets", "support-golden.jsonl")}, + {"empty outputDir falls back to the base", "", "support-quality", ".json", + filepath.Join("base", "support-quality.json")}, + {"yaml rubric file path", "generated/rubrics/quality.yaml", "ignored", ".json", + filepath.Join("base", "generated", "rubrics", "quality.yaml")}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, ArtifactPath("base", tc.outputDir, tc.resource, tc.ext)) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go new file mode 100644 index 00000000000..2cf1fe8e889 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// FingerprintGroup is covered in service_target_eval_test.go. These cover the +// file hash and the environment key it is stored under, which nothing did. + +// A fingerprint is compared against the one recorded at the last deploy, so +// identical content must hash identically and a single changed byte must not. +func TestFingerprint(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.jsonl") + b := filepath.Join(dir, "b.jsonl") + require.NoError(t, os.WriteFile(a, []byte(`{"query":"hi"}`), 0o600)) + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hi"}`), 0o600)) + + sumA, err := Fingerprint(a) + require.NoError(t, err) + sumB, err := Fingerprint(b) + require.NoError(t, err) + + assert.Equal(t, sumA, sumB, "same content, same fingerprint") + assert.Len(t, sumA, 64, "sha-256 as hex") + + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hI"}`), 0o600)) + sumB, err = Fingerprint(b) + require.NoError(t, err) + assert.NotEqual(t, sumA, sumB, "one changed byte has to show") +} + +// A missing file names itself, because the usual cause is a catalog entry +// pointing at something that was moved or never generated. +func TestFingerprint_MissingFileNamesIt(t *testing.T) { + _, err := Fingerprint(filepath.Join(t.TempDir(), "gone.jsonl")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "gone.jsonl") +} + +// The key goes into an azd environment file, which accepts only uppercase +// letters, digits and underscores. A name that reached it unmapped would +// produce a key azd cannot round-trip, and the artifact would look changed on +// every deploy. +func TestFingerprintKey_IsAValidEnvironmentKey(t *testing.T) { + tests := []struct { + kind, name, want string + }{ + {"dataset", "support-regression", "DATASET_SUPPORT_REGRESSION"}, + {"evaluator", "quality.v2", "EVALUATOR_QUALITY_V2"}, + {"dataset", "Mixed Case Name", "DATASET_MIXED_CASE_NAME"}, + // One rune maps to one underscore, so a multi-byte character does not + // widen the key. + {"eval", "unicode-caf\u00e9", "EVAL_UNICODE_CAF_"}, + } + + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + key := FingerprintKey(tt.kind, tt.name) + + assert.Equal(t, EnvKeyFingerprintPrefix+tt.want, key) + for _, r := range strings.TrimPrefix(key, EnvKeyFingerprintPrefix) { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } + }) + } +} + +// Two artifacts of different kinds can share a name, and they must not share a +// key — one would overwrite the other's recorded fingerprint. +func TestFingerprintKey_KindSeparatesTheNamespaces(t *testing.T) { + assert.NotEqual(t, + FingerprintKey("dataset", "quality"), + FingerprintKey("evaluator", "quality")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go new file mode 100644 index 00000000000..2097621ed6d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -0,0 +1,360 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/azure/azure-dev/cli/azd/pkg/foundry" + "google.golang.org/protobuf/types/known/structpb" +) + +// EvalHost is the azure.yaml host this provider serves. +const EvalHost = "azure.ai.eval" + +// azd environment keys owned by this extension. +const ( + EnvKeyEvalID = "EVAL_ID" + EnvKeyDatasetVersion = "EVAL_DATASET_VERSION" + EnvKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) + +// Reconciler applies the eval configuration to the service. It is satisfied by +// the command layer, which owns the data-plane clients. +type Reconciler interface { + // EnsureDataset registers a new dataset version when the local content + // changed, returning the resolved version and whether anything was written. + EnsureDataset(ctx context.Context, decl DatasetDecl, localPath string) (version string, changed bool, err error) + // EnsureEvaluator registers a new evaluator version when the definition + // differs from what the service already holds. + EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) + // EnsureEval creates the group when it is absent or its resolved + // evaluators or options changed, returning its id. datasetPath is the local + // dataset backing the group, or empty when it is already registered; it lets + // the reconciler bind criteria to the columns that actually exist. + EnsureEval(ctx context.Context, group Eval, datasetPath string, recreate bool) (id string, err error) +} + +// EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns +// ordering across services through `uses:`; this provider owns only the order +// within the eval service itself. +type EvalServiceTargetProvider struct { + azdClient *azdext.AzdClient + newReconciler func(ctx context.Context) (Reconciler, error) + + serviceConfig *azdext.ServiceConfig + envName string +} + +// NewEvalServiceTargetProvider builds the provider. The reconciler is supplied +// lazily so the data-plane clients are only created when a deploy actually runs. +func NewEvalServiceTargetProvider( + azdClient *azdext.AzdClient, + newReconciler func(ctx context.Context) (Reconciler, error), +) *EvalServiceTargetProvider { + return &EvalServiceTargetProvider{azdClient: azdClient, newReconciler: newReconciler} +} + +func (p *EvalServiceTargetProvider) Initialize( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, +) error { + p.serviceConfig = serviceConfig + return nil +} + +// Endpoints reports no endpoints: eval resources are not addressable. +func (p *EvalServiceTargetProvider) Endpoints( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + targetResource *azdext.TargetResource, +) ([]string, error) { + return nil, nil +} + +func (p *EvalServiceTargetProvider) GetTargetResource( + ctx context.Context, + subscriptionId string, + serviceConfig *azdext.ServiceConfig, + defaultResolver func() (*azdext.TargetResource, error), +) (*azdext.TargetResource, error) { + if defaultResolver != nil { + if target, err := defaultResolver(); err == nil { + return target, nil + } + } + // Eval resources live on the project data plane, so there is no ARM + // resource of our own to resolve. + return &azdext.TargetResource{SubscriptionId: subscriptionId}, nil +} + +// Package is a no-op: eval artifacts are plain files already on disk. +func (p *EvalServiceTargetProvider) Package( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + progress azdext.ProgressReporter, +) (*azdext.ServicePackageResult, error) { + return &azdext.ServicePackageResult{}, nil +} + +// Publish is a no-op: there is no artifact registry step for eval resources. +func (p *EvalServiceTargetProvider) Publish( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + publishOptions *azdext.PublishOptions, + progress azdext.ProgressReporter, +) (*azdext.ServicePublishResult, error) { + return &azdext.ServicePublishResult{}, nil +} + +// Deploy reconciles the eval configuration in a fixed order — datasets, then +// evaluators, then evals — because a group references the versions the +// first two resolve to. It fails fast; the next `azd up` resumes from wherever +// it stopped. +func (p *EvalServiceTargetProvider) Deploy( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + progress azdext.ProgressReporter, +) (*azdext.ServiceDeployResult, error) { + cfg, err := EvalConfigFromService(serviceConfig, p.projectRoot(ctx)) + if err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("eval config is invalid: %w", err) + } + + reconciler, err := p.newReconciler(ctx) + if err != nil { + return nil, err + } + + baseDir := serviceRelativeDir(serviceConfig) + + // 1. Datasets the configuration owns. Paths are kept so an eval that names + // one can derive its columns without reading the blob back. + anyChanged := false + datasetPaths := map[string]string{} + for _, decl := range cfg.Datasets { + if decl.Source == "" { + continue + } + report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + datasetPaths[decl.Name] = localPath + version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) + if err != nil { + return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) + } + anyChanged = anyChanged || changed + report(progress, describeResult("dataset", decl.Name, version, changed)) + } + + // 2. Evaluators this configuration owns. Built-ins and already-registered + // ones need no publish. + for _, decl := range cfg.CustomEvaluators() { + report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) + if err != nil { + return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) + } + anyChanged = anyChanged || changed + report(progress, describeResult("evaluator", decl.Name, version, changed)) + } + + // 3. The evals. Evals are immutable, so a change upstream means a new one + // must be created and the stored id replaced. + for i := range cfg.Evals { + eval := cfg.Evals[i] + report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) + id, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset], anyChanged) + if err != nil { + return nil, fmt.Errorf("eval %q: %w", eval.Name, err) + } + report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) + } + + return &azdext.ServiceDeployResult{}, nil +} + +// projectRoot is the directory `$ref` paths resolve against. It is the +// directory holding azure.yaml, which only azd can report. +func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { + if p.azdClient == nil { + return "" + } + resp, err := p.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "" + } + return resp.GetProject().GetPath() +} + +// describeResult reports whether a version was published or reused, so a +// no-op deploy is visibly a no-op. +func describeResult(kind, name, version string, changed bool) string { + if changed { + return fmt.Sprintf("Published %s %s version %s", kind, name, version) + } + return fmt.Sprintf("%s %s is unchanged at version %s", strings.ToUpper(kind[:1])+kind[1:], name, version) +} + +func report(progress azdext.ProgressReporter, message string) { + if progress != nil { + progress(message) + } +} + +// EvalConfigFromService reads the eval configuration carried inline on the +// service entry. azd captures unknown keys into AdditionalProperties and hands +// them to the extension untouched. +// +// azd core deliberately does not resolve `$ref` includes for extensions — it +// strips the ServiceConfig fields it owns and leaves `$ref` at the top of the +// map for the owning extension to resolve. Without this call a service written +// as `host: azure.ai.eval` + `$ref: ./evals/azure.yaml` deploys nothing at all, +// because the config parses to an empty set of datasets and groups. +func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*EvalConfig, error) { + props := serviceProps(svc) + if props == nil || len(props.GetFields()) == 0 { + return nil, fmt.Errorf( + "service %q carries no eval configuration; expected evaluators, datasets, or evals", + svc.GetName()) + } + + values := props.AsMap() + if projectRoot != "" { + resolved, err := foundry.ResolveFileRefs(values, projectRoot) + if err != nil { + return nil, fmt.Errorf("resolving $ref in the eval service configuration: %w", err) + } + values = resolved + } + + raw, err := json.Marshal(values) + if err != nil { + return nil, fmt.Errorf("reading the eval service configuration: %w", err) + } + + var cfg EvalConfig + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, fmt.Errorf("parsing the eval service configuration: %w", err) + } + return &cfg, nil +} + +// serviceProps prefers the inline properties, falling back to the nested +// config block. +func serviceProps(svc *azdext.ServiceConfig) *structpb.Struct { + if s := svc.GetAdditionalProperties(); s != nil && len(s.GetFields()) > 0 { + return s + } + return svc.GetConfig() +} + +// serviceRelativeDir returns the directory that `source:` paths resolve against. +// +// When the service is authored as `host:` + `$ref: ./evals/azure.yaml`, the +// paths inside that file are written relative to the file itself, so the +// include's own directory is the base. ResolveFileRefs inlines the content +// without rebasing paths, so the base has to be recovered from the `$ref` +// value before resolution. +func serviceRelativeDir(svc *azdext.ServiceConfig) string { + if svc == nil { + return "." + } + if props := serviceProps(svc); props != nil { + if ref, ok := props.AsMap()["$ref"].(string); ok && ref != "" { + if dir := filepath.Dir(filepath.FromSlash(ref)); dir != "" { + return dir + } + } + } + if p := svc.GetRelativePath(); p != "" { + return p + } + return "." +} + +// resolveSource joins a declared source against the service directory, leaving +// absolute paths and empty values alone. +func resolveSource(baseDir, source string) string { + if source == "" { + return "" + } + if filepath.IsAbs(source) { + return source + } + return filepath.Join(baseDir, source) +} + +// Fingerprint hashes a local artifact so a later deploy can tell whether the +// content changed without downloading anything from the service. +// +// The dataset API returns no content hash or etag, so comparing against the +// service would mean downloading the blob on every deploy. Every artifact this +// applies to — a dataset, a rubric, an evaluator script — is a single file. +func Fingerprint(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("hashing %q: %w", path, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintGroup hashes an eval's own declaration. +// +// Change detection on upstream artifacts is not sufficient: editing a group's +// evaluators, target, or options changes what the group means, and groups are +// immutable, so the group has to be recreated even when the dataset and +// evaluators are untouched. Without this a retargeted group keeps running +// against the old definition. +func FingerprintGroup(group Eval) (string, error) { + // Only substance is hashed. The id is server-assigned; name and description + // are what UpdateEvalParametersBody reaches, so an edit confined to them is + // pushed in place and must not cost the eval its id and its run history. + // Everything else — dataset, source, evaluators, target, level — is fixed at + // creation, so a change there is a new eval. + name := group.Name + group.ID = "" + group.Name = "" + group.Description = "" + + data, err := json.Marshal(group) + if err != nil { + return "", fmt.Errorf("hashing eval %q: %w", name, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintKey is the azd environment key holding an artifact's fingerprint. +func FingerprintKey(kind, name string) string { + safe := strings.Map(func(r rune) rune { + switch { + case r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return r + case r >= 'a' && r <= 'z': + return r - 32 + default: + return '_' + } + }, kind+"_"+name) + return EnvKeyFingerprintPrefix + safe +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go new file mode 100644 index 00000000000..e20ecd49009 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func propsFrom(t *testing.T, values map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(values) + require.NoError(t, err) + return s +} + +// A service authored as `host:` + `$ref: ./evals/azure.yaml` has its relative +// source paths written against the included file, not the project root. +// ResolveFileRefs inlines the content without rebasing them, so the base has to +// come from the $ref value. +func TestServiceRelativeDirUsesRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("evals"), serviceRelativeDir(svc)) +} + +// A nested include keeps its own directory. +func TestServiceRelativeDirUsesNestedRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./config/evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("config/evals"), serviceRelativeDir(svc)) +} + +// Without a $ref the service's own relative path is the base. +func TestServiceRelativeDirFallsBackToRelativePath(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + RelativePath: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{}, + }), + } + require.Equal(t, "evals", serviceRelativeDir(svc)) +} + +// With neither, sources resolve against the project root. +func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { + require.Equal(t, ".", serviceRelativeDir(&azdext.ServiceConfig{Name: "evals"})) + require.Equal(t, ".", serviceRelativeDir(nil)) +} + +// An inline config still parses when no project root is available to resolve +// includes against. +func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "support-agent-evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{ + map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + }, + "evals": []any{ + map[string]any{ + "name": "support-agent-smoke", + "dataset": "golden", + "evaluators": []any{ + map[string]any{"evaluator": "builtin.task_adherence"}, + }, + "target": map[string]any{"type": "agent", "name": "my-agent"}, + }, + }, + }), + } + + cfg, err := EvalConfigFromService(svc, "") + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Equal(t, "golden", cfg.Datasets[0].Name) + + // One service covers every eval in the file it pulled in, so the eval is + // selected by its own name rather than by the service key. + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, "golden", eval.Dataset) + require.Len(t, eval.Evaluators, 1) + require.Equal(t, "builtin.task_adherence", eval.Evaluators[0].Evaluator) + require.Equal(t, "my-agent", eval.Target.Name) +} + +func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { + _, err := EvalConfigFromService(&azdext.ServiceConfig{Name: "evals"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "no eval configuration") +} + +// Evals are immutable, so a change to an eval's own declaration has to be +// detectable. Upstream artifact fingerprints do not cover it: retargeting an +// eval at a different agent leaves the dataset and evaluators untouched. +func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { + base := Eval{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + Target: &Target{Type: "agent", Name: "agent-a"}, + EvaluationLevel: EvaluationLevelTurn, + } + + original, err := FingerprintGroup(base) + require.NoError(t, err) + + same, err := FingerprintGroup(base) + require.NoError(t, err) + require.Equal(t, original, same, "an unchanged eval must keep its fingerprint") + + cases := map[string]func(g *Eval){ + "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, + "evaluators": func(g *Eval) { + g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) + }, + "judge deployment": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Evaluator: "builtin.task_adherence", + InitializationParameters: map[string]any{"deployment_name": "gpt-4o-mini"}, + }} + }, + "version pin": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Evaluator: "builtin.task_adherence", Version: "2", + }} + }, + "evaluation level": func(g *Eval) { g.EvaluationLevel = EvaluationLevelConversation }, + "dataset": func(g *Eval) { g.Dataset = "other" }, + "source": func(g *Eval) { + g.Dataset = "" + g.Source = &SourceDecl{Type: SourceTypeTraces, AgentName: "agent-a"} + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + changed := base + changed.Evaluators = append(evalcore.EvaluatorList(nil), base.Evaluators...) + mutate(&changed) + + digest, err := FingerprintGroup(changed) + require.NoError(t, err) + require.NotEqual(t, original, digest, "changing %s must change the fingerprint", name) + }) + } +} + +// The fingerprint covers substance only. The id is server-assigned, and name +// and description are what UpdateEvalParametersBody reaches — an edit confined +// to those is pushed in place, so it must not fork the run history. +func TestFingerprintGroupIgnoresIdNameAndDescription(t *testing.T) { + base := Eval{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + original, err := FingerprintGroup(base) + require.NoError(t, err) + + noisy := base + noisy.ID = "eval_abc123" + noisy.Name = "quality-renamed" + noisy.Description = "reworded" + + digest, err := FingerprintGroup(noisy) + require.NoError(t, err) + require.Equal(t, original, digest) +} + +// Editing one eval must not recreate its siblings: the unit compared is the +// eval's own subtree, never the file. +func TestFingerprintGroupIsScopedToOneEval(t *testing.T) { + gate := Eval{ + Name: "support-agent-gate", + Dataset: "prod-golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + regression := Eval{ + Name: "support-agent-regression-eval", + Dataset: "support-agent-regression", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + + before, err := FingerprintGroup(regression) + require.NoError(t, err) + + gate.Evaluators = append(gate.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) + + after, err := FingerprintGroup(regression) + require.NoError(t, err) + require.Equal(t, before, after, "editing a sibling must leave this eval alone") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go new file mode 100644 index 00000000000..e7279d11fba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package version + +var ( + // Populated at build time. + Version = "dev" + Commit = "none" + BuildDate = "unknown" +) diff --git a/cli/azd/extensions/azure.ai.evaluations/main.go b/cli/azd/extensions/azure.ai.evaluations/main.go new file mode 100644 index 00000000000..993d2e8816e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/main.go @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package main + +import ( + "azureaieval/internal/cmd" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +func main() { + azdext.Run(cmd.NewRootCommand()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go new file mode 100644 index 00000000000..d82415517fc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +type datasetSummary struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` +} + +const datasetRows = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +` + +// registeredDataset is a dataset with more than one version, which is what +// makes --version on show and --name on list worth asserting. +type registeredDataset struct { + Name string + // Versions are read back from each registration rather than assumed to + // start at 1: the server assigns them, and a test that hardcoded the + // numbering would be asserting its own guess. + Versions []string +} + +var ( + readOnlyDatasetOnce sync.Once + readOnlyDataset *registeredDataset +) + +// sharedDataset is registered once for the tests that only read it. Each +// registration uploads a blob, so redoing it per test buys nothing. +func sharedDataset(t *testing.T) *registeredDataset { + t.Helper() + readOnlyDatasetOnce.Do(func() { + readOnlyDataset = registerDataset(t, 2) + }) + require.NotNil(t, readOnlyDataset, "the shared dataset could not be registered") + return readOnlyDataset +} + +// registerDataset publishes a dataset and removes every version it created. +func registerDataset(t *testing.T, versions int) *registeredDataset { + t.Helper() + require.Positive(t, versions) + + path := filepath.Join(t.TempDir(), "golden.jsonl") + require.NoError(t, os.WriteFile(path, []byte(datasetRows), 0o600)) + + ds := ®isteredDataset{Name: uniqueName("azdcli_ds")} + for i := range versions { + // The first publish is a create; every later one is an update, which is + // the only difference between them. + verb := "update" + if i == 0 { + verb = "create" + } + r := requireSuccess(t, run(t, "dataset", verb, + ds.Name, "--from-file", path, "-o", "json")) + + var created datasetSummary + r.JSON(t, &created) + require.NotEmpty(t, created.Version, "the service assigns the version") + ds.Versions = append(ds.Versions, created.Version) + + version := created.Version + deferTeardown(func() { + runQuietly("dataset", "delete", ds.Name, "--version", version) + }) + } + require.Len(t, ds.Versions, versions) + require.NotEqual(t, ds.Versions[0], ds.Versions[len(ds.Versions)-1], + "updating must advance the version rather than overwrite") + return ds +} + +func TestCLIDatasetList(t *testing.T) { + ds := sharedDataset(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name)) + for _, header := range []string{"NAME", "VERSION", "FORMAT"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + // `versions list` is what makes the listing usable once a project holds more + // than a screenful: it narrows to one dataset's versions. + t.Run("versions list scopes to one dataset's versions", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name, "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.NotEmpty(t, listed) + + seen := map[string]bool{} + for _, v := range listed { + require.Equalf(t, ds.Name, v.Name, + "the listing must return only that dataset's versions; got %q", v.Name) + seen[v.Version] = true + } + for _, want := range ds.Versions { + require.Truef(t, seen[want], "version %s is missing from the listing", want) + } + }) + + // Unscoped, the listing is every dataset rather than every version, so the + // one just registered has to be in it. + t.Run("unscoped lists the project's datasets", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "-o", "json")) + var all []datasetSummary + r.JSON(t, &all) + require.NotEmpty(t, all) + + found := false + for _, d := range all { + if d.Name == ds.Name { + found = true + } + } + require.True(t, found, "a registered dataset must appear in the unscoped listing") + }) + + t.Run("an unknown name lists nothing rather than failing", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.Empty(t, listed) + }) +} + +func TestCLIDatasetShow(t *testing.T) { + ds := sharedDataset(t) + latest := ds.Versions[len(ds.Versions)-1] + + // Omitting the version means the latest, which is the only sensible + // default for a name that gains a version on every registration. + t.Run("defaults to the latest version", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", ds.Name, "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Name, shown.Name) + require.Equal(t, latest, shown.Version) + }) + + t.Run("version pins an earlier one", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", + ds.Name, "--version", ds.Versions[0], "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Versions[0], shown.Version) + require.NotEqual(t, latest, shown.Version) + }) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", ds.Name)) + for _, header := range []string{"NAME", "VERSION", "FORMAT", "URI"} { + require.Containsf(t, r.Stdout, header, "the table lost its %s column", header) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + t.Run("the name is required", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show")) + require.Contains(t, r.Combined(), "accepts 1 arg") + }) + + t.Run("an unknown dataset is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", "azdcli-no-such-dataset")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "azdcli-no-such-dataset") + }) + + t.Run("an unknown version of a real dataset is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", + ds.Name, "--version", "9999")) + require.Contains(t, r.Combined(), "9999") + require.Less(t, len(r.Combined()), 600, r.Combined()) + }) +} + +func TestCLIDatasetDelete(t *testing.T) { + t.Run("the name and version are both required", func(t *testing.T) { + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "--version", "1")).Combined(), + "accepts 1 arg") + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "whatever")).Combined(), + "--version is required") + }) + + // Deleting something that was never registered succeeds. The service + // treats DELETE as idempotent and answers 204 whatever the name, so the + // command reports a removal it did not perform — and the not-found branch + // in `dataset delete` cannot be reached this way. Asserted rather than + // wished away, because a caller scripting against the exit code is + // entitled to know it means "gone", not "was there and is now gone". + t.Run("deleting an unregistered dataset is idempotent, not an error", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "delete", + "azdcli-no-such-dataset", "--version", "1")) + require.Contains(t, r.Stdout, "Deleted dataset") + + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + require.Empty(t, remaining, "nothing was there to delete in the first place") + }) + + // A successful delete answers 204 No Content, so asserting the exit code + // is what catches a client that reads an empty body as a failure and + // reports a removal it just performed as an error. + t.Run("one version is removed and the other survives", func(t *testing.T) { + ds := registerDataset(t, 2) + gone, kept := ds.Versions[0], ds.Versions[1] + + r := requireSuccess(t, run(t, "dataset", "delete", + ds.Name, "--version", gone)) + require.Contains(t, r.Stdout, "Deleted dataset") + require.Contains(t, r.Stdout, ds.Name) + + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + ds.Name, "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + + versions := map[string]bool{} + for _, v := range remaining { + versions[v.Version] = true + } + require.False(t, versions[gone], "the deleted version must leave the listing") + require.True(t, versions[kept], "deleting one version must not remove the others") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go new file mode 100644 index 00000000000..e6c97dffd9e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestCLIEvaluatorListBuiltin is the cheapest proof the binary can reach the +// service on its own: no azd project, no config, just a flag. +func TestCLIEvaluatorListBuiltin(t *testing.T) { + r := requireSuccess(t, run(t, "evaluator", "list", "--builtin", "-o", "json")) + + var builtins []struct { + Name string `json:"name"` + EvaluatorType string `json:"evaluator_type"` + } + r.JSON(t, &builtins) + require.NotEmpty(t, builtins, "the project must expose built-in evaluators") + + for _, b := range builtins { + require.True(t, strings.HasPrefix(b.Name, "builtin."), + "--builtin must return only built-ins, got %q", b.Name) + } + + // The default rendering is a table, not JSON. A script reading stdout + // without -o json would otherwise silently parse a header row. + table := requireSuccess(t, run(t, "evaluator", "list", "--builtin")) + require.Contains(t, table.Stdout, "NAME") + require.Contains(t, table.Stdout, "VERSION") +} + +// TestCLIJSONListsAreBareArrays pins the envelope. +// +// The service wraps listings in {"value":[...]} or {"data":[...]} depending on +// the route. Leaking either would make every consumer special-case the +// command it came from, so the CLI unwraps them, and this is what says so. +func TestCLIJSONListsAreBareArrays(t *testing.T) { + for _, args := range [][]string{ + {"evaluator", "list", "--builtin", "-o", "json"}, + {"dataset", "list", "-o", "json"}, + } { + t.Run(strings.Join(args[:2], " "), func(t *testing.T) { + r := requireSuccess(t, run(t, args...)) + trimmed := strings.TrimSpace(r.Stdout) + require.True(t, strings.HasPrefix(trimmed, "["), + "a list must be a bare array, not an envelope; got:\n%s", firstLine(trimmed)) + + var out []any + r.JSON(t, &out) + }) + } +} + +// TestCLIUnknownEvaluatorIsBrief covers the failure a user hits by typo. +// +// The service answers with a long JSON body. Printing it verbatim buries the +// one useful sentence, so the CLI shortens it, and a regression here is the +// kind that only shows up in someone's terminal. +func TestCLIUnknownEvaluatorIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "evaluator", "show", "azdcli-does-not-exist-9999")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) +} + +// TestCLIInitNeedsAnAzdProject covers the whole of what `init` can be asked +// through this harness. +// +// init resolves the project over azd's gRPC channel, so it only works when azd +// is hosting the extension. Running the binary directly there is no host, and +// that is exactly the case a user hits by running the command outside a +// project — so what is asserted is the refusal: it must name `azd init` rather +// than surface a transport error. The scaffolding itself is covered by the +// unit tests, which can supply a fake azd client. +func TestCLIInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + r := requireFailure(t, runIn(t, dir, "init", + "--target", "probe-agent", + "--generation-model", "gpt-4o-mini", + "--no-prompt")) + + require.Contains(t, r.Combined(), "azd init", + "the refusal must name the command that makes a project") + require.NotContains(t, strings.ToLower(r.Combined()), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, + "a refused init must leave nothing behind") +} + +// TestCLINoPromptFailsInsteadOfHanging is what makes the CLI usable in CI: a +// missing required value must end the process, not wait on a terminal nobody +// is watching. +func TestCLINoPromptFailsInsteadOfHanging(t *testing.T) { + dir := t.TempDir() + r := requireFailure(t, runIn(t, dir, "init", "--no-prompt")) + require.NotEmpty(t, strings.TrimSpace(r.Combined()), + "--no-prompt must say what it could not resolve") +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go new file mode 100644 index 00000000000..e159f28ff4a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go @@ -0,0 +1,415 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + +// The command tests need an eval that has already been run, and building one +// through the CLI is not possible: there is no command that creates an eval +// from flags, only `run start`, which needs a config file and a deployed +// target. So the fixture is built with the client and every assertion is made +// against the binary. What is under test is the command surface; the eval is +// scenery. +// +// It is built once for the whole package because two completed runs cost +// minutes, and torn down in TestMain rather than t.Cleanup so that whichever +// test happened to trigger the build does not take the fixture away from the +// rest. +// +// It evaluates an agent with a built-in evaluator, because that is all M1 can +// run: a deterministic code grader over a target-less dataset would score the +// rows predictably, but code evaluators and no-target runs are both M2. The +// cost is that pass and fail are decided by a judge, so no test may assert how +// many rows failed — only that filtering by verdict is self-consistent. + +const fixtureAPIVersion = "2025-11-15-preview" + +const defaultFixtureModel = "gpt-4o-mini" + +// fixtureQueries are answered by the agent under evaluation. They are ordinary +// support questions: the fixture proves the command surface, not the agent. +var fixtureQueries = []string{ + "How do I reset my password?", + "How do I change my billing address?", + "What are your support hours?", +} + +// evalFixture is one eval with two completed runs. +type evalFixture struct { + EvaluatorName string + EvalID string + + // The agent the runs evaluate, so that a test needing a further run does + // not have to resolve one again. + AgentName string + + // Two runs of the same eval, so that listing, limiting and defaulting to + // the most recent all have something to distinguish. + FirstRunID string + SecondRunID string +} + +var ( + fixtureOnce sync.Once + fixture *evalFixture + fixtureErr error + + // teardown runs after the last test, in reverse order. + teardownMu sync.Mutex + teardown []func() +) + +func deferTeardown(fn func()) { + teardownMu.Lock() + defer teardownMu.Unlock() + teardown = append(teardown, fn) +} + +func runTeardown() { + teardownMu.Lock() + defer teardownMu.Unlock() + for i := len(teardown) - 1; i >= 0; i-- { + teardown[i]() + } + teardown = nil +} + +// runQuietly invokes the binary without a *testing.T. +// +// Teardown runs after the last test has reported, and logging or asserting +// against a finished test panics, so nothing here may touch one. +func runQuietly(args ...string) { + full := append(append([]string{}, args...), "--project-endpoint", endpoint) + _ = exec.Command(binaryPath, full...).Run() +} + +var ( + credOnce sync.Once + cred *azidentity.AzureDeveloperCLICredential + credErr error +) + +// liveClient builds the client the fixture is assembled with. One credential +// for the package, because azidentity caches tokens per instance and a fresh +// one per call makes every call shell out to azd again. +// +// The first token is fetched here rather than lazily on the first request: +// that call is the one that flakes, and paying for it up front means the rest +// of the fixture runs against a cached token. +func liveClient() (*eval_api.EvalClient, error) { + credOnce.Do(func() { + cred, credErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + if credErr != nil { + return + } + credErr = retryCredentialFlake(func() error { + _, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + return err + }) + }) + if credErr != nil { + return nil, credErr + } + return eval_api.NewEvalClient(endpoint, cred), nil +} + +// retryCredentialFlake reruns a request that failed only because azd's token +// helper exited non-zero. +// +// It is the same failure the harness retries around the binary, for the same +// reason: nothing was sent, and the alternative is a suite that fails on a +// different test each run for a reason unrelated to the code. Any other error +// is returned immediately. +func retryCredentialFlake(fn func() error) error { + var err error + for attempt := range 4 { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * 2 * time.Second) + } + if err = fn(); err == nil || !strings.Contains(err.Error(), credentialFlake) { + return err + } + } + return err +} + +// sharedEval returns the fixture, building it on first use. +// +// A failure here fails the calling test rather than skipping it: every test +// that asks for the fixture is testing something that cannot be exercised +// without one, and a suite that goes green because its subject was missing is +// worse than one that goes red. +func sharedEval(t *testing.T) *evalFixture { + t.Helper() + fixtureOnce.Do(func() { + start := time.Now() + fixture, fixtureErr = buildFixture(t.Logf) + t.Logf("fixture ready in %s", time.Since(start).Round(time.Second)) + }) + if fixtureErr != nil { + t.Fatalf("building the shared eval the command tests run against: %v", fixtureErr) + } + return fixture +} + +func fixtureModel() string { + if model := os.Getenv("AZURE_AI_EVAL_MODEL"); model != "" { + return model + } + return defaultFixtureModel +} + +// resolveFixtureAgent names the agent the fixture evaluates. +// +// It reads /agents, not /assistants: they are different collections, and an +// eval target resolves against the former. Naming an assistant is accepted by +// the create and then fails the run with "resources not found". +func resolveFixtureAgent(ctx context.Context) (string, error) { + if name := os.Getenv("AZURE_AI_EVAL_AGENT"); name != "" { + return name, nil + } + + // Builds the shared credential if it does not exist yet; the token below + // comes from it. + if _, err := liveClient(); err != nil { + return "", err + } + + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + if err != nil { + return "", fmt.Errorf("acquiring a token to list agents: %w", err) + } + + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, endpoint+"/agents?api-version="+fixtureAPIVersion, nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("listing the project's agents: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf( + "listing the project's agents returned %d; set AZURE_AI_EVAL_AGENT to name one", + resp.StatusCode) + } + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&listing); err != nil { + return "", err + } + for _, a := range listing.Data { + if a.Name != "" { + return a.Name, nil + } + } + return "", fmt.Errorf( + "this project has no agent in /agents, so an agent-target run cannot be built; " + + "deploy an agent or set AZURE_AI_EVAL_AGENT") +} + +func buildFixture(logf func(string, ...any)) (*evalFixture, error) { + ctx := context.Background() + + client, err := liveClient() + if err != nil { + return nil, fmt.Errorf("acquiring an azd credential: %w", err) + } + + agent, err := resolveFixtureAgent(ctx) + if err != nil { + return nil, err + } + logf("evaluating agent %q", agent) + + evaluatorName := "builtin.task_adherence" + evalID, err := createFixtureEval(ctx, client, evaluatorName) + if err != nil { + return nil, err + } + logf("created eval %s", evalID) + + first, err := startFixtureRun(ctx, client, evalID, agent, "first") + if err != nil { + return nil, err + } + second, err := startFixtureRun(ctx, client, evalID, agent, "second") + if err != nil { + return nil, err + } + logf("started runs %s and %s", first, second) + + // Polled together: they are independent, and serialising them doubles the + // slowest part of the suite for nothing. + errs := make(chan error, 2) + for _, runID := range []string{first, second} { + go func(id string) { errs <- awaitCompleted(ctx, client, evalID, id, logf) }(runID) + } + for range 2 { + if err := <-errs; err != nil { + return nil, err + } + } + + return &evalFixture{ + // The criterion is named without the builtin. prefix, and that is the + // name results are reported under. + EvaluatorName: strings.TrimPrefix(evaluatorName, "builtin."), + EvalID: evalID, + AgentName: agent, + FirstRunID: first, + SecondRunID: second, + }, nil +} + +func createFixtureEval( + ctx context.Context, + client *eval_api.EvalClient, + evaluatorName string, +) (string, error) { + criterionName := strings.TrimPrefix(evaluatorName, "builtin.") + + var group *eval_api.OpenAIEval + if err := retryCredentialFlake(func() error { + var err error + group, err = client.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azdcli-fixture"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: criterionName, + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": fixtureModel(), + "deployment_name": fixtureModel(), + }, + }}, + }) + return err + }); err != nil { + return "", fmt.Errorf("creating the fixture eval: %w", err) + } + deferTeardown(func() { + _ = client.DeleteOpenAIEval(context.Background(), group.ID) + }) + return group.ID, nil +} + +func startFixtureRun( + ctx context.Context, + client *eval_api.EvalClient, + evalID, agentName, label string, +) (string, error) { + rows := make([]map[string]any, 0, len(fixtureQueries)) + for _, q := range fixtureQueries { + rows = append(rows, map[string]any{"query": q}) + } + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent(rows) + + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("azdcli-" + label), + DataSource: ds, + }) + return err + }); err != nil { + return "", fmt.Errorf("starting the %s run: %w", label, err) + } + return run.ID, nil +} + +var terminalRunStatus = map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, +} + +// awaitCompleted requires the run to have scored something. +// +// A run whose every sample errors still reports completed, so the status alone +// would let the whole suite run against an eval that measured nothing. +func awaitCompleted( + ctx context.Context, + client *eval_api.EvalClient, + evalID, runID string, + logf func(string, ...any), +) error { + deadline := time.Now().Add(15 * time.Minute) + for { + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.GetOpenAIEvalRun(ctx, evalID, runID) + return err + }); err != nil { + return fmt.Errorf("polling run %s: %w", runID, err) + } + if terminalRunStatus[strings.ToLower(run.Status)] { + if strings.ToLower(run.Status) != "completed" { + return fmt.Errorf("run %s finished as %q: %s", runID, run.Status, run.Failure()) + } + if run.ResultCounts == nil { + return fmt.Errorf("run %s completed without reporting counts", runID) + } + if run.ResultCounts.Passed+run.ResultCounts.Failed == 0 { + return fmt.Errorf( + "run %s completed without scoring any row (errored=%d); the fixture "+ + "would prove nothing", runID, run.ResultCounts.Errored) + } + logf("run %s completed: passed=%d failed=%d errored=%d", + runID, run.ResultCounts.Passed, run.ResultCounts.Failed, run.ResultCounts.Errored) + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("run %s did not finish in time (last status %q)", runID, run.Status) + } + time.Sleep(10 * time.Second) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go new file mode 100644 index 00000000000..3b7c80b7cd4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// Generation submits a job that costs model time and takes minutes, so what is +// exercised here is everything up to that point: the flag combinations each +// command refuses and the spec it parses. No test here submits a job. +// +// There is one command per artifact, so nothing suppresses anything: a caller +// who already has a dataset simply does not run `dataset generate`. + +// TestCLIGenerateRefusesBadFlagCombinations covers the mistakes that must cost +// nothing to make. Each is decided locally, so a user finds out before a job is +// billed. +func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { + dir := t.TempDir() + instruction := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(instruction, []byte("test refunds"), 0o600)) + + cases := []struct { + name string + args []string + want string + }{{ + name: "the two instruction sources are mutually exclusive", + args: []string{"dataset", "generate", "d", "--target", "a", + "--agent-instruction", "inline", "--agent-instruction-file", instruction}, + want: "agent-instruction-file", + }, { + name: "below the minimum sample size", + args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "14"}, + want: "between 15 and 1000", + }, { + name: "above the maximum sample size", + args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "1001"}, + want: "between 15 and 1000", + }, { + name: "a missing instruction file names the flag", + args: []string{"dataset", "generate", "d", "--target", "a", + "--agent-instruction-file", filepath.Join(dir, "absent.md")}, + want: "--agent-instruction-file", + }, { + name: "generating a dataset needs a model deployment", + args: []string{"dataset", "generate", "d", "--target", "a", "--agent-instruction", "inline"}, + want: "--generation-model", + }, { + name: "generating an evaluator needs a model deployment", + args: []string{"evaluator", "generate", "e", "--target", "a", "--agent-instruction", "inline"}, + want: "--generation-model", + }} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := requireFailure(t, runIn(t, dir, tc.args...)) + require.Contains(t, r.Combined(), tc.want) + }) + } +} + +// TestCLIGenerateNamesTheArtifact pins the positional argument. Without it the +// name would come from the spec, and two runs would quietly overwrite the same +// artifact. +func TestCLIGenerateNamesTheArtifact(t *testing.T) { + for _, group := range []string{"dataset", "evaluator"} { + t.Run(group, func(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), group, "generate")) + require.Contains(t, r.Combined(), "accepts 1 arg") + }) + } +} + +// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with nothing to +// prompt with, the process has to end saying which flag to pass. +// +// The target is no longer among them — it is read from the eval's declaration — +// but the generation model has no other source, so it is the one input a bare +// directory cannot supply. +func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) + require.Contains(t, r.Combined(), "--generation-model") +} + +// TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not +// share settings that only one of them can honour. A sample count means nothing +// to a rubric, and a trace window means nothing to a synthetic dataset; either +// would be accepted and dropped. +func TestCLIGenerateFlagsAreScopedToTheirArtifact(t *testing.T) { + dir := t.TempDir() + + r := requireFailure(t, runIn(t, dir, "evaluator", "generate", "e", + "--target", "a", "--max-samples", "20")) + require.Contains(t, r.Combined(), "max-samples", + "--max-samples belongs to dataset generate") + + r = requireFailure(t, runIn(t, dir, "dataset", "generate", "d", + "--target", "a", "--trace-days", "7")) + require.Contains(t, r.Combined(), "trace-days", + "--trace-days belongs to evaluator generate") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go new file mode 100644 index 00000000000..6751ddd8bde --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// The CI path: start a run without waiting, read the handoff, come back for +// the result later. Everything here is what a pipeline does, so it is driven +// through the binary exactly the way a pipeline would. + +package cli + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCLIStartNoWaitEmitsTheHandoff pins the JSON a pipeline reads. +// +// A script captures the run id here and reattaches to it in a later step, so +// the field names are a contract. Emitting the service's run object instead +// would make that script depend on a shape this extension does not control. +func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, + "run", "start", "--eval", f.EvalID, "--no-wait", "-o", "json")) + + var handoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + } + r.JSON(t, &handoff) + + require.NotEmpty(t, handoff.RunID, "a pipeline has nothing to reattach to without run_id") + assert.Equal(t, f.EvalID, handoff.EvalID) + assert.NotEmpty(t, handoff.Status) + + // Started, not finished: this is the whole point of --no-wait, and a + // command that quietly blocked would pass every other assertion here. + assert.NotEqual(t, "completed", handoff.Status) + + deferTeardown(func() { + runQuietly("run", "cancel", handoff.RunID, "--eval", f.EvalID) + }) + + // The id it handed back has to be one the next step can use. + shown := requireSuccess(t, run(t, + "run", "show", handoff.RunID, "--eval", f.EvalID, "-o", "json")) + var reattached struct { + ID string `json:"id"` + } + shown.JSON(t, &reattached) + assert.Equal(t, handoff.RunID, reattached.ID, + "the run id in the handoff must be the one `run show` resolves") +} + +// TestCLIStartNoWaitTellsAPersonHowToReattach covers the same path without +// -o json, where what matters is that the printed command is one that works +// rather than a sentence containing a placeholder. +func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "run", "start", "--eval", f.EvalID, "--no-wait")) + + assert.Contains(t, r.Stdout, "Reattach with: azd ai eval run show") + assert.Contains(t, r.Stdout, f.EvalID, + "the reattach line must carry the eval id, not a placeholder for it") + assert.NotContains(t, r.Stdout, "<", + "nothing printed for a person to copy may contain a placeholder") + + var runID string + for _, field := range strings.Fields(r.Stdout) { + if strings.HasPrefix(field, "evalrun_") { + runID = field + break + } + } + require.NotEmpty(t, runID, "the run id must be printed:\n%s", r.Stdout) + deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval", f.EvalID) }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go new file mode 100644 index 00000000000..1557ea4aa0d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package cli drives the built binary as a subprocess. +// +// The other live tests call the client layer directly, which proves the API +// paths work but says nothing about the command surface on top of them: flag +// parsing, mutual exclusion, prompting, --no-prompt, exit codes, the rendered +// tables, and whether -o json emits what a script can actually consume. Those +// are the parts a user touches, and until now nothing exercised them against a +// real service. +// +// go test -tags live -v ./tests/cli/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var ( + binaryPath string + endpoint string +) + +// TestMain builds the extension once so every test runs the same binary a user +// would, rather than an in-process command tree that skips main's wiring. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + fmt.Fprintln(os.Stderr, "set AZURE_AI_EVAL_E2E_LIVE=1 to run the CLI tests") + os.Exit(0) + } + + endpoint = strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + fmt.Fprintln(os.Stderr, "FOUNDRY_PROJECT_ENDPOINT is required") + os.Exit(1) + } + + dir, err := os.MkdirTemp("", "azdeval-cli") + if err != nil { + fmt.Fprintf(os.Stderr, "creating a temp dir: %v\n", err) + os.Exit(1) + } + defer os.RemoveAll(dir) + + binaryPath = filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binaryPath, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "building the extension: %v\n%s\n", err, out) + os.Exit(1) + } + + code := m.Run() + // The shared eval outlives any single test, so it cannot be released with + // t.Cleanup without taking it away from the tests that run after. + runTeardown() + os.RemoveAll(dir) + os.Exit(code) +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// result is one invocation of the binary. +type result struct { + Args []string + Stdout string + Stderr string + ExitCode int +} + +// Combined is stdout and stderr together, for assertions that do not care +// which stream carried the message. +func (r result) Combined() string { return r.Stdout + r.Stderr } + +// JSON decodes stdout, failing the test when the command did not emit +// something a script could consume. +func (r result) JSON(t *testing.T, into any) { + t.Helper() + require.NoError(t, json.Unmarshal([]byte(r.Stdout), into), + "-o json must emit parseable JSON on stdout; got:\n%s", r.Stdout) +} + +// run invokes the binary with the project endpoint already supplied. +func run(t *testing.T, args ...string) result { + t.Helper() + return runIn(t, "", args...) +} + +// credentialFlake is azd's token helper failing under rapid sequential calls. +// +// Every invocation here is a fresh process, so each one shells out to azd for +// a token, and azd intermittently exits non-zero doing it. Retrying is safe +// because no request was made, and the alternative is a suite that fails on a +// different test each run for a reason that has nothing to do with the code. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// runIn invokes the binary with a working directory, for commands that write +// files. +func runIn(t *testing.T, dir string, args ...string) result { + t.Helper() + + res := invoke(t, dir, args...) + for attempt := 0; attempt < 2 && strings.Contains(res.Combined(), credentialFlake); attempt++ { + t.Logf("azd credential flaked; retrying `%s`", strings.Join(args, " ")) + time.Sleep(2 * time.Second) + res = invoke(t, dir, args...) + } + require.NotContains(t, res.Combined(), credentialFlake, + "azd could not produce a token after retries; run `azd auth login` and try again") + return res +} + +func invoke(t *testing.T, dir string, args ...string) result { + t.Helper() + + full := append([]string{}, args...) + if !hasFlag(args, "--project-endpoint") && needsEndpoint(args) { + full = append(full, "--project-endpoint", endpoint) + } + + cmd := exec.Command(binaryPath, full...) + if dir != "" { + cmd.Dir = dir + } + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + code := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("could not run %v: %v", full, err) + } + + res := result{Args: full, Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: code} + t.Logf("$ azd ai eval %s -> exit %d", strings.Join(args, " "), res.ExitCode) + return res +} + +func hasFlag(args []string, flag string) bool { + for _, a := range args { + if a == flag { + return true + } + } + return false +} + +// needsEndpoint keeps --project-endpoint off the commands that reject it. +func needsEndpoint(args []string) bool { + for _, a := range args { + switch a { + case "init", "--help", "-h": + return false + } + } + return true +} + +// requireSuccess fails with the command's own output, which is what a user +// would have seen. +func requireSuccess(t *testing.T, r result) result { + t.Helper() + require.Equalf(t, 0, r.ExitCode, + "expected `%s` to succeed\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +// requireFailure asserts a non-zero exit, so a command that silently succeeds +// where it should refuse is caught. +func requireFailure(t *testing.T, r result) result { + t.Helper() + require.NotEqualf(t, 0, r.ExitCode, + "expected `%s` to fail\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go new file mode 100644 index 00000000000..c104f369e46 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// A rubric is the other kind of evaluator: a JSON file of weighted dimensions, +// graded by a judge model rather than by code. It shares nothing with the code +// path on the wire beyond the route, so publishing one had never been +// exercised against a real project. + +// writeRubric lays down a rubric file and returns its path. +func writeRubric(t *testing.T, dimensions string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, + []byte(`{"dimensions":`+dimensions+`}`), 0o600)) + return path +} + +// evaluatorDocument is what `evaluator show` prints. +type evaluatorDocument struct { + Name string `json:"name"` + Version string `json:"version"` + EvaluatorType string `json:"evaluator_type"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Description string `json:"description"` + Weight int `json:"weight"` + } `json:"dimensions"` + DataSchema map[string]any `json:"data_schema"` + InitParameters map[string]any `json:"init_parameters"` + Metrics map[string]any `json:"metrics"` + } `json:"definition"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels"` +} + +// TestCLIRubricRoundTrip publishes a rubric, reads it back, and republishes it. +func TestCLIRubricRoundTrip(t *testing.T) { + name := uniqueName("azdcli_rubric") + rubric := writeRubric(t, `[ + {"id":"tone","description":"Is the answer polite?","weight":5}, + {"id":"accuracy","description":"Is the answer correct?","weight":10} + ]`) + + created := requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", rubric)) + require.Contains(t, created.Stdout, "version 1") + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", name)) + var doc evaluatorDocument + shown.JSON(t, &doc) + + require.Equal(t, name, doc.Name) + require.Equal(t, "1", doc.Version) + require.Equal(t, "custom", doc.EvaluatorType) + require.Equal(t, "rubric", doc.Definition.Type, + "the discriminator is what tells the service which definition kind it holds") + + require.Len(t, doc.Definition.Dimensions, 2) + byID := map[string]int{} + for _, d := range doc.Definition.Dimensions { + byID[d.ID] = d.Weight + require.NotEmpty(t, d.Description, "a dimension's description is what the judge grades against") + } + require.Equal(t, 5, byID["tone"]) + require.Equal(t, 10, byID["accuracy"]) + + // The rubric named only dimensions. Everything else is filled in by the + // service, and a caller reading the definition back gets those defaults + // rather than what was sent — including the judge model the evaluator will + // require at run time. + require.NotEmpty(t, doc.Definition.DataSchema, + "the service supplies a rubric's data schema; the author never writes one") + require.NotEmpty(t, doc.Definition.InitParameters) + require.NotEmpty(t, doc.Definition.Metrics) + require.NotEmpty(t, doc.SupportedEvaluationLevels) + + // Every registration publishes a new immutable version, which is what + // `update` means for an evaluator. + republished := requireSuccess(t, run(t, "evaluator", "update", name, "--from-file", rubric)) + require.Contains(t, republished.Stdout, "version 2", + "updating must advance the version rather than overwrite") + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "2") + }) + + // The earlier version stays reachable, which is what makes a published + // version safe to reference from a config. + pinned := requireSuccess(t, run(t, "evaluator", "show", name, "--version", "1")) + var first evaluatorDocument + pinned.JSON(t, &first) + require.Equal(t, "1", first.Version) +} + +// TestCLIRubricWeightMustBeAnIntegerFromOneToTen covers the validation a +// hand-authored rubric is most likely to trip. +// +// The service runs two separate checks and they answer differently: a +// fractional weight is rejected for not being an integer, an out-of-range one +// for being out of range. Both are asserted because a caller only ever sees +// one of them, and both have to say what a legal weight is. +func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { + cases := []struct { + name string + weight string + }{ + {"fractional", "2.5"}, + {"zero", "0"}, + {"above ten", "11"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rubric := writeRubric(t, + `[{"id":"tone","description":"Is the answer polite?","weight":`+tc.weight+`}]`) + + r := requireFailure(t, run(t, "evaluator", "create", + uniqueName("azdcli_badweight"), "--from-file", rubric)) + require.Contains(t, r.Combined(), "between 1 and 10", + "the refusal must say what a legal weight is") + }) + } + + // A weight the service accepts, so the cases above are failing on the + // weight rather than on the rubric shape they share. + name := uniqueName("azdcli_goodweight") + ok := writeRubric(t, `[{"id":"tone","description":"Is the answer polite?","weight":1}]`) + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", ok)) + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "1") + }) +} + +// TestCLIRubricNeedsDimensions covers the local check, which costs nothing and +// names the field the service would not. +func TestCLIRubricNeedsDimensions(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte(`{"criteria":[]}`), 0o600)) + + r := requireFailure(t, run(t, "evaluator", "create", + uniqueName("azdcli_nodims"), "--from-file", path)) + require.Contains(t, r.Combined(), "dimensions") +} + +// TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show` emits JSON a +// script can consume, whatever the definition kind. It renders the service's +// body rather than a typed struct, so nothing else pins that it stays parseable. +func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { + name := uniqueName("azdcli_rubricdoc") + + // The wrapped form: a whole evaluator document rather than a bare + // definition. Both are accepted, and generated rubrics arrive wrapped. + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte( + `{"name":"ignored","definition":{"dimensions":[{"id":"tone","description":"polite","weight":3}]}}`, + ), 0o600)) + + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", path)) + t.Cleanup(func() { + run(t, "evaluator", "delete", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", name)) + var raw map[string]any + require.NoError(t, json.Unmarshal([]byte(shown.Stdout), &raw), + "evaluator show must emit parseable JSON:\n%s", shown.Stdout) + + // The flag names the evaluator, so a name inside the file must not win. + require.Equal(t, name, raw["name"], + "--name must decide the evaluator's name, not the document's own field") + require.NotContains(t, strings.ToLower(shown.Stdout), `"name": "ignored"`) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go new file mode 100644 index 00000000000..3d68a583083 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +type runSummary struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + ResultCounts *struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` +} + +func TestCLIRunList(t *testing.T) { + f := sharedEval(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID)) + for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, f.SecondRunID) + require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout, + "the listing must summarise each run's counts, not just its status") + }) + + t.Run("json", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "-o", "json")) + require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), + "a list must be a bare array, not the service's envelope") + + var runs []runSummary + r.JSON(t, &runs) + require.GreaterOrEqual(t, len(runs), 2) + + byID := map[string]runSummary{} + for _, entry := range runs { + byID[entry.ID] = entry + } + first, ok := byID[f.FirstRunID] + require.True(t, ok, "the eval's own run is missing from its listing") + require.Equal(t, "completed", first.Status) + require.NotNil(t, first.ResultCounts) + require.Equal(t, len(fixtureQueries), + first.ResultCounts.Passed+first.ResultCounts.Failed, + "every dataset row must be accounted for by a verdict") + }) + + // The client has always taken a limit; until recently the command did not + // expose one, so a service-side truncation would have passed unnoticed. + t.Run("limit", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) + var runs []runSummary + r.JSON(t, &runs) + require.Len(t, runs, 1, "--limit must reach the service") + }) + + t.Run("unknown eval is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "list", "--eval", "eval_azdcli_no_such_eval")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") + }) +} + +func TestCLIRunShow(t *testing.T) { + f := sharedEval(t) + + t.Run("by run id", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval", f.EvalID)) + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, "status") + require.Contains(t, r.Stdout, "completed") + require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout) + require.Contains(t, r.Stdout, "report") + }) + + // Without --run-id the command has to pick one, and outside an azd + // environment there is no remembered id to fall back on, so what is + // exercised is the listing path. + t.Run("defaults to the most recent run", func(t *testing.T) { + listed := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) + var newest []runSummary + listed.JSON(t, &newest) + require.Len(t, newest, 1) + + r := requireSuccess(t, run(t, "run", "show", "--eval", f.EvalID, "-o", "json")) + var shown runSummary + r.JSON(t, &shown) + require.Equal(t, newest[0].ID, shown.ID, + "the default must be the run the listing puts first") + }) + + // A remembered run that no longer resolves falls through to the eval's + // latest, but one named explicitly must not: silently showing a different + // run than the one asked for is worse than saying it is gone. + // + // Only the substitution is asserted. Unlike `run list` and `run delete`, + // this path does not shorten the service's body, so the message runs to + // about 1700 characters of raw JSON — recorded in the report rather than + // pinned here, since pinning it would make the length a requirement. + t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope", + "the failure must name the run that was asked for") + require.NotContains(t, r.Combined(), f.FirstRunID, + "an explicit --run-id must not fall back to another run") + }) +} + +// TestCLIRunCancelAndDelete covers both halves of cancel, and the delete that +// follows it, against a single in-flight run: each run costs a minute of +// service time, so the two happy paths share one. +// +// The service answers a cancel on a finished run with success, so without the +// guard the command would tell a user it had stopped something it had not. +func TestCLIRunCancelAndDelete(t *testing.T) { + f := sharedEval(t) + + t.Run("a finished run is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "already finished") + require.Contains(t, r.Combined(), "completed") + }) + + // Delete is covered as far as the service honours it. + // + // The removal itself is not asserted, because it does not happen: the + // service accepts the DELETE and the run is still readable by id and still + // in the listing minutes later. What is asserted instead is that the + // command reaches the right resource — a real run is accepted, an unknown + // one is refused — which is the part that would break if the route or the + // id handling regressed. + t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { + runID := startCancellableRun(t, f) + + cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval", f.EvalID)) + require.Contains(t, cancelled.Stdout, runID) + require.Contains(t, cancelled.Stdout, "is now") + + shown := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) + var after runSummary + shown.JSON(t, &after) + require.NotEqual(t, "completed", after.Status, + "a cancelled run must not go on to complete") + + deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval", f.EvalID)) + require.Contains(t, deleted.Stdout, "Deleted run") + require.Contains(t, deleted.Stdout, runID) + + still := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) + var survivor runSummary + still.JSON(t, &survivor) + t.Logf("the run is still readable after a successful delete (status %q); "+ + "the service accepts the request without removing anything", survivor.Status) + }) + + // Deleting is not undoable, so the id is required rather than defaulted to + // whichever run happens to be newest. + t.Run("delete requires the run id", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "accepts 1 arg") + }) + + t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval", f.EvalID)) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope") + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + }) +} + +// startCancellableRun adds a run to the fixture's eval and returns it before it +// can finish. +// +// An agent-target run invokes the agent once per row and is judged after that, +// which takes far longer than the second it takes to issue the cancel; a run +// that finished first would turn the cancel test into an assertion about the +// guard it is not testing. +func startCancellableRun(t *testing.T, f *evalFixture) string { + t.Helper() + + client, err := liveClient() + require.NoError(t, err) + + runID, err := startFixtureRun(context.Background(), client, f.EvalID, f.AgentName, "cancelme") + require.NoError(t, err, "starting a run to cancel") + t.Cleanup(func() { + _ = client.DeleteOpenAIEvalRun(context.Background(), f.EvalID, runID) + }) + return runID +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go new file mode 100644 index 00000000000..181cc5df69b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -0,0 +1,226 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/csv" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// The fixture is judged by a model, so no test here may assert how many rows +// passed. What is under test is the command, and the properties that hold +// whatever the judge decided: every dataset row comes back, every row carries a +// verdict and a score, and filtering by verdict returns a subset that agrees +// with the totals. + +// resultsPayload is what `results show -o json` emits: the run and the rows. +type resultsPayload struct { + Run struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` + PerTestingCriteria []struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"per_testing_criteria_results"` + } `json:"run"` + OutputItems []struct { + ID string `json:"id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item"` + Results []struct { + Name string `json:"name"` + Score *float64 `json:"score"` + Passed bool `json:"passed"` + } `json:"results"` + } `json:"output_items"` +} + +// TestCLIResultsShowRendersTheRows is the difference between `results show` and +// `run show`: the totals say how many failed, these say which. +func TestCLIResultsShowRendersTheRows(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval", f.EvalID)) + + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, "Totals:") + require.Contains(t, r.Stdout, "CRITERION") + require.Contains(t, r.Stdout, f.EvaluatorName) + + // One row per evaluated sample, which is what makes "how many should I go + // and look at" answerable by counting lines. + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.NotContains(t, r.Stdout, "EVALUATOR ", + "a per-verdict table would list a sample once per evaluator") + + // The fixture's rows all pass, so every row names no failing evaluator. + require.Contains(t, r.Stdout, "Report:") +} + +func TestCLIResultsShowJSON(t *testing.T) { + f := sharedEval(t) + + payload := resultsFor(t, f.EvalID, f.FirstRunID) + + require.Equal(t, f.FirstRunID, payload.Run.ID) + require.Equal(t, "completed", payload.Run.Status) + require.Equal(t, len(fixtureQueries), payload.Run.ResultCounts.Total) + require.Zero(t, payload.Run.ResultCounts.Errored, + "an errored row means the fixture measured nothing") + + require.Len(t, payload.Run.PerTestingCriteria, 1) + require.Equal(t, f.EvaluatorName, payload.Run.PerTestingCriteria[0].TestingCriteria) + + // The rows are the reason this command exists, and a run reporting counts + // while returning none would still satisfy everything above. + require.Len(t, payload.OutputItems, len(fixtureQueries), + "every dataset row must come back as an item") + + passed := 0 + for _, item := range payload.OutputItems { + require.NotEmpty(t, item.DataSourceItem["query"], + "each row must carry the column it was evaluated on") + require.Len(t, item.Results, 1) + require.Equal(t, f.EvaluatorName, item.Results[0].Name) + require.NotNil(t, item.Results[0].Score, "a scored row must report its score") + if item.Results[0].Passed { + passed++ + } + } + require.Equal(t, payload.Run.ResultCounts.Passed, passed, + "the per-row verdicts must agree with the totals") +} + +// TestCLIResultsShowFailedOnly asserts the filter removes rows rather than +// merely relabelling them. +// +// The service has no verdict filter — its `status` selects on execution status, +// so `status=failed` returns errored rows, not failing ones — which makes this +// entirely the CLI's own work and worth testing directly. +func TestCLIResultsShowFailedOnly(t *testing.T) { + f := sharedEval(t) + + payload := resultsFor(t, f.EvalID, f.FirstRunID) + + // One rendered row is one evaluator's verdict on one sample, so the count + // to expect is failing *results*, not failing rows: a sample that fails two + // evaluators is two lines. `ResultCounts.Failed` answers the other question. + failing := 0 + for _, item := range payload.OutputItems { + for _, r := range item.Results { + if !r.Passed { + failing++ + } + } + } + + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, + "--eval", f.EvalID, "--failed-only")) + + if failing == 0 { + // Saying so is not the same as printing an empty table. + require.Contains(t, r.Stdout, "No failing rows.") + return + } + + require.NotContains(t, r.Stdout, " pass ", + "--failed-only must drop the rows that passed") + + // Matched on a word boundary so the per-criterion table's FAILED column + // header is not counted as a verdict. + verdicts := regexp.MustCompile(`\bFAIL\b`).FindAllString(r.Stdout, -1) + require.Equal(t, failing, len(verdicts), + "every failing verdict must appear exactly once:\n%s", r.Stdout) +} + +// resultsFor reads a run's results as JSON, which several tests need before +// they can decide what the rendered output should say. +func resultsFor(t *testing.T, evalID, runID string) resultsPayload { + t.Helper() + r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval", evalID, "-o", "json")) + var payload resultsPayload + r.JSON(t, &payload) + return payload +} + +func TestCLIResultsExport(t *testing.T) { + f := sharedEval(t) + + t.Run("json to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "json")) + + var exported struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"result_counts"` + } + r.JSON(t, &exported) + require.Equal(t, f.FirstRunID, exported.ID) + require.Equal(t, "completed", exported.Status) + require.Equal(t, len(fixtureQueries), exported.ResultCounts.Total) + }) + + t.Run("csv to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "csv")) + + rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() + require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) + require.Len(t, rows, 2, "a header and one row per criterion") + require.Equal(t, + []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + require.Equal(t, f.FirstRunID, rows[1][0]) + require.Equal(t, "completed", rows[1][1]) + require.Equal(t, f.EvaluatorName, rows[1][2]) + }) + + t.Run("output-file writes the path instead of stdout", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "results.csv") + + r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "csv", "--output-file", path)) + require.Empty(t, strings.TrimSpace(r.Stdout), + "--output-file redirects the payload; leaving it on stdout too would double it") + + body, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(body), "run_id,status,criterion,passed,failed") + require.Contains(t, string(body), f.FirstRunID) + }) + + t.Run("an unknown format is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval", f.EvalID, "--format", "xml")) + require.Contains(t, r.Combined(), "json or csv") + }) +} + +func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "run", "output", "list", "--eval", "eval_does_not_exist")) + require.Contains(t, r.Combined(), "eval_does_not_exist") + require.NotContains(t, r.Combined(), "RESPONSE 404") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go new file mode 100644 index 00000000000..6c02b3b8ecd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -0,0 +1,388 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build hero + +// Package hero drives the hero scenarios through real azd, with the extension +// installed the way a user installs it. +// +// The CLI suite in ../cli runs the extension binary directly, which covers the +// command surface but cannot reach `init`: `init` resolves the project and +// edits azure.yaml over azd's gRPC channel, so without azd hosting the process +// there is nothing on the other end. That is not a detail — it is the first +// command in Scenario 1 and the one that produces the local diff every later +// step depends on, and until now the only thing asserting its output was a +// unit test calling the scaffold function directly. A unit test cannot see the +// service entry azd writes, the detection that reads the project, or the +// terminal output the spec pins line for line. +// +// azd x pack --rebuild +// azd extension install azure.ai.evaluations --source local +// go test -tags hero -v ./tests/hero/... +package hero + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// reinstall is what to run when the installed extension is not this code. +// +// `azd x pack` rewrites the artifacts but leaves the checksum in the local +// registry alone when the version has not changed, so a plain reinstall then +// fails validation. Bumping the version in extension.yaml is the way through. +const reinstall = " azd x pack --rebuild\n" + + " azd extension uninstall azure.ai.evaluations\n" + + " azd extension install azure.ai.evaluations --source local\n" + +// TestMain refuses to run against an azd that cannot reach the extension, or +// that is hosting a different build of it. +// +// Skipping would be worse than failing here: these tests exist because nothing +// else covers the azd-hosted path, so a silent skip returns the suite to the +// state it was in before they were written. Running against a stale install is +// worse still — it reports on code that is not the code under test, which is +// the one outcome a test must never produce. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_HERO") != "1" { + fmt.Fprintf(os.Stderr, + "set AZURE_AI_EVAL_HERO=1 to run the hero scenarios. They need azd "+ + "hosting this extension:\n%s", reinstall) + os.Exit(0) + } + + hosted, err := exec.Command("azd", "ai", "eval", "init", "--help").CombinedOutput() + if err != nil || !strings.Contains(string(hosted), "Scaffold evaluation config") { + fmt.Fprintf(os.Stderr, + "azd cannot reach the evaluations extension. Install it first:\n%s\n%s\n", + reinstall, hosted) + os.Exit(1) + } + + if err := requireCurrentInstall(string(hosted)); err != nil { + fmt.Fprintf(os.Stderr, "%v\n\n%s", err, reinstall) + os.Exit(1) + } + + os.Exit(m.Run()) +} + +// requireCurrentInstall compares the installed extension's help against this +// working tree's, so a stale install fails loudly instead of quietly reporting +// on the wrong binary. +// +// Help text is the cheapest available fingerprint that actually moves: it +// carries every command and flag, which is what these tests assert on, and it +// costs one build rather than a version stamp nobody remembers to bump. +func requireCurrentInstall(hosted string) error { + dir, err := os.MkdirTemp("", "azdeval-hero") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + binary := filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binary, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + return fmt.Errorf("building this working tree to compare against: %v\n%s", err, out) + } + + local, err := exec.Command(binary, "init", "--help").CombinedOutput() + if err != nil { + return fmt.Errorf("reading this working tree's help: %w", err) + } + + if normalize(string(local)) != normalize(hosted) { + return fmt.Errorf( + "azd is hosting a different build of this extension.\n"+ + "installed:\n%s\nthis working tree:\n%s", + normalize(hosted), normalize(string(local))) + } + return nil +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// project writes a minimal azd project for `init` to attach to. +// +// It declares the two services detection reads — the Foundry project and the +// agent — because what `init` writes into azure.yaml depends on which of them +// exist, and a project with neither would exercise only the fallback. +func project(t *testing.T, agent string) string { + t.Helper() + dir := t.TempDir() + body := fmt.Sprintf(`name: support-app +services: + ai-project: + host: azure.ai.project + %s: + host: azure.ai.agent +`, agent) + require.NoError(t, os.WriteFile(filepath.Join(dir, "azure.yaml"), []byte(body), 0o600)) + return dir +} + +// azdEval runs the extension through azd, in dir. +func azdEval(t *testing.T, dir string, args ...string) (string, int) { + t.Helper() + + cmd := exec.Command("azd", append([]string{"ai", "eval"}, args...)...) + cmd.Dir = dir + var out strings.Builder + cmd.Stdout = &out + cmd.Stderr = &out + + code := 0 + if err := cmd.Run(); err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("could not run azd ai eval %v: %v", args, err) + } + code = exitErr.ExitCode() + } + + // azd prints its own upgrade notice to stderr, which is not the command's + // output and would break an exact comparison. + text := dropUpgradeNotice(out.String()) + t.Logf("$ azd ai eval %s -> exit %d\n%s", strings.Join(args, " "), code, text) + return text, code +} + +// dropUpgradeNotice removes azd's "Update available" banner and everything +// after it, which azd appends regardless of the command. +func dropUpgradeNotice(s string) string { + if i := strings.Index(s, "Update available:"); i >= 0 { + s = s[:i] + } + return strings.TrimRight(s, " \r\n\t") +} + +// normalize makes terminal output comparable across platforms. +func normalize(s string) string { + return strings.ReplaceAll(dropUpgradeNotice(s), "\r\n", "\n") +} + +// TestHeroScenario1ColdStart is the first half of Scenario 1: the offline +// baseline `init` writes, asserted against the terminal block the spec shows. +// +// The output is compared whole rather than by keyword. Every line of it is a +// promise the spec makes to a reader deciding whether to adopt this — which +// files appear, what was detected, what to run next — and a keyword assertion +// would pass while the reader's terminal said something else. +func TestHeroScenario1ColdStart(t *testing.T) { + const ( + agent = "support-agent" + judge = "gpt-5.6-luna" + ) + dir := project(t, agent) + + // The evaluator and judge are passed rather than prompted for, because the + // spec's two `?` lines are answers to prompts and a test has no terminal to + // answer them at. + out, code := azdEval(t, dir, "init", + "--target", agent, "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", judge) + require.Zero(t, code, "init makes no service calls, so nothing can fail it here") + + want := `(✓) Done: Detected agent target: support-agent +(✓) Done: Using data source: traces (Application Insights) +(✓) Done: Judge model deployment: gpt-5.6-luna + +Created + evals/eval.yaml evaluation configuration + azure.yaml added service 'support-agent-evals' + +Next: azd up + azd ai eval run start` + + require.Equal(t, want, normalize(out)) +} + +// Scenario 1's second half: the eval.yaml the terminal block promised. The spec +// prints this file, so its shape is as much a promise as the output above — +// and it is the file a reader reviews before running `azd up`. +func TestHeroScenario1WritesTheDocumentedConfig(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", + "--target", "support-agent", "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", "gpt-5.6-luna") + require.Zero(t, code) + + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + require.NoError(t, err) + text := string(body) + + require.Contains(t, text, "name: support-agent-trace-eval") + require.Contains(t, text, "type: traces") + require.Contains(t, text, "agent_name: support-agent", + "a trace run has no target, so agent_name is what scopes it") + require.Contains(t, text, "max_traces: 20", + "a first run is bounded rather than taking the service default of 1000") + require.Contains(t, text, "evaluator: builtin.task_adherence") + require.Contains(t, text, "model: gpt-5.6-luna", + "the judge is written per evaluator reference as initialization_parameters.model") + + require.NotContains(t, text, "datasets:", + "there is no file to register, so the catalog is absent rather than empty") + require.NotContains(t, text, "target:", + "a trace run invokes nothing") +} + +// `init` is offline, and being offline is the property that makes its output a +// reviewable local diff. A service call here would also make the command fail +// for a user who has not authenticated yet, which is exactly when they run it. +func TestHeroInitMakesNoServiceCalls(t *testing.T) { + dir := project(t, "support-agent") + + cmd := exec.Command("azd", "ai", "eval", "init", + "--target", "support-agent", "--evaluator", "builtin.task_adherence", + "--judge-model", "m") + cmd.Dir = dir + // A proxy pointing nowhere fails any outbound request, so a command that + // stays offline is unaffected and one that does not cannot be mistaken for + // working. + cmd.Env = append(os.Environ(), + "HTTPS_PROXY=http://127.0.0.1:9", + "HTTP_PROXY=http://127.0.0.1:9", + "NO_PROXY=", + ) + + out, err := cmd.CombinedOutput() + require.NoError(t, err, "init must not need the network:\n%s", out) +} + +// The eval service has to be declared in azure.yaml before azd will act on it. +// Printing the block and leaving the edit to the reader was enough, once, to +// make the documented flow stop working between `init` and `azd up`. +func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m") + require.Zero(t, code) + + root, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + text := string(root) + + require.Contains(t, text, "support-agent-evals:", + "the service is named for the agent it evaluates") + require.Contains(t, text, "host: azure.ai.eval") + require.Contains(t, text, "$ref: ./evals/eval.yaml") + + // azd owns the edit, so everything the project already declared survives it. + require.Contains(t, text, "name: support-app") + require.Contains(t, text, "host: azure.ai.project") + require.Contains(t, text, "host: azure.ai.agent") + + // The eval reads both, so azd has to deploy both first. + require.Regexp(t, `(?s)support-agent-evals:.*uses:.*ai-project.*support-agent`, text) +} + +// Running `init` twice must not deploy the same eval twice. The service key is +// the eval's name, so the second run recognizes its own work. +func TestHeroInitIsIdempotent(t *testing.T) { + dir := project(t, "support-agent") + args := []string{"init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m"} + + _, code := azdEval(t, dir, args...) + require.Zero(t, code) + first, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + + out, code := azdEval(t, dir, args...) + require.NotZero(t, code, "the scaffold already exists, so a second run must refuse") + require.Contains(t, out, "--force", "the refusal has to say how to proceed") + + second, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, string(first), string(second), + "a refused init must not have edited the project") + + // With --force the files are rewritten, and the service is still declared + // exactly once. + out, code = azdEval(t, dir, append(args, "--force")...) + require.Zero(t, code, out) + + third, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, 1, strings.Count(string(third), "host: azure.ai.eval"), + "a second eval service would deploy the same eval twice") + require.Contains(t, normalize(out), "already declares service 'support-agent-evals'") +} + +// Evals attach to a project; they do not create one. Naming the command that +// makes a project is more use than a transport error from the gRPC channel +// that was not there. +func TestHeroInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + out, code := azdEval(t, dir, "init", "--target", "support-agent", "--no-prompt") + require.NotZero(t, code) + require.Contains(t, out, "azd init") + require.NotContains(t, strings.ToLower(out), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "a refused init must leave nothing behind") +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation — so the "next" steps must stop offering to generate one. +func TestHeroInitExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--judge-model", "m", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + + text := normalize(out) + require.NotContains(t, text, "evaluator generate", + "nothing was scheduled to be generated, so nothing should be suggested") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + require.NoError(t, err) + require.Contains(t, string(body), "evaluator: builtin.task_adherence") + require.NotContains(t, string(body), "support-agent-quality", + "the default rubric was replaced, not added to") +} + +// A supplied dataset is not generated either, so `init` has nothing left to +// suggest and must not send the reader to a command that would submit a job +// for an artifact they already have. +func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--judge-model", "m", + "--dataset", "prod-golden", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + + text := normalize(out) + require.NotContains(t, text, "dataset generate") + require.Contains(t, text, "Next: azd up", + "with nothing left to generate, the next step is the deploy") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + require.NoError(t, err) + require.Contains(t, string(body), "dataset: prod-golden") + require.NotContains(t, string(body), "source:", + "a registered dataset has nothing to upload") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go new file mode 100644 index 00000000000..32dda623b17 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -0,0 +1,422 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package live holds integration tests that talk to a real Foundry project. +// They are excluded from the default build by the `live` tag and additionally +// gated on AZURE_AI_EVAL_E2E_LIVE so an accidental run cannot create resources. +// +// go test -tags live -v ./tests/live/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +// +// Optional: +// +// AZURE_AI_EVAL_MODEL= (default gpt-4.1-nano) +// AZURE_AI_EVAL_AGENT= (enables the run phase) +package live + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +const ( + projectAPIVersion = "2025-11-15-preview" + defaultJudgeModel = "gpt-4.1-nano" + sampleDatasetContent = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +{"query":"Can I change my shipping address after ordering?"} +` +) + +type liveEnv struct { + endpoint string + judgeModel string + agentName string + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient +} + +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + // Works non-interactively when azd already holds a refresh token, + // which is what makes an unattended run possible. + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + }) + return sharedCred, sharedCredErr +} + +func setup(t *testing.T) *liveEnv { + t.Helper() + + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + + cred, err := liveCredential() + require.NoError(t, err, "acquiring an azd credential") + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = defaultJudgeModel + } + + return &liveEnv{ + endpoint: endpoint, + judgeModel: judge, + agentName: os.Getenv("AZURE_AI_EVAL_AGENT"), + evalClient: eval_api.NewEvalClient(endpoint, cred), + datasetClient: dataset_api.NewDatasetClient(endpoint, cred), + } +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().Unix()) +} + +// pickQualityEvaluator selects a built-in whose required inputs match the +// agent-target data mapping this extension sends. +// +// Built-ins do not share one input contract: builtin.ifeval, for example, +// requires an `instruction_id_list` field, and creating a group with it under +// the agent-target mapping fails with MissingRequiredDataMapping. The +// agent-target mapping supplies query, response, tool_calls and +// tool_definitions, so the evaluators below are the compatible set. +func pickQualityEvaluator(t *testing.T, available []eval_api.EvaluatorSummary) string { + t.Helper() + + preferred := []string{ + "builtin.task_adherence", + "builtin.task_completion", + "builtin.tool_call_accuracy", + } + present := map[string]bool{} + for _, e := range available { + present[e.Name] = true + } + for _, name := range preferred { + if present[name] { + return name + } + } + + names := make([]string, 0, len(available)) + for _, e := range available { + names = append(names, e.Name) + } + t.Skipf("no agent-target compatible evaluator found; available: %s", strings.Join(names, ", ")) + return "" +} + +// TestLiveBuiltinEvaluators is the cheapest reachability check: it proves the +// endpoint, credential, api-version, and auth scope are all correct without +// creating anything. +func TestLiveBuiltinEvaluators(t *testing.T) { + env := setup(t) + ctx := context.Background() + + list, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion, + ) + require.NoError(t, err, "listing built-in evaluators") + require.NotEmpty(t, list.Value, "the project should expose built-in evaluators") + + t.Logf("found %d built-in evaluators; first: %s", len(list.Value), list.Value[0].Name) +} + +// TestLiveDatasetLifecycle exercises the full pending-upload flow and asserts +// that re-registering the same name yields the next version rather than an error. +func TestLiveDatasetLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + dir := t.TempDir() + require.NoError(t, + os.WriteFile(filepath.Join(dir, "golden.jsonl"), []byte(sampleDatasetContent), 0o600)) + + name := uniqueName("azd-eval-e2e") + + first, err := env.datasetClient.UploadNewVersion(ctx, name, "", dir, projectAPIVersion) + require.NoError(t, err, "registering the first dataset version") + require.Equal(t, name, first.Name) + require.NotEmpty(t, first.Version) + t.Logf("registered %s version %s", first.Name, first.Version) + + t.Cleanup(func() { + // Best effort: leave nothing behind even if the test fails midway. + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, first.Version, projectAPIVersion) + }) + + fetched, err := env.datasetClient.GetDataset(ctx, name, first.Version, projectAPIVersion) + require.NoError(t, err, "reading the dataset back") + t.Logf("dataset uri: %q (empty means a credential call is required)", fetched.ResolvedBlobURI()) + + // The version listing is eventually consistent: it returns nothing for a + // second or two after a version is created, even though the version itself + // reads back fine. Poll rather than asserting on the first response. + var versions *dataset_api.DatasetList + require.Eventually(t, func() bool { + var err error + versions, err = env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) + return err == nil && versions != nil && len(versions.Value) > 0 + }, 30*time.Second, 2*time.Second, "the version listing never caught up") + require.Equal(t, first.Version, dataset_api.LatestVersion(versions.Value)) + + // A second upload must advance the version, not conflict. + second, err := env.datasetClient.UploadNewVersion( + ctx, name, first.Version, dir, projectAPIVersion) + require.NoError(t, err, "registering a second dataset version") + require.NotEqual(t, first.Version, second.Version, + "re-registering the same name must produce the next version") + t.Cleanup(func() { + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, second.Version, projectAPIVersion) + }) +} + +// TestLiveEvalLifecycle proves the create request this extension builds is +// accepted, which is the single most important contract to get right. +func TestLiveEvalLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value, "need at least one built-in evaluator") + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + threshold := 3.0 + req := &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-group"), + Metadata: map[string]string{"azd_source": "e2e"}, + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + "threshold": threshold, + }, + }}, + } + + group, err := env.evalClient.CreateOpenAIEval(ctx, req) + require.NoError(t, err, "creating the eval") + require.NotEmpty(t, group.ID, "the service assigns the id; name is not unique") + t.Logf("created eval %s (name %q)", group.ID, group.Name) + + fetched, err := env.evalClient.GetOpenAIEval(ctx, group.ID) + require.NoError(t, err, "reading the eval back") + require.Equal(t, group.ID, fetched.ID) +} + +// resolveAgent names the agent the run phase evaluates. +// +// AZURE_AI_EVAL_AGENT wins when set. Otherwise one is discovered, and failing +// to find one is a failure rather than a skip: skipping by default is how the +// agent-target path went unverified for weeks while the suite reported green. +// +// The listing is /agents, not /assistants. They are different collections and +// a project can have plenty of the latter and none of the former — an eval +// target resolves against /agents, so an assistant name is accepted by the +// request and then fails the run with "resources not found". +func resolveAgent(t *testing.T, env *liveEnv) string { + t.Helper() + + if env.agentName != "" { + return env.agentName + } + + cred, err := liveCredential() + require.NoError(t, err) + token, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + require.NoError(t, err, "acquiring a token to list agents") + + req, err := http.NewRequest(http.MethodGet, env.endpoint+"/agents?api-version="+projectAPIVersion, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err, "listing the project's agents") + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, + "could not list agents; set AZURE_AI_EVAL_AGENT to name one directly") + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&listing)) + + for _, a := range listing.Data { + if a.Name != "" { + t.Logf("no AZURE_AI_EVAL_AGENT set; evaluating %q", a.Name) + return a.Name + } + } + + t.Fatal("this project has no agent in /agents, so the agent-target run path " + + "cannot be verified here. Assistants do not count: an eval target " + + "resolves against /agents, and naming an assistant fails the run with " + + "\"resources not found\". Deploy an agent, or set AZURE_AI_EVAL_AGENT " + + "to one in another project") + return "" +} + +// TestLiveRun invokes a real agent, which is the only cover the agent-target +// run path has. +func TestLiveRun(t *testing.T) { + env := setup(t) + agentName := resolveAgent(t, env) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-run"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval for the run") + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("run"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + t.Cleanup(func() { + _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) + }) + + // A single sample is roughly 40 seconds; allow generous headroom. + deadline := time.Now().Add(10 * time.Minute) + terminal := map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, + } + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the run") + if terminal[strings.ToLower(current.Status)] { + t.Logf("run reached %s", current.Status) + if current.ResultCounts != nil { + t.Logf("counts: passed=%d failed=%d errored=%d", + current.ResultCounts.Passed, + current.ResultCounts.Failed, + current.ResultCounts.Errored) + } + body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") + t.Logf("per-criteria results: %s", string(body)) + + // Reaching a terminal state is not the same as having evaluated + // anything. A run whose every sample errors still reports + // "completed", so asserting only on the status would let the target + // or the evaluator break without the test noticing. + require.Equal(t, "completed", strings.ToLower(current.Status), + "the run must complete rather than fail or cancel") + require.NotNil(t, current.ResultCounts, "a completed run must report counts") + require.Zero(t, current.ResultCounts.Errored, + "an errored sample means the target or the evaluator did not run") + require.Positive(t, + current.ResultCounts.Passed+current.ResultCounts.Failed, + "the run must score at least one sample; a pass or a fail are both fine, "+ + "but scoring nothing means the data never reached the evaluator") + return + } + if time.Now().After(deadline) { + t.Fatalf("run %s did not finish within the deadline (last status %q)", + run.ID, current.Status) + } + time.Sleep(10 * time.Second) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go new file mode 100644 index 00000000000..a6b01729a5e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package live + +import ( + "context" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveRunCancel covers the one route whose meaning depends on the request +// body. POST on the run cancels it when the body is empty and updates its +// status and counters when it is not, so a stray body here would silently +// overwrite a run instead of stopping it. Only a live call can tell the two +// apart: both are the same method on the same path, and both return 200. +func TestLiveRunCancel(t *testing.T) { + env := setup(t) + agentName := resolveAgent(t, env) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-cancel"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval to cancel a run from") + + t.Cleanup(func() { + _ = env.evalClient.DeleteOpenAIEval(context.Background(), group.ID) + }) + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("cancel"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run to cancel") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + canceled, err := env.evalClient.CancelOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "cancelling the run") + require.NotNil(t, canceled) + t.Logf("cancel returned status %s", canceled.Status) + + // A sample takes roughly 40 seconds, so a run cancelled immediately after + // it starts should never reach completed. + deadline := time.Now().Add(5 * time.Minute) + var status string + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the cancelled run") + status = strings.ToLower(current.Status) + if status == "canceled" || status == "cancelled" { + break + } + require.NotEqual(t, "completed", status, + "the run completed instead of cancelling, so the empty-body POST did not cancel it") + require.False(t, time.Now().After(deadline), + "the run never reached a cancelled state; last status was %s", status) + time.Sleep(5 * time.Second) + } + + t.Logf("run reached %s", status) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt new file mode 100644 index 00000000000..ffbc9939864 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -0,0 +1 @@ +1.0.0-beta.1 diff --git a/eng/pipelines/release-ext-azure-ai-evaluations.yml b/eng/pipelines/release-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..329c4a2181e --- /dev/null +++ b/eng/pipelines/release-ext-azure-ai-evaluations.yml @@ -0,0 +1,45 @@ +# Continuous deployment trigger +trigger: + branches: + include: + - main + paths: + include: + - cli/azd/extensions/azure.ai.evaluations + - /eng/pipelines/templates/stages/release-azd-extension.yml + - /eng/pipelines/templates/jobs/build-azd-extension.yml + - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml + - /eng/pipelines/templates/variables/image.yml + +pr: + paths: + include: + - cli/azd/extensions/azure.ai.evaluations + - eng/pipelines/release-ext-azure-ai-evaluations.yml + - /eng/pipelines/templates/stages/release-azd-extension.yml + - eng/pipelines/templates/steps/publish-cli.yml + exclude: + - cli/azd/docs/** + +parameters: + - name: PublishToRegistry + displayName: Publish to registry + type: string + # Scheduled (nightly) runs override this in the shared templates; the runtime + # parameter default must be a literal because it renders before variables exist. + default: stable + values: + - stable + - dev + - nightly + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - template: /eng/pipelines/templates/stages/release-azd-extension.yml + parameters: + AzdExtensionId: azure.ai.evaluations + SanitizedExtensionId: azure-ai-evaluations + AzdExtensionDirectory: cli/azd/extensions/azure.ai.evaluations + PublishToRegistry: ${{ parameters.PublishToRegistry }}