From d68e564961ae3797f021e598fb52779a6e7e7cc4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 04:48:28 +0300 Subject: [PATCH 001/114] feat(azure.ai.evaluations): scaffold the Foundry evaluations extension New azd extension exposing azd ai eval, registering the azure.ai.eval service-target provider. Scaffold only: manifest, entrypoint, root command. --- .../azure.ai.evaluations/extension.yaml | 29 ++ .../extensions/azure.ai.evaluations/go.mod | 104 ++++++ .../extensions/azure.ai.evaluations/go.sum | 314 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 29 ++ .../extensions/azure.ai.evaluations/main.go | 14 + .../azure.ai.evaluations/version.txt | 1 + 6 files changed, 491 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/extension.yaml create mode 100644 cli/azd/extensions/azure.ai.evaluations/go.mod create mode 100644 cli/azd/extensions/azure.ai.evaluations/go.sum create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/main.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/version.txt diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml new file mode 100644 index 00000000000..2b8e8bbb58f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=../extension.schema.json +id: azure.ai.evaluations +namespace: ai.eval +displayName: Foundry evaluations (Beta) +description: Define and run Foundry evaluations from your terminal. (Beta) +usage: azd ai eval [options] +# NOTE: Make sure version.txt is in sync with this version. +version: 1.0.0-beta.1 +requiredAzdVersion: ">=1.27.1" +dependencies: + - id: azure.ai.projects + version: "~1.0.0-beta.3" +language: go +capabilities: + - custom-commands + - lifecycle-events + - service-target-provider + - metadata +providers: + - name: azure.ai.eval + type: service-target + description: Deploys evaluation datasets, evaluators, and eval groups to Foundry +examples: + - name: init + description: Scaffold evaluation config for an agent. + usage: azd ai eval init + - name: run + description: Run an evaluation and summarize the results. + usage: azd ai eval run diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod new file mode 100644 index 00000000000..d1099806c54 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -0,0 +1,104 @@ +module azureaieval + +go 1.26.4 + +require ( + github.com/azure/azure-dev/cli/azd v1.28.0 + github.com/fatih/color v1.18.0 + github.com/spf13/cobra v1.10.1 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b // indirect + github.com/alecthomas/chroma/v2 v2.20.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/braydonk/yaml v0.9.0 // indirect + github.com/buger/goterm v1.0.4 // indirect + github.com/buger/jsonparser v1.1.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.3.2 // indirect + github.com/charmbracelet/glamour v0.10.0 // indirect + github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 // indirect + github.com/charmbracelet/x/ansi v0.10.2 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cli/browser v1.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect + github.com/drone/envsubst v1.0.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gofrs/flock v0.12.1 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golobby/container/v3 v3.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jmespath-community/go-jmespath v1.1.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mailru/easyjson v0.9.1 // indirect + github.com/mark3labs/mcp-go v0.41.1 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect + github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect + github.com/microsoft/go-deviceid v1.0.0 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/theckman/yacspin v0.13.12 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yuin/goldmark v1.7.13 // indirect + github.com/yuin/goldmark-emoji v1.0.6 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cli/azd/extensions/azure.ai.evaluations/go.sum b/cli/azd/extensions/azure.ai.evaluations/go.sum new file mode 100644 index 00000000000..6d0434c4767 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/go.sum @@ -0,0 +1,314 @@ +code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= +github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 h1:0g4UTtvRA9goC37cmD9ZHdW6CCNJR4cOXBnHz0r4ubM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3/go.mod h1:fEiHi0sbYqbo3shUkIF1SNxm8GyeEJl+Poc/djOvbdE= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 h1:nnQ9vXH039UrEFxi08pPuZBE7VfqSJt343uJLw0rhWI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0/go.mod h1:4YIVtzMFVsPwBvitCDX7J9sqthSj43QD1sP6fYc1egc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b h1:g9SuFmxM/WucQFKTMSP+irxyf5m0RiUJreBDhGI6jSA= +github.com/adam-lavrik/go-imath v0.0.0-20210910152346-265a42a96f0b/go.mod h1:XjvqMUpGd3Xn9Jtzk/4GEBCSoBX0eB2RyriXgne0IdM= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw= +github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA= +github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg= +github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/azure/azure-dev/cli/azd v1.28.0 h1:mqqyV85m7A1XfWJFjV/Ut0QoIEImFeF++1Ruq/cRp0s= +github.com/azure/azure-dev/cli/azd v1.28.0/go.mod h1:Ge7QaU9PoJM7i6J0xArDoQCf2tUn6O7OIKkoItxFTA8= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/braydonk/yaml v0.9.0 h1:ewGMrVmEVpsm3VwXQDR388sLg5+aQ8Yihp6/hc4m+h4= +github.com/braydonk/yaml v0.9.0/go.mod h1:hcm3h581tudlirk8XEUPDBAimBPbmnL0Y45hCRl47N4= +github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= +github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/colorprofile v0.3.2 h1:9J27WdztfJQVAQKX2WOlSSRB+5gaKqqITmrvb1uTIiI= +github.com/charmbracelet/colorprofile v0.3.2/go.mod h1:mTD5XzNeWHj8oqHb+S1bssQb7vIHbepiebQ2kPKVKbI= +github.com/charmbracelet/glamour v0.10.0 h1:MtZvfwsYCx8jEPFJm3rIBFIMZUfUJ765oX8V6kXldcY= +github.com/charmbracelet/glamour v0.10.0/go.mod h1:f+uf+I/ChNmqo087elLnVdCiVgjSKWuXa/l6NU2ndYk= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834 h1:ZR7e0ro+SZZiIZD7msJyA+NjkCNNavuiPBLgerbOziE= +github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834/go.mod h1:aKC/t2arECF6rNOnaKaVU6y4t4ZeHQzqfxedE/VkVhA= +github.com/charmbracelet/x/ansi v0.10.2 h1:ith2ArZS0CJG30cIUfID1LXN7ZFXRCww6RUvAPA+Pzw= +github.com/charmbracelet/x/ansi v0.10.2/go.mod h1:HbLdJjQH4UH4AqA2HpRWuWNluRE6zxJH/yteYEYCFa8= +github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= +github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= +github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489 h1:a5q2sWiet6kgqucSGjYN1jhT2cn4bMKUwprtm2IGRto= +github.com/charmbracelet/x/exp/slice v0.0.0-20251008171431-5d3777519489/go.mod h1:vqEfX6xzqW1pKKZUUiFOKg0OQ7bCh54Q2vR/tserrRA= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/cli/browser v1.3.0 h1:LejqCrpWr+1pRqmEPDGnTZOjsMe7sehifLynZJuqJpo= +github.com/cli/browser v1.3.0/go.mod h1:HH8s+fOAxjhQoBUAsKuPCbqUuxZDhQ2/aD+SzsEfBTk= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golobby/container/v3 v3.3.2 h1:7u+RgNnsdVlhGoS8gY4EXAG601vpMMzLZlYqSp77Quw= +github.com/golobby/container/v3 v3.3.2/go.mod h1:RDdKpnKpV1Of11PFBe7Dxc2C1k2KaLE4FD47FflAmj0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= +github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= +github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mark3labs/mcp-go v0.41.1 h1:w78eWfiQam2i8ICL7AL0WFiq7KHNJQ6UB53ZVtH4KGA= +github.com/mark3labs/mcp-go v0.41.1/go.mod h1:T7tUa2jO6MavG+3P25Oy/jR7iCeJPHImCZHRymCn39g= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= +github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= +github.com/microsoft/go-deviceid v1.0.0 h1:i5AQ654Xk9kfvwJeKQm3w2+eT1+ImBDVEpAR0AjpP40= +github.com/microsoft/go-deviceid v1.0.0/go.mod h1:KY13FeVdHkzD8gy+6T8+kVmD/7RMpTaWW75K+T4uZWg= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d h1:NqRhLdNVlozULwM1B3VaHhcXYSgrOAv8V5BE65om+1Q= +github.com/nathan-fiscaletti/consolesize-go v0.0.0-20220204101620-317176b6684d/go.mod h1:cxIIfNMTwff8f/ZvRouvWYF6wOoO7nj99neWSx2q/Es= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRzJr4= +github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9HTHs= +github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= +golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go new file mode 100644 index 00000000000..6654435f055 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +// NewRootCommand builds the `azd ai eval` command tree. +func NewRootCommand() *cobra.Command { + rootCmd, _ := azdext.NewExtensionRootCommand(azdext.ExtensionCommandOptions{ + Name: "eval", + Use: "eval [options]", + Short: fmt.Sprintf( + "Define and run Foundry evaluations from your terminal. %s", + color.YellowString("(Beta)"), + ), + }) + rootCmd.SilenceUsage = true + rootCmd.SilenceErrors = true + rootCmd.CompletionOptions.DisableDefaultCmd = true + + return rootCmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/main.go b/cli/azd/extensions/azure.ai.evaluations/main.go new file mode 100644 index 00000000000..993d2e8816e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/main.go @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package main + +import ( + "azureaieval/internal/cmd" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +func main() { + azdext.Run(cmd.NewRootCommand()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/version.txt b/cli/azd/extensions/azure.ai.evaluations/version.txt new file mode 100644 index 00000000000..ffbc9939864 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/version.txt @@ -0,0 +1 @@ +1.0.0-beta.1 From a9607265b07dc9fd8ee20d3d4f03619730b62ee5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 04:54:49 +0300 Subject: [PATCH 002/114] feat(azure.ai.evaluations): port the eval and dataset data-plane clients Lifted eval_api (models, operations, poller, generation, portal_urls) and dataset_api from azure.ai.agents, de-agent-scoped. Added evalcore with IsTransientError and an EvaluatorList that accepts either a bare string or a mapping with a threshold. Skipped artifacts.go and eval_config.go since the config model differs. --- .../extensions/azure.ai.evaluations/go.mod | 6 +- .../extensions/azure.ai.evaluations/go.sum | 4 + .../internal/pkg/dataset_api/models.go | 192 ++++++++ .../internal/pkg/dataset_api/operations.go | 442 ++++++++++++++++++ .../internal/pkg/eval_api/generation.go | 140 ++++++ .../internal/pkg/eval_api/models.go | 412 ++++++++++++++++ .../internal/pkg/eval_api/operations.go | 290 ++++++++++++ .../internal/pkg/eval_api/poller.go | 195 ++++++++ .../internal/pkg/eval_api/portal_urls.go | 79 ++++ .../internal/pkg/evalcore/evaluator.go | 89 ++++ .../internal/pkg/evalcore/transient.go | 29 ++ .../internal/version/version.go | 11 + 12 files changed, 1887 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/version/version.go diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod index d1099806c54..c63d3067dfb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.mod +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -3,17 +3,20 @@ module azureaieval go 1.26.4 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 github.com/azure/azure-dev/cli/azd v1.28.0 github.com/fatih/color v1.18.0 + github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.1 + go.yaml.in/yaml/v3 v3.0.4 ) require ( dario.cat/mergo v1.0.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 // indirect @@ -47,7 +50,6 @@ require ( github.com/gofrs/flock v0.12.1 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golobby/container/v3 v3.3.2 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect diff --git a/cli/azd/extensions/azure.ai.evaluations/go.sum b/cli/azd/extensions/azure.ai.evaluations/go.sum index 6d0434c4767..81c10e45793 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.sum +++ b/cli/azd/extensions/azure.ai.evaluations/go.sum @@ -11,6 +11,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 h1:JI8PcWOImyvIUEZ0Bbmfe05FOlWkMi2KhjG+cAKaUms= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0/go.mod h1:nJLFPGJkyKfDDyJiPuHIXsCi/gpJkm07EvRgiX7SGlI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw= @@ -247,6 +249,8 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go new file mode 100644 index 00000000000..cee6bbcd886 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "fmt" + "math" + "os" + "path/filepath" + "strconv" + "strings" +) + +// CreateDatasetRequest is the request body for creating (uploading) a dataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +// Note: The GET /datasets API returns snake_case field names (data_uri, blob_uri, +// content_uri), while the POST /finalize API accepts camelCase (dataUri). +// Both conventions are correct for their respective endpoints. +type Dataset struct { + Name string `json:"name"` + Version string `json:"version"` + BlobURI string `json:"blob_uri,omitempty"` + Format string `json:"format,omitempty"` + DataURI string `json:"data_uri,omitempty"` + ContentURI string `json:"content_uri,omitempty"` +} + +// ResolvedBlobURI returns the best available blob URI. Prefers blob_uri, +// falls back to data_uri, then content_uri. +func (d *Dataset) ResolvedBlobURI() string { + if d.BlobURI != "" { + return d.BlobURI + } + if d.DataURI != "" { + return d.DataURI + } + return d.ContentURI +} + +// DatasetCredential is the response for dataset credential (SAS token) requests. +// The API returns a nested structure with blobReference and blobReferenceForConsumption. +type DatasetCredential struct { + // Flat fields (legacy format). + BlobURI string `json:"blob_uri,omitempty"` + SAS string `json:"sas,omitempty"` + SASUri string `json:"sas_uri,omitempty"` + + // Nested fields (current API format). + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` +} + +// BlobReference represents a blob storage reference with credentials. +type BlobReference struct { + BlobURI string `json:"blobUri,omitempty"` + StorageAccountARM string `json:"storageAccountArmId,omitempty"` + Credential *BlobCredential `json:"credential,omitempty"` +} + +// BlobCredential holds SAS credential details for blob access. +type BlobCredential struct { + Type string `json:"type,omitempty"` + SASUri string `json:"sasUri,omitempty"` + SASPath string `json:"sas,omitempty"` +} + +// ResolvedDownloadURI returns the URL to download the dataset. +// Prefers blobReferenceForConsumption.credential.sasUri (current API), +// then blobReference.credential.sasUri, then flat sas_uri, then blob_uri + sas. +func (c *DatasetCredential) ResolvedDownloadURI() string { + // Current API format: nested blob references. + if c.BlobReferenceConsumption != nil && c.BlobReferenceConsumption.Credential != nil { + if uri := c.BlobReferenceConsumption.Credential.SASUri; uri != "" { + return uri + } + } + if c.BlobReference != nil && c.BlobReference.Credential != nil { + if uri := c.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + // Legacy flat format. + if c.SASUri != "" { + return c.SASUri + } + if c.BlobURI != "" && c.SAS != "" { + return c.BlobURI + "?" + c.SAS + } + return c.BlobURI +} + +// PendingUploadResponse is returned by the startPendingUpload endpoint. +// It contains a SAS URI for uploading blob data and the blob container URI. +type PendingUploadResponse struct { + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` + PendingUploadID *string `json:"pendingUploadId,omitempty"` + PendingUploadType string `json:"pendingUploadType,omitempty"` + Version string `json:"version,omitempty"` +} + +// ResolvedUploadURI returns the SAS URI for uploading blobs. +func (p *PendingUploadResponse) ResolvedUploadURI() string { + if p.BlobReference != nil && p.BlobReference.Credential != nil { + if uri := p.BlobReference.Credential.SASUri; uri != "" { + return uri + } + } + return "" +} + +// ResolvedBlobURI returns the blob container URI (without SAS) for the finalize request. +func (p *PendingUploadResponse) ResolvedBlobURI() string { + if p.BlobReference != nil { + return p.BlobReference.BlobURI + } + return "" +} + +// FinalizeDatasetRequest is the request body for finalizing a dataset version +// after blob upload. +type FinalizeDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Type string `json:"type"` + IsReference bool `json:"isReference"` + DataURI string `json:"dataUri"` +} + +// NextVersion computes the next dataset version string. +// +// Rules: +// 1. Empty → "1.0" +// 2. Parsable as a decimal number → increment by 1, format as "N.0" +// 3. Ends with trailing digits → increment the trailing numeric part +// 4. Otherwise → append ".1" +func NextVersion(current string) string { + current = strings.TrimSpace(current) + if current == "" { + return "1.0" + } + + // Try parsing as a decimal number (e.g. "1", "1.0", "2.0"). + if f, err := strconv.ParseFloat(current, 64); err == nil { + return strconv.FormatFloat(math.Floor(f)+1, 'f', 1, 64) + } + + // Find trailing digits and increment them. + i := len(current) - 1 + for i >= 0 && current[i] >= '0' && current[i] <= '9' { + i-- + } + if i < len(current)-1 { + prefix := current[:i+1] + n, err := strconv.Atoi(current[i+1:]) + if err == nil { + return prefix + strconv.Itoa(n+1) + } + } + + return current + ".1" +} + +// ReadFirstJSONLFile finds and reads the first .jsonl file in a directory. +func ReadFirstJSONLFile(dir string) (string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", fmt.Errorf("reading directory: %w", err) + } + for _, e := range entries { + if e.IsDir() { + continue + } + if filepath.Ext(e.Name()) == ".jsonl" { + data, err := os.ReadFile(filepath.Join(dir, e.Name())) //nolint:gosec // local artifact path + if err != nil { + return "", fmt.Errorf("reading %s: %w", e.Name(), err) + } + return string(data), nil + } + } + return "", fmt.Errorf("no .jsonl file found in %s", dir) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go new file mode 100644 index 00000000000..e29b6710f12 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -0,0 +1,442 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefix for dataset endpoints. +const pathDatasets = "/datasets" + +// DatasetClient provides methods for dataset upload, download, and metadata retrieval. +type DatasetClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewDatasetClient creates a new DatasetClient. +func NewDatasetClient(endpoint string, cred azcore.TokenCredential) *DatasetClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-datasets", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewDatasetClientFromPipeline creates a DatasetClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewDatasetClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *DatasetClient { + return &DatasetClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataset registers a dataset with inline content (upload). +func (c *DatasetClient) CreateDataset( + ctx context.Context, + request *CreateDatasetRequest, + apiVersion string, +) (*Dataset, error) { + return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) +} + +// UploadNewVersion reads the first JSONL file from localDir, computes the next +// version from currentVersion, and uploads it as a new dataset version using +// the 3-step pending upload flow: +// 1. startPendingUpload → get SAS URI +// 2. Upload blob to SAS URI +// 3. Finalize dataset version with dataUri +func (c *DatasetClient) UploadNewVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + content, err := ReadFirstJSONLFile(localDir) + if err != nil { + return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) + } + + newVersion := NextVersion(currentVersion) + + // Step 1: Start pending upload to get a SAS URI. + pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) + if err != nil { + return nil, fmt.Errorf("starting pending upload: %w", err) + } + + uploadURI := pending.ResolvedUploadURI() + if uploadURI == "" { + return nil, fmt.Errorf("no upload SAS URI returned from startPendingUpload") + } + + // Step 2: Upload the JSONL file to blob storage. + blobName := name + ".jsonl" + if err := c.UploadBlob(ctx, uploadURI, blobName, []byte(content)); err != nil { + return nil, fmt.Errorf("uploading blob: %w", err) + } + + // Step 3: Finalize the dataset version with the full blob URI. + dataURI := strings.TrimSuffix(pending.ResolvedBlobURI(), "/") + "/" + blobName + return c.FinalizeDatasetVersion(ctx, name, newVersion, dataURI, apiVersion) +} + +// StartPendingUpload initiates a pending upload for a dataset version. +// Returns the SAS URI and blob reference for uploading data. +func (c *DatasetClient) StartPendingUpload( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*PendingUploadResponse, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/startPendingUpload", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[PendingUploadResponse](c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + +// UploadBlob uploads data to a container SAS URI as a block blob. +func (c *DatasetClient) UploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { + u, err := url.Parse(containerSASUri) + if err != nil { + return fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("failed to create upload request: %w", err) + } + req.Header.Set("x-ms-blob-type", "BlockBlob") + req.Header.Set("Content-Type", "application/octet-stream") + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("blob upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// FinalizeDatasetVersion completes the dataset version after blob upload +// by sending the metadata (name, version, dataUri) to the API. +func (c *DatasetClient) FinalizeDatasetVersion( + ctx context.Context, + name string, + version string, + dataURI string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + request := &FinalizeDatasetRequest{ + Name: name, + Version: version, + Type: "uri_file", + DataURI: dataURI, + } + return doRequestTyped[Dataset](c, ctx, http.MethodPut, path, nil, request, apiVersion) +} + +// GetDataset retrieves metadata for a dataset by name and version. +func (c *DatasetClient) GetDataset( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*Dataset, error) { + path := fmt.Sprintf("%s/%s/versions/%s", pathDatasets, url.PathEscape(name), url.PathEscape(version)) + return doRequestTyped[Dataset](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// GetDatasetCredential retrieves a SAS credential for downloading a dataset from blob storage. +func (c *DatasetClient) GetDatasetCredential( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*DatasetCredential, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/credentials", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) +} + +// DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. +// Returns the raw content as bytes. The downloadURL should be the full URL with SAS token +// (e.g., from DatasetCredential.ResolvedDownloadURI()). +func (c *DatasetClient) DownloadDataset(ctx context.Context, downloadURL string) ([]byte, error) { + req, err := runtime.NewRequest(ctx, http.MethodGet, downloadURL) + if err != nil { + return nil, fmt.Errorf("failed to create download request: %w", err) + } + + // Use a plain HTTP client for blob downloads — the SAS token in the URL provides + // authentication, and Azure SDK pipeline policies (bearer token, correlation ID) + // should not be sent to Azure Blob Storage endpoints. + httpClient := &http.Client{} + resp, err := httpClient.Do(req.Raw()) + if err != nil { + return nil, fmt.Errorf("failed to download dataset from blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d", resp.StatusCode) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read dataset content: %w", err) + } + + log.Printf("[dataset_api] downloaded %d bytes", len(data)) + return data, nil +} + +// ListContainerBlobs lists blobs in a container using a container-level SAS URI. +// The containerSASUri should include the SAS token (e.g., from credential.sasUri with sr=c). +// Returns a list of blob names found in the container. +func (c *DatasetClient) ListContainerBlobs(ctx context.Context, containerSASUri string) ([]string, error) { + // Parse the container URI and append list query parameters. + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + q := u.Query() + q.Set("restype", "container") // cspell:ignore restype — Azure Storage API query parameter + q.Set("comp", "list") + u.RawQuery = q.Encode() + + log.Printf("[dataset_api] listing blobs: %s", u.Redacted()) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create list request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to list container blobs: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("container list failed with status %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read list response: %w", err) + } + + // Parse XML blob listing to extract blob names. + names := parseBlobNames(string(body)) + log.Printf("[dataset_api] found %d blobs in container", len(names)) + return names, nil +} + +// DownloadBlob downloads a single blob from a container using the container SAS URI +// and the blob name. Returns the blob content as bytes. +func (c *DatasetClient) DownloadBlob(ctx context.Context, containerSASUri, blobName string) ([]byte, error) { + u, err := url.Parse(containerSASUri) + if err != nil { + return nil, fmt.Errorf("invalid container SAS URI: %w", err) + } + + // Append blob name to the container path. + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create blob download request: %w", err) + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("blob download failed with status %d for %s", resp.StatusCode, blobName) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read blob content: %w", err) + } + + log.Printf("[dataset_api] downloaded blob %s (%d bytes)", blobName, len(data)) + return data, nil +} + +// parseBlobNames extracts blob names from the Azure Blob Storage XML list response +// using proper XML parsing against the EnumerationResults schema. +func parseBlobNames(xmlBody string) []string { + type blob struct { + Name string `xml:"Name"` + } + type blobs struct { + Blob []blob `xml:"Blob"` + } + type enumerationResults struct { + Blobs blobs `xml:"Blobs"` + } + + var result enumerationResults + if err := xml.Unmarshal([]byte(xmlBody), &result); err != nil { + return nil + } + + names := make([]string, 0, len(result.Blobs.Blob)) + for _, b := range result.Blobs.Blob { + if b.Name != "" { + names = append(names, b.Name) + } + } + return names +} + +// doRequest performs an HTTP request against the dataset API and returns the raw response body. +func (c *DatasetClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + log.Printf("[dataset_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[dataset_api] response status: %d", resp.StatusCode) + + if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *DatasetClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go new file mode 100644 index 00000000000..2b12968065a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "path/filepath" + "strings" + "time" + + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// Generation source building +// --------------------------------------------------------------------------- + +// TraceOptions holds optional trace inclusion parameters for generation sources. +type TraceOptions struct { + Days int +} + +// BuildGenerationSources constructs the sources array for generation jobs. +// A prompt source is included when instruction is non-empty, along with the +// agent source. When traces is non-nil and Days > 0, a traces source is +// appended with start_time computed from the current time. +func BuildGenerationSources(agentKind, agentName, version, instruction string, traces *TraceOptions) []GenerationSource { + var sources []GenerationSource + + if instruction != "" { + sources = append(sources, GenerationSource{ + Type: "prompt", + Prompt: instruction, + }) + } + + agentSource := GenerationSource{ + Type: "agent", + AgentName: agentName, + } + if version != "" { + agentSource.AgentVersion = version + } + sources = append(sources, agentSource) + + if traces != nil && traces.Days > 0 { + startTime := time.Now().AddDate(0, 0, -traces.Days).Unix() + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + StartTime: startTime, + }) + } + + return sources +} + +// --------------------------------------------------------------------------- +// Request builders +// --------------------------------------------------------------------------- + +// NewDataGenerationJobRequest builds a DataGenerationJobRequest from the +// provided parameters. Currently, it's always "simple_qna" type with multiple sources +func NewDataGenerationJobRequest( + name, evalModel string, + maxSamples int, + sources []GenerationSource, +) *DataGenerationJobRequest { + return &DataGenerationJobRequest{ + Inputs: DataGenerationInputs{ + Name: name, + Scenario: "evaluation", + Options: DataGenerationOptions{ + Type: "simple_qna", + MaxSamples: maxSamples, + ModelOptions: ModelOptions{ + Model: evalModel, + }, + }, + Sources: sources, + }, + } +} + +// NewEvaluatorGenerationJobRequest builds an EvaluatorGenerationJobRequest +// from the provided parameters. +func NewEvaluatorGenerationJobRequest( + name, evalModel string, + sources []GenerationSource, +) *EvaluatorGenerationJobRequest { + return &EvaluatorGenerationJobRequest{ + Inputs: EvaluatorGenerationInputs{ + Name: name, + EvaluatorName: name, + Model: evalModel, + Sources: sources, + }, + } +} + +// --------------------------------------------------------------------------- +// Evaluator classification +// --------------------------------------------------------------------------- + +// IsBuiltinEvaluator returns true when the evaluator name has the "builtin." +// prefix. +func IsBuiltinEvaluator(name string) bool { + return strings.HasPrefix(name, "builtin.") +} + +// SplitEvaluators partitions evaluators into generated (non-builtin) and +// built-in lists. +func SplitEvaluators(evaluators evalcore.EvaluatorList) (generated, builtin evalcore.EvaluatorList) { + for _, e := range evaluators { + if IsBuiltinEvaluator(e.Name) { + builtin = append(builtin, e) + } else { + generated = append(generated, e) + } + } + return generated, builtin +} + +// --------------------------------------------------------------------------- +// Dataset name detection +// --------------------------------------------------------------------------- + +// IsDatasetName returns true when the value looks like a registered dataset +// name rather than a local file path. A name has no path separators and no +// common data-file extension (.jsonl, .json, .csv). +func IsDatasetName(value string) bool { + if value == "" { + return false + } + if strings.ContainsAny(value, "/\\") { + return false + } + ext := strings.ToLower(filepath.Ext(value)) + return ext != ".jsonl" && ext != ".json" && ext != ".csv" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go new file mode 100644 index 00000000000..cf51017746c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -0,0 +1,412 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import "encoding/json" + +// --------------------------------------------------------------------------- +// Data Generation Jobs +// --------------------------------------------------------------------------- + +// DataGenerationJobRequest is the request body for CreateDataGenerationJob. +type DataGenerationJobRequest struct { + Inputs DataGenerationInputs `json:"inputs"` +} + +// DataGenerationInputs holds the inputs for a data generation job. +type DataGenerationInputs struct { + Name string `json:"name"` + Scenario string `json:"scenario"` + Options DataGenerationOptions `json:"options"` + Sources []GenerationSource `json:"sources"` +} + +// DataGenerationOptions holds configuration for data generation. +type DataGenerationOptions struct { + Type string `json:"type"` + MaxSamples int `json:"max_samples"` + ModelOptions ModelOptions `json:"model_options"` +} + +// ModelOptions holds the model selection for generation. +type ModelOptions struct { + Model string `json:"model"` +} + +// GenerationSource describes a source used for dataset or evaluator generation. +type GenerationSource struct { + Type string `json:"type"` + Prompt string `json:"prompt,omitempty"` + AgentName string `json:"agent_name,omitempty"` + AgentVersion string `json:"agent_version,omitempty"` + StartTime int64 `json:"start_time,omitempty"` +} + +// GenerationJob is the response for data and evaluator generation job operations. +type GenerationJob struct { + ID string `json:"id"` + Status string `json:"status"` + Result json.RawMessage `json:"result,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// JobError captures error details from a failed generation job. +type JobError struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +// OperationID returns the job's operation identifier. +func (j *GenerationJob) OperationID() string { + return j.ID +} + +// NormalizedStatus returns the lowercase status, defaulting to "running". +func (j *GenerationJob) NormalizedStatus() string { + if j.Status == "" { + return "running" + } + return j.Status +} + +// ResolvedNameVersion extracts the name and version from the generation job result. +// If name is empty, both return values are empty (caller should treat as no result). +// If version is empty, it defaults to "latest". +func (j *GenerationJob) ResolvedNameVersion() (string, string) { + name := j.resultStringField("name") + if name == "" { + return "", "" + } + version := j.resultStringField("version") + if version == "" { + version = "latest" + } + return name, version +} + +// resultStringField extracts a string field from the raw Result JSON. +// It first checks for a top-level key, then falls back to outputs[0].key +// to handle the nested response format. +func (j *GenerationJob) resultStringField(key string) string { + if len(j.Result) == 0 { + return "" + } + var m map[string]json.RawMessage + if err := json.Unmarshal(j.Result, &m); err != nil { + return "" + } + + // Try top-level field first. + if raw, ok := m[key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil && s != "" { + return s + } + } + + // Fall back to outputs[0].key for nested response format. + if rawOutputs, ok := m["outputs"]; ok { + var outputs []map[string]json.RawMessage + if err := json.Unmarshal(rawOutputs, &outputs); err == nil && len(outputs) > 0 { + if raw, ok := outputs[0][key]; ok { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + } + } + } + + return "" +} + +// --------------------------------------------------------------------------- +// Evaluator Generation Jobs +// --------------------------------------------------------------------------- + +// EvaluatorGenerationJobRequest is the request body for CreateEvaluatorGenerationJob. +type EvaluatorGenerationJobRequest struct { + Inputs EvaluatorGenerationInputs `json:"inputs"` +} + +// EvaluatorGenerationInputs holds the inputs for an evaluator generation job. +type EvaluatorGenerationInputs struct { + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + Category string `json:"category,omitempty"` + Model string `json:"model"` + Sources []GenerationSource `json:"sources"` +} + +// --------------------------------------------------------------------------- +// Evaluator Versions +// --------------------------------------------------------------------------- + +// EvaluatorVersion is the response for evaluator version operations. +type EvaluatorVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// Evaluator Definition (Rubric) +// --------------------------------------------------------------------------- + +// EvaluatorResult is the top-level response from evaluator generation, +// containing the evaluator's definition. +type EvaluatorResult struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Definition EvaluatorDefinition `json:"definition"` +} + +// EvaluatorDefinition describes an evaluator's scoring rubric. +type EvaluatorDefinition struct { + Type string `json:"type"` + Dimensions []EvaluatorDimension `json:"dimensions"` +} + +// EvaluatorDimension is a single scoring dimension within a rubric evaluator. +type EvaluatorDimension struct { + ID string `json:"id"` + Description string `json:"description,omitempty"` + Weight int `json:"weight"` + AlwaysApplicable bool `json:"always_applicable,omitempty"` +} + +// ParseEvaluatorResult parses a GenerationJob result into a structured EvaluatorResult. +// Returns nil if the result cannot be parsed. +func ParseEvaluatorResult(result json.RawMessage) *EvaluatorResult { + if len(result) == 0 { + return nil + } + var r EvaluatorResult + if err := json.Unmarshal(result, &r); err != nil { + return nil + } + if len(r.Definition.Dimensions) == 0 { + return nil + } + return &r +} + +// --------------------------------------------------------------------------- +// Datasets +// --------------------------------------------------------------------------- + +// CreateDatasetRequest is the request body for CreateDataset. +type CreateDatasetRequest struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` + Content string `json:"content"` +} + +// Dataset is the response for dataset operations. +type Dataset struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// --------------------------------------------------------------------------- +// OpenAI Evals +// --------------------------------------------------------------------------- + +// DataSourceConfig describes the data source for an OpenAI eval. +type DataSourceConfig struct { + Type string `json:"type"` + ItemSchema map[string]any `json:"item_schema"` + IncludeSampleSchema bool `json:"include_sample_schema"` +} + +// DataSourceSchema defines the item and sample schemas for an eval data source. +type DataSourceSchema struct { + Item map[string]any `json:"item,omitempty"` + Sample map[string]any `json:"sample,omitempty"` +} + +// TestingCriterion describes a single evaluator in testing_criteria. +type TestingCriterion struct { + Type string `json:"type"` + Name string `json:"name"` + EvaluatorName string `json:"evaluator_name"` + InitializationParameters map[string]any `json:"initialization_parameters,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` +} + +// CreateOpenAIEvalRequest is the request body for CreateOpenAIEval. +type CreateOpenAIEvalRequest struct { + Name string `json:"name"` + Metadata map[string]string `json:"metadata,omitempty"` + DataSourceConfig *DataSourceConfig `json:"data_source_config,omitempty"` + TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` +} + +// OpenAIEval is the response for an OpenAI eval definition. +type OpenAIEval struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// ResolvedID returns the eval's ID, falling back to name. +func (e *OpenAIEval) ResolvedID() string { + if e.ID != "" { + return e.ID + } + return e.Name +} + +// OpenAIEvalList is the response for listing OpenAI eval definitions. +type OpenAIEvalList struct { + Data []OpenAIEval `json:"data"` +} + +// --------------------------------------------------------------------------- +// OpenAI Eval Runs +// --------------------------------------------------------------------------- + +// CreateOpenAIEvalRunRequest is the request body for CreateOpenAIEvalRun. +type CreateOpenAIEvalRunRequest struct { + Name string `json:"name"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +// EvalRunDataSourceType defines the type for an eval run data source. +type EvalRunDataSourceType string + +const ( + // EvalRunDataSourceTypeAgentTarget is the data source type for agent target completions. + EvalRunDataSourceTypeAgentTarget EvalRunDataSourceType = "azure_ai_target_completions" +) + +// EvalRunDataContentType defines the source type for eval run data content. +type EvalRunDataContentType string + +const ( + EvalRunDataContentTypeFileContent EvalRunDataContentType = "file_content" + EvalRunDataContentTypeFileID EvalRunDataContentType = "file_id" +) + +// EvalRunDataSource describes the data source for an eval run with agent target completions. +type EvalRunDataSource struct { + Type EvalRunDataSourceType `json:"type"` + InputMessages *EvalRunInputMessages `json:"input_messages,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` + Target *EvalRunTarget `json:"target,omitempty"` +} + +// EvalRunInputMessages describes how input messages are constructed from dataset items. +type EvalRunInputMessages struct { + Type string `json:"type"` + Template []EvalRunMessageTemplate `json:"template"` +} + +// EvalRunMessageTemplate describes a single message in the input template. +type EvalRunMessageTemplate struct { + Role string `json:"role"` + Content string `json:"content"` + Type string `json:"type"` +} + +// EvalRunTarget describes the agent target for completions. +type EvalRunTarget struct { + Type string `json:"type"` + Name string `json:"name"` + Version *string `json:"version"` + ToolDescriptions []string `json:"tool_descriptions"` +} + +// EvalRunDataContent holds the source reference within an EvalRunDataSource. +type EvalRunDataContent struct { + Type EvalRunDataContentType `json:"type"` + ID string `json:"id,omitempty"` + Content []map[string]any `json:"content,omitempty"` +} + +// NewAgentTargetDataSource builds an EvalRunDataSource configured for agent target completions. +// The source field must be set separately via SetFileContent or SetFileID. +func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_agent", + Name: agentName, + Version: agentVersion, + ToolDescriptions: []string{}, + }, + } +} + +// SetFileContent sets the data source to use inline file content. +func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { + ds.Source = &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: items, + } +} + +// SetFileID sets the data source to reference a remote dataset by ID. +func (ds *EvalRunDataSource) SetFileID(fileID string) { + ds.Source = &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileID, + ID: fileID, + } +} + +// OpenAIEvalRun is the response for an OpenAI eval run. +type OpenAIEvalRun struct { + ID string `json:"id"` + EvalID string `json:"eval_id,omitempty"` + Name string `json:"name,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt any `json:"created_at,omitempty"` + ModifiedAt any `json:"modified_at,omitempty"` + CreatedBy string `json:"created_by,omitempty"` + DataSource *EvalRunDataSource `json:"data_source,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + ReportURL string `json:"report_url,omitempty"` + + // Result summary + ResultCounts *EvalRunResultCounts `json:"result_counts,omitempty"` + PerTestingCriteria []EvalRunCriteriaResult `json:"per_testing_criteria_results,omitempty"` + Error any `json:"error,omitempty"` +} + +// EvalRunResultCounts holds pass/fail/error/skip counts for a run. +type EvalRunResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// EvalRunCriteriaResult holds per-testing-criteria pass/fail counts. +type EvalRunCriteriaResult struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + Skipped int `json:"skipped"` +} + +// OpenAIEvalRunList is the response for listing OpenAI eval runs. +type OpenAIEvalRunList struct { + Data []OpenAIEvalRun `json:"data"` +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go new file mode 100644 index 00000000000..510b2680b31 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -0,0 +1,290 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strconv" + + "azureaieval/internal/version" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "github.com/azure/azure-dev/cli/azd/pkg/azsdk" +) + +// API path prefixes for eval service endpoints. +const ( + pathDataGenerationJobs = "/data_generation_jobs" + pathEvaluatorGenerationJobs = "/evaluator_generation_jobs" + pathEvaluators = "/evaluators" + pathDatasets = "/datasets" + pathOpenAIEvals = "/openai/v1/evals" +) + +// EvalClient provides methods for interacting with the Azure AI eval APIs. +type EvalClient struct { + endpoint string + pipeline runtime.Pipeline +} + +// NewEvalClient creates a new EvalClient. +func NewEvalClient(endpoint string, cred azcore.TokenCredential) *EvalClient { + userAgent := fmt.Sprintf("azd-ext-azure-ai-evaluations/%s", version.Version) + + clientOptions := &policy.ClientOptions{ + Logging: policy.LogOptions{ + AllowedHeaders: []string{"X-Ms-Correlation-Request-Id", "X-Request-Id"}, + IncludeBody: false, + }, + PerCallPolicies: []policy.Policy{ + runtime.NewBearerTokenPolicy(cred, []string{"https://ai.azure.com/.default"}, nil), + azsdk.NewMsCorrelationPolicy(), + azsdk.NewUserAgentPolicy(userAgent), + }, + } + + pipeline := runtime.NewPipeline( + "azure-ai-evals", + "v1.0.0", + runtime.PipelineOptions{}, + clientOptions, + ) + + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// NewEvalClientFromPipeline creates an EvalClient with a pre-built pipeline. +// This is intended for tests that need to bypass auth policies. +func NewEvalClientFromPipeline(endpoint string, pipeline runtime.Pipeline) *EvalClient { + return &EvalClient{ + endpoint: endpoint, + pipeline: pipeline, + } +} + +// CreateDataGenerationJob starts a dataset generation job for eval onboarding. +func (c *EvalClient) CreateDataGenerationJob( + ctx context.Context, + request *DataGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathDataGenerationJobs, nil, request, apiVersion) +} + +// GetDataGenerationJob gets the current state of a dataset generation job. +func (c *EvalClient) GetDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathDataGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorGenerationJob starts an evaluator generation job for eval onboarding. +func (c *EvalClient) CreateEvaluatorGenerationJob( + ctx context.Context, + request *EvaluatorGenerationJobRequest, + apiVersion string, +) (*GenerationJob, error) { + return doRequestTyped[GenerationJob](c, ctx, http.MethodPost, pathEvaluatorGenerationJobs, nil, request, apiVersion) +} + +// GetEvaluatorGenerationJob gets the current state of an evaluator generation job. +func (c *EvalClient) GetEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + path := pathEvaluatorGenerationJobs + "/" + url.PathEscape(operationID) + return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateEvaluatorVersion creates a new version of a named evaluator. +// The body should be the full evaluator JSON with the definition field updated. +func (c *EvalClient) CreateEvaluatorVersion( + ctx context.Context, + name string, + body json.RawMessage, + apiVersion string, +) (*EvaluatorVersion, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) +} + +// GetEvaluatorRaw gets an evaluator by name and version as raw JSON. +// If version is empty, the latest version is fetched. +func (c *EvalClient) GetEvaluatorRaw( + ctx context.Context, + name string, + version string, + apiVersion string, +) (json.RawMessage, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + if version != "" { + path += "/versions/" + url.PathEscape(version) + } + return c.doRequest(ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// CreateOpenAIEval creates an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEval( + ctx context.Context, + request *CreateOpenAIEvalRequest, +) (*OpenAIEval, error) { + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, pathOpenAIEvals, nil, request, "") +} + +// ListOpenAIEvals lists OpenAI eval definitions. +func (c *EvalClient) ListOpenAIEvals(ctx context.Context, limit int) (*OpenAIEvalList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + return doRequestTyped[OpenAIEvalList](c, ctx, http.MethodGet, pathOpenAIEvals, query, nil, "") +} + +// GetOpenAIEval gets an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEval(ctx context.Context, evalID string) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodGet, path, nil, nil, "") +} + +// CreateOpenAIEvalRun starts a run for an OpenAI eval definition. +func (c *EvalClient) CreateOpenAIEvalRun( + ctx context.Context, + evalID string, + request *CreateOpenAIEvalRunRequest, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, request, "") +} + +// ListOpenAIEvalRuns lists runs for an OpenAI eval definition. +func (c *EvalClient) ListOpenAIEvalRuns( + ctx context.Context, + evalID string, + limit int, +) (*OpenAIEvalRunList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + path := fmt.Sprintf("%s/%s/runs", pathOpenAIEvals, url.PathEscape(evalID)) + return doRequestTyped[OpenAIEvalRunList](c, ctx, http.MethodGet, path, query, nil, "") +} + +// GetOpenAIEvalRun gets a run for an OpenAI eval definition. +func (c *EvalClient) GetOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf("%s/%s/runs/%s", pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID)) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodGet, path, nil, nil, "") +} + +func (c *EvalClient) doRequest( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) ([]byte, error) { + u, err := url.Parse(c.endpoint) + if err != nil { + return nil, fmt.Errorf("invalid endpoint URL: %w", err) + } + + u.Path += path + q := u.Query() + if apiVersion != "" { + q.Set("api-version", apiVersion) + } + for k, v := range query { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + req, err := runtime.NewRequest(ctx, method, u.String()) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + log.Printf("[eval_api] %s %s", method, u.Redacted()) + + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + log.Printf("[eval_api] response status: %d", resp.StatusCode) + + if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + // Restore the body so runtime.NewResponseError can read it. + resp.Body = io.NopCloser(bytes.NewReader(respBody)) + return nil, runtime.NewResponseError(resp) + } + + return respBody, nil +} + +// doRequestTyped performs an HTTP request and unmarshals the response into T. +func doRequestTyped[T any]( + c *EvalClient, + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, +) (*T, error) { + respBody, err := c.doRequest(ctx, method, path, query, body, apiVersion) + if err != nil { + return nil, err + } + + if len(respBody) == 0 { + return new(T), nil + } + + var result T + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &result, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go new file mode 100644 index 00000000000..4976248d87a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/poller.go @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "log" + "strings" + "time" + + "azureaieval/internal/pkg/evalcore" +) + +// --------------------------------------------------------------------------- +// JobStatus — typed status with terminal/failed semantics +// --------------------------------------------------------------------------- + +// JobStatus represents the normalized status of a generation job. +type JobStatus string + +const ( + JobStatusRunning JobStatus = "running" + JobStatusCompleted JobStatus = "completed" + JobStatusSucceeded JobStatus = "succeeded" + JobStatusFailed JobStatus = "failed" + JobStatusCancelled JobStatus = "cancelled" + JobStatusCanceled JobStatus = "canceled" +) + +// ParseJobStatus normalizes a raw status string into a JobStatus. +// An empty string is treated as "running". +func ParseJobStatus(s string) JobStatus { + if s == "" { + return JobStatusRunning + } + return JobStatus(strings.ToLower(s)) +} + +// IsTerminal returns true when the status represents a final state. +func (s JobStatus) IsTerminal() bool { + switch s { + case JobStatusCompleted, JobStatusSucceeded, JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// IsFailed returns true when the status represents a failure or cancellation. +func (s JobStatus) IsFailed() bool { + switch s { + case JobStatusFailed, JobStatusCancelled, JobStatusCanceled: + return true + } + return false +} + +// String returns the status as a plain string. +func (s JobStatus) String() string { + return string(s) +} + +// --------------------------------------------------------------------------- +// JobFailedError — returned when a polled job reaches a failed state +// --------------------------------------------------------------------------- + +// JobFailedError is returned when a generation job reaches a failed terminal state. +type JobFailedError struct { + Job *GenerationJob + Status JobStatus +} + +func (e *JobFailedError) Error() string { + if e.Job != nil && e.Job.Error != nil && e.Job.Error.Message != "" { + return fmt.Sprintf("job failed with status %q: %s", e.Status, e.Job.Error.Message) + } + return fmt.Sprintf("job failed with status %q", e.Status) +} + +// --------------------------------------------------------------------------- +// PollerTimeoutError — returned when polling exhausts all attempts +// --------------------------------------------------------------------------- + +// PollerTimeoutError is returned when a generation job has not reached a +// terminal state within the configured number of polling attempts. +type PollerTimeoutError struct { + OperationID string + Attempts int +} + +func (e *PollerTimeoutError) Error() string { + return fmt.Sprintf( + "operation %s did not complete within %d attempts", + e.OperationID, e.Attempts, + ) +} + +// --------------------------------------------------------------------------- +// GetJobFunc — callback type for fetching job state +// --------------------------------------------------------------------------- + +// GetJobFunc fetches the current state of a generation job by operation ID. +type GetJobFunc func(ctx context.Context, operationID, apiVersion string) (*GenerationJob, error) + +// --------------------------------------------------------------------------- +// PollerOptions — configurable polling behavior +// --------------------------------------------------------------------------- + +// PollerOptions configures the polling interval and attempt limit. +type PollerOptions struct { + Interval time.Duration + MaxAttempts int +} + +// DefaultPollerOptions returns sensible defaults: 2 s interval, 300 attempts (~10 min). +func DefaultPollerOptions() PollerOptions { + return PollerOptions{ + Interval: 2 * time.Second, + MaxAttempts: 300, + } +} + +// --------------------------------------------------------------------------- +// Poller — polls a generation job until it reaches a terminal state +// --------------------------------------------------------------------------- + +// Poller polls a GenerationJob until it reaches a terminal status. +type Poller struct { + OperationID string + APIVersion string + GetJob GetJobFunc + Options PollerOptions + // OnPoll is called after each successful poll with the latest status. + // Callers can use this for progress reporting (e.g. debug logging). + OnPoll func(status JobStatus) +} + +// NewPoller creates a Poller with default options. +func NewPoller(operationID, apiVersion string, getJob GetJobFunc) *Poller { + return &Poller{ + OperationID: operationID, + APIVersion: apiVersion, + GetJob: getJob, + Options: DefaultPollerOptions(), + } +} + +// Poll blocks until the job reaches a terminal state, the context is +// cancelled, or the maximum number of attempts is exhausted. +// +// On success it returns the completed GenerationJob. +// On failure it returns a *JobFailedError (which wraps the job for inspection). +// On timeout it returns a plain error. +func (p *Poller) Poll(ctx context.Context) (*GenerationJob, error) { + if p.OperationID == "" { + return nil, fmt.Errorf("operation ID is empty") + } + + for range p.Options.MaxAttempts { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(p.Options.Interval): + } + + job, err := p.GetJob(ctx, p.OperationID, p.APIVersion) + if err != nil { + if evalcore.IsTransientError(err) { + log.Printf("[poller] transient error polling %s, will retry: %v", p.OperationID, err) + continue + } + return nil, err + } + + status := ParseJobStatus(job.Status) + log.Printf("[poller] operationID=%s status=%s", p.OperationID, status) + + if p.OnPoll != nil { + p.OnPoll(status) + } + + if status.IsTerminal() { + if status.IsFailed() { + return nil, &JobFailedError{Job: job, Status: status} + } + return job, nil + } + } + + return nil, &PollerTimeoutError{ + OperationID: p.OperationID, + Attempts: p.Options.MaxAttempts, + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go new file mode 100644 index 00000000000..8b1ccd0fd5d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls.go @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/base64" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/google/uuid" +) + +// PortalPrefix holds the parsed project context needed to construct Foundry portal URLs. +type PortalPrefix struct { + prefix string // e.g. "https://ai.azure.com/nextgen/r/,,,," +} + +// NewPortalPrefix parses an ARM project resource ID and returns a PortalPrefix +// that can be reused to build multiple portal URLs. +// Returns an error if the resource ID is invalid or not a Foundry project. +func NewPortalPrefix(projectResourceID string) (*PortalPrefix, error) { + resourceID, err := arm.ParseResourceID(projectResourceID) + if err != nil { + return nil, fmt.Errorf("failed to parse project resource ID: %w", err) + } + + encodedSub, err := encodeSubscriptionForURL(resourceID.SubscriptionID) + if err != nil { + return nil, fmt.Errorf("failed to encode subscription ID: %w", err) + } + + if resourceID.Parent == nil || + !strings.Contains(string(resourceID.ResourceType.Type), "/") { + return nil, fmt.Errorf( + "resource ID does not represent a Foundry project (missing parent account): %s", + projectResourceID, + ) + } + + prefix := fmt.Sprintf( + "https://ai.azure.com/nextgen/r/%s,%s,,%s,%s", + encodedSub, resourceID.ResourceGroupName, + resourceID.Parent.Name, resourceID.Name, + ) + return &PortalPrefix{prefix: prefix}, nil +} + +// EvalRunURL returns the portal URL for an eval run report. +func (p *PortalPrefix) EvalRunURL(evalID, runID string) string { + return fmt.Sprintf("%s/build/evaluations/%s/run/%s", p.prefix, evalID, runID) +} + +// EvaluatorURL returns the portal URL for a generated evaluator. +func (p *PortalPrefix) EvaluatorURL(evaluatorName, version string) string { + return fmt.Sprintf("%s/build/evaluations/catalog/%s/%s", p.prefix, evaluatorName, version) +} + +// DatasetURL returns the portal URL for a dataset. +func (p *PortalPrefix) DatasetURL(datasetName, version string) string { + return fmt.Sprintf("%s/build/data/datasets/%s/%s", p.prefix, datasetName, version) +} + +// OptimizationURL returns the portal URL for an optimization job. +func (p *PortalPrefix) OptimizationURL(agentName, operationID string) string { + return fmt.Sprintf("%s/build/agents/%s/optimization/%s", + p.prefix, agentName, operationID) +} + +// encodeSubscriptionForURL encodes a subscription ID GUID as base64 without padding. +func encodeSubscriptionForURL(subscriptionID string) (string, error) { + guid, err := uuid.Parse(subscriptionID) + if err != nil { + return "", fmt.Errorf("invalid subscription ID format: %w", err) + } + guidBytes, _ := guid.MarshalBinary() + return strings.TrimRight(base64.URLEncoding.EncodeToString(guidBytes), "="), nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go new file mode 100644 index 00000000000..0a4847ad4bd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "fmt" + "strings" + + "go.yaml.in/yaml/v3" +) + +// BuiltinPrefix marks an evaluator provided by the platform. The prefix is +// stripped before the name is sent as testing_criteria[].evaluator_name. +const BuiltinPrefix = "builtin." + +// EvaluatorRef references an evaluator from an eval group. It accepts either a +// bare string or a mapping carrying a pass threshold: +// +// evaluators: +// - builtin.task_adherence +// - { name: support-quality, threshold: 4.0 } +type EvaluatorRef struct { + Name string `yaml:"name" json:"name"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` + // Threshold maps to testing_criteria[].initialization_parameters.threshold. + Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"` +} + +// IsBuiltin reports whether the reference names a platform evaluator, which +// needs no declaration and is never uploaded. +func (e EvaluatorRef) IsBuiltin() bool { + return strings.HasPrefix(e.Name, BuiltinPrefix) +} + +// APIName is the name the service expects, with the builtin prefix removed. +func (e EvaluatorRef) APIName() string { + return strings.TrimPrefix(e.Name, BuiltinPrefix) +} + +// EvaluatorList is a sequence of EvaluatorRef supporting mixed string and +// mapping entries. +type EvaluatorList []EvaluatorRef + +func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { + if value.Kind != yaml.SequenceNode { + return fmt.Errorf("evaluators must be a sequence, got %v", value.Kind) + } + + result := make([]EvaluatorRef, 0, len(value.Content)) + for _, node := range value.Content { + switch node.Kind { + case yaml.ScalarNode: + var name string + if err := node.Decode(&name); err != nil { + return fmt.Errorf("decoding evaluator name: %w", err) + } + result = append(result, EvaluatorRef{Name: name}) + case yaml.MappingNode: + var ref EvaluatorRef + if err := node.Decode(&ref); err != nil { + return fmt.Errorf("decoding evaluator: %w", err) + } + if ref.Name == "" { + return fmt.Errorf("evaluator entry is missing 'name'") + } + result = append(result, ref) + default: + return fmt.Errorf("evaluator entry must be a string or a mapping, got %v", node.Kind) + } + } + + *el = result + return nil +} + +// MarshalYAML emits the compact string form when an entry carries nothing but a +// name, so round-tripping a hand-written config does not rewrite it. +func (el EvaluatorList) MarshalYAML() (any, error) { + out := make([]any, 0, len(el)) + for _, ref := range el { + if ref.Threshold == nil && ref.Version == "" { + out = append(out, ref.Name) + continue + } + out = append(out, ref) + } + return out, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go new file mode 100644 index 00000000000..029af83ffc0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/transient.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "errors" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsTransientError reports whether err is worth retrying: throttling, a server +// fault, or a dropped connection. +func IsTransientError(err error) bool { + if err == nil { + return false + } + + var respErr *azcore.ResponseError + if errors.As(err, &respErr) { + return respErr.StatusCode == 429 || respErr.StatusCode >= 500 + } + + msg := err.Error() + return strings.Contains(msg, "connection reset") || + strings.Contains(msg, "connection refused") || + strings.Contains(msg, "EOF") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go new file mode 100644 index 00000000000..e7279d11fba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/version/version.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package version + +var ( + // Populated at build time. + Version = "dev" + Commit = "none" + BuildDate = "unknown" +) From 2a8dbed08b945a270530af0360a198f5907a5d8d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:02:58 +0300 Subject: [PATCH 003/114] feat(azure.ai.evaluations): config model, client context, and dataset commands - EvalConfig and GenerateConfig types with validation covering cross-references, duplicate names, unsupported target types, and evaluation levels - ResolveGroup picks the only group or errors with the available names - ArtifactPath accepts a directory or an explicit file path for local_dir - evalContext resolves the project endpoint (flag, azd env, host env) and builds both clients against the azd developer CLI credential - dataset create/update/list/show/delete with -o json - Tier-0 tests for parsing, validation, group resolution, and path handling --- .../internal/cmd/apiversions.go | 17 ++ .../internal/cmd/context.go | 145 ++++++++++ .../internal/cmd/dataset.go | 253 ++++++++++++++++++ .../internal/cmd/output.go | 72 +++++ .../azure.ai.evaluations/internal/cmd/root.go | 2 + .../internal/pkg/dataset_api/list.go | 91 +++++++ .../internal/project/eval_config.go | 222 +++++++++++++++ .../internal/project/eval_config_test.go | 235 ++++++++++++++++ .../internal/project/generate_config.go | 163 +++++++++++ 9 files changed, 1200 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go new file mode 100644 index 00000000000..89b1d125f34 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +// API versions used by the Foundry data plane. +const ( + // ProjectEndpointAPIVersion covers datasets, evaluators, and evaluator + // generation jobs on the project endpoint. + ProjectEndpointAPIVersion = "2025-11-15-preview" + + // DataGenerationAPIVersion covers dataset generation jobs. + DataGenerationAPIVersion = "v1" + + // OpenAI-compatible eval group and run calls send no api-version, so there + // is deliberately no constant for them. +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go new file mode 100644 index 00000000000..f9ba6d0ec8d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "os" + "strings" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" +) + +// projectEndpointEnvKey is the azd environment key holding the Foundry project +// endpoint the data-plane clients target. +const projectEndpointEnvKey = "FOUNDRY_PROJECT_ENDPOINT" + +// evalContext carries everything the commands need to reach the data plane. +type evalContext struct { + azdClient *azdext.AzdClient + endpoint string + envName string + cred azcore.TokenCredential + + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient +} + +// newEvalContext resolves the project endpoint and builds the data-plane +// clients. Endpoint resolution order: +// +// 1. --project-endpoint +// 2. the active azd environment's FOUNDRY_PROJECT_ENDPOINT +// 3. the host environment variable of the same name +func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, error) { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return nil, fmt.Errorf("connecting to azd: %w", err) + } + + ec := &evalContext{azdClient: azdClient} + + if endpointFlag != "" { + ec.endpoint = endpointFlag + } else { + ec.endpoint, ec.envName = lookupEndpointFromAzd(ctx, azdClient) + } + if ec.endpoint == "" { + ec.endpoint = os.Getenv(projectEndpointEnvKey) + } + if ec.endpoint == "" { + return nil, fmt.Errorf( + "no Foundry project endpoint found; pass --project-endpoint or set %s "+ + "in the azd environment (azd env set %s )", + projectEndpointEnvKey, projectEndpointEnvKey) + } + ec.endpoint = strings.TrimSuffix(ec.endpoint, "/") + + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + if err != nil { + return nil, fmt.Errorf("creating Azure credential: %w", err) + } + ec.cred = cred + + ec.evalClient = eval_api.NewEvalClient(ec.endpoint, cred) + ec.datasetClient = dataset_api.NewDatasetClient(ec.endpoint, cred) + + return ec, nil +} + +// lookupEndpointFromAzd reads the endpoint from the active azd environment, +// returning empty strings when azd has no current environment. +func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (endpoint, envName string) { + envResp, err := azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return "", "" + } + val, err := azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: envResp.Environment.Name, + Key: projectEndpointEnvKey, + }) + if err != nil || val == nil || val.Value == "" { + return "", envResp.Environment.Name + } + return val.Value, envResp.Environment.Name +} + +// setEnvValue persists a value into the active azd environment. azd itself +// writes none of these keys — the extension owns them. +func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { + if ec.envName == "" { + envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) + if err != nil || envResp == nil || envResp.Environment == nil { + return fmt.Errorf("no active azd environment to write %s into", key) + } + ec.envName = envResp.Environment.Name + } + _, err := ec.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ + EnvName: ec.envName, + Key: key, + Value: value, + }) + if err != nil { + return fmt.Errorf("writing %s to the azd environment: %w", key, err) + } + return nil +} + +// getEnvValue reads a value from the active azd environment, returning empty +// when it is unset. +func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { + if ec.envName == "" { + return "" + } + val, err := ec.azdClient.Environment().GetValue(ctx, &azdext.GetEnvRequest{ + EnvName: ec.envName, + Key: key, + }) + if err != nil || val == nil { + return "" + } + return val.Value +} + +func (ec *evalContext) Close() { + if ec.azdClient != nil { + ec.azdClient.Close() + } +} + +// azd environment keys written by this extension. +const ( + envKeyEvalGroupID = "EVAL_GROUP_ID" + envKeyEvalRunID = "EVAL_RUN_ID" + envKeyDatasetVersion = "EVAL_DATASET_VERSION" + envKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go new file mode 100644 index 00000000000..41685ae7e18 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/spf13/cobra" +) + +func newDatasetCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "dataset", + Short: "Manage evaluation datasets.", + } + cmd.AddCommand( + newDatasetCreateCommand(false), + newDatasetCreateCommand(true), + newDatasetListCommand(), + newDatasetShowCommand(), + newDatasetDeleteCommand(), + ) + return cmd +} + +// newDatasetCreateCommand builds `dataset create` and `dataset update`. Both +// publish a new immutable version; the server auto-increments. +func newDatasetCreateCommand(update bool) *cobra.Command { + var ( + name string + file string + version string + endpointFlg string + ) + + use, short := "create", "Register a dataset, creating its first version." + if update { + use, short = "update", "Publish a new version of an existing dataset." + } + + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if file == "" { + return requireFlag("file") + } + + info, err := os.Stat(file) + if err != nil { + return fmt.Errorf("reading --file %q: %w", file, err) + } + // The upload helper scans a directory for the first .jsonl, so pass + // the containing directory when given a file path. + localDir := file + if !info.IsDir() { + if !strings.EqualFold(filepath.Ext(file), ".jsonl") { + return fmt.Errorf("--file must be a .jsonl file or a directory containing one, got %q", file) + } + localDir = filepath.Dir(file) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + ds, err := ec.datasetClient.UploadNewVersion( + ctx, name, version, localDir, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("registering dataset %q: %w", name, err) + } + + if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { + // Persisting is a convenience; do not fail the command over it. + fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + fmt.Fprintf(cmd.OutOrStdout(), "Registered dataset %s version %s\n", ds.Name, ds.Version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") + cmd.Flags().StringVar(&file, "file", "", "Path to a .jsonl file, or a directory containing one.") + cmd.Flags().StringVar(&version, "version", "", + "Current version to increment from. Omit to let the server assign the next version.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetListCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List registered datasets, or the versions of one dataset.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + var list *dataset_api.DatasetList + if name != "" { + list, err = ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + } else { + list, err = ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) + } + if err != nil { + return fmt.Errorf("listing datasets: %w", err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), list) + } + rows := make([][]string, 0, len(list.Value)) + for _, d := range list.Value { + rows = append(rows, []string{d.Name, d.Version, d.Format}) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") + return nil + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this dataset.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetShowCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show", + Short: "Show a dataset version.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if version == "" { + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("resolving the latest version of %q: %w", name, err) + } + if len(list.Value) == 0 { + return fmt.Errorf("dataset %q has no versions", name) + } + version = dataset_api.LatestVersion(list.Value) + } + + ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), ds) + } + return emitTable(cmd.OutOrStdout(), + []string{"NAME", "VERSION", "FORMAT", "URI"}, + [][]string{{ds.Name, ds.Version, ds.Format, ds.ResolvedBlobURI()}}, + ) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newDatasetDeleteCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete a dataset version.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.datasetClient.DeleteDatasetVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted dataset %s version %s\n", name, version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go new file mode 100644 index 00000000000..ee6a1928fdb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "text/tabwriter" + + "github.com/spf13/cobra" +) + +const outputJSON = "json" + +// outputFormat reads the inherited -o/--output flag. +func outputFormat(cmd *cobra.Command) string { + if cmd == nil { + return "" + } + v, err := cmd.Flags().GetString("output") + if err != nil { + return "" + } + return strings.ToLower(v) +} + +// isJSON reports whether the command should emit machine-readable output. +func isJSON(cmd *cobra.Command) bool { + return outputFormat(cmd) == outputJSON +} + +// emitJSON writes v as indented JSON. +func emitJSON(w io.Writer, v any) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +// emitTable writes a simple aligned table. Rows must match the header width. +func emitTable(w io.Writer, headers []string, rows [][]string) error { + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + if _, err := fmt.Fprintln(tw, strings.Join(headers, "\t")); err != nil { + return err + } + for _, row := range rows { + if _, err := fmt.Fprintln(tw, strings.Join(row, "\t")); err != nil { + return err + } + } + return tw.Flush() +} + +// noPrompt reports whether the command must run without any interaction. +func noPrompt(cmd *cobra.Command) bool { + if cmd == nil { + return false + } + v, err := cmd.Flags().GetBool("no-prompt") + if err != nil { + return false + } + return v +} + +// requireFlag returns an error naming the missing flag, used when --no-prompt +// prevents asking for a required value. +func requireFlag(name string) error { + return fmt.Errorf("--%s is required (running with --no-prompt)", name) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 6654435f055..9901d141836 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -25,5 +25,7 @@ func NewRootCommand() *cobra.Command { rootCmd.SilenceErrors = true rootCmd.CompletionOptions.DisableDefaultCmd = true + rootCmd.AddCommand(newDatasetCommand()) + return rootCmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go new file mode 100644 index 00000000000..f81f372131d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" +) + +// DatasetList is the paged response returned when listing datasets or the +// versions of one dataset. +type DatasetList struct { + Value []Dataset `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ListDatasets returns the datasets registered on the project. +func (c *DatasetClient) ListDatasets(ctx context.Context, apiVersion string) (*DatasetList, error) { + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, pathDatasets, nil, nil, apiVersion) +} + +// ListDatasetVersions returns every version of a single dataset. +func (c *DatasetClient) ListDatasetVersions( + ctx context.Context, + name string, + apiVersion string, +) (*DatasetList, error) { + path := fmt.Sprintf("%s/%s/versions", pathDatasets, url.PathEscape(name)) + return doRequestTyped[DatasetList](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// DeleteDatasetVersion removes a single dataset version. +func (c *DatasetClient) DeleteDatasetVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathDatasets, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// VersionOrder returns a sortable value for a version string, matching the +// decimal convention NextVersion produces ("1.0", "2.0"). Unparseable versions +// sort lowest. +func VersionOrder(version string) float64 { + v := strings.TrimSpace(version) + if v == "" { + return -1 + } + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + // Fall back to trailing digits, e.g. "v3" -> 3. + i := len(v) + for i > 0 && v[i-1] >= '0' && v[i-1] <= '9' { + i-- + } + if i == len(v) { + return -1 + } + if n, err := strconv.Atoi(v[i:]); err == nil { + return float64(n) + } + return -1 +} + +// LatestVersion returns the highest version in the list, falling back to the +// last entry when none of the versions can be ordered. +func LatestVersion(datasets []Dataset) string { + best := "" + bestOrder := -2.0 + for _, d := range datasets { + if o := VersionOrder(d.Version); o > bestOrder { + bestOrder, best = o, d.Version + } + } + if best == "" && len(datasets) > 0 { + return datasets[len(datasets)-1].Version + } + return best +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go new file mode 100644 index 00000000000..37a0777448a --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Package project models the eval configuration carried by the +// `host: azure.ai.eval` service entry in azure.yaml. +package project + +import ( + "fmt" + "os" + "strings" + + "azureaieval/internal/pkg/evalcore" + + "go.yaml.in/yaml/v3" +) + +// EvalConfig is the deployment spec — the body of the azure.ai.eval service +// entry, normally kept in evals/azure.yaml and pulled in with $ref. +type EvalConfig struct { + Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` + EvalGroups []EvalGroup `yaml:"evalGroups,omitempty" json:"evalGroups,omitempty"` +} + +// DatasetDecl declares a dataset. A local Source is uploaded on deploy; without +// one the name must already resolve to a registered dataset. +type DatasetDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly +// from an eval group and never declared here. +type EvaluatorDecl struct { + Name string `yaml:"name" json:"name"` + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` +} + +// EvalGroup is a run definition: evaluators plus options, bound to a dataset. +type EvalGroup struct { + Name string `yaml:"name" json:"name"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` + Options *Options `yaml:"options,omitempty" json:"options,omitempty"` +} + +// Target names what the run invokes. Only type "agent" is supported today. +type Target struct { + Type string `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` +} + +const TargetTypeAgent = "agent" + +// Options are run settings carried on the group. +type Options struct { + EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` + MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` + EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` +} + +// Evaluation levels accepted by the service. The service default is turn. +const ( + EvaluationLevelTurn = "turn" + EvaluationLevelConversation = "conversation" +) + +// LoadEvalConfig reads a deployment spec from disk. The path is used verbatim, +// relative to the process working directory — never re-rooted. +func LoadEvalConfig(path string) (*EvalConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading eval config %q: %w", path, err) + } + + var cfg EvalConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing eval config %q: %w", path, err) + } + return &cfg, nil +} + +// Validate checks the invariants the provider relies on before it calls the +// service, so failures surface as config errors rather than opaque 4xx. +func (c *EvalConfig) Validate() error { + datasets := map[string]bool{} + for i, d := range c.Datasets { + if d.Name == "" { + return fmt.Errorf("datasets[%d]: 'name' is required", i) + } + if datasets[d.Name] { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) + } + datasets[d.Name] = true + } + + evaluators := map[string]bool{} + for i, e := range c.Evaluators { + if e.Name == "" { + return fmt.Errorf("evaluators[%d]: 'name' is required", i) + } + if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { + return fmt.Errorf( + "evaluators[%d]: built-in evaluator %q must not be declared; "+ + "reference it directly from an eval group", i, e.Name) + } + if evaluators[e.Name] { + return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) + } + evaluators[e.Name] = true + } + + groups := map[string]bool{} + for i, g := range c.EvalGroups { + if g.Name == "" { + return fmt.Errorf("evalGroups[%d]: 'name' is required", i) + } + if groups[g.Name] { + return fmt.Errorf("evalGroups[%d]: duplicate eval group name %q", i, g.Name) + } + groups[g.Name] = true + + if g.Dataset != "" && !datasets[g.Dataset] { + return fmt.Errorf( + "evalGroups[%d] (%s): dataset %q is not declared in datasets", + i, g.Name, g.Dataset) + } + if len(g.Evaluators) == 0 { + return fmt.Errorf("evalGroups[%d] (%s): at least one evaluator is required", i, g.Name) + } + for _, ref := range g.Evaluators { + if ref.IsBuiltin() { + continue + } + if !evaluators[ref.Name] { + return fmt.Errorf( + "evalGroups[%d] (%s): evaluator %q is not declared in evaluators "+ + "(built-ins need the %q prefix)", + i, g.Name, ref.Name, evalcore.BuiltinPrefix) + } + } + if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { + return fmt.Errorf( + "evalGroups[%d] (%s): target.type %q is not supported; only %q is available today", + i, g.Name, g.Target.Type, TargetTypeAgent) + } + if g.Options != nil { + switch g.Options.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: + return fmt.Errorf( + "evalGroups[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + i, g.Name, g.Options.EvaluationLevel, + EvaluationLevelTurn, EvaluationLevelConversation) + } + } + } + + return nil +} + +// Dataset returns the declaration with the given name. +func (c *EvalConfig) Dataset(name string) (*DatasetDecl, bool) { + for i := range c.Datasets { + if c.Datasets[i].Name == name { + return &c.Datasets[i], true + } + } + return nil, false +} + +// Evaluator returns the declaration with the given name. +func (c *EvalConfig) Evaluator(name string) (*EvaluatorDecl, bool) { + for i := range c.Evaluators { + if c.Evaluators[i].Name == name { + return &c.Evaluators[i], true + } + } + return nil, false +} + +// Group returns the eval group with the given name. +func (c *EvalConfig) Group(name string) (*EvalGroup, bool) { + for i := range c.EvalGroups { + if c.EvalGroups[i].Name == name { + return &c.EvalGroups[i], true + } + } + return nil, false +} + +// ResolveGroup picks the group to act on: the named one, or the only one when +// the config declares exactly one. +func (c *EvalConfig) ResolveGroup(name string) (*EvalGroup, error) { + if name != "" { + g, ok := c.Group(name) + if !ok { + return nil, fmt.Errorf("eval group %q is not declared in the config", name) + } + return g, nil + } + switch len(c.EvalGroups) { + case 0: + return nil, fmt.Errorf("no eval groups are declared in the config") + case 1: + return &c.EvalGroups[0], nil + default: + names := make([]string, 0, len(c.EvalGroups)) + for _, g := range c.EvalGroups { + names = append(names, g.Name) + } + return nil, fmt.Errorf( + "the config declares %d eval groups (%s); choose one with --eval-group", + len(c.EvalGroups), strings.Join(names, ", ")) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go new file mode 100644 index 00000000000..172154ce714 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -0,0 +1,235 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +const sampleDeployConfig = ` +evaluators: + - name: support-quality + source: ./evaluators/support-quality/rubric_dimensions.json + - name: safety-check + source: ./evaluators/safety-check.json + +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + +evalGroups: + - name: pr-gate + description: Quality gate for the support agent + dataset: support-golden + evaluators: + - builtin.task_adherence + - { name: support-quality, threshold: 4.0 } + - safety-check + target: + type: agent + name: support-agent + options: + eval_model: gpt-4.1-nano + max_samples: 100 + evaluation_level: conversation +` + +func loadFromString(t *testing.T, body string) *EvalConfig { + t.Helper() + path := filepath.Join(t.TempDir(), "azure.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + return cfg +} + +func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { + cfg := loadFromString(t, sampleDeployConfig) + + require.Len(t, cfg.Evaluators, 2) + require.Len(t, cfg.Datasets, 1) + require.Len(t, cfg.EvalGroups, 1) + + ds, ok := cfg.Dataset("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) + require.Equal(t, "1", ds.Version) + + g, ok := cfg.Group("pr-gate") + require.True(t, ok) + require.Equal(t, "support-golden", g.Dataset) + require.Equal(t, TargetTypeAgent, g.Target.Type) + require.Equal(t, "support-agent", g.Target.Name) + require.Equal(t, EvaluationLevelConversation, g.Options.EvaluationLevel) +} + +// Evaluator entries accept a bare string or a mapping carrying a threshold. +func TestEvaluatorList_MixedForms(t *testing.T) { + cfg := loadFromString(t, sampleDeployConfig) + g, ok := cfg.Group("pr-gate") + require.True(t, ok) + require.Len(t, g.Evaluators, 3) + + require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) + require.True(t, g.Evaluators[0].IsBuiltin()) + require.Equal(t, "task_adherence", g.Evaluators[0].APIName(), + "the builtin prefix must be stripped before it reaches the service") + require.Nil(t, g.Evaluators[0].Threshold) + + require.Equal(t, "support-quality", g.Evaluators[1].Name) + require.False(t, g.Evaluators[1].IsBuiltin()) + require.NotNil(t, g.Evaluators[1].Threshold) + require.InDelta(t, 4.0, *g.Evaluators[1].Threshold, 0.0001) + + require.Equal(t, "safety-check", g.Evaluators[2].Name) + require.Nil(t, g.Evaluators[2].Threshold) +} + +// Round-tripping must not rewrite bare names into mappings. +func TestEvaluatorList_RoundTripKeepsCompactForm(t *testing.T) { + threshold := 4.0 + list := evalcore.EvaluatorList{ + {Name: "builtin.relevance"}, + {Name: "support-quality", Threshold: &threshold}, + } + + out, err := yaml.Marshal(list) + require.NoError(t, err) + + var back evalcore.EvaluatorList + require.NoError(t, yaml.Unmarshal(out, &back)) + require.Len(t, back, 2) + require.Equal(t, "builtin.relevance", back[0].Name) + require.Nil(t, back[0].Threshold) + require.NotNil(t, back[1].Threshold) + require.Contains(t, string(out), "- builtin.relevance", + "an evaluator with only a name should stay a plain string") +} + +func TestValidate_Accepts(t *testing.T) { + require.NoError(t, loadFromString(t, sampleDeployConfig).Validate()) +} + +func TestValidate_Rejects(t *testing.T) { + cases := []struct { + name string + body string + wantErr string + }{ + { + name: "dataset referenced but not declared", + body: "evalGroups:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", + wantErr: "is not declared in datasets", + }, + { + name: "custom evaluator referenced but not declared", + body: "datasets:\n - name: d\n" + + "evalGroups:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", + wantErr: "is not declared in evaluators", + }, + { + name: "built-in declared as a custom evaluator", + body: "evaluators:\n - name: builtin.relevance\n", + wantErr: "must not be declared", + }, + { + name: "group without evaluators", + body: "evalGroups:\n - name: g\n evaluators: []\n", + wantErr: "at least one evaluator is required", + }, + { + name: "unsupported target type", + body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: model\n", + wantErr: "is not supported", + }, + { + name: "invalid evaluation level", + body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n" + + " options:\n evaluation_level: sentence\n", + wantErr: "evaluation_level", + }, + { + name: "duplicate dataset", + body: "datasets:\n - name: d\n - name: d\n", + wantErr: "duplicate dataset name", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := loadFromString(t, tc.body).Validate() + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +func TestResolveGroup(t *testing.T) { + single := loadFromString(t, sampleDeployConfig) + + t.Run("only group is used when unnamed", func(t *testing.T) { + g, err := single.ResolveGroup("") + require.NoError(t, err) + require.Equal(t, "pr-gate", g.Name) + }) + + t.Run("named group", func(t *testing.T) { + g, err := single.ResolveGroup("pr-gate") + require.NoError(t, err) + require.Equal(t, "pr-gate", g.Name) + }) + + t.Run("unknown name is an error", func(t *testing.T) { + _, err := single.ResolveGroup("nope") + require.ErrorContains(t, err, "is not declared") + }) + + t.Run("ambiguous without a name", func(t *testing.T) { + multi := loadFromString(t, + "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ + " - name: nightly\n evaluators: [builtin.relevance]\n") + _, err := multi.ResolveGroup("") + require.ErrorContains(t, err, "--eval-group") + require.ErrorContains(t, err, "nightly") + }) + + t.Run("empty config", func(t *testing.T) { + _, err := (&EvalConfig{}).ResolveGroup("") + require.ErrorContains(t, err, "no eval groups") + }) +} + +// local_dir accepts a directory or an explicit file path. +func TestArtifactPath(t *testing.T) { + cases := []struct { + name string + localDir string + resource string + ext string + want string + }{ + {"directory derives the file name", "datasets", "support-golden", ".jsonl", + filepath.Join("base", "datasets", "support-golden.jsonl")}, + {"explicit file path is used as-is", "generated/datasets/support-golden.jsonl", "ignored", ".jsonl", + filepath.Join("base", "generated", "datasets", "support-golden.jsonl")}, + {"empty local_dir falls back to the base", "", "support-quality", ".json", + filepath.Join("base", "support-quality.json")}, + {"yaml rubric file path", "generated/rubrics/quality.yaml", "ignored", ".json", + filepath.Join("base", "generated", "rubrics", "quality.yaml")}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, ArtifactPath("base", tc.localDir, tc.resource, tc.ext)) + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go new file mode 100644 index 00000000000..4e1d647387d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "go.yaml.in/yaml/v3" +) + +// Conventional locations. Both are relative to the working directory and are +// used verbatim — never re-rooted under the agent or project directory. +const ( + DefaultEvalDir = "evals" + DefaultGenerateConfig = "evals/eval_generate.yaml" + DefaultDeployConfig = "evals/azure.yaml" + DefaultDatasetsDir = "datasets" + DefaultEvaluatorsDir = "evaluators" +) + +// GenerateConfig is the generation spec — input to `azd ai eval generate`. It +// is never deployed. +type GenerateConfig struct { + Agent AgentSpec `yaml:"agent" json:"agent"` + Generate GenerateSpec `yaml:"generate" json:"generate"` +} + +// AgentSpec identifies the agent and the context the generator reads. +type AgentSpec struct { + Name string `yaml:"name" json:"name"` + Context AgentContext `yaml:"context,omitempty" json:"context,omitempty"` +} + +// AgentContext points at the material used to synthesize a rubric and dataset. +type AgentContext struct { + Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` + Tools string `yaml:"tools,omitempty" json:"tools,omitempty"` + Traces *TraceSpec `yaml:"traces,omitempty" json:"traces,omitempty"` +} + +// TraceSpec seeds rubric generation from recent traces. Traces are a generation +// input only; they cannot be a run's data source. +type TraceSpec struct { + Source string `yaml:"source,omitempty" json:"source,omitempty"` + Window string `yaml:"window,omitempty" json:"window,omitempty"` + Sample int `yaml:"sample,omitempty" json:"sample,omitempty"` +} + +// GenerateSpec configures what gets produced. +type GenerateSpec struct { + Rubric *RubricSpec `yaml:"rubric,omitempty" json:"rubric,omitempty"` + Dataset *DatasetSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` +} + +// RubricSpec configures rubric (LLM-graded evaluator) generation. +type RubricSpec struct { + Name string `yaml:"name" json:"name"` + Model string `yaml:"model,omitempty" json:"model,omitempty"` + LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +} + +// DatasetSpec configures synthetic dataset generation. +type DatasetSpec struct { + Name string `yaml:"name" json:"name"` + Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"` + SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` + LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +} + +// Generation strategies. +const ( + StrategySynthetic = "synthetic" + StrategyFromTraces = "from-traces" +) + +// Sample-count bounds enforced by the generation service. +const ( + MinSampleSize = 15 + MaxSampleSize = 1000 + DefaultSampleSize = 15 +) + +// LoadGenerateConfig reads a generation spec from disk. +func LoadGenerateConfig(path string) (*GenerateConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading generation config %q: %w", path, err) + } + + var cfg GenerateConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing generation config %q: %w", path, err) + } + return &cfg, nil +} + +// Validate reports configuration errors before any generation job is submitted. +func (c *GenerateConfig) Validate() error { + if c.Agent.Name == "" { + return fmt.Errorf("agent.name is required") + } + if c.Generate.Rubric == nil && c.Generate.Dataset == nil { + return fmt.Errorf("generate must declare a rubric, a dataset, or both") + } + if r := c.Generate.Rubric; r != nil && r.Name == "" { + return fmt.Errorf("generate.rubric.name is required") + } + if d := c.Generate.Dataset; d != nil { + if d.Name == "" { + return fmt.Errorf("generate.dataset.name is required") + } + switch d.Strategy { + case "", StrategySynthetic, StrategyFromTraces: + default: + return fmt.Errorf( + "generate.dataset.strategy %q is invalid; expected %q or %q", + d.Strategy, StrategySynthetic, StrategyFromTraces) + } + if d.SampleSize != 0 && (d.SampleSize < MinSampleSize || d.SampleSize > MaxSampleSize) { + return fmt.Errorf( + "generate.dataset.sampleSize must be between %d and %d, got %d", + MinSampleSize, MaxSampleSize, d.SampleSize) + } + } + return nil +} + +// ArtifactPath resolves a local_dir value against baseDir. The value may be a +// directory, in which case the file name is derived from resourceName and ext, +// or an explicit file path, which is used as-is. +func ArtifactPath(baseDir, localDir, resourceName, ext string) string { + if localDir == "" { + return filepath.Join(baseDir, resourceName+ext) + } + candidate := localDir + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(baseDir, candidate) + } + if looksLikeFile(localDir, ext) { + return candidate + } + return filepath.Join(candidate, resourceName+ext) +} + +// looksLikeFile treats a trailing recognised extension as an explicit file path. +func looksLikeFile(p, ext string) bool { + got := strings.ToLower(filepath.Ext(p)) + if got == "" { + return false + } + if got == strings.ToLower(ext) { + return true + } + switch got { + case ".json", ".jsonl", ".yaml", ".yml": + return true + } + return false +} From 125cd7a9b2462f722621fda773fd2edbc4fbf9cb Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:05:40 +0300 Subject: [PATCH 004/114] feat(azure.ai.evaluations): eval group builder and run command - buildEvalGroupRequest maps evaluators to testing criteria, keeping the builtin prefix on evaluator_name while stripping it from name, and carries per-evaluator thresholds in initialization_parameters - run resolves the group from --eval-id, a pinned id, or the azd environment, creating it when absent, then binds the dataset to the run since the group has no dataset binding today - Local datasets are sent inline with optional truncation; registered datasets are referenced by id --- .../internal/cmd/build.go | 83 +++++ .../azure.ai.evaluations/internal/cmd/root.go | 5 +- .../azure.ai.evaluations/internal/cmd/run.go | 313 ++++++++++++++++++ 3 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go new file mode 100644 index 00000000000..77a3ef8918c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" +) + +// dataMapping is the template binding the service uses to feed each evaluator. +// It pairs with the item schema below, which declares a single `query` field. +func dataMapping() map[string]string { + return map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + } +} + +// agentItemSchema mirrors the shape the agent-target runner expects. It is a +// fixed schema, not inferred from the dataset. +func agentItemSchema() map[string]any { + return map[string]any{ + "type": "object", + "properties": map[string]any{ + "query": map[string]any{"type": "string"}, + }, + } +} + +// buildEvalGroupRequest converts an eval group declaration into the create +// request. Evaluators become testing criteria; a per-evaluator threshold is +// carried in initialization_parameters alongside the judge model. +func buildEvalGroupRequest(group *project.EvalGroup) *eval_api.CreateOpenAIEvalRequest { + metadata := map[string]string{} + if group.Target != nil && group.Target.Name != "" { + metadata["azd_agent"] = group.Target.Name + } + metadata["azd_eval_group"] = group.Name + + req := &eval_api.CreateOpenAIEvalRequest{ + Name: group.Name, + Metadata: metadata, + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: agentItemSchema(), + }, + } + + evalModel := "" + if group.Options != nil { + evalModel = group.Options.EvalModel + } + + for _, ref := range group.Evaluators { + criterion := eval_api.TestingCriterion{ + Type: "azure_ai_evaluator", + // Name drops the builtin prefix; EvaluatorName keeps it. + Name: ref.APIName(), + EvaluatorName: ref.Name, + DataMapping: dataMapping(), + } + + params := map[string]any{} + if evalModel != "" { + params["model"] = evalModel + params["deployment_name"] = evalModel + } + if ref.Threshold != nil { + params["threshold"] = *ref.Threshold + } + if len(params) > 0 { + criterion.InitializationParameters = params + } + + req.TestingCriteria = append(req.TestingCriteria, criterion) + } + + return req +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 9901d141836..40f666b2f90 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -25,7 +25,10 @@ func NewRootCommand() *cobra.Command { rootCmd.SilenceErrors = true rootCmd.CompletionOptions.DisableDefaultCmd = true - rootCmd.AddCommand(newDatasetCommand()) + rootCmd.AddCommand( + newDatasetCommand(), + newRunCommand(), + ) return rootCmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go new file mode 100644 index 00000000000..7085d5b7ad7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -0,0 +1,313 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Terminal run states reported by the service. +var terminalRunStates = map[string]bool{ + "completed": true, + "failed": true, + "canceled": true, + "cancelled": true, + "error": true, +} + +func newRunCommand() *cobra.Command { + var ( + configPath string + groupName string + evalID string + runName string + level string + maxSamples int + wait bool + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "run", + Short: "Run an evaluation, creating the eval group if it does not exist yet.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // --eval-id bypasses the config entirely. + var group *project.EvalGroup + if evalID == "" { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + group, err = cfg.ResolveGroup(groupName) + if err != nil { + return err + } + + evalID, err = ec.resolveEvalGroupID(ctx, group, out, isJSON(cmd)) + if err != nil { + return err + } + } + + dataSource, err := buildRunDataSource(group, configPath, maxSamples) + if err != nil { + return err + } + + if runName == "" { + base := "eval" + if group != nil { + base = group.Name + } + runName = fmt.Sprintf("%s-%s", base, time.Now().UTC().Format("20060102-150405")) + } + + metadata := map[string]string{} + if lvl := resolveLevel(level, group); lvl != "" { + metadata["evaluation_level"] = lvl + } + + run, err := ec.evalClient.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: runName, + DataSource: dataSource, + Metadata: metadata, + }) + if err != nil { + return fmt.Errorf("starting the evaluation run: %w", err) + } + + if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + } + + if !wait { + if isJSON(cmd) { + return emitJSON(out, run) + } + fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) + fmt.Fprintf(out, "Check progress with: azd ai eval results show %s --run-id %s\n", evalID, run.ID) + return nil + } + + final, err := ec.pollRun(ctx, evalID, run.ID, out, isJSON(cmd)) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(out, final) + } + return renderRun(out, final) + }, + } + + cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, + "Path to the eval deployment config.") + cmd.Flags().StringVar(&groupName, "eval-group", "", + "Which evalGroups entry to run. Defaults to the only one.") + cmd.Flags().StringVar(&evalID, "eval-id", "", + "Run against an existing eval group by id, ignoring the config.") + cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the group name plus a timestamp.") + cmd.Flags().StringVar(&level, "level", "", + "Scoring granularity: turn or conversation. Defaults to the service default (turn).") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + "Cap the rows sent from a local dataset file. Ignored for registered datasets.") + cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveEvalGroupID finds the eval group to run against, creating it when it +// has never been deployed. Resolution order: an id pinned on the group, then +// the azd environment, then create. +func (ec *evalContext) resolveEvalGroupID( + ctx context.Context, + group *project.EvalGroup, + out interface{ Write([]byte) (int, error) }, + jsonMode bool, +) (string, error) { + if group.ID != "" { + return group.ID, nil + } + + if cached := ec.getEnvValue(ctx, envKeyEvalGroupID); cached != "" { + // Confirm it still exists; a deleted group should fall through to create. + if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + return cached, nil + } + } + + if !jsonMode { + fmt.Fprintf(out, "Creating eval group %q...\n", group.Name) + } + created, err := ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(group)) + if err != nil { + return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) + } + if err := ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID); err != nil { + fmt.Fprintf(out, "warning: %v\n", err) + } + return created.ID, nil +} + +// buildRunDataSource binds the dataset to the run. The eval group carries no +// dataset today, so it is supplied here. +func buildRunDataSource( + group *project.EvalGroup, + configPath string, + maxSamples int, +) (*eval_api.EvalRunDataSource, error) { + if group == nil || group.Target == nil { + return nil, fmt.Errorf( + "the eval group must declare target.type: agent so the run knows what to invoke") + } + + ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + + if group.Dataset == "" { + return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) + } + + // A local source is sent inline; anything else is a registered dataset. + localPath := localDatasetPath(configPath, group) + if localPath == "" { + ds.SetFileID(group.Dataset) + return ds, nil + } + + items, err := readJSONL(localPath, maxSamples) + if err != nil { + return nil, err + } + if len(items) == 0 { + return nil, fmt.Errorf("dataset file %q has no rows", localPath) + } + ds.SetFileContent(items) + return ds, nil +} + +// localDatasetPath resolves the dataset's local source relative to the config +// file, returning empty when the dataset is registered rather than local. +func localDatasetPath(configPath string, group *project.EvalGroup) string { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { + return "" + } + decl, ok := cfg.Dataset(group.Dataset) + if !ok || decl.Source == "" { + return "" + } + if filepath.IsAbs(decl.Source) { + return decl.Source + } + return filepath.Join(filepath.Dir(configPath), decl.Source) +} + +// readJSONL reads newline-delimited JSON, optionally truncating to limit rows. +func readJSONL(path string, limit int) ([]map[string]any, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + defer f.Close() + + var items []map[string]any + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + line := 0 + for scanner.Scan() { + line++ + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return nil, fmt.Errorf("%s line %d is not valid JSON: %w", path, line, err) + } + items = append(items, row) + if limit > 0 && len(items) >= limit { + break + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + return items, nil +} + +// resolveLevel prefers the flag, then the group's options. +func resolveLevel(flag string, group *project.EvalGroup) string { + if flag != "" { + return flag + } + if group != nil && group.Options != nil { + return group.Options.EvaluationLevel + } + return "" +} + +// pollRun waits for the run to reach a terminal state, reporting status changes. +func (ec *evalContext) pollRun( + ctx context.Context, + evalID, runID string, + out interface{ Write([]byte) (int, error) }, + jsonMode bool, +) (*eval_api.OpenAIEvalRun, error) { + const interval = 5 * time.Second + lastStatus := "" + + for { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err != nil { + return nil, fmt.Errorf("polling run %s: %w", runID, err) + } + if run.Status != lastStatus { + lastStatus = run.Status + if !jsonMode { + fmt.Fprintf(out, " status: %s\n", run.Status) + } + } + if terminalRunStates[strings.ToLower(run.Status)] { + return run, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(interval): + } + } +} + +func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { + fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) + if run.ReportURL != "" { + fmt.Fprintf(out, "Report: %s\n", run.ReportURL) + } + return nil +} From 035f6bb85fe4b3db522a2bce5c133b4c5ddcd97d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:11:15 +0300 Subject: [PATCH 005/114] feat(azure.ai.evaluations): evaluator and results commands - evaluator upload/update/list/show/builtins/delete; rubric evaluators only in M1, code evaluators deferred to M2 with the folder walk and RBAC they require - normalizeRubricBody accepts a bare definition or a full document - results show/export with per-criteria pass and fail counts, --failed-only, and JSON or CSV output, replacing the counts-only view - Added ListEvaluators, ListEvaluatorVersions, DeleteEvaluatorVersion, and CancelOpenAIEvalRun to the eval client --- .../internal/cmd/evaluator.go | 298 ++++++++++++++++++ .../internal/cmd/results.go | 239 ++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 2 + .../internal/pkg/eval_api/evaluators.go | 84 +++++ 4 files changed, 623 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go new file mode 100644 index 00000000000..e374079dd9e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -0,0 +1,298 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "fmt" + "os" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +func newEvaluatorCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "evaluator", + Short: "Manage custom evaluators.", + } + cmd.AddCommand( + newEvaluatorUploadCommand(false), + newEvaluatorUploadCommand(true), + newEvaluatorListCommand(), + newEvaluatorShowCommand(), + newEvaluatorBuiltinsCommand(), + newEvaluatorDeleteCommand(), + ) + return cmd +} + +// newEvaluatorUploadCommand builds `evaluator upload` and `evaluator update`. +// Both publish a new immutable version. +// +// M1 supports rubric evaluators only. Code evaluators need a folder walk, +// multi-blob upload, and the Azure AI User role assignment, so they land in M2. +func newEvaluatorUploadCommand(update bool) *cobra.Command { + var ( + name string + rubric string + endpointFlg string + ) + + use, short := "upload", "Register a rubric evaluator, creating its first version." + if update { + use, short = "update", "Publish a new version of an existing rubric evaluator." + } + + cmd := &cobra.Command{ + Use: use, + Short: short, + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if rubric == "" { + return requireFlag("rubric") + } + + raw, err := os.ReadFile(rubric) + if err != nil { + return fmt.Errorf("reading rubric %q: %w", rubric, err) + } + + body, err := normalizeRubricBody(name, raw) + if err != nil { + return fmt.Errorf("rubric %q: %w", rubric, err) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + created, err := ec.evalClient.CreateEvaluatorVersion( + ctx, name, body, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("registering evaluator %q: %w", name, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), created) + } + fmt.Fprintf(cmd.OutOrStdout(), + "Registered evaluator %s version %s\n", created.Name, created.Version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") + cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or +// a full evaluator document ({name, definition}) and returns the request body. +func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { + var probe map[string]json.RawMessage + if err := json.Unmarshal(raw, &probe); err != nil { + return nil, fmt.Errorf("not valid JSON: %w", err) + } + + if _, hasDefinition := probe["definition"]; hasDefinition { + // Already a full document; make sure the name matches the flag. + probe["name"] = json.RawMessage(fmt.Sprintf("%q", name)) + out, err := json.Marshal(probe) + if err != nil { + return nil, err + } + return out, nil + } + + if _, hasDimensions := probe["dimensions"]; !hasDimensions { + return nil, fmt.Errorf( + "expected a rubric definition with 'dimensions', or a document with 'definition'") + } + + doc := map[string]any{ + "name": name, + "definition": json.RawMessage(raw), + } + out, err := json.Marshal(doc) + if err != nil { + return nil, err + } + return out, nil +} + +func newEvaluatorListCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List evaluators, or the versions of one evaluator.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + var list *eval_api.EvaluatorListResponse + if name != "" { + list, err = ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + } else { + list, err = ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + } + if err != nil { + return fmt.Errorf("listing evaluators: %w", err) + } + return renderEvaluators(cmd, list) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this evaluator.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvaluatorBuiltinsCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "builtins", + Short: "List the platform's built-in evaluators.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("listing built-in evaluators: %w", err) + } + return renderEvaluators(cmd, list) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), list) + } + if len(list.Value) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No evaluators found.") + return nil + } + rows := make([][]string, 0, len(list.Value)) + for _, e := range list.Value { + rows = append(rows, []string{e.Name, e.Version, e.Type}) + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) +} + +func newEvaluatorShowCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show", + Short: "Show an evaluator definition.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + raw, err := ec.evalClient.GetEvaluatorRaw(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("reading evaluator %q: %w", name, err) + } + + var pretty any + if err := json.Unmarshal(raw, &pretty); err != nil { + fmt.Fprintln(cmd.OutOrStdout(), string(raw)) + return nil + } + return emitJSON(cmd.OutOrStdout(), pretty) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") + cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvaluatorDeleteCommand() *cobra.Command { + var ( + name string + version string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete an evaluator version.", + RunE: func(cmd *cobra.Command, args []string) error { + if name == "" { + return requireFlag("name") + } + if version == "" { + return requireFlag("version") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.evalClient.DeleteEvaluatorVersion( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return fmt.Errorf("deleting evaluator %q version %q: %w", name, version, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "name": name, "version": version, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted evaluator %s version %s\n", name, version) + return nil + }, + } + + cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") + cmd.Flags().StringVar(&version, "version", "", "Version to delete.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go new file mode 100644 index 00000000000..3eb79a1bc22 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +func newResultsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "results", + Short: "Inspect evaluation results.", + } + cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) + return cmd +} + +func newResultsShowCommand() *cobra.Command { + var ( + runID string + failedOnly bool + outFile string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show per-sample results for a run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return fmt.Errorf("creating %q: %w", outFile, err) + } + defer f.Close() + return emitJSON(f, run) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), run) + } + return renderResults(cmd.OutOrStdout(), run, failedOnly) + }, + } + + cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") + cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") + cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newResultsExportCommand() *cobra.Command { + var ( + runID string + format string + outFile string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "export ", + Short: "Export run results as JSON or CSV.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + format = strings.ToLower(format) + if format != "json" && format != "csv" { + return fmt.Errorf("--format must be json or csv, got %q", format) + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + + var w io.Writer = cmd.OutOrStdout() + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return fmt.Errorf("creating %q: %w", outFile, err) + } + defer f.Close() + w = f + } + + if format == "json" { + return emitJSON(w, run) + } + return writeResultsCSV(w, run) + }, + } + + cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") + cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") + cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveEvalID takes the eval group id from the argument, falling back to the +// id cached in the azd environment. +func resolveEvalID(cmd *cobra.Command, ec *evalContext, args []string) (string, error) { + if len(args) > 0 && args[0] != "" { + return args[0], nil + } + if cached := ec.getEnvValue(cmd.Context(), envKeyEvalGroupID); cached != "" { + return cached, nil + } + return "", fmt.Errorf( + "no eval group id given; pass it as an argument or set %s in the azd environment", + envKeyEvalGroupID) +} + +// latestOrNamedRun returns the named run, or the most recent one for the group. +func (ec *evalContext) latestOrNamedRun( + cmd *cobra.Command, + evalID, runID string, +) (*eval_api.OpenAIEvalRun, error) { + ctx := cmd.Context() + + if runID == "" { + if cached := ec.getEnvValue(ctx, envKeyEvalRunID); cached != "" { + runID = cached + } + } + if runID != "" { + run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + if err != nil { + return nil, fmt.Errorf("reading run %s: %w", runID, err) + } + return run, nil + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + return nil, fmt.Errorf("listing runs for eval group %s: %w", evalID, err) + } + if len(list.Data) == 0 { + return nil, fmt.Errorf("eval group %s has no runs yet", evalID) + } + return &list.Data[0], nil +} + +func renderResults(w io.Writer, run *eval_api.OpenAIEvalRun, failedOnly bool) error { + fmt.Fprintf(w, "Run %s status: %s\n", run.ID, run.Status) + + if c := run.ResultCounts; c != nil { + fmt.Fprintf(w, "Totals: %d passed, %d failed, %d errored\n\n", + c.Passed, c.Failed, c.Errored) + } + + if len(run.PerTestingCriteria) == 0 { + fmt.Fprintln(w, "No per-criteria results are available yet.") + return nil + } + + rows := make([][]string, 0, len(run.PerTestingCriteria)) + for _, cr := range run.PerTestingCriteria { + if failedOnly && cr.Failed == 0 { + continue + } + rows = append(rows, []string{ + cr.TestingCriteria, + strconv.Itoa(cr.Passed), + strconv.Itoa(cr.Failed), + }) + } + if len(rows) == 0 { + fmt.Fprintln(w, "No failing criteria.") + return nil + } + if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { + return err + } + if run.ReportURL != "" { + fmt.Fprintf(w, "\nReport: %s\n", run.ReportURL) + } + return nil +} + +func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { + cw := csv.NewWriter(w) + defer cw.Flush() + + if err := cw.Write([]string{"run_id", "status", "criterion", "passed", "failed"}); err != nil { + return err + } + if len(run.PerTestingCriteria) == 0 { + return cw.Write([]string{run.ID, run.Status, "", "", ""}) + } + for _, cr := range run.PerTestingCriteria { + if err := cw.Write([]string{ + run.ID, run.Status, cr.TestingCriteria, + strconv.Itoa(cr.Passed), strconv.Itoa(cr.Failed), + }); err != nil { + return err + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 40f666b2f90..64c6d63c568 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -28,6 +28,8 @@ func NewRootCommand() *cobra.Command { rootCmd.AddCommand( newDatasetCommand(), newRunCommand(), + newEvaluatorCommand(), + newResultsCommand(), ) return rootCmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go new file mode 100644 index 00000000000..771f894b03d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +// EvaluatorTypeBuiltin selects the platform-provided evaluators. +const EvaluatorTypeBuiltin = "Builtin" + +// EvaluatorSummary is a single entry in an evaluator listing. +type EvaluatorSummary struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` +} + +// EvaluatorListResponse is the paged response for an evaluator listing. +type EvaluatorListResponse struct { + Value []EvaluatorSummary `json:"value"` + NextLink string `json:"nextLink,omitempty"` +} + +// ListEvaluators returns the evaluators visible to the project. Pass +// EvaluatorTypeBuiltin to list only the platform's built-ins. +func (c *EvalClient) ListEvaluators( + ctx context.Context, + evaluatorType string, + apiVersion string, +) (*EvaluatorListResponse, error) { + var query map[string]string + if evaluatorType != "" { + query = map[string]string{"type": evaluatorType} + } + return doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, pathEvaluators, query, nil, apiVersion, + ) +} + +// ListEvaluatorVersions returns every version of one evaluator. +func (c *EvalClient) ListEvaluatorVersions( + ctx context.Context, + name string, + apiVersion string, +) (*EvaluatorListResponse, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorListResponse]( + c, ctx, http.MethodGet, path, nil, nil, apiVersion, + ) +} + +// DeleteEvaluatorVersion removes a single evaluator version. +func (c *EvalClient) DeleteEvaluatorVersion( + ctx context.Context, + name string, + version string, + apiVersion string, +) error { + path := fmt.Sprintf( + "%s/%s/versions/%s", + pathEvaluators, url.PathEscape(name), url.PathEscape(version), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// CancelOpenAIEvalRun stops an in-flight run. +func (c *EvalClient) CancelOpenAIEvalRun( + ctx context.Context, + evalID string, + runID string, +) (*OpenAIEvalRun, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s/cancel", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") +} From 4f8a4eb60a641ec32f0e493fd709c9a249882baa Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:13:10 +0300 Subject: [PATCH 006/114] feat(azure.ai.evaluations): offline init command with round-trip tests init scaffolds both YAML files and the artifact directories without any service call, so it works offline and unauthenticated. Built-ins are referenced from the group but never declared as custom evaluators. A dataset flag containing a path becomes a local source; a bare name references a registered dataset. Tests assert the scaffold loads and validates, and that paths are used verbatim rather than re-rooted. --- .../azure.ai.evaluations/internal/cmd/init.go | 218 ++++++++++++++++++ .../internal/cmd/init_test.go | 143 ++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 + 3 files changed, 362 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go new file mode 100644 index 00000000000..047257edd2e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +// newInitCommand scaffolds the eval configuration. It makes no service calls at +// all, so it works offline and unauthenticated. +func newInitCommand() *cobra.Command { + var ( + target string + dataset string + evaluators []string + evalModel string + outDir string + force bool + ) + + cmd := &cobra.Command{ + Use: "init", + Short: "Scaffold evaluation config for an agent. Makes no service calls.", + RunE: func(cmd *cobra.Command, args []string) error { + out := cmd.OutOrStdout() + + if target == "" { + return requireFlag("target") + } + if outDir == "" { + outDir = project.DefaultEvalDir + } + + genPath := filepath.Join(outDir, "eval_generate.yaml") + depPath := filepath.Join(outDir, "azure.yaml") + + for _, p := range []string{genPath, depPath} { + if _, err := os.Stat(p); err == nil && !force { + return fmt.Errorf("%s already exists; pass --force to overwrite", p) + } + } + + if err := os.MkdirAll(filepath.Join(outDir, project.DefaultDatasetsDir), 0o750); err != nil { + return fmt.Errorf("creating the datasets directory: %w", err) + } + if err := os.MkdirAll(filepath.Join(outDir, project.DefaultEvaluatorsDir), 0o750); err != nil { + return fmt.Errorf("creating the evaluators directory: %w", err) + } + + rubricName := fmt.Sprintf("%s-quality", target) + + genCfg := buildGenerateScaffold(target, rubricName, evalModel) + if err := writeYAML(genPath, genCfg); err != nil { + return err + } + + depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel) + if err := writeYAML(depPath, depCfg); err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(out, map[string]any{ + "generateConfig": genPath, + "deployConfig": depPath, + "datasetsDir": filepath.Join(outDir, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(outDir, project.DefaultEvaluatorsDir), + }) + } + + fmt.Fprintf(out, "Wrote %s\n", genPath) + fmt.Fprintf(out, "Wrote %s\n", depPath) + fmt.Fprintln(out, "\nNext:") + fmt.Fprintf(out, " 1. Reference %s from your root azure.yaml:\n", depPath) + fmt.Fprintln(out, " services:") + fmt.Fprintln(out, " evals:") + fmt.Fprintln(out, " host: azure.ai.eval") + fmt.Fprintln(out, " uses: [ai-project]") + fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) + fmt.Fprintln(out, " 2. azd ai eval generate (or supply your own dataset)") + fmt.Fprintln(out, " 3. azd up") + fmt.Fprintln(out, " 4. azd ai eval run") + return nil + }, + } + + cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") + cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, + "Evaluator reference, repeatable. Use builtin. for a built-in.") + cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used as the LLM judge.") + cmd.Flags().StringVar(&outDir, "out-dir", project.DefaultEvalDir, + "Directory to write the config into. Used verbatim, never re-rooted.") + cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") + return cmd +} + +func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { + return &project.GenerateConfig{ + Agent: project.AgentSpec{ + Name: target, + Context: project.AgentContext{ + Instructions: "./agent/instructions.md", + Tools: "./agent/tools.json", + }, + }, + Generate: project.GenerateSpec{ + Rubric: &project.RubricSpec{ + Name: rubricName, + Model: evalModel, + LocalDir: "./" + project.DefaultEvaluatorsDir, + }, + Dataset: &project.DatasetSpec{ + Name: fmt.Sprintf("%s-golden", target), + Strategy: project.StrategySynthetic, + SampleSize: project.DefaultSampleSize, + LocalDir: "./" + project.DefaultDatasetsDir, + }, + }, + } +} + +func buildDeployScaffold( + target, rubricName, dataset string, + evaluators []string, + evalModel string, +) *project.EvalConfig { + cfg := &project.EvalConfig{} + + datasetName := fmt.Sprintf("%s-golden", target) + datasetSource := "" + if dataset != "" { + if looksLikeLocalDataset(dataset) { + datasetSource = dataset + datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) + } else { + // A bare name references an already-registered dataset. + datasetName = dataset + } + } else { + datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) + } + cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + Name: datasetName, + Source: datasetSource, + }) + + // Evaluators supplied on the command line win; otherwise scaffold the + // generated rubric so `generate` has somewhere to write its reference. + refs := evalcore.EvaluatorList{} + if len(evaluators) == 0 { + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), + }) + refs = append(refs, evalcore.EvaluatorRef{Name: rubricName}) + } else { + for _, e := range evaluators { + ref := evalcore.EvaluatorRef{Name: e} + if !ref.IsBuiltin() { + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: e, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), + }) + } + refs = append(refs, ref) + } + } + + group := project.EvalGroup{ + Name: fmt.Sprintf("%s-quality", target), + Description: fmt.Sprintf("Quality gate for %s", target), + Dataset: datasetName, + Evaluators: refs, + Target: &project.Target{ + Type: project.TargetTypeAgent, + Name: target, + }, + } + if evalModel != "" { + group.Options = &project.Options{EvalModel: evalModel} + } + cfg.EvalGroups = append(cfg.EvalGroups, group) + + return cfg +} + +// looksLikeLocalDataset distinguishes a path from a registered dataset name. +func looksLikeLocalDataset(v string) bool { + if strings.ContainsAny(v, `/\`) { + return true + } + return strings.EqualFold(filepath.Ext(v), ".jsonl") +} + +func writeYAML(path string, v any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + data, err := yaml.Marshal(v) + if err != nil { + return fmt.Errorf("serializing %q: %w", path, err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("writing %q: %w", path, err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go new file mode 100644 index 00000000000..55820051efb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// The scaffold must load and validate cleanly, otherwise `azd up` fails on a +// config the tool itself produced. +func TestScaffold_RoundTripsAndValidates(t *testing.T) { + dir := t.TempDir() + depPath := filepath.Join(dir, "azure.yaml") + + cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano") + require.NoError(t, writeYAML(depPath, cfg)) + + loaded, err := project.LoadEvalConfig(depPath) + require.NoError(t, err) + require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") + + g, err := loaded.ResolveGroup("") + require.NoError(t, err) + require.Equal(t, project.TargetTypeAgent, g.Target.Type) + require.Equal(t, "support-agent", g.Target.Name) + require.Equal(t, "gpt-4.1-nano", g.Options.EvalModel) + require.Len(t, g.Evaluators, 1) +} + +func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { + dir := t.TempDir() + genPath := filepath.Join(dir, "eval_generate.yaml") + + cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") + require.NoError(t, writeYAML(genPath, cfg)) + + loaded, err := project.LoadGenerateConfig(genPath) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) + require.Equal(t, "support-agent", loaded.Agent.Name) + require.Equal(t, project.StrategySynthetic, loaded.Generate.Dataset.Strategy) + require.Equal(t, project.DefaultSampleSize, loaded.Generate.Dataset.SampleSize) +} + +// Built-ins are referenced from the group but never declared as custom +// evaluators; declaring one is a validation error. +func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { + cfg := buildDeployScaffold( + "support-agent", "unused", "", + []string{"builtin.task_adherence", "my-custom"}, "", + ) + + require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") + require.Equal(t, "my-custom", cfg.Evaluators[0].Name) + + require.Len(t, cfg.EvalGroups[0].Evaluators, 2) + require.True(t, cfg.EvalGroups[0].Evaluators[0].IsBuiltin()) + require.False(t, cfg.EvalGroups[0].Evaluators[1].IsBuiltin()) + + path := filepath.Join(t.TempDir(), "azure.yaml") + require.NoError(t, writeYAML(path, cfg)) + loaded, err := project.LoadEvalConfig(path) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A bare name means an already-registered dataset; a path means a local file. +func TestScaffold_DatasetReferenceForms(t *testing.T) { + t.Run("local path becomes a source", func(t *testing.T) { + cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "") + require.Equal(t, "./tests/golden.jsonl", cfg.Datasets[0].Source) + require.Equal(t, "golden", cfg.Datasets[0].Name) + }) + + t.Run("bare name references a registered dataset", func(t *testing.T) { + cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "") + require.Equal(t, "prod-sample", cfg.Datasets[0].Name) + require.Empty(t, cfg.Datasets[0].Source, + "a registered dataset must not get a local source") + }) + + t.Run("no dataset flag scaffolds a local path", func(t *testing.T) { + cfg := buildDeployScaffold("support-agent", "r", "", nil, "") + require.Contains(t, cfg.Datasets[0].Source, "support-agent-golden.jsonl") + }) +} + +func TestLooksLikeLocalDataset(t *testing.T) { + require.True(t, looksLikeLocalDataset("./data/golden.jsonl")) + require.True(t, looksLikeLocalDataset("golden.jsonl")) + require.True(t, looksLikeLocalDataset(`data\golden.jsonl`)) + require.False(t, looksLikeLocalDataset("prod-sample")) +} + +// Paths are used verbatim relative to the working directory; the doubling bug +// in the agent-scoped command must not reappear. +func TestWriteYAML_UsesPathVerbatim(t *testing.T) { + dir := t.TempDir() + nested := filepath.Join(dir, "evals", "azure.yaml") + + require.NoError(t, writeYAML(nested, &project.EvalConfig{})) + _, err := os.Stat(nested) + require.NoError(t, err, "the file must land exactly at the requested path") + + doubled := filepath.Join(dir, "evals", "evals", "azure.yaml") + _, err = os.Stat(doubled) + require.Error(t, err, "the path must not be re-rooted under itself") +} + +// normalizeRubricBody accepts a bare definition or a full document. +func TestNormalizeRubricBody(t *testing.T) { + t.Run("bare definition is wrapped", func(t *testing.T) { + body, err := normalizeRubricBody("quality", + []byte(`{"type":"rubric","dimensions":[{"id":"q","weight":10}]}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"quality"`) + require.Contains(t, string(body), `"definition"`) + }) + + t.Run("full document keeps its definition and takes the flag name", func(t *testing.T) { + body, err := normalizeRubricBody("renamed", + []byte(`{"name":"old","definition":{"type":"rubric","dimensions":[]}}`)) + require.NoError(t, err) + require.Contains(t, string(body), `"name":"renamed"`) + }) + + t.Run("rejects a document with neither", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"unrelated":true}`)) + require.ErrorContains(t, err, "dimensions") + }) + + t.Run("rejects invalid JSON", func(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`not json`)) + require.ErrorContains(t, err, "not valid JSON") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 64c6d63c568..bb606a49303 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -26,6 +26,7 @@ func NewRootCommand() *cobra.Command { rootCmd.CompletionOptions.DisableDefaultCmd = true rootCmd.AddCommand( + newInitCommand(), newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), From 6b576e3581e623d4e5396c0d395c976e4453ff98 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:17:43 +0300 Subject: [PATCH 007/114] feat(azure.ai.evaluations): service-target provider with change detection Registers the azure.ai.eval service target so azd up and azd deploy reach this extension; the extension ships no deploy command of its own. - Reads the eval config from the service entry's inline properties, the same AdditionalProperties channel the agents extension uses - Deploy reconciles datasets, then evaluators, then eval groups, since a group references the versions the first two resolve to; it fails fast and the next deploy resumes - Datasets are change-detected with a local SHA-256 digest kept in the azd environment, because the dataset API returns no content hash and comparing against the service would mean downloading the blob every deploy - Evaluator definitions come back inline, so those are compared directly - Package and Publish are no-ops; eval artifacts are plain files already on disk --- .../extensions/azure.ai.evaluations/go.mod | 6 +- .../internal/cmd/listen.go | 34 +++ .../internal/cmd/reconciler.go | 216 ++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 + .../internal/project/service_target_eval.go | 281 ++++++++++++++++++ 5 files changed, 535 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go diff --git a/cli/azd/extensions/azure.ai.evaluations/go.mod b/cli/azd/extensions/azure.ai.evaluations/go.mod index c63d3067dfb..ab57a735e7f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/go.mod +++ b/cli/azd/extensions/azure.ai.evaluations/go.mod @@ -4,17 +4,19 @@ go 1.26.4 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 github.com/azure/azure-dev/cli/azd v1.28.0 github.com/fatih/color v1.18.0 github.com/google/uuid v1.6.0 github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 + google.golang.org/protobuf v1.36.11 ) require ( dario.cat/mergo v1.0.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0-beta.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.5.0 // indirect @@ -77,7 +79,6 @@ require ( github.com/sethvargo/go-retry v0.3.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/theckman/yacspin v0.13.12 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -101,6 +102,5 @@ require ( golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go new file mode 100644 index 00000000000..9cef816cec2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/listen.go @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + + "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/spf13/cobra" +) + +// newListenCommand registers the service-target provider with azd. It is hidden +// and invoked by azd itself, not by users. +func newListenCommand() *cobra.Command { + return azdext.NewListenCommand(configureExtensionHost) +} + +// configureExtensionHost wires the azure.ai.eval service target so `azd up` and +// `azd deploy` reach this extension. The provider name must match the manifest. +func configureExtensionHost(host *azdext.ExtensionHost) { + azdClient := host.Client() + + host.WithServiceTarget(project.EvalHost, func() azdext.ServiceTargetProvider { + return project.NewEvalServiceTargetProvider( + azdClient, + func(ctx context.Context) (project.Reconciler, error) { + return newEvalReconciler(ctx) + }, + ) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go new file mode 100644 index 00000000000..e2f433fe84f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/project" +) + +// evalReconciler applies the eval configuration to the data plane. It is the +// deploy half of the provider; the provider owns ordering, this owns the calls. +type evalReconciler struct { + ec *evalContext +} + +var _ project.Reconciler = (*evalReconciler)(nil) + +func newEvalReconciler(ctx context.Context) (project.Reconciler, error) { + ec, err := newEvalContext(ctx, "") + if err != nil { + return nil, err + } + return &evalReconciler{ec: ec}, nil +} + +// EnsureDataset registers a new version only when the local content changed. +// +// The dataset API exposes no content hash, so comparing against the service +// would mean downloading the blob on every deploy. Instead the local file is +// hashed and the digest kept in the azd environment. +func (r *evalReconciler) EnsureDataset( + ctx context.Context, + decl project.DatasetDecl, + localPath string, +) (string, bool, error) { + // No local source means the dataset is already registered; just confirm it. + if localPath == "" { + version := decl.Version + if version == "" { + list, err := r.ec.datasetClient.ListDatasetVersions( + ctx, decl.Name, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, fmt.Errorf( + "dataset %q has no local source and could not be found on the project: %w", + decl.Name, err) + } + if len(list.Value) == 0 { + return "", false, fmt.Errorf( + "dataset %q has no local source and is not registered on the project", decl.Name) + } + version = dataset_api.LatestVersion(list.Value) + } + return version, false, nil + } + + if _, err := os.Stat(localPath); err != nil { + return "", false, fmt.Errorf("dataset source %q: %w", localPath, err) + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + return "", false, err + } + + key := project.FingerprintKey("dataset", decl.Name) + if prior := r.ec.getEnvValue(ctx, key); prior == digest { + // Unchanged since the last deploy; reuse the recorded version. + if version := r.ec.getEnvValue(ctx, versionKey("dataset", decl.Name)); version != "" { + return version, false, nil + } + } + + // The upload helper scans a directory for the first .jsonl. + dir := localPath + if info, err := os.Stat(localPath); err == nil && !info.IsDir() { + dir = filepath.Dir(localPath) + } + + ds, err := r.ec.datasetClient.UploadNewVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) + _ = r.ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version) + + return ds.Version, true, nil +} + +// EnsureEvaluator publishes a new version when the local definition differs +// from what the service holds. Evaluator definitions come back inline, so this +// compares content directly rather than relying on a cached digest. +func (r *evalReconciler) EnsureEvaluator( + ctx context.Context, + decl project.EvaluatorDecl, + localPath string, +) (string, bool, error) { + if localPath == "" { + raw, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, decl.Version, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, fmt.Errorf( + "evaluator %q has no local source and could not be found on the project: %w", + decl.Name, err) + } + return versionFromRaw(raw, decl.Version), false, nil + } + + raw, err := os.ReadFile(localPath) + if err != nil { + return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + } + + body, err := normalizeRubricBody(decl.Name, raw) + if err != nil { + return "", false, fmt.Errorf("evaluator %q: %w", decl.Name, err) + } + + // Compare against the definition already on the service. + if existing, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, decl.Name, "", ProjectEndpointAPIVersion, + ); err == nil { + if sameDefinition(existing, body) { + return versionFromRaw(existing, decl.Version), false, nil + } + } + + created, err := r.ec.evalClient.CreateEvaluatorVersion( + ctx, decl.Name, body, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + return created.Version, true, nil +} + +// EnsureEvalGroup creates the group when it has never been deployed, or when an +// upstream artifact changed. Groups are immutable, so a change means a new +// group and a new id. +func (r *evalReconciler) EnsureEvalGroup( + ctx context.Context, + group project.EvalGroup, + recreate bool, +) (string, error) { + if group.ID != "" { + return group.ID, nil + } + + cached := r.ec.getEnvValue(ctx, envKeyEvalGroupID) + if cached != "" && !recreate { + if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + return cached, nil + } + } + + created, err := r.ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(&group)) + if err != nil { + return "", err + } + _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) + return created.ID, nil +} + +// sameDefinition compares only the definition body, ignoring server-assigned +// fields such as version and timestamps. +func sameDefinition(existing, candidate []byte) bool { + extract := func(raw []byte) string { + var doc map[string]json.RawMessage + if err := json.Unmarshal(raw, &doc); err != nil { + return "" + } + def, ok := doc["definition"] + if !ok { + return "" + } + var normalized any + if err := json.Unmarshal(def, &normalized); err != nil { + return "" + } + out, err := json.Marshal(normalized) + if err != nil { + return "" + } + return string(out) + } + + a, b := extract(existing), extract(candidate) + return a != "" && a == b +} + +func versionFromRaw(raw []byte, fallback string) string { + var doc struct { + Version string `json:"version"` + } + if err := json.Unmarshal(raw, &doc); err == nil && doc.Version != "" { + return doc.Version + } + return fallback +} + +// versionKey holds the version resolved for an artifact at the last deploy. +func versionKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_VERSION" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index bb606a49303..671c10fe0ad 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -31,6 +31,7 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), + newListenCommand(), ) return rootCmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go new file mode 100644 index 00000000000..ce8a1439b5d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -0,0 +1,281 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "google.golang.org/protobuf/types/known/structpb" +) + +// EvalHost is the azure.yaml host this provider serves. +const EvalHost = "azure.ai.eval" + +// azd environment keys owned by this extension. +const ( + EnvKeyEvalGroupID = "EVAL_GROUP_ID" + EnvKeyDatasetVersion = "EVAL_DATASET_VERSION" + EnvKeyFingerprintPrefix = "EVAL_FINGERPRINT_" +) + +// Reconciler applies the eval configuration to the service. It is satisfied by +// the command layer, which owns the data-plane clients. +type Reconciler interface { + // EnsureDataset registers a new dataset version when the local content + // changed, returning the resolved version and whether anything was written. + EnsureDataset(ctx context.Context, decl DatasetDecl, localPath string) (version string, changed bool, err error) + // EnsureEvaluator registers a new evaluator version when the definition + // differs from what the service already holds. + EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) + // EnsureEvalGroup creates the group when it is absent or its resolved + // evaluators or options changed, returning its id. + EnsureEvalGroup(ctx context.Context, group EvalGroup, recreate bool) (id string, err error) +} + +// EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns +// ordering across services through `uses:`; this provider owns only the order +// within the eval service itself. +type EvalServiceTargetProvider struct { + azdClient *azdext.AzdClient + newReconciler func(ctx context.Context) (Reconciler, error) + + serviceConfig *azdext.ServiceConfig + envName string +} + +// NewEvalServiceTargetProvider builds the provider. The reconciler is supplied +// lazily so the data-plane clients are only created when a deploy actually runs. +func NewEvalServiceTargetProvider( + azdClient *azdext.AzdClient, + newReconciler func(ctx context.Context) (Reconciler, error), +) *EvalServiceTargetProvider { + return &EvalServiceTargetProvider{azdClient: azdClient, newReconciler: newReconciler} +} + +func (p *EvalServiceTargetProvider) Initialize( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, +) error { + p.serviceConfig = serviceConfig + return nil +} + +// Endpoints reports no endpoints: eval resources are not addressable. +func (p *EvalServiceTargetProvider) Endpoints( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + targetResource *azdext.TargetResource, +) ([]string, error) { + return nil, nil +} + +func (p *EvalServiceTargetProvider) GetTargetResource( + ctx context.Context, + subscriptionId string, + serviceConfig *azdext.ServiceConfig, + defaultResolver func() (*azdext.TargetResource, error), +) (*azdext.TargetResource, error) { + if defaultResolver != nil { + if target, err := defaultResolver(); err == nil { + return target, nil + } + } + // Eval resources live on the project data plane, so there is no ARM + // resource of our own to resolve. + return &azdext.TargetResource{SubscriptionId: subscriptionId}, nil +} + +// Package is a no-op: eval artifacts are plain files already on disk. +func (p *EvalServiceTargetProvider) Package( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + progress azdext.ProgressReporter, +) (*azdext.ServicePackageResult, error) { + return &azdext.ServicePackageResult{}, nil +} + +// Publish is a no-op: there is no artifact registry step for eval resources. +func (p *EvalServiceTargetProvider) Publish( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + publishOptions *azdext.PublishOptions, + progress azdext.ProgressReporter, +) (*azdext.ServicePublishResult, error) { + return &azdext.ServicePublishResult{}, nil +} + +// Deploy reconciles the eval configuration in a fixed order — datasets, then +// evaluators, then eval groups — because a group references the versions the +// first two resolve to. It fails fast; the next `azd up` resumes from wherever +// it stopped. +func (p *EvalServiceTargetProvider) Deploy( + ctx context.Context, + serviceConfig *azdext.ServiceConfig, + serviceContext *azdext.ServiceContext, + targetResource *azdext.TargetResource, + progress azdext.ProgressReporter, +) (*azdext.ServiceDeployResult, error) { + cfg, err := EvalConfigFromService(serviceConfig) + if err != nil { + return nil, err + } + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("eval config is invalid: %w", err) + } + + reconciler, err := p.newReconciler(ctx) + if err != nil { + return nil, err + } + + baseDir := serviceRelativeDir(serviceConfig) + + // 1. Datasets. + anyChanged := false + for _, decl := range cfg.Datasets { + report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) + if err != nil { + return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) + } + anyChanged = anyChanged || changed + report(progress, describeResult("dataset", decl.Name, version, changed)) + } + + // 2. Evaluators. + for _, decl := range cfg.Evaluators { + report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) + if err != nil { + return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) + } + anyChanged = anyChanged || changed + report(progress, describeResult("evaluator", decl.Name, version, changed)) + } + + // 3. Eval groups. Groups are immutable, so a change upstream means a new + // group must be created and the stored id replaced. + for _, group := range cfg.EvalGroups { + report(progress, fmt.Sprintf("Reconciling eval group %s", group.Name)) + id, err := reconciler.EnsureEvalGroup(ctx, group, anyChanged) + if err != nil { + return nil, fmt.Errorf("eval group %q: %w", group.Name, err) + } + report(progress, fmt.Sprintf("Eval group %s is %s", group.Name, id)) + } + + return &azdext.ServiceDeployResult{}, nil +} + +// describeResult reports whether a version was published or reused, so a +// no-op deploy is visibly a no-op. +func describeResult(kind, name, version string, changed bool) string { + if changed { + return fmt.Sprintf("Published %s %s version %s", kind, name, version) + } + return fmt.Sprintf("%s %s is unchanged at version %s", strings.ToUpper(kind[:1])+kind[1:], name, version) +} + +func report(progress azdext.ProgressReporter, message string) { + if progress != nil { + progress(message) + } +} + +// EvalConfigFromService reads the eval configuration carried inline on the +// service entry. azd captures unknown keys into AdditionalProperties and hands +// them to the extension untouched. +func EvalConfigFromService(svc *azdext.ServiceConfig) (*EvalConfig, error) { + props := serviceProps(svc) + if props == nil || len(props.GetFields()) == 0 { + return nil, fmt.Errorf( + "service %q carries no eval configuration; expected evaluators, datasets, or evalGroups", + svc.GetName()) + } + + raw, err := props.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("reading the eval service configuration: %w", err) + } + + var cfg EvalConfig + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, fmt.Errorf("parsing the eval service configuration: %w", err) + } + return &cfg, nil +} + +// serviceProps prefers the inline properties, falling back to the nested +// config block. +func serviceProps(svc *azdext.ServiceConfig) *structpb.Struct { + if s := svc.GetAdditionalProperties(); s != nil && len(s.GetFields()) > 0 { + return s + } + return svc.GetConfig() +} + +// serviceRelativeDir returns the directory that `source:` paths resolve against. +func serviceRelativeDir(svc *azdext.ServiceConfig) string { + if svc == nil { + return "." + } + if p := svc.GetRelativePath(); p != "" { + return p + } + return "." +} + +// resolveSource joins a declared source against the service directory, leaving +// absolute paths and empty values alone. +func resolveSource(baseDir, source string) string { + if source == "" { + return "" + } + if filepath.IsAbs(source) { + return source + } + return filepath.Join(baseDir, source) +} + +// Fingerprint hashes a local artifact so a later deploy can tell whether the +// content changed without downloading anything from the service. +// +// The dataset API returns no content hash or etag, so comparing against the +// service would mean downloading the blob on every deploy. +func Fingerprint(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("hashing %q: %w", path, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// FingerprintKey is the azd environment key holding an artifact's fingerprint. +func FingerprintKey(kind, name string) string { + safe := strings.Map(func(r rune) rune { + switch { + case r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return r + case r >= 'a' && r <= 'z': + return r - 32 + default: + return '_' + } + }, kind+"_"+name) + return EnvKeyFingerprintPrefix + safe +} From 065e84f442077aebb22a50ac0325c1fe0cdb7a2c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:23:27 +0300 Subject: [PATCH 008/114] feat(azure.ai.evaluations): generate command with YAML write-back - generate submits the rubric and dataset generation jobs, downloads the artifacts locally, and writes source references into the deployment spec - MergeArtifactRefs edits through the yaml Node API so comments, key order, and hand-edited sibling keys survive; matching is by name and merging is idempotent - Raised the client poll budget from 2s x 300 to 5s x 720. The old 10 minute limit gave up while the service was still working, which is the timeout that forced a second command - A supplied --evaluator or a local --dataset is honored and its generation is skipped - Tests cover comment preservation, section creation, idempotence, and fingerprinting --- .../internal/cmd/generate.go | 360 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 + .../internal/project/merge.go | 171 +++++++++ .../internal/project/merge_test.go | 142 +++++++ 4 files changed, 674 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go new file mode 100644 index 00000000000..3ce03cbfd65 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -0,0 +1,360 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. +// The generation job is not gateway-capped; the old limit simply gave up while +// the service was still working, forcing a second command. +var generatePollBudget = eval_api.PollerOptions{ + Interval: 5 * time.Second, + MaxAttempts: 720, // one hour +} + +func newGenerateCommand() *cobra.Command { + var ( + configPath string + deployPath string + target string + instruction string + datasetFlag string + evaluators []string + maxSamples int + traceDays int + evalModel string + noWait bool + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "generate", + Short: "Generate a rubric and dataset, download them, and reference them from the deployment spec.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + cfg, err := resolveGenerateConfig( + configPath, target, evalModel, datasetFlag, maxSamples, traceDays, + ) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + baseDir := filepath.Dir(deployPath) + var datasetRefs, evaluatorRefs []project.ArtifactRef + + // Supplied evaluators are honored: their generation is skipped. + if len(evaluators) > 0 { + fmt.Fprintf(out, "Using the supplied evaluators; skipping rubric generation.\n") + } else if cfg.Generate.Rubric != nil { + ref, err := ec.generateRubric(ctx, cfg, instruction, baseDir, out, noWait) + if err != nil { + return err + } + if ref != nil { + evaluatorRefs = append(evaluatorRefs, *ref) + } + } + + if datasetFlag != "" && looksLikeLocalDataset(datasetFlag) { + fmt.Fprintf(out, "Using the supplied dataset; skipping data generation.\n") + } else if cfg.Generate.Dataset != nil { + ref, err := ec.generateDataset(ctx, cfg, instruction, baseDir, out, noWait) + if err != nil { + return err + } + if ref != nil { + datasetRefs = append(datasetRefs, *ref) + } + } + + if len(datasetRefs) == 0 && len(evaluatorRefs) == 0 { + fmt.Fprintln(out, "Nothing was generated.") + return nil + } + + if err := project.MergeArtifactRefs(deployPath, datasetRefs, evaluatorRefs); err != nil { + return err + } + fmt.Fprintf(out, "\nUpdated %s\n", deployPath) + fmt.Fprintln(out, "Review the generated artifacts, then run: azd up && azd ai eval run") + return nil + }, + } + + cmd.Flags().StringVar(&configPath, "config", project.DefaultGenerateConfig, + "Path to the generation spec. Optional; flags alone are sufficient.") + cmd.Flags().StringVar(&deployPath, "deploy-config", project.DefaultDeployConfig, + "Deployment spec to write source references into.") + cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") + cmd.Flags().StringVar(&instruction, "gen-instruction", "", + "What the agent does and what to test.") + cmd.Flags().StringVar(&datasetFlag, "dataset", "", + "Use this dataset instead of generating one.") + cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, + "Use these evaluators instead of generating a rubric; repeatable.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Days of traces to seed rubric generation. 0 disables.") + cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used for generation.") + cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the jobs and return without polling.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveGenerateConfig loads the spec when present, then layers flags on top. +// A missing file is not an error: flags alone are sufficient. +func resolveGenerateConfig( + path, target, evalModel, datasetFlag string, + maxSamples, traceDays int, +) (*project.GenerateConfig, error) { + cfg := &project.GenerateConfig{} + + if _, err := os.Stat(path); err == nil { + loaded, err := project.LoadGenerateConfig(path) + if err != nil { + return nil, err + } + cfg = loaded + } + + if target != "" { + cfg.Agent.Name = target + } + if cfg.Agent.Name == "" { + return nil, requireFlag("target") + } + + if cfg.Generate.Rubric == nil { + cfg.Generate.Rubric = &project.RubricSpec{ + Name: cfg.Agent.Name + "-quality", + LocalDir: "./" + project.DefaultEvaluatorsDir, + } + } + if cfg.Generate.Dataset == nil && !looksLikeLocalDataset(datasetFlag) { + cfg.Generate.Dataset = &project.DatasetSpec{ + Name: cfg.Agent.Name + "-golden", + Strategy: project.StrategySynthetic, + SampleSize: project.DefaultSampleSize, + LocalDir: "./" + project.DefaultDatasetsDir, + } + } + + if evalModel != "" { + cfg.Generate.Rubric.Model = evalModel + } + if maxSamples > 0 && cfg.Generate.Dataset != nil { + cfg.Generate.Dataset.SampleSize = maxSamples + } + if cfg.Generate.Dataset != nil && cfg.Generate.Dataset.SampleSize == 0 { + cfg.Generate.Dataset.SampleSize = project.DefaultSampleSize + } + if traceDays > 0 { + if cfg.Agent.Context.Traces == nil { + cfg.Agent.Context.Traces = &project.TraceSpec{} + } + cfg.Agent.Context.Traces.Window = fmt.Sprintf("%dd", traceDays) + } + + return cfg, nil +} + +// generateRubric submits the evaluator generation job and saves the rubric. +func (ec *evalContext) generateRubric( + ctx context.Context, + cfg *project.GenerateConfig, + instruction, baseDir string, + out io.Writer, + noWait bool, +) (*project.ArtifactRef, error) { + spec := cfg.Generate.Rubric + fmt.Fprintf(out, "Generating rubric %s...\n", spec.Name) + + sources := eval_api.BuildGenerationSources( + "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + ) + req := eval_api.NewEvaluatorGenerationJobRequest(spec.Name, spec.Model, sources) + + job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the rubric generation job: %w", err) + } + if noWait { + fmt.Fprintf(out, " submitted job %s\n", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, ProjectEndpointAPIVersion, + ec.evalClient.GetEvaluatorGenerationJob) + if err != nil { + return nil, fmt.Errorf("rubric generation: %w", err) + } + + path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".json") + if err := writeRubric(path, completed.Result); err != nil { + return nil, err + } + fmt.Fprintf(out, " wrote %s\n", path) + + return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil +} + +// generateDataset submits the data generation job and downloads the result. +func (ec *evalContext) generateDataset( + ctx context.Context, + cfg *project.GenerateConfig, + instruction, baseDir string, + out io.Writer, + noWait bool, +) (*project.ArtifactRef, error) { + spec := cfg.Generate.Dataset + fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", spec.Name, spec.SampleSize) + + sources := eval_api.BuildGenerationSources( + "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + ) + model := "" + if cfg.Generate.Rubric != nil { + model = cfg.Generate.Rubric.Model + } + req := eval_api.NewDataGenerationJobRequest(spec.Name, model, spec.SampleSize, sources) + + job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the data generation job: %w", err) + } + if noWait { + fmt.Fprintf(out, " submitted job %s\n", job.ID) + return nil, nil + } + + completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + if err != nil { + return nil, fmt.Errorf("data generation: %w", err) + } + + name, version := completed.ResolvedNameVersion() + if name == "" { + return nil, fmt.Errorf("the data generation job returned no dataset reference") + } + + ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading the generated dataset %q: %w", name, err) + } + content, err := ec.datasetClient.DownloadDataset(ctx, ds.ResolvedBlobURI()) + if err != nil { + return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) + } + + path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".jsonl") + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return nil, fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + if err := os.WriteFile(path, content, 0o600); err != nil { + return nil, fmt.Errorf("writing %q: %w", path, err) + } + fmt.Fprintf(out, " wrote %s\n", path) + + return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil +} + +// pollGeneration waits for a generation job using the raised budget. +func (ec *evalContext) pollGeneration( + ctx context.Context, + operationID, apiVersion string, + get eval_api.GetJobFunc, +) (*eval_api.GenerationJob, error) { + poller := eval_api.NewPoller(operationID, apiVersion, get) + poller.Options = generatePollBudget + return poller.Poll(ctx) +} + +// traceOptions converts the config's trace window into the generation client's +// day count. Traces seed rubric generation only; they are never a run's data +// source. +func traceOptions(cfg *project.GenerateConfig) *eval_api.TraceOptions { + t := cfg.Agent.Context.Traces + if t == nil { + return nil + } + days := parseWindowDays(t.Window) + if days <= 0 { + return nil + } + return &eval_api.TraceOptions{Days: days} +} + +// parseWindowDays reads a window such as "30d" or a bare day count. +func parseWindowDays(window string) int { + w := strings.TrimSpace(strings.ToLower(window)) + if w == "" { + return 0 + } + w = strings.TrimSuffix(w, "d") + days, err := strconv.Atoi(w) + if err != nil { + return 0 + } + return days +} + +// writeRubric persists only the rubric dimensions so the developer can edit +// weights and descriptions and publish a new version. +func writeRubric(path string, result json.RawMessage) error { + if len(result) == 0 { + return fmt.Errorf("the rubric generation job returned no result") + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("creating %q: %w", filepath.Dir(path), err) + } + + var parsed eval_api.EvaluatorResult + if err := json.Unmarshal(result, &parsed); err == nil && len(parsed.Definition.Dimensions) > 0 { + body, err := json.MarshalIndent(parsed.Definition, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, body, 0o600) + } + + // Fall back to the raw payload rather than losing the result. + return os.WriteFile(path, result, 0o600) +} + +// relativeSource expresses an artifact path relative to the deployment spec. +func relativeSource(baseDir, path string) string { + rel, err := filepath.Rel(baseDir, path) + if err != nil { + return filepath.ToSlash(path) + } + return "./" + filepath.ToSlash(rel) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 671c10fe0ad..3a289d6347d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -27,6 +27,7 @@ func NewRootCommand() *cobra.Command { rootCmd.AddCommand( newInitCommand(), + newGenerateCommand(), newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go new file mode 100644 index 00000000000..c8329356c23 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "os" + + "go.yaml.in/yaml/v3" +) + +// ArtifactRef is a name/source pair written back into the deployment spec after +// generation. +type ArtifactRef struct { + Name string + Source string +} + +// MergeArtifactRefs writes `source:` references for generated artifacts into +// the deployment spec, matching entries by name and appending when absent. +// +// It edits the document through the yaml Node API rather than round-tripping +// through structs, so comments, key order, and formatting survive. Only the +// `source` key of a matched entry is touched; anything the developer hand-edited +// is left alone. +func MergeArtifactRefs(path string, datasets, evaluators []ArtifactRef) error { + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("reading %q: %w", path, err) + } + + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return fmt.Errorf("parsing %q: %w", path, err) + } + + root := documentRoot(&doc) + if root == nil { + return fmt.Errorf("%q is not a YAML mapping", path) + } + + if err := mergeSection(root, "datasets", datasets); err != nil { + return fmt.Errorf("%q: %w", path, err) + } + if err := mergeSection(root, "evaluators", evaluators); err != nil { + return fmt.Errorf("%q: %w", path, err) + } + + out, err := yaml.Marshal(&doc) + if err != nil { + return fmt.Errorf("serializing %q: %w", path, err) + } + if err := os.WriteFile(path, out, 0o600); err != nil { + return fmt.Errorf("writing %q: %w", path, err) + } + return nil +} + +// documentRoot unwraps the document node to the top-level mapping. +func documentRoot(doc *yaml.Node) *yaml.Node { + if doc.Kind == yaml.DocumentNode && len(doc.Content) > 0 { + doc = doc.Content[0] + } + if doc.Kind != yaml.MappingNode { + return nil + } + return doc +} + +// mergeSection updates or appends entries in a top-level sequence. +func mergeSection(root *yaml.Node, key string, refs []ArtifactRef) error { + if len(refs) == 0 { + return nil + } + + seq := findOrCreateSequence(root, key) + if seq == nil { + return fmt.Errorf("%q is present but is not a sequence", key) + } + + for _, ref := range refs { + if entry := findEntryByName(seq, ref.Name); entry != nil { + setMappingValue(entry, "source", ref.Source) + continue + } + seq.Content = append(seq.Content, newArtifactNode(ref)) + } + return nil +} + +// findOrCreateSequence returns the sequence node for key, creating an empty one +// when the key is absent. +func findOrCreateSequence(root *yaml.Node, key string) *yaml.Node { + for i := 0; i+1 < len(root.Content); i += 2 { + if root.Content[i].Value != key { + continue + } + value := root.Content[i+1] + if value.Kind == yaml.SequenceNode { + return value + } + // An explicit null is treated as an empty sequence. + if value.Tag == "!!null" { + value.Kind = yaml.SequenceNode + value.Tag = "!!seq" + value.Value = "" + return value + } + return nil + } + + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key} + seqNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + root.Content = append(root.Content, keyNode, seqNode) + return seqNode +} + +// findEntryByName locates a mapping entry whose `name` matches. +func findEntryByName(seq *yaml.Node, name string) *yaml.Node { + for _, item := range seq.Content { + if item.Kind != yaml.MappingNode { + continue + } + if mappingValue(item, "name") == name { + return item + } + } + return nil +} + +// mappingValue reads a scalar value from a mapping node. +func mappingValue(node *yaml.Node, key string) string { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + return node.Content[i+1].Value + } + } + return "" +} + +// setMappingValue updates a scalar in place, or appends it when absent. Only +// the targeted key is touched. +func setMappingValue(node *yaml.Node, key, value string) { + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + node.Content[i+1].Kind = yaml.ScalarNode + node.Content[i+1].Tag = "!!str" + node.Content[i+1].Value = value + return + } + } + node.Content = append(node.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, + ) +} + +// newArtifactNode builds a fresh `{name, source}` entry. +func newArtifactNode(ref ArtifactRef) *yaml.Node { + return &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "name"}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Name}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "source"}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Source}, + }, + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go new file mode 100644 index 00000000000..b253c8ad1f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +const handAuthored = `# Eval deployment spec +# Edited by hand - comments must survive generate. +evaluators: + - name: safety-check # hand-authored + source: ./evaluators/safety-check.json + +datasets: + - name: support-golden + source: ./datasets/old.jsonl + version: "3" + +evalGroups: + - name: pr-gate + dataset: support-golden + evaluators: + - builtin.task_adherence + - { name: safety-check, threshold: 4.0 } +` + +func writeTemp(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "azure.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// Regenerating must not destroy a hand-edited file. +func TestMergeArtifactRefs_PreservesCommentsAndSiblings(t *testing.T) { + path := writeTemp(t, handAuthored) + + require.NoError(t, MergeArtifactRefs(path, + []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}}, + []ArtifactRef{{Name: "support-quality", Source: "./evaluators/support-quality.json"}}, + )) + + out, err := os.ReadFile(path) + require.NoError(t, err) + text := string(out) + + require.Contains(t, text, "# Eval deployment spec", "leading comments must survive") + require.Contains(t, text, "# hand-authored", "inline comments must survive") + require.Contains(t, text, "./datasets/new.jsonl", "the matched source must be updated") + require.NotContains(t, text, "./datasets/old.jsonl", "the old source must be replaced") + require.Contains(t, text, "support-quality", "a new evaluator must be appended") + require.Contains(t, text, "safety-check", "existing entries must be kept") + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + ds, ok := cfg.Dataset("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/new.jsonl", ds.Source) + require.Equal(t, "3", ds.Version, "sibling keys must not be disturbed") + require.Len(t, cfg.Evaluators, 2) +} + +// The eval group's evaluator list must be left exactly as written. +func TestMergeArtifactRefs_DoesNotTouchEvalGroups(t *testing.T) { + path := writeTemp(t, handAuthored) + require.NoError(t, MergeArtifactRefs(path, nil, + []ArtifactRef{{Name: "support-quality", Source: "./evaluators/q.json"}})) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + g, ok := cfg.Group("pr-gate") + require.True(t, ok) + require.Len(t, g.Evaluators, 2) + require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) + require.NotNil(t, g.Evaluators[1].Threshold) +} + +// Sections absent from the file are created rather than erroring. +func TestMergeArtifactRefs_CreatesMissingSections(t *testing.T) { + path := writeTemp(t, "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") + + require.NoError(t, MergeArtifactRefs(path, + []ArtifactRef{{Name: "d1", Source: "./datasets/d1.jsonl"}}, + []ArtifactRef{{Name: "e1", Source: "./evaluators/e1.json"}}, + )) + + cfg, err := LoadEvalConfig(path) + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "d1", cfg.Datasets[0].Name) +} + +// Running generate twice must be idempotent. +func TestMergeArtifactRefs_IsIdempotent(t *testing.T) { + path := writeTemp(t, handAuthored) + refs := []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}} + + require.NoError(t, MergeArtifactRefs(path, refs, nil)) + first, err := os.ReadFile(path) + require.NoError(t, err) + + require.NoError(t, MergeArtifactRefs(path, refs, nil)) + second, err := os.ReadFile(path) + require.NoError(t, err) + + require.Equal(t, string(first), string(second), + "merging the same references twice must not change the file") +} + +func TestFingerprint_DetectsChange(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "data.jsonl") + + require.NoError(t, os.WriteFile(path, []byte(`{"query":"a"}`), 0o600)) + first, err := Fingerprint(path) + require.NoError(t, err) + + again, err := Fingerprint(path) + require.NoError(t, err) + require.Equal(t, first, again, "unchanged content must hash the same") + + require.NoError(t, os.WriteFile(path, []byte(`{"query":"b"}`), 0o600)) + changed, err := Fingerprint(path) + require.NoError(t, err) + require.NotEqual(t, first, changed, "changed content must hash differently") +} + +func TestFingerprintKey_IsEnvSafe(t *testing.T) { + require.Equal(t, "EVAL_FINGERPRINT_DATASET_SUPPORT_GOLDEN", + FingerprintKey("dataset", "support-golden")) + require.Equal(t, "EVAL_FINGERPRINT_EVALUATOR_MY_EVAL_1", + FingerprintKey("evaluator", "my.eval-1")) +} From 38fb856537621e85fc3657b8817ed41922c5ecb2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:25:03 +0300 Subject: [PATCH 009/114] build(azure.ai.evaluations): release pipeline and foundry bundle entry Adds the templated release pipeline for the new extension and lists it as a dependency of the microsoft.foundry meta-package. The registry.json artifact entries are generated by the release, so they are not hand-authored here. --- .../microsoft.foundry/extension.yaml | 2 + .../release-ext-azure-ai-evaluations.yml | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 eng/pipelines/release-ext-azure-ai-evaluations.yml diff --git a/cli/azd/extensions/microsoft.foundry/extension.yaml b/cli/azd/extensions/microsoft.foundry/extension.yaml index 336bc749643..7c96b52bd9f 100644 --- a/cli/azd/extensions/microsoft.foundry/extension.yaml +++ b/cli/azd/extensions/microsoft.foundry/extension.yaml @@ -12,6 +12,8 @@ dependencies: version: "~1.0.0-beta.7" - id: azure.ai.connections version: "~1.0.0-beta.1" + - id: azure.ai.evaluations + version: "~1.0.0-beta.1" - id: azure.ai.inspector version: "~1.0.0-beta.1" - id: azure.ai.projects diff --git a/eng/pipelines/release-ext-azure-ai-evaluations.yml b/eng/pipelines/release-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..d8b60935909 --- /dev/null +++ b/eng/pipelines/release-ext-azure-ai-evaluations.yml @@ -0,0 +1,46 @@ +# Continuous deployment trigger +trigger: + branches: + include: + - main + paths: + include: + - go.mod + - cli/azd/extensions/azure.ai.evaluations + - eng/pipelines/release-azd-extension.yml + - /eng/pipelines/templates/jobs/build-azd-extension.yml + - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml + - /eng/pipelines/templates/variables/image.yml + +pr: + paths: + include: + - cli/azd/extensions/azure.ai.evaluations + - eng/pipelines/release-ext-azure-ai-evaluations.yml + - eng/pipelines/release-azd-extension.yml + - eng/pipelines/templates/steps/publish-cli.yml + exclude: + - cli/azd/docs/** + +parameters: + - name: PublishToRegistry + displayName: Publish to registry + type: string + # Scheduled (nightly) runs override this in the shared templates; the runtime + # parameter default must be a literal because it renders before variables exist. + default: stable + values: + - stable + - dev + - nightly + +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + - template: /eng/pipelines/templates/stages/release-azd-extension.yml + parameters: + AzdExtensionId: azure.ai.evaluations + SanitizedExtensionId: azure-ai-evaluations + AzdExtensionDirectory: cli/azd/extensions/azure.ai.evaluations + PublishToRegistry: ${{ parameters.PublishToRegistry }} From 3b9cb935fad16ac33ddfc96c95904c3792797e99 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:29:44 +0300 Subject: [PATCH 010/114] fix(azure.ai.evaluations): bind dataset URIs and add live integration tests Live testing against a real project found two issues. 1. The dataset model only bound snake_case URIs (data_uri, blob_uri), but the project endpoint returns camelCase (dataUri). ResolvedBlobURI therefore returned empty, which would have failed the generate download much later with no useful error. Both spellings are now accepted. 2. Built-in evaluators do not share one input contract. builtin.ifeval requires instruction_id_list and is rejected under the agent-target data mapping with MissingRequiredDataMapping. The live tests now select an evaluator whose inputs match, and the helper documents why. Live tests are gated behind the live build tag and AZURE_AI_EVAL_E2E_LIVE, and clean up every resource they create. Verified: builtin listing, the full dataset pending-upload lifecycle with version auto-increment, and eval group creation returning 201. --- .../internal/pkg/dataset_api/models.go | 46 ++- .../tests/live/live_test.go | 325 ++++++++++++++++++ 2 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index cee6bbcd886..7e6c8633c09 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -21,28 +21,46 @@ type CreateDatasetRequest struct { } // Dataset is the response for dataset operations. -// Note: The GET /datasets API returns snake_case field names (data_uri, blob_uri, -// content_uri), while the POST /finalize API accepts camelCase (dataUri). -// Both conventions are correct for their respective endpoints. +// +// The field spelling is not consistent across the surface: the live +// project-endpoint GET returns camelCase (dataUri, isSingleFile), while other +// paths have used snake_case (data_uri, blob_uri, content_uri). Both spellings +// are accepted here because binding only one silently yields an empty URI, +// which then fails much later at download time. type Dataset struct { - Name string `json:"name"` - Version string `json:"version"` + ID string `json:"id,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + Type string `json:"type,omitempty"` + Format string `json:"format,omitempty"` + + // camelCase spellings (project endpoint). + DataURICamel string `json:"dataUri,omitempty"` + BlobURICamel string `json:"blobUri,omitempty"` + ContentURICamel string `json:"contentUri,omitempty"` + IsSingleFile bool `json:"isSingleFile,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` + + // snake_case spellings. BlobURI string `json:"blob_uri,omitempty"` - Format string `json:"format,omitempty"` DataURI string `json:"data_uri,omitempty"` ContentURI string `json:"content_uri,omitempty"` } -// ResolvedBlobURI returns the best available blob URI. Prefers blob_uri, -// falls back to data_uri, then content_uri. +// ResolvedBlobURI returns the first URI the service supplied, across both +// spellings. An empty result means the dataset carries no downloadable URI and +// the caller must fetch a credential instead. func (d *Dataset) ResolvedBlobURI() string { - if d.BlobURI != "" { - return d.BlobURI - } - if d.DataURI != "" { - return d.DataURI + for _, candidate := range []string{ + d.BlobURI, d.BlobURICamel, + d.DataURI, d.DataURICamel, + d.ContentURI, d.ContentURICamel, + } { + if candidate != "" { + return candidate + } } - return d.ContentURI + return "" } // DatasetCredential is the response for dataset credential (SAS token) requests. diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go new file mode 100644 index 00000000000..c04af2d47a6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -0,0 +1,325 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package live holds integration tests that talk to a real Foundry project. +// They are excluded from the default build by the `live` tag and additionally +// gated on AZURE_AI_EVAL_E2E_LIVE so an accidental run cannot create resources. +// +// go test -tags live -v ./tests/live/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +// +// Optional: +// +// AZURE_AI_EVAL_MODEL= (default gpt-4.1-nano) +// AZURE_AI_EVAL_AGENT= (enables the run phase) +package live + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +const ( + projectAPIVersion = "2025-11-15-preview" + defaultJudgeModel = "gpt-4.1-nano" + sampleDatasetContent = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +{"query":"Can I change my shipping address after ordering?"} +` +) + +type liveEnv struct { + endpoint string + judgeModel string + agentName string + evalClient *eval_api.EvalClient + datasetClient *dataset_api.DatasetClient +} + +func setup(t *testing.T) *liveEnv { + t.Helper() + + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + + // The azd developer CLI credential works non-interactively when azd already + // holds a refresh token, which is what makes an unattended run possible. + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + require.NoError(t, err, "acquiring an azd credential") + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = defaultJudgeModel + } + + return &liveEnv{ + endpoint: endpoint, + judgeModel: judge, + agentName: os.Getenv("AZURE_AI_EVAL_AGENT"), + evalClient: eval_api.NewEvalClient(endpoint, cred), + datasetClient: dataset_api.NewDatasetClient(endpoint, cred), + } +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s-%d", prefix, time.Now().UTC().Unix()) +} + +// pickQualityEvaluator selects a built-in whose required inputs match the +// agent-target data mapping this extension sends. +// +// Built-ins do not share one input contract: builtin.ifeval, for example, +// requires an `instruction_id_list` field, and creating a group with it under +// the agent-target mapping fails with MissingRequiredDataMapping. The +// agent-target mapping supplies query, response, tool_calls and +// tool_definitions, so the evaluators below are the compatible set. +func pickQualityEvaluator(t *testing.T, available []eval_api.EvaluatorSummary) string { + t.Helper() + + preferred := []string{ + "builtin.task_adherence", + "builtin.task_completion", + "builtin.tool_call_accuracy", + } + present := map[string]bool{} + for _, e := range available { + present[e.Name] = true + } + for _, name := range preferred { + if present[name] { + return name + } + } + + names := make([]string, 0, len(available)) + for _, e := range available { + names = append(names, e.Name) + } + t.Skipf("no agent-target compatible evaluator found; available: %s", strings.Join(names, ", ")) + return "" +} + +// TestLiveBuiltinEvaluators is the cheapest reachability check: it proves the +// endpoint, credential, api-version, and auth scope are all correct without +// creating anything. +func TestLiveBuiltinEvaluators(t *testing.T) { + env := setup(t) + ctx := context.Background() + + list, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion, + ) + require.NoError(t, err, "listing built-in evaluators") + require.NotEmpty(t, list.Value, "the project should expose built-in evaluators") + + t.Logf("found %d built-in evaluators; first: %s", len(list.Value), list.Value[0].Name) +} + +// TestLiveDatasetLifecycle exercises the full pending-upload flow and asserts +// that re-registering the same name yields the next version rather than an error. +func TestLiveDatasetLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + dir := t.TempDir() + require.NoError(t, + os.WriteFile(filepath.Join(dir, "golden.jsonl"), []byte(sampleDatasetContent), 0o600)) + + name := uniqueName("azd-eval-e2e") + + first, err := env.datasetClient.UploadNewVersion(ctx, name, "", dir, projectAPIVersion) + require.NoError(t, err, "registering the first dataset version") + require.Equal(t, name, first.Name) + require.NotEmpty(t, first.Version) + t.Logf("registered %s version %s", first.Name, first.Version) + + t.Cleanup(func() { + // Best effort: leave nothing behind even if the test fails midway. + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, first.Version, projectAPIVersion) + }) + + fetched, err := env.datasetClient.GetDataset(ctx, name, first.Version, projectAPIVersion) + require.NoError(t, err, "reading the dataset back") + t.Logf("dataset uri: %q (empty means a credential call is required)", fetched.ResolvedBlobURI()) + + versions, err := env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) + require.NoError(t, err, "listing dataset versions") + require.NotEmpty(t, versions.Value) + require.Equal(t, first.Version, dataset_api.LatestVersion(versions.Value)) + + // A second upload must advance the version, not conflict. + second, err := env.datasetClient.UploadNewVersion( + ctx, name, first.Version, dir, projectAPIVersion) + require.NoError(t, err, "registering a second dataset version") + require.NotEqual(t, first.Version, second.Version, + "re-registering the same name must produce the next version") + t.Cleanup(func() { + _ = env.datasetClient.DeleteDatasetVersion( + context.Background(), name, second.Version, projectAPIVersion) + }) +} + +// TestLiveEvalGroupLifecycle proves the create request this extension builds is +// accepted, which is the single most important contract to get right. +func TestLiveEvalGroupLifecycle(t *testing.T) { + env := setup(t) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value, "need at least one built-in evaluator") +evaluatorName := pickQualityEvaluator(t, builtins.Value) + + threshold := 3.0 + req := &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-group"), + Metadata: map[string]string{"azd_source": "e2e"}, + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + "threshold": threshold, + }, + }}, + } + + group, err := env.evalClient.CreateOpenAIEval(ctx, req) + require.NoError(t, err, "creating the eval group") + require.NotEmpty(t, group.ID, "the service assigns the id; name is not unique") + t.Logf("created eval group %s (name %q)", group.ID, group.Name) + + fetched, err := env.evalClient.GetOpenAIEval(ctx, group.ID) + require.NoError(t, err, "reading the eval group back") + require.Equal(t, group.ID, fetched.ID) +} + +// TestLiveRun invokes a real agent, so it only runs when one is named. +func TestLiveRun(t *testing.T) { + env := setup(t) + if env.agentName == "" { + t.Skip("set AZURE_AI_EVAL_AGENT to a deployed agent to exercise the run phase") + } + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) +evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-run"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval group for the run") + + ds := eval_api.NewAgentTargetDataSource(env.agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("run"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + t.Cleanup(func() { + _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) + }) + + // A single sample is roughly 40 seconds; allow generous headroom. + deadline := time.Now().Add(10 * time.Minute) + terminal := map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, + } + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the run") + if terminal[strings.ToLower(current.Status)] { + t.Logf("run reached %s", current.Status) + if current.ResultCounts != nil { + t.Logf("counts: passed=%d failed=%d errored=%d", + current.ResultCounts.Passed, + current.ResultCounts.Failed, + current.ResultCounts.Errored) + } + body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") + t.Logf("per-criteria results: %s", string(body)) + return + } + if time.Now().After(deadline) { + t.Fatalf("run %s did not finish within the deadline (last status %q)", + run.ID, current.Status) + } + time.Sleep(10 * time.Second) + } +} From b7aba9dae3cca902e6793320ae247e6cb5272841 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:45:03 +0300 Subject: [PATCH 011/114] fix(azure.ai.evaluations): build data mappings from each evaluator's published contract The builder sent one fixed data mapping and one fixed set of initialization parameters to every evaluator. That only suited agent-target quality evaluators, and the service rejected the rest. The evaluator listing publishes a full contract per evaluator: definition.data_schema (accepted and required inputs), definition.init_parameters, and supported_evaluation_levels. The builder now reads it and shapes each testing criterion accordingly. This fixes four concrete defects. - Required inputs were never honoured, so builtin.ifeval (instruction_id_list), builtin.similarity (ground_truth) and builtin.retrieval (context) all failed with MissingRequiredDataMapping. Fields not supplied by the agent target are now bound to dataset columns, and the item schema declares them. - Inputs an evaluator does not accept were sent anyway. - initialization_parameters always carried model, deployment_name and threshold. No evaluator accepts 'model', and builtin.ifeval accepts nothing at all. Parameters are now filtered to the declared properties, and a required one that is missing is reported locally. - evaluation_level was sent as run metadata, where it has no effect. It is an initialization parameter on the evaluators that declare it. It also encodes an exclusivity rule the service enforces: 'messages' and 'query'/'response' cannot both appear in a mapping, so the evaluation level selects between the conversation and turn shapes. A missing dataset column is now caught before the request is sent and names the column, rather than surfacing as a 400 pointing at testing_criteria[0].data_mapping. An evaluator with no published contract keeps the previous agent-target shape, so custom evaluators are unaffected. Verified against a live project: all ten built-ins are accepted, where two previously failed. The live test exercises the shipping builder rather than a hand-rolled request, so a regression in this logic fails the suite. Adds DeleteOpenAIEval so those tests clean up after themselves. --- .../internal/cmd/build.go | 320 +++++++++++++++--- .../internal/cmd/build_live_test.go | 142 ++++++++ .../internal/cmd/build_test.go | 277 +++++++++++++++ .../internal/cmd/reconciler.go | 11 +- .../azure.ai.evaluations/internal/cmd/run.go | 57 +++- .../internal/pkg/dataset_api/models.go | 10 +- .../internal/pkg/eval_api/evaluators.go | 90 +++++ .../internal/pkg/eval_api/operations.go | 7 + .../internal/project/service_target_eval.go | 12 +- .../tests/live/live_test.go | 4 +- 10 files changed, 876 insertions(+), 54 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 77a3ef8918c..7e880001488 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -4,80 +4,318 @@ package cmd import ( + "context" + "fmt" + "sort" + "strings" + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" ) -// dataMapping is the template binding the service uses to feed each evaluator. -// It pairs with the item schema below, which declares a single `query` field. -func dataMapping() map[string]string { - return map[string]string{ - "query": "{{item.query}}", - "response": "{{sample.output_items}}", - "tool_calls": "{{sample.tool_calls}}", - "tool_definitions": "{{sample.tool_definitions}}", +// evaluatorSchemas indexes the published evaluator contracts by name. +// +// A failure is deliberately not fatal: without schemas the builder falls back +// to the agent-target shape, which is what it always used to send. +func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { + list, err := ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + if err != nil { + return nil } + return list.ByName() } -// agentItemSchema mirrors the shape the agent-target runner expects. It is a -// fixed schema, not inferred from the dataset. -func agentItemSchema() map[string]any { - return map[string]any{ - "type": "object", - "properties": map[string]any{ - "query": map[string]any{"type": "string"}, - }, +// sampleBindings are the fields an agent target produces at run time. Anything +// an evaluator accepts that is not in this set has to come from a dataset +// column instead. +var sampleBindings = map[string]string{ + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", +} + +// legacyInputs is the mapping used when the service publishes no schema for an +// evaluator, which is the case for freshly uploaded custom evaluators. It +// matches the agent-target shape. +var legacyInputs = []string{"query", "response", "tool_calls", "tool_definitions"} + +// criterionPlan is the resolved binding for one evaluator. +type criterionPlan struct { + dataMapping map[string]string + initParams map[string]any + // itemFields are the fields sourced from dataset columns; they have to be + // declared in the item schema. + itemFields []string +} + +// conversationField carries a whole conversation. The service rejects a +// mapping that pairs it with the turn-level fields: +// +// Evaluator 'builtin.task_completion' has both 'messages' and +// 'query'/'response' in data_mapping. Use 'messages' for conversation-level +// evaluation or 'query'/'response' for turn-level evaluation, but not both. +const conversationField = "messages" + +// turnFields are the per-turn counterparts to conversationField. +var turnFields = []string{"query", "response"} + +// selectLevelFields resolves the conversation/turn exclusivity for evaluators +// that accept both shapes, keeping whichever matches the evaluation level. +// Required fields are never dropped, so a genuine conflict still surfaces as a +// missing-field error rather than being silently reshaped. +func selectLevelFields(accepted, required []string, level string) []string { + isRequired := make(map[string]bool, len(required)) + for _, name := range required { + isRequired[name] = true + } + + acceptsConversation := false + acceptsTurn := false + for _, field := range accepted { + if field == conversationField { + acceptsConversation = true + } + for _, turn := range turnFields { + if field == turn { + acceptsTurn = true + } + } + } + if !acceptsConversation || !acceptsTurn { + return accepted + } + + drop := map[string]bool{} + if strings.EqualFold(level, project.EvaluationLevelConversation) { + for _, turn := range turnFields { + drop[turn] = true + } + } else { + drop[conversationField] = true + } + + kept := make([]string, 0, len(accepted)) + for _, field := range accepted { + if drop[field] && !isRequired[field] { + continue + } + kept = append(kept, field) + } + return kept +} + +// planCriterion shapes one evaluator's bindings from its published contract. +// +// Evaluators do not share an input contract: builtin.similarity needs +// ground_truth, builtin.retrieval needs context, and builtin.ifeval needs +// instruction_id_list. Sending one fixed mapping to all of them earns a +// service-side MissingRequiredDataMapping rejection, so the mapping is derived +// per evaluator and anything unsatisfiable is reported before the request is +// sent. +func planCriterion( + ref evalcore.EvaluatorRef, + schema *eval_api.EvaluatorSummary, + hasTarget bool, + datasetColumns map[string]bool, + evalModel string, + level string, +) (*criterionPlan, error) { + accepted := legacyInputs + var required []string + // A published schema is authoritative even when it is empty: an empty + // property set means the evaluator accepts nothing, which is different from + // publishing no schema at all. + if dataSchema := schema.DataSchema(); dataSchema != nil { + accepted = dataSchema.PropertyNames() + required = dataSchema.Required + } + accepted = selectLevelFields(accepted, required, level) + + plan := &criterionPlan{ + dataMapping: map[string]string{}, + initParams: map[string]any{}, + } + + for _, field := range accepted { + if binding, ok := sampleBindings[field]; ok && hasTarget { + plan.dataMapping[field] = binding + continue + } + // Everything else comes from the dataset. When the columns are known, + // bind only the ones that exist so optional fields stay unbound rather + // than resolving to nothing at run time. + if datasetColumns != nil && !datasetColumns[field] { + continue + } + plan.dataMapping[field] = fmt.Sprintf("{{item.%s}}", field) + plan.itemFields = append(plan.itemFields, field) + } + + var missing []string + for _, field := range required { + if _, ok := plan.dataMapping[field]; !ok { + missing = append(missing, field) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf( + "evaluator %q requires %s, which the dataset does not provide; "+ + "add %s to the dataset, or choose an evaluator that matches the data", + ref.Name, quoteList(missing), pluralColumns(missing), + ) + } + + if !schema.SupportsLevel(level) { + return nil, fmt.Errorf( + "evaluator %q does not support evaluation level %q; it supports %s", + ref.Name, level, quoteList(schema.SupportedEvaluationLevels), + ) + } + + initSchema := schema.InitSchema() + accepts := func(name string) bool { + // Only an absent schema falls back to the historical parameters; + // builtin.ifeval publishes an empty one and takes none. + if initSchema == nil { + return name == "deployment_name" || name == "threshold" + } + return initSchema.Accepts(name) + } + + if evalModel != "" && accepts("deployment_name") { + plan.initParams["deployment_name"] = evalModel + } + if ref.Threshold != nil && accepts("threshold") { + plan.initParams["threshold"] = *ref.Threshold + } + if level != "" && accepts("evaluation_level") { + plan.initParams["evaluation_level"] = level + } + + if initSchema != nil { + var missingInit []string + for _, name := range initSchema.Required { + if _, ok := plan.initParams[name]; !ok { + missingInit = append(missingInit, name) + } + } + if len(missingInit) > 0 { + return nil, fmt.Errorf( + "evaluator %q requires %s; set the judge model on the eval group", + ref.Name, quoteList(missingInit), + ) + } } + + return plan, nil } // buildEvalGroupRequest converts an eval group declaration into the create -// request. Evaluators become testing criteria; a per-evaluator threshold is -// carried in initialization_parameters alongside the judge model. -func buildEvalGroupRequest(group *project.EvalGroup) *eval_api.CreateOpenAIEvalRequest { +// request. Each evaluator becomes a testing criterion bound to its own +// contract, and the item schema declares every dataset column those bindings +// reference. +// +// schemas may be nil or partial; an evaluator with no published contract falls +// back to the agent-target shape. datasetColumns may be nil, meaning the +// columns are unknown and every accepted field is assumed present. +func buildEvalGroupRequest( + group *project.EvalGroup, + schemas map[string]*eval_api.EvaluatorSummary, + datasetColumns map[string]bool, +) (*eval_api.CreateOpenAIEvalRequest, error) { metadata := map[string]string{} - if group.Target != nil && group.Target.Name != "" { + hasTarget := group.Target != nil && group.Target.Name != "" + if hasTarget { metadata["azd_agent"] = group.Target.Name } metadata["azd_eval_group"] = group.Name + evalModel := "" + level := "" + if group.Options != nil { + evalModel = group.Options.EvalModel + level = group.Options.EvaluationLevel + } + req := &eval_api.CreateOpenAIEvalRequest{ Name: group.Name, Metadata: metadata, - DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", - IncludeSampleSchema: true, - ItemSchema: agentItemSchema(), - }, } - evalModel := "" - if group.Options != nil { - evalModel = group.Options.EvalModel - } + itemFields := map[string]bool{} for _, ref := range group.Evaluators { + schema := schemas[ref.Name] + if schema == nil { + schema = &eval_api.EvaluatorSummary{Name: ref.Name} + } + + plan, err := planCriterion(ref, schema, hasTarget, datasetColumns, evalModel, level) + if err != nil { + return nil, err + } + criterion := eval_api.TestingCriterion{ Type: "azure_ai_evaluator", // Name drops the builtin prefix; EvaluatorName keeps it. Name: ref.APIName(), EvaluatorName: ref.Name, - DataMapping: dataMapping(), - } - - params := map[string]any{} - if evalModel != "" { - params["model"] = evalModel - params["deployment_name"] = evalModel + DataMapping: plan.dataMapping, } - if ref.Threshold != nil { - params["threshold"] = *ref.Threshold + if len(plan.initParams) > 0 { + criterion.InitializationParameters = plan.initParams } - if len(params) > 0 { - criterion.InitializationParameters = params + for _, field := range plan.itemFields { + itemFields[field] = true } req.TestingCriteria = append(req.TestingCriteria, criterion) } - return req + req.DataSourceConfig = &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: hasTarget, + ItemSchema: itemSchema(itemFields), + } + + return req, nil +} + +// itemSchema declares the dataset columns the criteria bind to. It always +// declares at least `query`, the column an agent target reads. +func itemSchema(fields map[string]bool) map[string]any { + if len(fields) == 0 { + fields = map[string]bool{"query": true} + } + properties := map[string]any{} + for field := range fields { + properties[field] = map[string]any{"type": "string"} + } + return map[string]any{ + "type": "object", + "properties": properties, + } +} + +func quoteList(values []string) string { + if len(values) == 0 { + return "nothing" + } + quoted := make([]string, 0, len(values)) + for _, value := range values { + quoted = append(quoted, fmt.Sprintf("%q", value)) + } + sort.Strings(quoted) + if len(quoted) == 1 { + return quoted[0] + } + return strings.Join(quoted[:len(quoted)-1], ", ") + " and " + quoted[len(quoted)-1] +} + +func pluralColumns(values []string) string { + if len(values) == 1 { + return "that column" + } + return "those columns" } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go new file mode 100644 index 00000000000..0eefc25a34c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// This file proves the request buildEvalGroupRequest produces is accepted by +// the real service. It lives in the cmd package on purpose: the tests under +// tests/live can only hand-roll a request, which validates the API but not the +// code that ships. +// +// go test -tags live -v ./internal/cmd/ -run TestLiveBuild +// +// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. + +package cmd + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/stretchr/testify/require" +) + +func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") + } + cred, err := azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + require.NoError(t, err) + + judge := os.Getenv("AZURE_AI_EVAL_MODEL") + if judge == "" { + judge = "gpt-4.1-nano" + } + return eval_api.NewEvalClient(endpoint, cred), judge +} + +// TestLiveBuildAcceptedForEveryBuiltin walks every built-in the project +// exposes, builds a group with the shipping builder, and posts it. +// +// Each evaluator declares a different input contract, so this is the test that +// would have caught the fixed data mapping: it previously produced a +// MissingRequiredDataMapping rejection for builtin.ifeval and would do so +// again for any evaluator whose contract the builder stops honouring. +func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, listed.Value) + schemas := listed.ByName() + + for _, summary := range listed.Value { + summary := summary + t.Run(summary.Name, func(t *testing.T) { + // Give the builder a dataset carrying every column the evaluator + // accepts, so a rejection means the request shape is wrong rather + // than the data being genuinely absent. + columns := map[string]bool{"query": true} + if ds := summary.DataSchema(); ds != nil { + for _, name := range ds.PropertyNames() { + columns[name] = true + } + } + + level := "" + if len(summary.SupportedEvaluationLevels) > 0 { + level = summary.SupportedEvaluationLevels[0] + } + + group := &project.EvalGroup{ + Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, + Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, + } + + req, err := buildEvalGroupRequest(group, schemas, columns) + require.NoError(t, err, "the builder must satisfy every published contract") + + created, err := client.CreateOpenAIEval(ctx, req) + require.NoError(t, err, + "the service rejected the request this extension builds for %s", summary.Name) + require.NotEmpty(t, created.ID) + t.Cleanup(func() { + _ = client.DeleteOpenAIEval(context.Background(), created.ID) + }) + t.Logf("%s accepted as %s", summary.Name, created.ID) + }) + } +} + +// TestLiveBuildRejectsMissingColumnsLocally proves the pre-flight check fires +// before the network call, so a user sees which column is missing instead of a +// service error naming an internal field path. +func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + require.NoError(t, err) + schemas := listed.ByName() + + target, ok := schemas["builtin.ifeval"] + if !ok { + t.Skip("builtin.ifeval is not available in this project") + } + require.NotNil(t, target.DataSchema()) + require.NotEmpty(t, target.DataSchema().Required, + "this test relies on ifeval declaring required inputs") + + group := &project.EvalGroup{ + Name: "azd-live-negative", + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, + Options: &project.Options{EvalModel: judge}, + } + + // A dataset with only `query` cannot satisfy ifeval. + _, err = buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + t.Logf("pre-flight error: %v", err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go new file mode 100644 index 00000000000..f4b9fa2104e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -0,0 +1,277 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// schema builds an evaluator contract the way the service publishes one. +func schema(name string, dataRequired, dataProps, initRequired, initProps []string, levels ...string) *eval_api.EvaluatorSummary { + toProps := func(names []string) map[string]any { + if names == nil { + return nil + } + out := map[string]any{} + for _, n := range names { + out[n] = map[string]any{"type": "string"} + } + return out + } + return &eval_api.EvaluatorSummary{ + Name: name, + SupportedEvaluationLevels: levels, + Definition: &eval_api.EvaluatorContract{ + DataSchema: &eval_api.JSONSchema{Required: dataRequired, Properties: toProps(dataProps)}, + InitParameters: &eval_api.JSONSchema{Required: initRequired, Properties: toProps(initProps)}, + }, + } +} + +func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.EvalGroup { + return &project.EvalGroup{ + Name: "g", + Dataset: "d", + Target: &project.Target{Type: "agent", Name: "my-agent"}, + Evaluators: evaluators, + Options: opts, + } +} + +// An agent evaluator takes its response from the sample and its query from the +// dataset. +func TestBuildBindsAgentFieldsFromSample(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_adherence": schema("builtin.task_adherence", + nil, []string{"query", "response", "tool_definitions", "messages"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold", "evaluation_level"}, + "turn"), + } + group := groupWith( + []evalcore.EvaluatorRef{{Name: "builtin.task_adherence"}}, + &project.Options{EvalModel: "gpt-4.1-nano"}, + ) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Len(t, req.TestingCriteria, 1) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + // `messages` is not a dataset column here, so it stays unbound. + require.NotContains(t, mapping, "messages") +} + +// A required field the dataset does not carry is reported before the request +// is sent, naming the field. +func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + + _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "instruction_id_list") + require.Contains(t, err.Error(), "instruction_kwargs") +} + +// The same evaluator succeeds once the dataset supplies the columns. +func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + []string{"response", "instruction_id_list", "instruction_kwargs"}, + []string{"response", "instruction_id_list", "instruction_kwargs"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + "instruction_id_list": true, + "instruction_kwargs": true, + }) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + // response is satisfied by the agent target. + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{item.instruction_id_list}}", mapping["instruction_id_list"]) + + // The item schema has to declare the columns the criteria reference. + props := req.DataSourceConfig.ItemSchema["properties"].(map[string]any) + require.Contains(t, props, "instruction_id_list") + require.Contains(t, props, "instruction_kwargs") +} + +// Initialization parameters are filtered to what the evaluator accepts. +// builtin.ifeval takes none, so nothing is sent even when a model is set. +func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { + threshold := 4.0 + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.ifeval": schema("builtin.ifeval", + nil, []string{"response"}, nil, nil, "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response", "ground_truth"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{ + {Name: "builtin.ifeval", Threshold: &threshold}, + {Name: "builtin.similarity", Threshold: &threshold}, + }, &project.Options{EvalModel: "gpt-4.1-nano"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + "query": true, "ground_truth": true, + }) + require.NoError(t, err) + + // ifeval accepts no init parameters at all. + require.Empty(t, req.TestingCriteria[0].InitializationParameters) + + // similarity accepts both, and never the `model` alias. + params := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", params["deployment_name"]) + require.InDelta(t, 4.0, params["threshold"], 0.0001) + require.NotContains(t, params, "model") +} + +// evaluation_level is an initialization parameter, not run metadata, and only +// on evaluators that declare it. +func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{ + {Name: "builtin.task_completion"}, + {Name: "builtin.similarity"}, + }, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + require.Equal(t, "turn", req.TestingCriteria[0].InitializationParameters["evaluation_level"]) + require.NotContains(t, req.TestingCriteria[1].InitializationParameters, "evaluation_level") +} + +// An evaluator that does not support the requested level is rejected with the +// levels it does support. +func TestBuildRejectsUnsupportedLevel(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, + &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + + _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "conversation") + require.Contains(t, err.Error(), "turn") +} + +// A required init parameter with no judge model configured is caught locally. +func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + + _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.Error(t, err) + require.Contains(t, err.Error(), "deployment_name") +} + +// An evaluator with no published contract keeps the historical agent-target +// shape, so custom evaluators still deploy. +func TestBuildFallsBackWithoutSchema(t *testing.T) { + group := groupWith([]evalcore.EvaluatorRef{{Name: "my-custom-evaluator"}}, + &project.Options{EvalModel: "m"}) + + req, err := buildEvalGroupRequest(group, nil, nil) + require.NoError(t, err) + + mapping := req.TestingCriteria[0].DataMapping + require.Equal(t, "{{item.query}}", mapping["query"]) + require.Equal(t, "{{sample.output_items}}", mapping["response"]) + require.Equal(t, "{{sample.tool_calls}}", mapping["tool_calls"]) + require.Equal(t, "{{sample.tool_definitions}}", mapping["tool_definitions"]) + require.Equal(t, "m", req.TestingCriteria[0].InitializationParameters["deployment_name"]) +} + +// `messages` and `query`/`response` are mutually exclusive; the evaluation +// level picks which shape is bound. Sending both is rejected by the service. +func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.task_completion": schema("builtin.task_completion", + nil, []string{"query", "response", "messages", "tool_definitions"}, + []string{"deployment_name"}, []string{"deployment_name", "evaluation_level"}, + "conversation", "turn"), + } + columns := map[string]bool{"query": true, "messages": true, "response": true} + + // Turn level keeps query/response and drops messages. + turn := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, + &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + req, err := buildEvalGroupRequest(turn, schemas, columns) + require.NoError(t, err) + mapping := req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "query") + require.NotContains(t, mapping, "messages") + + // Conversation level keeps messages and drops query/response. + conv := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, + &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + req, err = buildEvalGroupRequest(conv, schemas, columns) + require.NoError(t, err) + mapping = req.TestingCriteria[0].DataMapping + require.Contains(t, mapping, "messages") + require.NotContains(t, mapping, "query") + require.NotContains(t, mapping, "response") + + // An unset level behaves as turn, matching the service default. + dflt := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, + &project.Options{EvalModel: "m"}) + req, err = buildEvalGroupRequest(dflt, schemas, columns) + require.NoError(t, err) + require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") +} + +// Without an agent target the sample bindings are unavailable, so every field +// has to come from the dataset and the sample schema is not requested. +func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + []string{"query", "response", "ground_truth"}, + []string{"query", "response", "ground_truth"}, + nil, nil, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + group.Target = nil + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + "query": true, "response": true, "ground_truth": true, + }) + require.NoError(t, err) + require.False(t, req.DataSourceConfig.IncludeSampleSchema) + require.Equal(t, "{{item.response}}", req.TestingCriteria[0].DataMapping["response"]) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index e2f433fe84f..cc3fd2f9a33 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -152,6 +152,7 @@ func (r *evalReconciler) EnsureEvaluator( func (r *evalReconciler) EnsureEvalGroup( ctx context.Context, group project.EvalGroup, + datasetPath string, recreate bool, ) (string, error) { if group.ID != "" { @@ -165,7 +166,15 @@ func (r *evalReconciler) EnsureEvalGroup( } } - created, err := r.ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(&group)) + req, err := buildEvalGroupRequest( + &group, + r.ec.evaluatorSchemas(ctx), + datasetColumnsFromPath(datasetPath), + ) + if err != nil { + return "", err + } + created, err := r.ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { return "", err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 7085d5b7ad7..abae99924ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -68,7 +68,8 @@ func newRunCommand() *cobra.Command { return err } - evalID, err = ec.resolveEvalGroupID(ctx, group, out, isJSON(cmd)) + evalID, err = ec.resolveEvalGroupID( + ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) if err != nil { return err } @@ -148,6 +149,8 @@ func newRunCommand() *cobra.Command { func (ec *evalContext) resolveEvalGroupID( ctx context.Context, group *project.EvalGroup, + configPath string, + level string, out interface{ Write([]byte) (int, error) }, jsonMode bool, ) (string, error) { @@ -165,7 +168,28 @@ func (ec *evalContext) resolveEvalGroupID( if !jsonMode { fmt.Fprintf(out, "Creating eval group %q...\n", group.Name) } - created, err := ec.evalClient.CreateOpenAIEval(ctx, buildEvalGroupRequest(group)) + + // The level from the flag wins over the group's own options, so it has to + // reach the criteria that accept evaluation_level. + effective := *group + if level != "" { + opts := project.Options{} + if group.Options != nil { + opts = *group.Options + } + opts.EvaluationLevel = level + effective.Options = &opts + } + + req, err := buildEvalGroupRequest( + &effective, + ec.evaluatorSchemas(ctx), + datasetColumns(configPath, group), + ) + if err != nil { + return "", err + } + created, err := ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) } @@ -211,6 +235,35 @@ func buildRunDataSource( return ds, nil } +// datasetColumns reports the columns a group's dataset provides, so criteria +// bind only to fields that exist and a missing required field is caught +// locally rather than as a service rejection. +// +// A nil result means the columns are unknown, which is the case for a dataset +// already registered in the project. The builder then assumes every field an +// evaluator accepts is present. +func datasetColumns(configPath string, group *project.EvalGroup) map[string]bool { + return datasetColumnsFromPath(localDatasetPath(configPath, group)) +} + +// datasetColumnsFromPath reads one row to learn the dataset's shape. An empty +// path, or an unreadable file, yields nil. +func datasetColumnsFromPath(localPath string) map[string]bool { + if localPath == "" { + return nil + } + // One row is enough to learn the shape. + items, err := readJSONL(localPath, 1) + if err != nil || len(items) == 0 { + return nil + } + columns := make(map[string]bool, len(items[0])) + for name := range items[0] { + columns[name] = true + } + return columns +} + // localDatasetPath resolves the dataset's local source relative to the config // file, returning empty when the dataset is registered rather than local. func localDatasetPath(configPath string, group *project.EvalGroup) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go index 7e6c8633c09..13fe6da2d9d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/models.go @@ -35,11 +35,11 @@ type Dataset struct { Format string `json:"format,omitempty"` // camelCase spellings (project endpoint). - DataURICamel string `json:"dataUri,omitempty"` - BlobURICamel string `json:"blobUri,omitempty"` - ContentURICamel string `json:"contentUri,omitempty"` - IsSingleFile bool `json:"isSingleFile,omitempty"` - ConnectionName string `json:"connectionName,omitempty"` + DataURICamel string `json:"dataUri,omitempty"` + BlobURICamel string `json:"blobUri,omitempty"` + ContentURICamel string `json:"contentUri,omitempty"` + IsSingleFile bool `json:"isSingleFile,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` // snake_case spellings. BlobURI string `json:"blob_uri,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 771f894b03d..dfa0abf60ca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -8,17 +8,95 @@ import ( "fmt" "net/http" "net/url" + "sort" + "strings" ) // EvaluatorTypeBuiltin selects the platform-provided evaluators. const EvaluatorTypeBuiltin = "Builtin" +// JSONSchema is the subset of JSON Schema the evaluator contract uses. +type JSONSchema struct { + Type string `json:"type,omitempty"` + Required []string `json:"required,omitempty"` + Properties map[string]any `json:"properties,omitempty"` +} + +// PropertyNames returns the accepted property names, sorted for stable output. +func (s *JSONSchema) PropertyNames() []string { + if s == nil { + return nil + } + names := make([]string, 0, len(s.Properties)) + for name := range s.Properties { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Accepts reports whether the schema declares the named property. +func (s *JSONSchema) Accepts(name string) bool { + if s == nil || s.Properties == nil { + return false + } + _, ok := s.Properties[name] + return ok +} + +// EvaluatorContract is the published input contract for an evaluator: which +// data fields it consumes and which initialization parameters it takes. +type EvaluatorContract struct { + Type string `json:"type,omitempty"` + DataSchema *JSONSchema `json:"data_schema,omitempty"` + InitParameters *JSONSchema `json:"init_parameters,omitempty"` +} + // EvaluatorSummary is a single entry in an evaluator listing. +// +// The listing carries the full contract, so callers can shape a request to +// match an evaluator instead of guessing and taking a service-side rejection. type EvaluatorSummary struct { Name string `json:"name"` Version string `json:"version,omitempty"` Type string `json:"type,omitempty"` Description string `json:"description,omitempty"` + + Categories []string `json:"categories,omitempty"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels,omitempty"` + Definition *EvaluatorContract `json:"definition,omitempty"` +} + +// SupportsLevel reports whether the evaluator runs at the given evaluation +// level. An evaluator that declares no levels is treated as unconstrained. +func (e *EvaluatorSummary) SupportsLevel(level string) bool { + if level == "" || len(e.SupportedEvaluationLevels) == 0 { + return true + } + for _, supported := range e.SupportedEvaluationLevels { + if strings.EqualFold(supported, level) { + return true + } + } + return false +} + +// DataSchema returns the evaluator's input schema, or nil when the listing +// did not describe one. +func (e *EvaluatorSummary) DataSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.DataSchema +} + +// InitSchema returns the evaluator's initialization-parameter schema, or nil +// when the listing did not describe one. +func (e *EvaluatorSummary) InitSchema() *JSONSchema { + if e == nil || e.Definition == nil { + return nil + } + return e.Definition.InitParameters } // EvaluatorListResponse is the paged response for an evaluator listing. @@ -27,6 +105,18 @@ type EvaluatorListResponse struct { NextLink string `json:"nextLink,omitempty"` } +// ByName indexes the listing by evaluator name. +func (r *EvaluatorListResponse) ByName() map[string]*EvaluatorSummary { + if r == nil { + return nil + } + index := make(map[string]*EvaluatorSummary, len(r.Value)) + for i := range r.Value { + index[r.Value[i].Name] = &r.Value[i] + } + return index +} + // ListEvaluators returns the evaluators visible to the project. Pass // EvaluatorTypeBuiltin to list only the platform's built-ins. func (c *EvalClient) ListEvaluators( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 510b2680b31..795e8301e0c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -165,6 +165,13 @@ func (c *EvalClient) GetOpenAIEval(ctx context.Context, evalID string) (*OpenAIE return doRequestTyped[OpenAIEval](c, ctx, http.MethodGet, path, nil, nil, "") } +// DeleteOpenAIEval removes an eval definition and its runs. +func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + // CreateOpenAIEvalRun starts a run for an OpenAI eval definition. func (c *EvalClient) CreateOpenAIEvalRun( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index ce8a1439b5d..734e163195b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -37,8 +37,10 @@ type Reconciler interface { // differs from what the service already holds. EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) // EnsureEvalGroup creates the group when it is absent or its resolved - // evaluators or options changed, returning its id. - EnsureEvalGroup(ctx context.Context, group EvalGroup, recreate bool) (id string, err error) + // evaluators or options changed, returning its id. datasetPath is the local + // dataset backing the group, or empty when it is already registered; it lets + // the reconciler bind criteria to the columns that actually exist. + EnsureEvalGroup(ctx context.Context, group EvalGroup, datasetPath string, recreate bool) (id string, err error) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -171,7 +173,11 @@ func (p *EvalServiceTargetProvider) Deploy( // group must be created and the stored id replaced. for _, group := range cfg.EvalGroups { report(progress, fmt.Sprintf("Reconciling eval group %s", group.Name)) - id, err := reconciler.EnsureEvalGroup(ctx, group, anyChanged) + datasetPath := "" + if decl, ok := cfg.Dataset(group.Dataset); ok { + datasetPath = resolveSource(baseDir, decl.Source) + } + id, err := reconciler.EnsureEvalGroup(ctx, group, datasetPath, anyChanged) if err != nil { return nil, fmt.Errorf("eval group %q: %w", group.Name, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index c04af2d47a6..9174ff294f4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -195,7 +195,7 @@ func TestLiveEvalGroupLifecycle(t *testing.T) { ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) require.NoError(t, err) require.NotEmpty(t, builtins.Value, "need at least one built-in evaluator") -evaluatorName := pickQualityEvaluator(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) threshold := 3.0 req := &eval_api.CreateOpenAIEvalRequest{ @@ -249,7 +249,7 @@ func TestLiveRun(t *testing.T) { ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) require.NoError(t, err) require.NotEmpty(t, builtins.Value) -evaluatorName := pickQualityEvaluator(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ Name: uniqueName("azd-eval-e2e-run"), From 4d265585679572afffcf97bb3483f8983482c0ba Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:50:23 +0300 Subject: [PATCH 012/114] build(azure.ai.evaluations): add the build scripts the extension kit invokes Without build.ps1 and build.sh the dev kit reported a successful build in under a second and produced no binary, so azd x pack had nothing to package and azd x publish failed with 'Artifacts not found'. Copied from azure.ai.agents with the version package path retargeted, plus its golangci config. Verified end to end: build, pack, publish, install from the local registry, and 'azd ai eval --help' listing every command. --- .../azure.ai.evaluations/.golangci.yaml | 21 +++++ .../extensions/azure.ai.evaluations/build.ps1 | 78 +++++++++++++++++++ .../extensions/azure.ai.evaluations/build.sh | 66 ++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/.golangci.yaml create mode 100644 cli/azd/extensions/azure.ai.evaluations/build.ps1 create mode 100644 cli/azd/extensions/azure.ai.evaluations/build.sh diff --git a/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml new file mode 100644 index 00000000000..9777522d023 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.golangci.yaml @@ -0,0 +1,21 @@ +version: "2" + +linters: + default: none + enable: + - gosec + - lll + - unused + - errorlint + settings: + lll: + line-length: 220 + tab-width: 4 + gosec: + excludes: + - G204 # Subprocess launched with variable (bicep build invoked in tests) + - G304 # Potential file inclusion via variable + +formatters: + enable: + - gofmt diff --git a/cli/azd/extensions/azure.ai.evaluations/build.ps1 b/cli/azd/extensions/azure.ai.evaluations/build.ps1 new file mode 100644 index 00000000000..f37f80cabf0 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.ps1 @@ -0,0 +1,78 @@ +# Ensure script fails on any error +$ErrorActionPreference = 'Stop' + +# Get the directory of the script +$EXTENSION_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path + +# Change to the script directory +Set-Location -Path $EXTENSION_DIR + +# Create a safe version of EXTENSION_ID replacing dots with dashes +$EXTENSION_ID_SAFE = $env:EXTENSION_ID -replace '\.', '-' + +# Define output directory +$OUTPUT_DIR = if ($env:OUTPUT_DIR) { $env:OUTPUT_DIR } else { Join-Path $EXTENSION_DIR "bin" } + +# Create output directory if it doesn't exist +if (-not (Test-Path -Path $OUTPUT_DIR)) { + New-Item -ItemType Directory -Path $OUTPUT_DIR | Out-Null +} + +# Get Git commit hash and build date +$COMMIT = git rev-parse HEAD +if ($LASTEXITCODE -ne 0) { + Write-Host "Error: Failed to get git commit hash" + exit 1 +} +$BUILD_DATE = (Get-Date -Format "yyyy-MM-ddTHH:mm:ssZ") + +# List of OS and architecture combinations +if ($env:EXTENSION_PLATFORM) { + $PLATFORMS = @($env:EXTENSION_PLATFORM) +} +else { + $PLATFORMS = @( + "windows/amd64", + "windows/arm64", + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm64" + ) +} + +$VERSION_PATH = "azureaieval/internal/version" + +# Loop through platforms and build +foreach ($PLATFORM in $PLATFORMS) { + $OS, $ARCH = $PLATFORM -split '/' + + $OUTPUT_NAME = Join-Path $OUTPUT_DIR "$EXTENSION_ID_SAFE-$OS-$ARCH" + + if ($OS -eq "windows") { + $OUTPUT_NAME += ".exe" + } + + Write-Host "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + if (Test-Path -Path $OUTPUT_NAME) { + Remove-Item -Path $OUTPUT_NAME -Force + } + + # Set environment variables for Go build + $env:GOOS = $OS + $env:GOARCH = $ARCH + + go build ` + -ldflags="-X '$VERSION_PATH.Version=$env:EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" ` + -o $OUTPUT_NAME + + if ($LASTEXITCODE -ne 0) { + Write-Host "An error occurred while building for $OS/$ARCH" + exit 1 + } +} + +Write-Host "Build completed successfully!" +Write-Host "Binaries are located in the $OUTPUT_DIR directory." diff --git a/cli/azd/extensions/azure.ai.evaluations/build.sh b/cli/azd/extensions/azure.ai.evaluations/build.sh new file mode 100644 index 00000000000..4165a516ac4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Get the directory of the script +EXTENSION_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Change to the script directory +cd "$EXTENSION_DIR" || exit + +# Create a safe version of EXTENSION_ID replacing dots with dashes +EXTENSION_ID_SAFE="${EXTENSION_ID//./-}" + +# Define output directory +OUTPUT_DIR="${OUTPUT_DIR:-$EXTENSION_DIR/bin}" + +# Create output and target directories if they don't exist +mkdir -p "$OUTPUT_DIR" + +# Get Git commit hash and build date +COMMIT=$(git rev-parse HEAD) +BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +# List of OS and architecture combinations +if [ -n "$EXTENSION_PLATFORM" ]; then + PLATFORMS=("$EXTENSION_PLATFORM") +else + PLATFORMS=( + "windows/amd64" + "windows/arm64" + "darwin/amd64" + "darwin/arm64" + "linux/amd64" + "linux/arm64" + ) +fi + +VERSION_PATH="azureaieval/internal/version" + +# Loop through platforms and build +for PLATFORM in "${PLATFORMS[@]}"; do + OS=$(echo "$PLATFORM" | cut -d'/' -f1) + ARCH=$(echo "$PLATFORM" | cut -d'/' -f2) + + OUTPUT_NAME="$OUTPUT_DIR/$EXTENSION_ID_SAFE-$OS-$ARCH" + + if [ "$OS" = "windows" ]; then + OUTPUT_NAME+='.exe' + fi + + echo "Building for $OS/$ARCH..." + + # Delete the output file if it already exists + [ -f "$OUTPUT_NAME" ] && rm -f "$OUTPUT_NAME" + + # Set environment variables for Go build + GOOS=$OS GOARCH=$ARCH go build \ + -ldflags="-X '$VERSION_PATH.Version=$EXTENSION_VERSION' -X '$VERSION_PATH.Commit=$COMMIT' -X '$VERSION_PATH.BuildDate=$BUILD_DATE'" \ + -o "$OUTPUT_NAME" + + if [ $? -ne 0 ]; then + echo "An error occurred while building for $OS/$ARCH" + exit 1 + fi +done + +echo "Build completed successfully!" +echo "Binaries are located in the $OUTPUT_DIR directory." From f92e69be38391477314a7e72bcc38deaadbba455 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 05:52:36 +0300 Subject: [PATCH 013/114] fix(azure.ai.evaluations): read the evaluator kind the listing actually returns The listing spells it evaluator_type, so the TYPE column in 'evaluator list' and 'evaluator builtins' was always blank. Both spellings are now accepted. --- .../azure.ai.evaluations/internal/cmd/evaluator.go | 2 +- .../internal/pkg/eval_api/evaluators.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index e374079dd9e..9bbc7e41b1a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -203,7 +203,7 @@ func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) } rows := make([][]string, 0, len(list.Value)) for _, e := range list.Value { - rows = append(rows, []string{e.Name, e.Version, e.Type}) + rows = append(rows, []string{e.Name, e.Version, e.Type()}) } return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "TYPE"}, rows) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index dfa0abf60ca..e03a7059efd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -59,14 +59,26 @@ type EvaluatorContract struct { type EvaluatorSummary struct { Name string `json:"name"` Version string `json:"version,omitempty"` - Type string `json:"type,omitempty"` Description string `json:"description,omitempty"` + // The listing spells this evaluator_type; `type` is accepted too because + // other evaluator payloads use it. + EvaluatorType string `json:"evaluator_type,omitempty"` + TypeAlias string `json:"type,omitempty"` + Categories []string `json:"categories,omitempty"` SupportedEvaluationLevels []string `json:"supported_evaluation_levels,omitempty"` Definition *EvaluatorContract `json:"definition,omitempty"` } +// Type reports the evaluator kind across both spellings. +func (e *EvaluatorSummary) Type() string { + if e.EvaluatorType != "" { + return e.EvaluatorType + } + return e.TypeAlias +} + // SupportsLevel reports whether the evaluator runs at the given evaluation // level. An evaluator that declares no levels is treated as unconstrained. func (e *EvaluatorSummary) SupportsLevel(level string) bool { From 5a670ca81162602aaf87022e5a624f17c9aef518 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:02:33 +0300 Subject: [PATCH 014/114] fix(azure.ai.evaluations): make azd deploy work end to end Running a real 'azd deploy' against the service-target provider surfaced two failures that no unit test covered. Evaluator references only decoded from YAML. azd hands the service entry to the extension as JSON, so a group written as '- builtin.task_adherence' -- the form the CLI's own init command writes -- failed with 'cannot unmarshal string into EvaluatorRef'. EvaluatorList now decodes and encodes the mixed string-or-mapping form through JSON as well, and a test asserts the two decoders agree. The dataset reconciler passed the declared version straight to UploadNewVersion, which derives the next version from it. A declaration without an explicit version passed empty, so every deploy retried 1.0 and the service returned 409 TemporaryDataReferencesForExistingAsset once that version existed. It now looks up the latest registered version first. Verified against a live project: first deploy publishes the dataset at 1.0 and creates the group; an unchanged redeploy reports 'unchanged at version 1.0' and uploads nothing; and editing the dataset publishes 2.0 and recreates the group, since groups are immutable. --- .../internal/cmd/reconciler.go | 21 ++++++- .../internal/pkg/evalcore/evaluator.go | 57 +++++++++++++++++ .../internal/pkg/evalcore/evaluator_test.go | 62 +++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index cc3fd2f9a33..92a55f808f4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -84,8 +84,17 @@ func (r *evalReconciler) EnsureDataset( dir = filepath.Dir(localPath) } + // UploadNewVersion derives the next version from the one passed in, so it + // has to be told what is already registered. Passing an empty version + // restarts at 1.0 and the service rejects the pending upload with + // TemporaryDataReferencesForExistingAsset once that version exists. + currentVersion := decl.Version + if currentVersion == "" { + currentVersion = r.latestDatasetVersion(ctx, decl.Name) + } + ds, err := r.ec.datasetClient.UploadNewVersion( - ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ctx, decl.Name, currentVersion, dir, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err @@ -98,6 +107,16 @@ func (r *evalReconciler) EnsureDataset( return ds.Version, true, nil } +// latestDatasetVersion reports the newest registered version, or empty when the +// dataset does not exist yet so the first upload starts at 1.0. +func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { + list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return dataset_api.LatestVersion(list.Value) +} + // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. Evaluator definitions come back inline, so this // compares content directly rather than relying on a cached digest. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 0a4847ad4bd..4832d52c42d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -4,6 +4,8 @@ package evalcore import ( + "bytes" + "encoding/json" "fmt" "strings" @@ -87,3 +89,58 @@ func (el EvaluatorList) MarshalYAML() (any, error) { } return out, nil } + +// UnmarshalJSON accepts the same mixed string-or-mapping form as the YAML +// decoder. +// +// This matters for the service-target provider: azd hands the service entry to +// the extension as JSON, so a config written as `- builtin.task_adherence` +// arrives as a bare string and would otherwise fail to decode. +func (el *EvaluatorList) UnmarshalJSON(data []byte) error { + var entries []json.RawMessage + if err := json.Unmarshal(data, &entries); err != nil { + return fmt.Errorf("evaluators must be a list: %w", err) + } + + result := make([]EvaluatorRef, 0, len(entries)) + for _, entry := range entries { + trimmed := bytes.TrimSpace(entry) + if len(trimmed) > 0 && trimmed[0] == '"' { + var name string + if err := json.Unmarshal(trimmed, &name); err != nil { + return fmt.Errorf("decoding evaluator name: %w", err) + } + result = append(result, EvaluatorRef{Name: name}) + continue + } + + var ref EvaluatorRef + if err := json.Unmarshal(trimmed, &ref); err != nil { + return fmt.Errorf("decoding evaluator: %w", err) + } + if ref.Name == "" { + return fmt.Errorf("evaluator entry is missing 'name'") + } + result = append(result, ref) + } + + *el = result + return nil +} + +// MarshalJSON mirrors MarshalYAML's compact form. +func (el EvaluatorList) MarshalJSON() ([]byte, error) { + out := make([]any, 0, len(el)) + for _, ref := range el { + if ref.Threshold == nil && ref.Version == "" { + out = append(out, ref.Name) + continue + } + out = append(out, struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + Threshold *float64 `json:"threshold,omitempty"` + }{ref.Name, ref.Version, ref.Threshold}) + } + return json.Marshal(out) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go new file mode 100644 index 00000000000..5408776bec7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// The service-target provider receives the config as JSON, not YAML, so the +// mixed string-or-mapping form has to decode through both paths. Supporting +// only YAML made `azd deploy` fail on a config the CLI itself writes. +func TestEvaluatorListDecodesMixedEntriesFromJSON(t *testing.T) { + const payload = `[ + "builtin.task_adherence", + {"name": "support-quality", "threshold": 4.0}, + {"name": "pinned", "version": "3"} + ]` + + var list EvaluatorList + require.NoError(t, json.Unmarshal([]byte(payload), &list)) + require.Len(t, list, 3) + + require.Equal(t, "builtin.task_adherence", list[0].Name) + require.Nil(t, list[0].Threshold) + + require.Equal(t, "support-quality", list[1].Name) + require.NotNil(t, list[1].Threshold) + require.InDelta(t, 4.0, *list[1].Threshold, 0.0001) + + require.Equal(t, "pinned", list[2].Name) + require.Equal(t, "3", list[2].Version) +} + +// The JSON and YAML decoders must agree, otherwise a config behaves one way +// through the CLI and another through `azd up`. +func TestEvaluatorListJSONMatchesYAML(t *testing.T) { + const doc = ` +- builtin.task_adherence +- { name: support-quality, threshold: 4.0 } +` + var fromYAML EvaluatorList + require.NoError(t, yaml.Unmarshal([]byte(doc), &fromYAML)) + + encoded, err := json.Marshal(fromYAML) + require.NoError(t, err) + + var fromJSON EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &fromJSON)) + require.Equal(t, fromYAML, fromJSON) +} + +func TestEvaluatorListRejectsEntryWithoutName(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`[{"threshold": 4.0}]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "name") +} From d21bbd3bce67735104d6c7f25544eee014957153 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:07:59 +0300 Subject: [PATCH 015/114] fix(azure.ai.evaluations): resolve $ref includes in the service entry azd core does not resolve $ref for extensions. It strips the ServiceConfig fields it owns and leaves $ref at the top of the map for the owning extension to resolve, so a service authored the way the spec documents it -- host: azure.ai.eval plus $ref: ./evals/azure.yaml -- parsed to an empty config. azd deploy then reported success in three seconds having created nothing, which is worse than failing. The provider now calls foundry.ResolveFileRefs with the project root from the azd project client. Relative source paths inside an included file are written against that file, but ResolveFileRefs inlines content without rebasing them, so the include's own directory is now the base for source resolution. Verified against a live project: the $ref form deploys, and the dataset fingerprint matches the one from the equivalent inline config, confirming both forms resolve to the same file. --- .../internal/project/service_target_eval.go | 48 ++++++++- .../project/service_target_eval_test.go | 98 +++++++++++++++++++ 2 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 734e163195b..ef24f4d00e2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/azure/azure-dev/cli/azd/pkg/foundry" "google.golang.org/protobuf/types/known/structpb" ) @@ -129,7 +130,7 @@ func (p *EvalServiceTargetProvider) Deploy( targetResource *azdext.TargetResource, progress azdext.ProgressReporter, ) (*azdext.ServiceDeployResult, error) { - cfg, err := EvalConfigFromService(serviceConfig) + cfg, err := EvalConfigFromService(serviceConfig, p.projectRoot(ctx)) if err != nil { return nil, err } @@ -187,6 +188,19 @@ func (p *EvalServiceTargetProvider) Deploy( return &azdext.ServiceDeployResult{}, nil } +// projectRoot is the directory `$ref` paths resolve against. It is the +// directory holding azure.yaml, which only azd can report. +func (p *EvalServiceTargetProvider) projectRoot(ctx context.Context) string { + if p.azdClient == nil { + return "" + } + resp, err := p.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "" + } + return resp.GetProject().GetPath() +} + // describeResult reports whether a version was published or reused, so a // no-op deploy is visibly a no-op. func describeResult(kind, name, version string, changed bool) string { @@ -205,7 +219,13 @@ func report(progress azdext.ProgressReporter, message string) { // EvalConfigFromService reads the eval configuration carried inline on the // service entry. azd captures unknown keys into AdditionalProperties and hands // them to the extension untouched. -func EvalConfigFromService(svc *azdext.ServiceConfig) (*EvalConfig, error) { +// +// azd core deliberately does not resolve `$ref` includes for extensions — it +// strips the ServiceConfig fields it owns and leaves `$ref` at the top of the +// map for the owning extension to resolve. Without this call a service written +// as `host: azure.ai.eval` + `$ref: ./evals/azure.yaml` deploys nothing at all, +// because the config parses to an empty set of datasets and groups. +func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*EvalConfig, error) { props := serviceProps(svc) if props == nil || len(props.GetFields()) == 0 { return nil, fmt.Errorf( @@ -213,7 +233,16 @@ func EvalConfigFromService(svc *azdext.ServiceConfig) (*EvalConfig, error) { svc.GetName()) } - raw, err := props.MarshalJSON() + values := props.AsMap() + if projectRoot != "" { + resolved, err := foundry.ResolveFileRefs(values, projectRoot) + if err != nil { + return nil, fmt.Errorf("resolving $ref in the eval service configuration: %w", err) + } + values = resolved + } + + raw, err := json.Marshal(values) if err != nil { return nil, fmt.Errorf("reading the eval service configuration: %w", err) } @@ -235,10 +264,23 @@ func serviceProps(svc *azdext.ServiceConfig) *structpb.Struct { } // serviceRelativeDir returns the directory that `source:` paths resolve against. +// +// When the service is authored as `host:` + `$ref: ./evals/azure.yaml`, the +// paths inside that file are written relative to the file itself, so the +// include's own directory is the base. ResolveFileRefs inlines the content +// without rebasing paths, so the base has to be recovered from the `$ref` +// value before resolution. func serviceRelativeDir(svc *azdext.ServiceConfig) string { if svc == nil { return "." } + if props := serviceProps(svc); props != nil { + if ref, ok := props.AsMap()["$ref"].(string); ok && ref != "" { + if dir := filepath.Dir(filepath.FromSlash(ref)); dir != "" { + return dir + } + } + } if p := svc.GetRelativePath(); p != "" { return p } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go new file mode 100644 index 00000000000..a709de919dc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func propsFrom(t *testing.T, values map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(values) + require.NoError(t, err) + return s +} + +// A service authored as `host:` + `$ref: ./evals/azure.yaml` has its relative +// source paths written against the included file, not the project root. +// ResolveFileRefs inlines the content without rebasing them, so the base has to +// come from the $ref value. +func TestServiceRelativeDirUsesRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("evals"), serviceRelativeDir(svc)) +} + +// A nested include keeps its own directory. +func TestServiceRelativeDirUsesNestedRefDirectory(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "$ref": "./config/evals/azure.yaml", + }), + } + require.Equal(t, filepath.FromSlash("config/evals"), serviceRelativeDir(svc)) +} + +// Without a $ref the service's own relative path is the base. +func TestServiceRelativeDirFallsBackToRelativePath(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + RelativePath: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{}, + }), + } + require.Equal(t, "evals", serviceRelativeDir(svc)) +} + +// With neither, sources resolve against the project root. +func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { + require.Equal(t, ".", serviceRelativeDir(&azdext.ServiceConfig{Name: "evals"})) + require.Equal(t, ".", serviceRelativeDir(nil)) +} + +// An inline config still parses when no project root is available to resolve +// includes against. +func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { + svc := &azdext.ServiceConfig{ + Name: "evals", + AdditionalProperties: propsFrom(t, map[string]any{ + "datasets": []any{ + map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + }, + "evalGroups": []any{ + map[string]any{ + "name": "quality", + "dataset": "golden", + "evaluators": []any{"builtin.task_adherence"}, + "target": map[string]any{"type": "agent", "name": "my-agent"}, + }, + }, + }), + } + + cfg, err := EvalConfigFromService(svc, "") + require.NoError(t, err) + require.Len(t, cfg.Datasets, 1) + require.Equal(t, "golden", cfg.Datasets[0].Name) + require.Len(t, cfg.EvalGroups, 1) + require.Len(t, cfg.EvalGroups[0].Evaluators, 1) + require.Equal(t, "builtin.task_adherence", cfg.EvalGroups[0].Evaluators[0].Name) +} + +func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { + _, err := EvalConfigFromService(&azdext.ServiceConfig{Name: "evals"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "no eval configuration") +} From ae6f43c549c054db7635abd807cc0548d7d89e37 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:18:24 +0300 Subject: [PATCH 016/114] fix(azure.ai.evaluations): recreate an eval group when its own declaration changes Change detection only covered upstream artifacts, so retargeting a group at a different agent, swapping an evaluator, or changing the judge model left the old group in place. Groups are immutable, so the edit silently had no effect and later runs kept evaluating the previous definition. The group's declaration is now fingerprinted alongside the dataset and evaluator artifacts. The id and description are excluded: one is server assigned and the other is cosmetic, so neither should force a recreate. The digest is recorded when an existing group is reused as well as when one is created. Recording it only on create meant a group deployed before this change never established a baseline, and the first edit after it would still go undetected. Verified against a live project: changing the target produced a new group id, and two further deploys with no change reused it. --- .../internal/cmd/reconciler.go | 17 ++++++ .../internal/project/service_target_eval.go | 21 +++++++ .../project/service_target_eval_test.go | 61 +++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 92a55f808f4..3b04655d768 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -178,9 +178,25 @@ func (r *evalReconciler) EnsureEvalGroup( return group.ID, nil } + // Groups are immutable, so a change to the group's own declaration — + // evaluators, target, or options — needs a new group just as much as a + // change to an upstream artifact does. + digest, err := project.FingerprintGroup(group) + if err != nil { + return "", err + } + key := project.FingerprintKey("evalgroup", group.Name) + if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { + recreate = true + } + cached := r.ec.getEnvValue(ctx, envKeyEvalGroupID) if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { + // Record the digest on reuse as well, otherwise a group deployed + // before fingerprinting existed never establishes a baseline and + // later edits go undetected. + _ = r.ec.setEnvValue(ctx, key, digest) return cached, nil } } @@ -197,6 +213,7 @@ func (r *evalReconciler) EnsureEvalGroup( if err != nil { return "", err } + _ = r.ec.setEnvValue(ctx, key, digest) _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) return created.ID, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index ef24f4d00e2..b6dd930ced4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -313,6 +313,27 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } +// FingerprintGroup hashes an eval group's own declaration. +// +// Change detection on upstream artifacts is not sufficient: editing a group's +// evaluators, target, or options changes what the group means, and groups are +// immutable, so the group has to be recreated even when the dataset and +// evaluators are untouched. Without this a retargeted group keeps running +// against the old definition. +func FingerprintGroup(group EvalGroup) (string, error) { + // The id is server-assigned and the description is cosmetic; neither + // changes what the group evaluates. + group.ID = "" + group.Description = "" + + data, err := json.Marshal(group) + if err != nil { + return "", fmt.Errorf("hashing eval group %q: %w", group.Name, err) + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + // FingerprintKey is the azd environment key holding an artifact's fingerprint. func FingerprintKey(kind, name string) string { safe := strings.Map(func(r rune) rune { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index a709de919dc..6052cb99c55 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "testing" + "azureaieval/internal/pkg/evalcore" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/structpb" @@ -96,3 +98,62 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "no eval configuration") } + +// Groups are immutable, so a change to the group's own declaration has to be +// detectable. Upstream artifact fingerprints do not cover it: retargeting a +// group at a different agent leaves the dataset and evaluators untouched. +func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { + base := EvalGroup{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, + Target: &Target{Type: "agent", Name: "agent-a"}, + Options: &Options{EvalModel: "gpt-4.1-nano"}, + } + + original, err := FingerprintGroup(base) + require.NoError(t, err) + + same, err := FingerprintGroup(base) + require.NoError(t, err) + require.Equal(t, original, same, "an unchanged group must keep its fingerprint") + + cases := map[string]func(g *EvalGroup){ + "target": func(g *EvalGroup) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, + "evaluators": func(g *EvalGroup) { + g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) + }, + "options": func(g *EvalGroup) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, + "dataset": func(g *EvalGroup) { g.Dataset = "other" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + changed := base + changed.Evaluators = append(evalcore.EvaluatorList(nil), base.Evaluators...) + mutate(&changed) + + digest, err := FingerprintGroup(changed) + require.NoError(t, err) + require.NotEqual(t, original, digest, "changing %s must change the fingerprint", name) + }) + } +} + +// Server-assigned and cosmetic fields must not force a recreate. +func TestFingerprintGroupIgnoresIdAndDescription(t *testing.T) { + base := EvalGroup{ + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, + } + original, err := FingerprintGroup(base) + require.NoError(t, err) + + noisy := base + noisy.ID = "eval_abc123" + noisy.Description = "reworded" + + digest, err := FingerprintGroup(noisy) + require.NoError(t, err) + require.Equal(t, original, digest) +} From e71fc44bb8122a9aef0adf67ea7a00a4019dc4e5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:25:17 +0300 Subject: [PATCH 017/114] fix(azure.ai.evaluations): keep HTTP traces out of normal output The data-plane clients trace every request and response through log.Printf, which Go writes to stderr by default, so a plain command interleaved raw URLs and status lines with its own output. A long generate run was mostly HTTP traces. Ports the debug setup from the agents extension: the standard logger is discarded unless --debug or AZD_EXT_DEBUG is set, and debug output goes to a dated file rather than the terminal. The hook chains the SDK PersistentPreRunE instead of replacing it. Assigning PersistentPreRun has no effect once the E variant is set, and overwriting the E variant would drop the SDK own setup. Also reports jobs as submitted when generate is given --no-wait, which is a successful submission rather than an empty result. --- .../internal/cmd/debug.go | 65 +++++++++++++++++++ .../internal/cmd/generate.go | 8 +++ .../azure.ai.evaluations/internal/cmd/root.go | 19 ++++++ 3 files changed, 92 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go new file mode 100644 index 00000000000..48fc0802aaf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/debug.go @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "io" + "log" + "os" + "strconv" + "time" + + azcorelog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" + "github.com/spf13/pflag" +) + +// setupDebugLogging silences the standard logger unless debug mode is on. +// +// The data-plane clients trace every request through log.Printf, which Go +// writes to stderr by default. Without this the CLI interleaves raw HTTP traces +// with its own output on every command. Returns a cleanup function the caller +// should defer. +func setupDebugLogging(flags *pflag.FlagSet) func() { + if !isDebug(flags) { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + return func() {} + } + + logFileName := fmt.Sprintf("azd-ai-eval-%s.log", time.Now().Format("2006-01-02")) + + //nolint:gosec // the name is generated locally from the date, not user input + logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + + var w io.Writer + var closeFile func() + if err != nil { + w = os.Stderr + closeFile = func() {} + } else { + w = logFile + closeFile = func() { logFile.Close() } //nolint:gosec // best-effort cleanup + } + + log.SetOutput(w) + azcorelog.SetListener(func(event azcorelog.Event, msg string) { + fmt.Fprintf(w, "[%s] %s: %s\n", time.Now().Format(time.RFC3339), event, msg) + }) + + return func() { + log.SetOutput(io.Discard) + azcorelog.SetListener(nil) + closeFile() + } +} + +// isDebug reports whether --debug or AZD_EXT_DEBUG is set. +func isDebug(flags *pflag.FlagSet) bool { + if debugFlag, err := flags.GetBool("debug"); err == nil && debugFlag { + return true + } + debug, _ := strconv.ParseBool(os.Getenv("AZD_EXT_DEBUG")) + return debug +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 3ce03cbfd65..334a95c6b61 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -95,6 +95,14 @@ func newGenerateCommand() *cobra.Command { } if len(datasetRefs) == 0 && len(evaluatorRefs) == 0 { + // With --no-wait the jobs were submitted and nothing was + // downloaded, which is success, not an empty result. + if noWait { + fmt.Fprintln(out, + "\nJobs submitted. Re-run without --no-wait to download the artifacts "+ + "and reference them from the deployment spec.") + return nil + } fmt.Fprintln(out, "Nothing was generated.") return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 3a289d6347d..a763a54b924 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -25,6 +25,25 @@ func NewRootCommand() *cobra.Command { rootCmd.SilenceErrors = true rootCmd.CompletionOptions.DisableDefaultCmd = true + // The data-plane clients trace requests through the standard logger, which + // Go writes to stderr, so it has to be silenced unless debug was asked for. + // + // The SDK's own hook is chained rather than replaced, and cobra ignores + // PersistentPreRun entirely once PersistentPreRunE is set. The SDK sets + // cobra.EnableTraverseRunHooks, so this still runs alongside subcommand + // hooks. The cleanup func is discarded on purpose: log writes are + // unbuffered and the OS closes the file at exit. + sdkPreRun := rootCmd.PersistentPreRunE + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if sdkPreRun != nil { + if err := sdkPreRun(cmd, args); err != nil { + return err + } + } + setupDebugLogging(cmd.Flags()) + return nil + } + rootCmd.AddCommand( newInitCommand(), newGenerateCommand(), From 02e9e953e24885df53fd6e5b3edceb068f7825f5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:34:41 +0300 Subject: [PATCH 018/114] feat(azure.ai.evaluations): explain why agent-seeded data generation fails Data generation with an agent source is accepted and then fails within seconds with DataGenerationJobSystemError, whose message says only that something went wrong and to try again. It is not transient: it reproduces for every agent tried, while the identical request without the agent source runs normally. The CLI now names the agent, says a retry will not help, and points at the two workarounds, instead of relaying advice that cannot succeed. --- .../internal/cmd/generate.go | 29 +++++++++++- .../internal/cmd/generate_test.go | 45 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 334a95c6b61..d3b2f4ce94f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -266,7 +266,7 @@ func (ec *evalContext) generateDataset( completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, ec.evalClient.GetDataGenerationJob) if err != nil { - return nil, fmt.Errorf("data generation: %w", err) + return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, cfg.Agent.Name)) } name, version := completed.ResolvedNameVersion() @@ -295,6 +295,33 @@ func (ec *evalContext) generateDataset( return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil } +// explainDataGenerationFailure adds context to the service's opaque system +// error. +// +// Seeding generation from an agent currently fails server-side with +// DataGenerationJobSystemError for every agent, within seconds, while the same +// request without the agent source runs normally. The raw message says only +// that something went wrong and to try again, which sends users into a retry +// loop against a deterministic failure. +func explainDataGenerationFailure(err error, agentName string) error { + if err == nil || agentName == "" { + return err + } + // The poller surfaces the service's message; the code is not always in it. + text := err.Error() + if !strings.Contains(text, "DataGenerationJobSystemError") && + !strings.Contains(text, "Something went wrong during data generation") { + return err + } + return fmt.Errorf( + "%w\n\n"+ + "This job seeded generation from agent %q. Agent-seeded data generation is "+ + "currently failing in the service for every agent, so retrying will not help.\n"+ + "Workarounds: supply your own dataset with --dataset, or run without --target "+ + "to generate from the instruction alone.", + err, agentName) +} + // pollGeneration waits for a generation job using the raised budget. func (ec *evalContext) pollGeneration( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go new file mode 100644 index 00000000000..e4c04baebf8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service's system error says only that something went wrong and to try +// again, but agent-seeded generation fails deterministically, so a bare retry +// suggestion sends users into a loop. +func TestExplainDataGenerationFailureAddsAgentContext(t *testing.T) { + err := errors.New( + `job failed with status "failed": Something went wrong during data generation. Please try again.`) + + explained := explainDataGenerationFailure(err, "my-agent") + require.Error(t, explained) + require.Contains(t, explained.Error(), "my-agent") + require.Contains(t, explained.Error(), "--dataset") + require.ErrorIs(t, explained, err, "the original error must stay in the chain") +} + +// The code spelling is matched as well, in case the poller starts surfacing it. +func TestExplainDataGenerationFailureMatchesErrorCode(t *testing.T) { + err := fmt.Errorf("job failed: DataGenerationJobSystemError") + explained := explainDataGenerationFailure(err, "my-agent") + require.Contains(t, explained.Error(), "Workarounds") +} + +// Unrelated failures are passed through untouched, and so is a job that had no +// agent source to blame. +func TestExplainDataGenerationFailureLeavesOthersAlone(t *testing.T) { + other := errors.New("submitting the data generation job: 403 Forbidden") + require.Equal(t, other, explainDataGenerationFailure(other, "my-agent")) + + systemErr := errors.New("Something went wrong during data generation") + require.Equal(t, systemErr, explainDataGenerationFailure(systemErr, "")) + + require.NoError(t, explainDataGenerationFailure(nil, "my-agent")) +} From a049fa64793a43c2ce451ddcf37e59a87321f9c0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:45:05 +0300 Subject: [PATCH 019/114] feat(azure.ai.evaluations): complete the atomic run operations The spec lists run start, list, show and cancel, and M1 requires every operation to be reachable atomically, but run was a single composite command with no subcommands. Listing runs, inspecting one, and cancelling an in-flight run were unreachable, even though the client already had the calls. Adds run list, run show and run cancel. Each takes the eval group id as an optional argument and otherwise falls back to the id recorded in the azd environment, matching results show. Cancelling a run that already reached a terminal state is refused locally, because the service reports success either way and the CLI would otherwise claim to have cancelled a finished run. Two related fixes. Passing --project-endpoint disabled the azd environment cache entirely: the environment name was only resolved when the endpoint came from azd, so every cached eval group and run id lookup returned empty. The name is now resolved independently of where the endpoint came from. The spec documents --wait and --no-wait, but cobra does not derive the negative form from a bool, so --no-wait was rejected as an unknown flag. Verified live: start with --no-wait, list, show, cancel, and the terminal-state guard on a second cancel. JSON output checked on the new subcommands. --- .../internal/cmd/context.go | 9 +- .../azure.ai.evaluations/internal/cmd/run.go | 11 ++ .../internal/cmd/run_ops.go | 180 ++++++++++++++++++ .../internal/cmd/run_ops_test.go | 52 +++++ 4 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index f9ba6d0ec8d..ef7562644fa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -46,10 +46,17 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err ec := &evalContext{azdClient: azdClient} + // The environment name is resolved regardless of where the endpoint comes + // from: it is what the cached eval group and run ids are read from and + // written to. Deriving it only when the endpoint came from azd meant + // --project-endpoint silently disabled that cache. + azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) + ec.envName = envName + if endpointFlag != "" { ec.endpoint = endpointFlag } else { - ec.endpoint, ec.envName = lookupEndpointFromAzd(ctx, azdClient) + ec.endpoint = azdEndpoint } if ec.endpoint == "" { ec.endpoint = os.Getenv(projectEndpointEnvKey) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index abae99924ac..30fe1c42100 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -139,7 +139,18 @@ func newRunCommand() *cobra.Command { cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from a local dataset file. Ignored for registered datasets.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + // The spec documents --no-wait, and cobra does not derive it from a bool. + var noWait bool + cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the run and return immediately.") + cmd.PreRun = func(*cobra.Command, []string) { + if noWait { + wait = false + } + } + cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + + addRunSubcommands(cmd) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go new file mode 100644 index 00000000000..c0eee5226cc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// addRunSubcommands attaches the atomic run operations. +// +// `azd ai eval run` stays the composite that creates the group if needed and +// starts a run; these expose the individual operations so every one is +// reachable without the config file. +func addRunSubcommands(cmd *cobra.Command) { + cmd.AddCommand( + newRunListCommand(), + newRunShowCommand(), + newRunCancelCommand(), + ) +} + +func newRunListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list [eval-id]", + Short: "List runs for an eval group.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) + if err != nil { + return fmt.Errorf("listing runs for %q: %w", evalID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), list) + } + if list == nil || len(list.Data) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "Eval group %s has no runs yet.\n", evalID) + return nil + } + + rows := make([][]string, 0, len(list.Data)) + for _, run := range list.Data { + rows = append(rows, []string{run.ID, run.Name, run.Status, summarizeCounts(run.ResultCounts)}) + } + return emitTable(cmd.OutOrStdout(), + []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) + }, + } + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunShowCommand() *cobra.Command { + var ( + runID string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show [eval-id]", + Short: "Show a single run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), run) + } + + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Run %s\n", run.ID) + fmt.Fprintf(out, " name : %s\n", run.Name) + fmt.Fprintf(out, " status : %s\n", run.Status) + if counts := summarizeCounts(run.ResultCounts); counts != "" { + fmt.Fprintf(out, " results : %s\n", counts) + } + if run.ReportURL != "" { + fmt.Fprintf(out, " report : %s\n", run.ReportURL) + } + return nil + }, + } + cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunCancelCommand() *cobra.Command { + var ( + runID string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "cancel [eval-id]", + Short: "Cancel an in-flight run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + target, err := ec.latestOrNamedRun(cmd, evalID, runID) + if err != nil { + return err + } + // Cancelling a run that already finished is a no-op worth naming, + // since the service reports success either way. + if terminalRunStates[target.Status] { + return fmt.Errorf("run %s already finished with status %q", + target.ID, target.Status) + } + + canceled, err := ec.evalClient.CancelOpenAIEvalRun(ctx, evalID, target.ID) + if err != nil { + return fmt.Errorf("cancelling run %s: %w", target.ID, err) + } + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + status := canceled.Status + if status == "" { + status = "cancelling" + } + fmt.Fprintf(cmd.OutOrStdout(), "Run %s is now %s\n", target.ID, status) + return nil + }, + } + cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func summarizeCounts(counts *eval_api.EvalRunResultCounts) string { + if counts == nil { + return "" + } + return fmt.Sprintf("%d passed, %d failed, %d errored", + counts.Passed, counts.Failed, counts.Errored) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go new file mode 100644 index 00000000000..01601ef023e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestSummarizeCounts(t *testing.T) { + require.Equal(t, "", summarizeCounts(nil)) + require.Equal(t, "3 passed, 1 failed, 0 errored", + summarizeCounts(&eval_api.EvalRunResultCounts{Total: 4, Passed: 3, Failed: 1})) +} + +// Cancelling a finished run is rejected locally. The service reports success +// either way, so without this the CLI would claim it cancelled a run that had +// already completed. +func TestTerminalRunStatesCoverServiceVocabulary(t *testing.T) { + for _, status := range []string{"completed", "failed", "canceled", "cancelled", "error"} { + require.True(t, terminalRunStates[status], "%q should be terminal", status) + } + for _, status := range []string{"in_progress", "queued", "running", ""} { + require.False(t, terminalRunStates[status], "%q should not be terminal", status) + } +} + +// The atomic run operations have to be reachable as subcommands; the spec +// requires start, list, show and cancel to exist alongside the composite. +func TestRunCommandExposesAtomicSubcommands(t *testing.T) { + cmd := newRunCommand() + + found := map[string]bool{} + for _, sub := range cmd.Commands() { + found[sub.Name()] = true + } + for _, name := range []string{"list", "show", "cancel"} { + require.True(t, found[name], "run should expose the %q subcommand", name) + } +} + +// --no-wait is documented in the spec, and cobra does not derive it from the +// --wait bool. +func TestRunCommandAcceptsNoWait(t *testing.T) { + cmd := newRunCommand() + require.NotNil(t, cmd.Flags().Lookup("no-wait"), "run should accept --no-wait") + require.NotNil(t, cmd.Flags().Lookup("wait"), "run should keep --wait") +} From 1ccb6d9cba252fd855e55d4b23d2181832aad641 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 06:51:29 +0300 Subject: [PATCH 020/114] feat(azure.ai.evaluations): add run start to complete the atomic surface The spec lists run start alongside list, show and cancel. The behaviour existed only as the composite `azd ai eval run`, so the atomic name in the spec did not resolve. Both forms are now built by one constructor, so their flags cannot drift apart, and a test asserts that. --- .../azure.ai.evaluations/internal/cmd/run.go | 19 ++++++++++++++--- .../internal/cmd/run_ops_test.go | 21 ++++++++++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 30fe1c42100..66cc843e103 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -28,7 +28,21 @@ var terminalRunStates = map[string]bool{ "error": true, } +// newRunCommand builds the composite `azd ai eval run` and attaches the atomic +// run operations, including `run start` which the spec lists as the atomic form +// of this same command. func newRunCommand() *cobra.Command { + cmd := buildRunCommand( + "run", "Run an evaluation, creating the eval group if it does not exist yet.") + addRunSubcommands(cmd) + cmd.AddCommand(buildRunCommand( + "start", "Start a run, creating the eval group if it does not exist yet.")) + return cmd +} + +// buildRunCommand is shared by `run` and `run start` so the two forms cannot +// drift apart. +func buildRunCommand(use, short string) *cobra.Command { var ( configPath string groupName string @@ -41,8 +55,8 @@ func newRunCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "run", - Short: "Run an evaluation, creating the eval group if it does not exist yet.", + Use: use, + Short: short, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() out := cmd.OutOrStdout() @@ -150,7 +164,6 @@ func newRunCommand() *cobra.Command { cmd.MarkFlagsMutuallyExclusive("wait", "no-wait") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - addRunSubcommands(cmd) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 01601ef023e..63ef3344e2e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -8,6 +8,7 @@ import ( "azureaieval/internal/pkg/eval_api" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) @@ -38,11 +39,29 @@ func TestRunCommandExposesAtomicSubcommands(t *testing.T) { for _, sub := range cmd.Commands() { found[sub.Name()] = true } - for _, name := range []string{"list", "show", "cancel"} { + for _, name := range []string{"start", "list", "show", "cancel"} { require.True(t, found[name], "run should expose the %q subcommand", name) } } +// `run start` is the atomic form of the composite and must accept the same +// flags, otherwise the two forms diverge. +func TestRunStartMirrorsCompositeFlags(t *testing.T) { + composite := newRunCommand() + + var start *cobra.Command + for _, sub := range composite.Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + + for _, flag := range []string{"eval-id", "eval-group", "name", "level", "max-samples", "wait", "no-wait"} { + require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) + } +} + // --no-wait is documented in the spec, and cobra does not derive it from the // --wait bool. func TestRunCommandAcceptsNoWait(t *testing.T) { From 45802246fbcfa60d3b038465afbb8cf7028088e0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:06:47 +0300 Subject: [PATCH 021/114] fix(azure.ai.evaluations): make the dataset and evaluator write paths work Exercising the atomic write commands against a live project found three failures. None were covered by tests, because none of these paths had been run end to end. dataset update always collided. It passed the --version flag straight to UploadNewVersion, which derives the next version from what it is given, so an omitted flag restarted at 1.0 and the service returned 409 TemporaryDataReferencesForExistingAsset. The flag help promised the opposite, that omitting it would take the next version. This is the same defect that was fixed in the deploy reconciler earlier, so the discovery is now centralised in DatasetClient.UploadNextVersion and both callers use it, rather than being fixed twice and available to be missed a third time. evaluator upload rejected every hand-authored rubric. The service needs a type discriminator on the definition, and without it fails the whole request with "The request field is required", which names a field that is present. Generated rubrics carry the type, so only the hand-authored path documented in the spec was affected. The type is now filled in when absent and left alone when set. evaluator show returned 404. It omitted the version segment from the path, but the service has no route for an unversioned evaluator, despite the doc comment claiming the latest would be fetched. The latest version is now resolved first, comparing numerically because versions are integers as strings and a lexical compare ranks "9" above "15" -- the service already publishes evaluators at version 15 and 17. Verified live: dataset create, show, update to 2.0, list and delete; evaluator upload, show resolving the latest, update to version 2, list and delete. Both suites leave nothing behind. --- .../internal/cmd/dataset.go | 4 +- .../internal/cmd/evaluator.go | 36 ++++++++- .../internal/cmd/evaluator_test.go | 81 +++++++++++++++++++ .../internal/cmd/reconciler.go | 25 +----- .../internal/pkg/dataset_api/operations.go | 25 ++++++ .../pkg/eval_api/evaluators_version_test.go | 47 +++++++++++ .../internal/pkg/eval_api/operations.go | 64 ++++++++++++++- 7 files changed, 253 insertions(+), 29 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 41685ae7e18..ebdc4439e83 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -76,7 +76,7 @@ func newDatasetCreateCommand(update bool) *cobra.Command { } defer ec.Close() - ds, err := ec.datasetClient.UploadNewVersion( + ds, err := ec.datasetClient.UploadNextVersion( ctx, name, version, localDir, ProjectEndpointAPIVersion, ) if err != nil { @@ -99,7 +99,7 @@ func newDatasetCreateCommand(update bool) *cobra.Command { cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") cmd.Flags().StringVar(&file, "file", "", "Path to a .jsonl file, or a directory containing one.") cmd.Flags().StringVar(&version, "version", "", - "Current version to increment from. Omit to let the server assign the next version.") + "Current version to increment from. Omit to increment from the latest registered version.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 9bbc7e41b1a..ca1d597eaa3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -98,14 +98,42 @@ func newEvaluatorUploadCommand(update bool) *cobra.Command { // normalizeRubricBody accepts either a bare definition ({type, dimensions}) or // a full evaluator document ({name, definition}) and returns the request body. +// rubricDefinitionType is the discriminator the service uses to deserialize a +// rubric definition. +const rubricDefinitionType = "rubric" + +// ensureDefinitionType adds the type discriminator when a definition omits it. +// +// Without it the service cannot tell which definition kind it is holding and +// rejects the whole request with "The request field is required", which points +// at the wrong field entirely. Generated rubrics carry the type; hand-authored +// ones written to the shape the spec documents — a bare list of weighted +// dimensions — do not. +func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { + var doc map[string]json.RawMessage + if err := json.Unmarshal(definition, &doc); err != nil { + return nil, fmt.Errorf("the definition is not a JSON object: %w", err) + } + if _, ok := doc["type"]; ok { + return definition, nil + } + doc["type"] = json.RawMessage(fmt.Sprintf("%q", rubricDefinitionType)) + return json.Marshal(doc) +} + func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { var probe map[string]json.RawMessage if err := json.Unmarshal(raw, &probe); err != nil { return nil, fmt.Errorf("not valid JSON: %w", err) } - if _, hasDefinition := probe["definition"]; hasDefinition { + if definition, hasDefinition := probe["definition"]; hasDefinition { // Already a full document; make sure the name matches the flag. + typed, err := ensureDefinitionType(definition) + if err != nil { + return nil, err + } + probe["definition"] = typed probe["name"] = json.RawMessage(fmt.Sprintf("%q", name)) out, err := json.Marshal(probe) if err != nil { @@ -119,9 +147,13 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { "expected a rubric definition with 'dimensions', or a document with 'definition'") } + typed, err := ensureDefinitionType(raw) + if err != nil { + return nil, err + } doc := map[string]any{ "name": name, - "definition": json.RawMessage(raw), + "definition": typed, } out, err := json.Marshal(doc) if err != nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go new file mode 100644 index 00000000000..e7ad1feab01 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// The service needs a type discriminator to deserialize a definition. Without +// it the whole request is rejected with "The request field is required", which +// names the wrong field, so a hand-authored rubric failed to upload. +func TestNormalizeRubricBodyAddsDefinitionType(t *testing.T) { + raw := []byte(`{"dimensions":[{"id":"accuracy","description":"Correct.","weight":5}]}`) + + body, err := normalizeRubricBody("support-quality", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "support-quality", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) + require.Len(t, doc.Definition.Dimensions, 1) + require.Equal(t, 5, doc.Definition.Dimensions[0].Weight) +} + +// A definition that already declares its type keeps it, so a generated rubric +// round-trips unchanged. +func TestNormalizeRubricBodyKeepsExistingType(t *testing.T) { + raw := []byte(`{"type":"custom_kind","dimensions":[{"id":"a","weight":1}]}`) + + body, err := normalizeRubricBody("x", raw) + require.NoError(t, err) + + var doc struct { + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "custom_kind", doc.Definition.Type) +} + +// A full document is normalized the same way, and the name follows the flag. +func TestNormalizeRubricBodyHandlesFullDocument(t *testing.T) { + raw := []byte(`{"name":"stale","definition":{"dimensions":[{"id":"a","weight":1}]}}`) + + body, err := normalizeRubricBody("actual-name", raw) + require.NoError(t, err) + + var doc struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(body, &doc)) + require.Equal(t, "actual-name", doc.Name) + require.Equal(t, "rubric", doc.Definition.Type) +} + +func TestNormalizeRubricBodyRejectsNonRubric(t *testing.T) { + _, err := normalizeRubricBody("x", []byte(`{"something":1}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "dimensions") + + _, err = normalizeRubricBody("x", []byte(`not json`)) + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 3b04655d768..965f67a4634 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -84,17 +84,10 @@ func (r *evalReconciler) EnsureDataset( dir = filepath.Dir(localPath) } - // UploadNewVersion derives the next version from the one passed in, so it - // has to be told what is already registered. Passing an empty version - // restarts at 1.0 and the service rejects the pending upload with - // TemporaryDataReferencesForExistingAsset once that version exists. - currentVersion := decl.Version - if currentVersion == "" { - currentVersion = r.latestDatasetVersion(ctx, decl.Name) - } - - ds, err := r.ec.datasetClient.UploadNewVersion( - ctx, decl.Name, currentVersion, dir, ProjectEndpointAPIVersion, + // UploadNextVersion discovers the currently registered version when none is + // declared, so the upload does not restart at 1.0 and collide. + ds, err := r.ec.datasetClient.UploadNextVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err @@ -107,16 +100,6 @@ func (r *evalReconciler) EnsureDataset( return ds.Version, true, nil } -// latestDatasetVersion reports the newest registered version, or empty when the -// dataset does not exist yet so the first upload starts at 1.0. -func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { - list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return "" - } - return dataset_api.LatestVersion(list.Value) -} - // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. Evaluator definitions come back inline, so this // compares content directly rather than relying on a cached digest. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index e29b6710f12..501f558408f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -80,6 +80,31 @@ func (c *DatasetClient) CreateDataset( return doRequestTyped[Dataset](c, ctx, http.MethodPost, pathDatasets, nil, request, apiVersion) } +// UploadNextVersion registers the next version of a dataset, discovering the +// current one from the service when currentVersion is empty. +// +// Prefer this over UploadNewVersion. That function derives the next version +// from whatever it is handed, so an empty value restarts at 1.0 and the +// service rejects the pending upload with a 409 +// TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers +// almost always mean "the version after whatever is registered", which is what +// this does. +func (c *DatasetClient) UploadNextVersion( + ctx context.Context, + name string, + currentVersion string, + localDir string, + apiVersion string, +) (*Dataset, error) { + if currentVersion == "" { + list, err := c.ListDatasetVersions(ctx, name, apiVersion) + if err == nil && list != nil && len(list.Value) > 0 { + currentVersion = LatestVersion(list.Value) + } + } + return c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) +} + // UploadNewVersion reads the first JSONL file from localDir, computes the next // version from currentVersion, and uploads it as a new dataset version using // the 3-step pending upload flow: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go new file mode 100644 index 00000000000..46f7524acbf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators_version_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Evaluator versions are integers rendered as strings, so a lexical compare +// ranks "9" above "15". The live service already has evaluators at version 15 +// and 17, so this is not hypothetical. +func TestPickLatestEvaluatorVersionIsNumeric(t *testing.T) { + cases := []struct { + name string + versions []string + want string + }{ + {"single", []string{"1"}, "1"}, + {"ascending", []string{"1", "2", "3"}, "3"}, + {"unordered", []string{"3", "1", "2"}, "3"}, + {"double digits beat single", []string{"9", "15"}, "15"}, + {"realistic", []string{"1", "9", "10", "17", "2"}, "17"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + entries := make([]EvaluatorSummary, 0, len(tc.versions)) + for _, v := range tc.versions { + entries = append(entries, EvaluatorSummary{Name: "e", Version: v}) + } + require.Equal(t, tc.want, pickLatestVersion(entries)) + }) + } +} + +// A non-numeric version is only used when nothing numeric exists, so one odd +// entry cannot mask the real latest. +func TestPickLatestEvaluatorVersionHandlesNonNumeric(t *testing.T) { + require.Equal(t, "2", pickLatestVersion([]EvaluatorSummary{ + {Version: "draft"}, {Version: "1"}, {Version: "2"}, + })) + require.Equal(t, "draft", pickLatestVersion([]EvaluatorSummary{{Version: "draft"}})) + require.Equal(t, "", pickLatestVersion(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 795e8301e0c..d04a3104636 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -127,20 +127,76 @@ func (c *EvalClient) CreateEvaluatorVersion( } // GetEvaluatorRaw gets an evaluator by name and version as raw JSON. -// If version is empty, the latest version is fetched. +// If version is empty, the latest version is resolved first. +// +// The service has no route for an unversioned evaluator: GET +// /evaluators/{name} returns 404 with no body, so the version cannot simply be +// left off the path. func (c *EvalClient) GetEvaluatorRaw( ctx context.Context, name string, version string, apiVersion string, ) (json.RawMessage, error) { - path := pathEvaluators + "/" + url.PathEscape(name) - if version != "" { - path += "/versions/" + url.PathEscape(version) + if version == "" { + latest, err := c.LatestEvaluatorVersion(ctx, name, apiVersion) + if err != nil { + return nil, err + } + version = latest } + path := pathEvaluators + "/" + url.PathEscape(name) + + "/versions/" + url.PathEscape(version) return c.doRequest(ctx, http.MethodGet, path, nil, nil, apiVersion) } +// LatestEvaluatorVersion returns the newest registered version of an evaluator. +func (c *EvalClient) LatestEvaluatorVersion( + ctx context.Context, + name string, + apiVersion string, +) (string, error) { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil { + return "", err + } + if list == nil || len(list.Value) == 0 { + return "", fmt.Errorf("evaluator %q has no versions", name) + } + latest := pickLatestVersion(list.Value) + if latest == "" { + return "", fmt.Errorf("evaluator %q has no usable version", name) + } + return latest, nil +} + +// pickLatestVersion selects the highest evaluator version. +// +// Versions are integers rendered as strings, so they are compared numerically: +// a lexical compare would rank "9" above "15", and the service already +// publishes evaluators at version 15 and 17. A non-numeric version is used +// only when nothing numeric is present. +func pickLatestVersion(entries []EvaluatorSummary) string { + best := "" + bestNum := -1 + for _, entry := range entries { + if entry.Version == "" { + continue + } + num, err := strconv.Atoi(entry.Version) + if err != nil { + if best == "" { + best = entry.Version + } + continue + } + if num > bestNum { + bestNum, best = num, entry.Version + } + } + return best +} + // CreateOpenAIEval creates an OpenAI eval definition. func (c *EvalClient) CreateOpenAIEval( ctx context.Context, From 331d3fe6c01a6fa00091737027a3a651d62b7036 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:13:00 +0300 Subject: [PATCH 022/114] fix(azure.ai.evaluations): support custom evaluators in a deployed config Deploying a config that declares a custom evaluator, rather than only built-in ones, failed in two ways. Every earlier test used built-ins, so neither showed up. The evaluator was republished on every deploy. The service enriches a definition when it stores it, so a rubric consisting of nothing but type and dimensions comes back carrying data_schema, init_parameters and metrics it was never given. Comparing whole documents therefore never matched. Only the keys the author actually wrote are compared now, structurally, so key order and formatting are not changes either. This is what the spec means by repeated azd up creating no redundant versions. The eval group was then rejected with a request for a model that had been set. Evaluators disagree on what the judge model is called: built-ins declare deployment_name, and a custom rubric declares model. The builder sent only deployment_name, so the custom evaluator saw its required parameter missing. The judge model is now bound under whichever name the evaluator declares. Verified live: first deploy publishes the evaluator and creates the group, two redeploys report it unchanged and publish nothing, editing the rubric publishes the next version and recreates the group, and a further redeploy is a no-op again. --- .../internal/cmd/build.go | 11 +++- .../internal/cmd/build_test.go | 29 +++++++++ .../internal/cmd/reconciler.go | 54 +++++++++++----- .../internal/cmd/reconciler_test.go | 63 +++++++++++++++++++ 4 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 7e880001488..b9c585c417c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -183,8 +183,15 @@ func planCriterion( return initSchema.Accepts(name) } - if evalModel != "" && accepts("deployment_name") { - plan.initParams["deployment_name"] = evalModel + // Evaluators disagree on what the judge model is called: built-ins declare + // deployment_name, custom rubrics declare model. Bind whichever the + // evaluator actually accepts rather than guessing one spelling. + if evalModel != "" { + for _, alias := range []string{"deployment_name", "model"} { + if accepts(alias) { + plan.initParams[alias] = evalModel + } + } } if ref.Threshold != nil && accepts("threshold") { plan.initParams["threshold"] = *ref.Threshold diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index f4b9fa2104e..89785096c8f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -256,6 +256,35 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") } +// Evaluators disagree on what the judge model is called. Built-ins declare +// deployment_name; a custom rubric declares model, and rejects the group with +// "requires model" if only deployment_name is sent. +func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + "my-rubric": schema("my-rubric", + nil, []string{"query", "response"}, + []string{"model"}, []string{"model"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{ + {Name: "builtin.similarity"}, + {Name: "my-rubric"}, + }, &project.Options{EvalModel: "gpt-4.1-nano"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + + builtin := req.TestingCriteria[0].InitializationParameters + require.Equal(t, "gpt-4.1-nano", builtin["deployment_name"]) + require.NotContains(t, builtin, "model") + + custom := req.TestingCriteria[1].InitializationParameters + require.Equal(t, "gpt-4.1-nano", custom["model"]) + require.NotContains(t, custom, "deployment_name") +} + // Without an agent target the sample bindings are unavailable, so every field // has to come from the dataset and the sample schema is not requested. func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 965f67a4634..6f010e4ecd8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/project" @@ -201,31 +202,56 @@ func (r *evalReconciler) EnsureEvalGroup( return created.ID, nil } -// sameDefinition compares only the definition body, ignoring server-assigned -// fields such as version and timestamps. +// sameDefinition reports whether the locally authored definition already +// matches what the service holds. +// +// Only the keys the candidate declares are compared. The service enriches a +// definition when it is created — a rubric of nothing but `type` and +// `dimensions` comes back carrying data_schema, init_parameters and metrics it +// was never given — so comparing whole documents never matches and every +// deploy publishes a redundant version. func sameDefinition(existing, candidate []byte) bool { - extract := func(raw []byte) string { + extract := func(raw []byte) map[string]json.RawMessage { var doc map[string]json.RawMessage if err := json.Unmarshal(raw, &doc); err != nil { - return "" + return nil } def, ok := doc["definition"] if !ok { - return "" + return nil } - var normalized any - if err := json.Unmarshal(def, &normalized); err != nil { - return "" + var fields map[string]json.RawMessage + if err := json.Unmarshal(def, &fields); err != nil { + return nil } - out, err := json.Marshal(normalized) - if err != nil { - return "" + return fields + } + + onService, authored := extract(existing), extract(candidate) + if onService == nil || authored == nil { + return false + } + + for key, want := range authored { + got, ok := onService[key] + if !ok || !equalJSON(got, want) { + return false } - return string(out) } + return true +} - a, b := extract(existing), extract(candidate) - return a != "" && a == b +// equalJSON compares two JSON values structurally, so key order and +// whitespace do not register as a change. +func equalJSON(a, b json.RawMessage) bool { + var left, right any + if err := json.Unmarshal(a, &left); err != nil { + return false + } + if err := json.Unmarshal(b, &right); err != nil { + return false + } + return reflect.DeepEqual(left, right) } func versionFromRaw(raw []byte, fallback string) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go new file mode 100644 index 00000000000..8086fd96c43 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_test.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// The service enriches a definition when it stores it: a rubric of nothing but +// type and dimensions comes back carrying data_schema, init_parameters and +// metrics. Comparing whole documents therefore never matched, and every deploy +// published a redundant version. +func TestSameDefinitionIgnoresServerAddedFields(t *testing.T) { + authored := []byte(`{ + "name": "r", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}] + } + }`) + + onService := []byte(`{ + "name": "r", + "version": "2", + "created_at": "2026-07-28T00:00:00Z", + "definition": { + "type": "rubric", + "dimensions": [{"id":"accuracy","description":"Correct.","weight":5}], + "data_schema": {"type":"object","properties":{"query":{"type":"string"}}}, + "init_parameters": {"required":["model"],"properties":{"model":{"type":"string"}}}, + "metrics": {"score":{"type":"number"}} + } + }`) + + require.True(t, sameDefinition(onService, authored), + "server-added fields must not count as a change") +} + +// A real edit still registers. +func TestSameDefinitionDetectsAuthoredChange(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":7}]}}`) + onService := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}],"metrics":{}}}`) + + require.False(t, sameDefinition(onService, authored)) +} + +// Key order and whitespace are not changes. +func TestSameDefinitionIsStructural(t *testing.T) { + authored := []byte(`{"definition":{"type":"rubric","dimensions":[{"id":"a","weight":5}]}}`) + onService := []byte("{\"definition\":{\n \"dimensions\": [ {\"weight\":5,\"id\":\"a\"} ],\n \"type\":\"rubric\"\n}}") + + require.True(t, sameDefinition(onService, authored)) +} + +func TestSameDefinitionRejectsMalformed(t *testing.T) { + good := []byte(`{"definition":{"type":"rubric"}}`) + require.False(t, sameDefinition([]byte(`not json`), good)) + require.False(t, sameDefinition(good, []byte(`not json`))) + require.False(t, sameDefinition([]byte(`{"no":"definition"}`), good)) +} From 4d1f8271454226af585b68880411248e9eb6c8e5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:23:22 +0300 Subject: [PATCH 023/114] fix(azure.ai.evaluations): honour --dataset when it names a registered dataset The flag is documented as taking a path or the name of a registered dataset, and means use this one instead of generating. It only suppressed generation when the value looked like a local path, so passing the name of an existing dataset still submitted a generation job and, since agent-seeded generation is currently broken server-side, failed the whole command. --evaluator already skipped unconditionally, so the two flags disagreed. Both the skip and the default-spec synthesis now key off whether the flag was supplied at all. This was the last thing standing between a generated config and the documented end-to-end flow. Verified live: init scaffolds a group referencing its own rubric, generate writes that rubric and merges the reference into the same file while preserving comments and ordering, azd up registers the dataset and evaluator and creates the group, and the run completes and scores against the generated rubric. --- .../internal/cmd/generate.go | 7 ++- .../internal/cmd/generate_defaults_test.go | 47 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index d3b2f4ce94f..2f3be4b9d38 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -82,7 +82,10 @@ func newGenerateCommand() *cobra.Command { } } - if datasetFlag != "" && looksLikeLocalDataset(datasetFlag) { + // --dataset means use this one, whether it names a local file or a + // dataset already registered on the project. Either way there is + // nothing to generate, which is how --evaluator behaves too. + if datasetFlag != "" { fmt.Fprintf(out, "Using the supplied dataset; skipping data generation.\n") } else if cfg.Generate.Dataset != nil { ref, err := ec.generateDataset(ctx, cfg, instruction, baseDir, out, noWait) @@ -166,7 +169,7 @@ func resolveGenerateConfig( LocalDir: "./" + project.DefaultEvaluatorsDir, } } - if cfg.Generate.Dataset == nil && !looksLikeLocalDataset(datasetFlag) { + if cfg.Generate.Dataset == nil && datasetFlag == "" { cfg.Generate.Dataset = &project.DatasetSpec{ Name: cfg.Agent.Name + "-golden", Strategy: project.StrategySynthetic, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go new file mode 100644 index 00000000000..97f141a2f53 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// --dataset is documented as taking a path or the name of a registered +// dataset, and means "use this one instead of generating". Only a local path +// used to suppress generation, so passing a registered name still submitted a +// generation job. +func TestGenerateScaffoldSkipsDatasetWhenSupplied(t *testing.T) { + cases := []struct { + name string + datasetFlag string + wantSpec bool + }{ + {"registered name", "prod-sample", false}, + {"relative path", "./data/golden.jsonl", false}, + {"bare filename", "golden.jsonl", false}, + {"not supplied", "", true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // A path that does not exist means flags alone drive the config. + cfg, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), + "my-agent", "gpt-4.1-nano", tc.datasetFlag, 0, 0) + require.NoError(t, err) + + if tc.wantSpec { + require.NotNil(t, cfg.Generate.Dataset, + "a dataset spec is needed when none was supplied") + } else { + require.Nil(t, cfg.Generate.Dataset, + "a supplied dataset must not produce a generation spec") + } + require.NotNil(t, cfg.Generate.Rubric, "the rubric spec is independent") + }) + } +} From 9e2797feb6555939446957962a02a33566337842 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:31:16 +0300 Subject: [PATCH 024/114] fix(azure.ai.evaluations): survive the eventually consistent version listing GET /datasets/{name}/versions returns nothing for a second or two after a version is created, even though the version itself reads back immediately. Measured: empty at 0s, populated at 2s. That undermines the version discovery added for dataset update, which reads the listing to decide what to increment from. An empty listing is ambiguous -- it means either a new dataset or a stale read -- so back-to-back create and update could still restart at 1.0 and take a 409. Rather than delaying every first upload to wait for the index, a conflict is now treated as the stale read it is: re-read the listing, which by then reflects reality, and retry once. The common path is unchanged. The live test asserted on the first listing response and was failing for the same reason. It now polls, and says why. Verified: create immediately followed by update produces 2.0 rather than a conflict, and the full live suite passes. --- .../internal/pkg/dataset_api/operations.go | 48 +++++++++++++++++-- .../tests/live/live_test.go | 12 +++-- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 501f558408f..2589981ed15 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "encoding/xml" + "errors" "fmt" "io" "log" @@ -89,6 +90,12 @@ func (c *DatasetClient) CreateDataset( // TemporaryDataReferencesForExistingAsset as soon as 1.0 exists. Callers // almost always mean "the version after whatever is registered", which is what // this does. +// +// The version listing is eventually consistent — it returns nothing for a +// second or two after a version is created — so an empty listing cannot be +// trusted to mean the dataset is new. A conflict is therefore treated as a +// stale read and retried once against a re-read listing, rather than adding a +// delay to every first upload. func (c *DatasetClient) UploadNextVersion( ctx context.Context, name string, @@ -97,12 +104,43 @@ func (c *DatasetClient) UploadNextVersion( apiVersion string, ) (*Dataset, error) { if currentVersion == "" { - list, err := c.ListDatasetVersions(ctx, name, apiVersion) - if err == nil && list != nil && len(list.Value) > 0 { - currentVersion = LatestVersion(list.Value) - } + currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) + } + + ds, err := c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + if err == nil || !isVersionConflict(err) { + return ds, err + } + + latest := c.latestRegisteredVersion(ctx, name, apiVersion) + if latest == "" || latest == currentVersion { + return nil, err + } + return c.UploadNewVersion(ctx, name, latest, localDir, apiVersion) +} + +// latestRegisteredVersion returns the newest registered version, or empty when +// the dataset is unknown or the listing has not caught up yet. +func (c *DatasetClient) latestRegisteredVersion( + ctx context.Context, + name string, + apiVersion string, +) string { + list, err := c.ListDatasetVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return LatestVersion(list.Value) +} + +// isVersionConflict reports whether the service refused the upload because the +// target version already exists. +func isVersionConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false } - return c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + return respErr.StatusCode == http.StatusConflict } // UploadNewVersion reads the first JSONL file from localDir, computes the next diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 9174ff294f4..9a4a209bffb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -168,9 +168,15 @@ func TestLiveDatasetLifecycle(t *testing.T) { require.NoError(t, err, "reading the dataset back") t.Logf("dataset uri: %q (empty means a credential call is required)", fetched.ResolvedBlobURI()) - versions, err := env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) - require.NoError(t, err, "listing dataset versions") - require.NotEmpty(t, versions.Value) + // The version listing is eventually consistent: it returns nothing for a + // second or two after a version is created, even though the version itself + // reads back fine. Poll rather than asserting on the first response. + var versions *dataset_api.DatasetList + require.Eventually(t, func() bool { + var err error + versions, err = env.datasetClient.ListDatasetVersions(ctx, name, projectAPIVersion) + return err == nil && versions != nil && len(versions.Value) > 0 + }, 30*time.Second, 2*time.Second, "the version listing never caught up") require.Equal(t, first.Version, dataset_api.LatestVersion(versions.Value)) // A second upload must advance the version, not conflict. From 2606b685f848f619c8043a6b73ddea6f9610d98e Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:40:20 +0300 Subject: [PATCH 025/114] build(azure.ai.evaluations): add the CI scripts the release pipeline requires The shared extension build template invokes ci-build.ps1 and ci-test.ps1 from the extension directory. Neither existed, so the release pipeline added alongside this extension would have failed on its first run. Both are modelled on the agents extension with two deliberate differences. ci-build.ps1 reads version.txt from the extension directory rather than its parent, where no such file exists, so the default works when the pipeline is not supplying -Version. It accepts -BuildRecordMode, which the template always passes, but builds nothing extra: this extension has no record/playback mode and no pipeline step consumes a record binary. ci-test.ps1 passes --junitfile explicitly. The pipeline publishes **/junitTestReport.xml from the extension directory, and the extension template does not set GOTESTSUM_JUNITFILE the way the CLI build does, so without this no test results would surface in the build. Verified locally with gotestsum installed: 97 tests across 7 suites reported. Also adds the README and CHANGELOG that 17 of the 21 extensions ship. The README documents the deployed shape, the command surface, the rubric weight constraint, and how to run the live tests. --- .../azure.ai.evaluations/CHANGELOG.md | 27 +++ .../extensions/azure.ai.evaluations/README.md | 158 ++++++++++++++++++ .../azure.ai.evaluations/ci-build.ps1 | 114 +++++++++++++ .../azure.ai.evaluations/ci-test.ps1 | 36 ++++ 4 files changed, 335 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md create mode 100644 cli/azd/extensions/azure.ai.evaluations/README.md create mode 100644 cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 create mode 100644 cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md new file mode 100644 index 00000000000..67a8c1d121e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -0,0 +1,27 @@ +# Release History + +## 1.0.0-beta.1 (Unreleased) + +### Features Added + +- Initial release of the Foundry evaluations extension, `azd ai eval`. +- `init` scaffolds `evals/eval_generate.yaml` and `evals/azure.yaml` next to an + agent, making no service calls. +- `generate` synthesizes a rubric and dataset from the agent's context, writes + them under `evals/`, and merges `source:` references into the deployment spec + while preserving comments, ordering and neighbouring entries. +- `run` creates the eval group when it does not exist, starts a run, and + summarizes the result. +- `azure.ai.eval` service-target provider deploys datasets, evaluators and eval + groups during `azd up`, reconciling them in dependency order. +- Change detection so a repeated `azd up` publishes no redundant versions: + datasets are fingerprinted locally, evaluator definitions are compared on the + keys the author wrote, and eval groups are recreated only when their own + declaration changes. +- Atomic commands for every operation: `dataset`, `evaluator`, `run` and + `results` subcommands, all supporting `-o json` and `--no-prompt`. +- Testing criteria are shaped from each evaluator's published contract, so + evaluators requiring inputs beyond the agent shape — `ground_truth`, + `context`, `instruction_id_list` — work by binding them to dataset columns. + A required column the dataset does not carry is reported before the request + is sent, naming the column. diff --git a/cli/azd/extensions/azure.ai.evaluations/README.md b/cli/azd/extensions/azure.ai.evaluations/README.md new file mode 100644 index 00000000000..2a5c80e62fe --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/README.md @@ -0,0 +1,158 @@ +# Azure Developer CLI (azd) Evaluations Extension + +Define Foundry evaluations alongside your agent in `azure.yaml`, deploy them +with `azd up`, and run them from the terminal. + +```bash +azd ai eval init # scaffold evals/ next to your agent +azd ai eval generate # synthesize a rubric and dataset from the agent +azd up # register datasets and evaluators, create the eval group +azd ai eval run # run the evaluation and summarize the results +``` + +## What gets deployed + +Eval resources are one service entry in `azure.yaml`, normally a `$ref` to a +file under `evals/`: + +```yaml +# azure.yaml +services: + ai-project: + host: azure.ai.project + evals: + host: azure.ai.eval + uses: [ai-project] + $ref: ./evals/azure.yaml +``` + +```yaml +# evals/azure.yaml +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + +evaluators: + - name: support-quality + source: ./evaluators/support-quality.json + +evalGroups: + - name: support-quality + dataset: support-golden + evaluators: + - builtin.task_adherence + - support-quality + target: + type: agent + name: support-agent + options: + eval_model: gpt-4.1-nano +``` + +`azd up` reconciles **datasets → evaluators → eval groups**, in that order, +because a group references the versions the first two resolve to. + +Relative paths inside a `$ref`'d file resolve against **that file's** +directory, so `./datasets/x.jsonl` above means `evals/datasets/x.jsonl`. + +### Repeated deploys do not create redundant versions + +Datasets are fingerprinted locally, because the dataset API exposes no content +hash and comparing against the service would mean downloading the blob on every +deploy. Evaluator definitions are compared against the service, but only on the +keys you authored — the service adds `data_schema`, `init_parameters` and +`metrics` of its own. + +Eval groups are immutable, so a change to a group's evaluators, target or +options creates a new group and a new id. The id is cached in the azd +environment so repeat runs stay comparable. + +## Commands + +| Group | Commands | +|---|---| +| `azd ai eval` | `init` · `generate` · `run` | +| `azd ai eval dataset` | `create` · `list` · `show` · `update` · `delete` | +| `azd ai eval evaluator` | `upload` · `list` · `show` · `update` · `delete` · `builtins` | +| `azd ai eval run` | `start` · `list` · `show` · `cancel` | +| `azd ai eval results` | `show` · `export` | + +`create` and `update` both publish a new immutable version; the server +auto-increments and nothing mutates in place. + +Every command supports `-o json` and `--no-prompt`, so the whole surface is +usable from CI. + +## Evaluators + +Built-ins need no declaration — reference them as `builtin.` and list +them with `azd ai eval evaluator builtins`. + +Evaluators do not share an input contract, so the CLI reads each one's +published contract and shapes the request to match. An evaluator needing an +input your dataset does not carry is reported before the request is sent, with +the column named, rather than as a service-side rejection. + +A custom rubric is a JSON list of weighted dimensions: + +```json +{ + "dimensions": [ + { "id": "accuracy", "description": "The answer is factually correct.", "weight": 5 }, + { "id": "tone", "description": "The answer is polite and professional.", "weight": 2 } + ] +} +``` + +`weight` is an **integer from 1 to 10**. Weights do not need to sum to +anything. + +## Choosing a project + +The project endpoint is resolved in this order: + +1. `--project-endpoint` +2. `FOUNDRY_PROJECT_ENDPOINT` in the active azd environment +3. the host environment variable of the same name + +## Local development + +### Prerequisites + +- Go (the version in `go.mod`; `GOTOOLCHAIN=auto` fetches it) +- [azd](https://aka.ms/azd) and the extension developer kit: + `azd ext install microsoft.azd.extensions` + +### Build, test, install + +```bash +azd x build # compile and install into the local azd +azd x pack # package the artifacts +azd x publish # register in the local extension source +azd ext install azure.ai.evaluations --source local +``` + +```bash +go test ./internal/... # unit tests +``` + +### Live integration tests + +These talk to a real Foundry project, so they are excluded from the default +build by the `live` tag and additionally gated on an environment variable: + +```bash +export AZURE_AI_EVAL_E2E_LIVE=1 +export FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +export AZURE_AI_EVAL_MODEL=gpt-4.1-nano # optional judge model +export AZURE_AI_EVAL_AGENT= # optional, enables the run phase + +go test -tags live ./internal/cmd/ ./tests/live/ +``` + +They clean up every resource they create. + +### Debug logging + +Request tracing is off by default. `--debug`, or `AZD_EXT_DEBUG=true`, writes +it to a dated log file rather than the terminal. diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 new file mode 100644 index 00000000000..403bc23b08d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-build.ps1 @@ -0,0 +1,114 @@ +param( + [string] $Version = (Get-Content "$PSScriptRoot/version.txt"), + [string] $SourceVersion = (git rev-parse HEAD), + [switch] $CodeCoverageEnabled, + # Accepted because the shared CI template always passes it. This extension + # has no record/playback mode, so there is no second binary to produce. + [switch] $BuildRecordMode, + [string] $MSYS2Shell, # path to msys2_shell.cmd + [string] $OutputFileName +) +$PSNativeCommandArgumentPassing = 'Legacy' + +# Remove any previously built binaries. +go clean + +if ($LASTEXITCODE) { + Write-Host "Error running go clean" + exit $LASTEXITCODE +} + +# Run `go help build` for detail on these flags. +$buildFlags = @( + # Remove file system paths from the binary. Recorded file names become a + # module path@version, or a plain import path for the standard library. + "-trimpath", + + # Position Independent Executable, for memory-corruption hardening across + # platforms. On Windows this enables ASLR and sets DYNAMICBASE and + # HIGH-ENTROPY-VA in the PE header. + "-buildmode=pie" +) + +if ($CodeCoverageEnabled) { + $buildFlags += "-cover" +} + +# cfi: Control Flow Integrity, cfg: Control Flow Guard, +# osusergo: use the pure Go user lookup. +$tagsFlag = "-tags=cfi,cfg,osusergo" + +# -s: omit the symbol table, -w: omit DWARF, -X: set a variable at link time. +$ldFlag = "-ldflags=-s -w " + + "-X 'azureaieval/internal/version.Version=$Version' " + + "-X 'azureaieval/internal/version.Commit=$SourceVersion' " + + "-X 'azureaieval/internal/version.BuildDate=$(Get-Date -Format o)' " + +if ($IsWindows) { + Write-Host "Building for Windows" +} +elseif ($IsLinux) { + Write-Host "Building for linux" + + # Disable cgo for the x64 Linux build. This also links statically, which + # widens compatibility with older Linux distributions. + if ($env:GOARCH -ne "arm64") { + $env:CGO_ENABLED = "0" + } +} +elseif ($IsMacOS) { + Write-Host "Building for macOS" +} + +$outputFlag = "-o=$OutputFileName" + +$buildFlags += @( + $tagsFlag, + $ldFlag, + $outputFlag +) + +function PrintFlags() { + param( + [string] $flags + ) + + # Format the flags so they can be pasted straight into pwsh. + $i = 0 + foreach ($buildFlag in $buildFlags) { + # Quote values so characters such as ',' survive a repaste. Not needed + # for the direct invocation below. + $argWithValue = $buildFlag.Split('=', 2) + if ($argWithValue.Length -eq 2 -and !$argWithValue[1].StartsWith("`"")) { + $buildFlag = "$($argWithValue[0])=`"$($argWithValue[1])`"" + } + + if ($i -eq $buildFlags.Length - 1) { + Write-Host " $buildFlag" + } + else { + Write-Host " $buildFlag ``" + } + $i++ + } +} + +$oldGOEXPERIMENT = $env:GOEXPERIMENT +# Opt into per-iteration loop variables, which is what most readers expect and +# what the Go team intends to make the default. +$env:GOEXPERIMENT = "loopvar" + +try { + Write-Host "Running: go build ``" + PrintFlags -flags $buildFlags + go build @buildFlags + if ($LASTEXITCODE) { + Write-Host "Error running go build" + exit $LASTEXITCODE + } + + Write-Host "go build succeeded" +} +finally { + $env:GOEXPERIMENT = $oldGOEXPERIMENT +} diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 new file mode 100644 index 00000000000..314095e75b3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -0,0 +1,36 @@ +# Runs the unit tests and writes a JUnit report. +# +# The pipeline publishes **/junitTestReport.xml from the extension directory, +# so the report has to be written under that name for results to show up in the +# build. gotestsum produces it; the go test fallback does not, so the fallback +# only runs when gotestsum is unavailable. +# +# The live integration tests are excluded: they carry the `live` build tag, so +# an untagged run does not compile them, and they additionally require +# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. + +$gopath = go env GOPATH +$gotestsumBinary = "gotestsum" +if ($IsWindows) { + $gotestsumBinary += ".exe" +} +$gotestsum = Join-Path $gopath "bin" $gotestsumBinary + +Write-Host "Running unit tests..." + +if (Test-Path $gotestsum) { + & $gotestsum --format testname --junitfile junitTestReport.xml -- ./... -count=1 +} else { + Write-Host "gotestsum not found; falling back to go test (no JUnit report)." -ForegroundColor Yellow + go test ./... -v -count=1 +} + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "Tests failed with exit code: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + +Write-Host "" +Write-Host "All tests passed!" -ForegroundColor Green +exit 0 From 288927f6b178b37cce3bcde74ce7dfb79877fe0f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 07:49:41 +0300 Subject: [PATCH 026/114] fix(azure.ai.evaluations): make the manifest and the binary agree The manifest declared two capabilities the extension did not back. metadata was declared but the command was never registered, so azd could not discover the command tree: azd ai eval metadata failed with unknown command while the same call against a peer extension returned its full tree. azd uses this for discovery, so the declaration was actively misleading. The command is now registered and reports nine commands. lifecycle-events was declared but no event handlers exist. The SDK only starts its event manager when handlers are registered, so the capability was an unused permission rather than a broken promise. It is removed; the listen command is still invoked because the service-target-provider capability triggers it, which a deploy after the change confirms. Adds tests over the manifest so neither can drift again: every declared capability must be backed by the command that implements it, the declared provider name must match the host the code registers, and version.txt must agree with the manifest version, which until now was only a comment asking for it. --- .../azure.ai.evaluations/extension.yaml | 1 - .../internal/cmd/manifest_test.go | 101 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 7 ++ 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index 2b8e8bbb58f..c73a8ad1db4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -13,7 +13,6 @@ dependencies: language: go capabilities: - custom-commands - - lifecycle-events - service-target-provider - metadata providers: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go new file mode 100644 index 00000000000..faf104043b9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/manifest_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// extensionManifest is the subset of extension.yaml this test asserts on. +type extensionManifest struct { + ID string `yaml:"id"` + Version string `yaml:"version"` + Capabilities []string `yaml:"capabilities"` + Providers []struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + } `yaml:"providers"` +} + +func loadManifest(t *testing.T) extensionManifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "extension.yaml")) + require.NoError(t, err, "reading extension.yaml") + + var manifest extensionManifest + require.NoError(t, yaml.Unmarshal(raw, &manifest)) + return manifest +} + +// A declared capability azd cannot reach is worse than an undeclared one: azd +// invokes `metadata` to discover the command tree, and it was declared without +// the command being registered, so discovery failed with "unknown command". +func TestDeclaredCapabilitiesAreImplemented(t *testing.T) { + manifest := loadManifest(t) + root := NewRootCommand() + + hasCommand := func(name string) bool { + for _, sub := range root.Commands() { + if sub.Name() == name { + return true + } + } + return false + } + + for _, capability := range manifest.Capabilities { + switch capability { + case "metadata": + require.True(t, hasCommand("metadata"), + "the metadata capability requires a metadata command") + case "service-target-provider": + require.True(t, hasCommand("listen"), + "a service-target provider is registered through the listen command") + require.NotEmpty(t, manifest.Providers, + "the manifest must name the provider it registers") + case "custom-commands": + require.NotEmpty(t, root.Commands()) + case "lifecycle-events": + // The SDK only starts the event manager when handlers are + // registered, so declaring this without any is an unused + // permission. Nothing here registers handlers today. + t.Fatalf("lifecycle-events is declared but no event handlers are registered") + } + } +} + +// The provider name in the manifest is what azd matches a service's `host` +// against, so a mismatch silently means the provider is never invoked. +func TestManifestProviderMatchesHostConstant(t *testing.T) { + manifest := loadManifest(t) + require.NotEmpty(t, manifest.Providers) + + names := make([]string, 0, len(manifest.Providers)) + for _, p := range manifest.Providers { + names = append(names, p.Name) + } + require.Contains(t, names, "azure.ai.eval", + "the manifest must declare the host the provider registers for") +} + +// extension.yaml carries a note asking that version.txt be kept in sync. The +// build stamps the binary from version.txt while the registry reads +// extension.yaml, so a drift ships a binary that misreports its own version. +func TestManifestVersionMatchesVersionFile(t *testing.T) { + manifest := loadManifest(t) + + raw, err := os.ReadFile(filepath.Join("..", "..", "version.txt")) + require.NoError(t, err, "reading version.txt") + + require.Equal(t, + strings.TrimSpace(string(raw)), + strings.TrimSpace(manifest.Version), + "version.txt and extension.yaml must agree") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index a763a54b924..40724c5a0a7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -54,5 +54,12 @@ func NewRootCommand() *cobra.Command { newListenCommand(), ) + // The manifest declares the `metadata` capability, which azd uses to + // discover this extension's command tree. Without the command registered, + // that discovery fails with "unknown command". + rootCmd.AddCommand(azdext.NewMetadataCommand("1.0", "azure.ai.evaluations", func() *cobra.Command { + return rootCmd + })) + return rootCmd } From 909ce14430ae5f19d613156b2cfa0b0b0d8d7aa2 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:03:18 +0300 Subject: [PATCH 027/114] feat(azure.ai.evaluations): detect artifact drift and read the instruction from a file Auditing every flag and API sequence the spec documents against the running extension turned up two gaps. The spec describes a drift check that was never implemented. It matters because of how change detection works: when local content is unchanged, the version recorded at the last deploy is reused, so a version published outside the repo would be silently ignored and the eval group pinned to older data. A deploy now fails when the service holds a newer version than the recorded one, naming both versions. An explicit version: on the declaration skips the check, because that is the author stating which version they want. This was added after testing the remedy the error message suggests and finding it did not work -- the message now describes something that does. --gen-instruction-file was documented but absent. A useful generation instruction is usually longer than fits on a command line, and putting it in a file makes it reviewable with the rest of the config. Verified live: publishing a version out-of-band fails the next deploy, and pinning that version lets it through. --- .../internal/cmd/generate.go | 52 +++++++++++++++---- .../internal/cmd/instruction_test.go | 50 ++++++++++++++++++ .../internal/cmd/reconciler.go | 48 ++++++++++++++++- .../internal/pkg/dataset_api/list.go | 12 +++++ .../internal/pkg/dataset_api/version_test.go | 41 +++++++++++++++ 5 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 2f3be4b9d38..0a574635044 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -30,17 +30,18 @@ var generatePollBudget = eval_api.PollerOptions{ func newGenerateCommand() *cobra.Command { var ( - configPath string - deployPath string - target string - instruction string - datasetFlag string - evaluators []string - maxSamples int - traceDays int - evalModel string - noWait bool - endpointFlg string + configPath string + deployPath string + target string + instruction string + instructionFile string + datasetFlag string + evaluators []string + maxSamples int + traceDays int + evalModel string + noWait bool + endpointFlg string ) cmd := &cobra.Command{ @@ -50,6 +51,11 @@ func newGenerateCommand() *cobra.Command { ctx := cmd.Context() out := cmd.OutOrStdout() + instruction, err := resolveInstruction(instruction, instructionFile) + if err != nil { + return err + } + cfg, err := resolveGenerateConfig( configPath, target, evalModel, datasetFlag, maxSamples, traceDays, ) @@ -126,6 +132,9 @@ func newGenerateCommand() *cobra.Command { cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") cmd.Flags().StringVar(&instruction, "gen-instruction", "", "What the agent does and what to test.") + cmd.Flags().StringVar(&instructionFile, "gen-instruction-file", "", + "Read the generation instruction from this file. Mutually exclusive with --gen-instruction.") + cmd.MarkFlagsMutuallyExclusive("gen-instruction", "gen-instruction-file") cmd.Flags().StringVar(&datasetFlag, "dataset", "", "Use this dataset instead of generating one.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, @@ -140,6 +149,27 @@ func newGenerateCommand() *cobra.Command { return cmd } +// resolveInstruction returns the generation instruction, reading it from a +// file when one is named. +// +// A useful instruction describes the agent and what to test, which is often +// more than fits comfortably on a command line, so it can live in a file that +// is reviewable alongside the rest of the config. +func resolveInstruction(inline, path string) (string, error) { + if path == "" { + return inline, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("reading --gen-instruction-file %q: %w", path, err) + } + text := strings.TrimSpace(string(raw)) + if text == "" { + return "", fmt.Errorf("--gen-instruction-file %q is empty", path) + } + return text, nil +} + // resolveGenerateConfig loads the spec when present, then layers flags on top. // A missing file is not an error: flags alone are sufficient. func resolveGenerateConfig( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go new file mode 100644 index 00000000000..4063a6ab1b4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A useful generation instruction is often longer than fits on a command +// line, so it can come from a file instead. +func TestResolveInstructionReadsFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(path, + []byte(" A customer support agent answering billing questions.\n\n"), 0o600)) + + got, err := resolveInstruction("", path) + require.NoError(t, err) + require.Equal(t, "A customer support agent answering billing questions.", got, + "surrounding whitespace should be trimmed") +} + +func TestResolveInstructionPrefersInlineWhenNoFile(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + + got, err = resolveInstruction("", "") + require.NoError(t, err) + require.Empty(t, got) +} + +// An unreadable or empty file is reported rather than silently generating from +// no instruction at all. +func TestResolveInstructionRejectsUnusableFile(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(t.TempDir(), "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "gen-instruction-file") + + empty := filepath.Join(t.TempDir(), "empty.md") + require.NoError(t, os.WriteFile(empty, []byte(" \n"), 0o600)) + _, err = resolveInstruction("", empty) + require.Error(t, err) + require.Contains(t, err.Error(), "empty") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 6f010e4ecd8..a00ce08711b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -73,8 +73,17 @@ func (r *evalReconciler) EnsureDataset( key := project.FingerprintKey("dataset", decl.Name) if prior := r.ec.getEnvValue(ctx, key); prior == digest { - // Unchanged since the last deploy; reuse the recorded version. + // Unchanged since the last deploy; reuse the recorded version, but only + // after confirming nobody published a newer one outside the repo. An + // explicit `version:` is the author saying which version they want, so + // it settles the question and the check does not apply. if version := r.ec.getEnvValue(ctx, versionKey("dataset", decl.Name)); version != "" { + if decl.Version != "" { + return decl.Version, false, nil + } + if err := r.checkDatasetDrift(ctx, decl.Name, version); err != nil { + return "", false, err + } return version, false, nil } } @@ -101,6 +110,43 @@ func (r *evalReconciler) EnsureDataset( return ds.Version, true, nil } +// checkDatasetDrift fails when the service holds a newer version than the one +// recorded at the last deploy. +// +// Local content being unchanged is not enough to reuse the recorded version: +// someone may have published a newer one outside the repo, and silently +// pinning the eval group to the older version would quietly evaluate against +// stale data. Publishing is not destructive — versions are immutable — so the +// remedy is to sync, not to overwrite. +func (r *evalReconciler) checkDatasetDrift( + ctx context.Context, + name, recorded string, +) error { + latest := r.latestDatasetVersion(ctx, name) + if latest == "" || latest == recorded { + return nil + } + if !dataset_api.VersionGreater(latest, recorded) { + return nil + } + return fmt.Errorf( + "dataset %q is at version %s on the project but %s was recorded at the last deploy; "+ + "someone published a version outside this repo. "+ + "Pin it with `version: %s` on the dataset, or pull the newer content locally, "+ + "then deploy again", + name, latest, recorded, latest) +} + +// latestDatasetVersion reports the newest registered version, or empty when the +// dataset is unknown or the listing has not caught up. +func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) string { + list, err := r.ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return "" + } + return dataset_api.LatestVersion(list.Value) +} + // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. Evaluator definitions come back inline, so this // compares content directly rather than relying on a cached digest. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go index f81f372131d..19b548c3e05 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/list.go @@ -74,6 +74,18 @@ func VersionOrder(version string) float64 { return -1 } +// VersionGreater reports whether a is a strictly newer version than b. +// +// Both must be orderable; when either is not, the answer is false so an +// unparseable version never triggers a drift failure on its own. +func VersionGreater(a, b string) bool { + orderA, orderB := VersionOrder(a), VersionOrder(b) + if orderA < 0 || orderB < 0 { + return false + } + return orderA > orderB +} + // LatestVersion returns the highest version in the list, falling back to the // last entry when none of the versions can be ordered. func LatestVersion(datasets []Dataset) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go new file mode 100644 index 00000000000..de822def3cd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// Drift detection compares the version on the service with the one recorded at +// the last deploy, so the ordering has to be numeric rather than lexical: +// "10.0" is newer than "9.0" even though it sorts earlier as a string. +func TestVersionGreater(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.0", "1.0", true}, + {"1.0", "2.0", false}, + {"1.0", "1.0", false}, + {"10.0", "9.0", true}, + {"9.0", "10.0", false}, + {"v3", "v2", true}, + } + + for _, tc := range cases { + require.Equal(t, tc.want, VersionGreater(tc.a, tc.b), + "VersionGreater(%q, %q)", tc.a, tc.b) + } +} + +// An unorderable version must never trigger a drift failure on its own: the +// deploy would be blocked with no way for the author to reason about it. +func TestVersionGreaterIgnoresUnorderable(t *testing.T) { + require.False(t, VersionGreater("draft", "1.0")) + require.False(t, VersionGreater("1.0", "draft")) + require.False(t, VersionGreater("", "1.0")) + require.False(t, VersionGreater("1.0", "")) +} From 800834e8fdf584b36f44107e236435bfe4cb6743 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:12:21 +0300 Subject: [PATCH 028/114] fix(azure.ai.evaluations): make --eval-id work and catch unregistered edits M1 exits on all the spec examples running end to end, so I ran them verbatim. Two did not. --eval-id could never work. It is meant to run an existing group ignoring the config, and appears in both the CI/CD example and the recovery advice, but a run needs a target and a dataset and an eval group carries neither: the group holds only its testing criteria, and the dataset travels on the run. Every --eval-id invocation failed asking for a target. The pairing survives in the group's previous run, so re-running a group now repeats what it last ran, and a group that has never run says so and points at the config-based path. The failure-and-recovery example promised an error that did not exist. A run sends a local dataset inline, so unregistered local edits were evaluated silently and the results could not be traced to any dataset version. That now fails with the message the spec documents, once a deploy has recorded a fingerprint to compare against. Before that there is nothing to have drifted from, and running is how a group first comes into existence. Verified live: the CI/CD example returns JSON with a run id, and with unregistered edits the config-based run fails while --eval-id succeeds, which is exactly the recovery the spec describes. --- .../azure.ai.evaluations/internal/cmd/run.go | 88 ++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 66cc843e103..db25185ecde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -82,6 +82,10 @@ func buildRunCommand(use, short string) *cobra.Command { return err } + if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { + return err + } + evalID, err = ec.resolveEvalGroupID( ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) if err != nil { @@ -89,7 +93,14 @@ func buildRunCommand(use, short string) *cobra.Command { } } - dataSource, err := buildRunDataSource(group, configPath, maxSamples) + // With --eval-id there is no config to read, so the pairing of + // target and dataset comes from the group's previous run. + var dataSource *eval_api.EvalRunDataSource + if group == nil { + dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) + } else { + dataSource, err = buildRunDataSource(group, configPath, maxSamples) + } if err != nil { return err } @@ -223,6 +234,81 @@ func (ec *evalContext) resolveEvalGroupID( return created.ID, nil } +// checkDatasetRegistered fails when the group's local dataset has edits that +// were never deployed. +// +// A run sends a local dataset inline, so without this the run would evaluate +// content that no registered version corresponds to: the results are attributed +// to the eval group but cannot be traced back to a dataset version, which +// makes them impossible to reproduce or compare. +// +// The check only applies once a deploy has recorded a fingerprint. Before that +// there is nothing to have drifted from, and running is how a group first comes +// into existence. +func (ec *evalContext) checkDatasetRegistered( + ctx context.Context, + cfg *project.EvalConfig, + group *project.EvalGroup, + configPath string, +) error { + localPath := localDatasetPath(configPath, group) + if localPath == "" { + return nil + } + + decl, ok := cfg.Dataset(group.Dataset) + if !ok { + return nil + } + + recorded := ec.getEnvValue(ctx, project.FingerprintKey("dataset", decl.Name)) + if recorded == "" { + return nil + } + + digest, err := project.Fingerprint(localPath) + if err != nil { + // Reading the file is the run's problem to report, not this check's. + return nil + } + if digest == recorded { + return nil + } + + return fmt.Errorf( + "dataset %q has local edits that are not registered.\n"+ + " Run `azd up` to register them, or `--eval-id ` to run against "+ + "an existing eval group", + decl.Name) +} + +// reuseDataSourceFromLastRun rebuilds a run's data source from the group's most +// recent run. +// +// `--eval-id` deliberately ignores the config, but a run still needs a target +// and a dataset, and an eval group carries neither: the group holds only its +// testing criteria, and the dataset travels on the run. The previous run is the +// only place that pairing survives, so re-running a group means repeating what +// it last ran. +func (ec *evalContext) reuseDataSourceFromLastRun( + ctx context.Context, + evalID string, +) (*eval_api.EvalRunDataSource, error) { + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) + if err != nil { + return nil, fmt.Errorf("reading previous runs of eval group %s: %w", evalID, err) + } + if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { + return nil, fmt.Errorf( + "eval group %s has no previous run to repeat, so there is no target or dataset "+ + "to reuse.\n"+ + " Run it from the config once with `azd ai eval run`, or pass a config that "+ + "declares the group", + evalID) + } + return list.Data[0].DataSource, nil +} + // buildRunDataSource binds the dataset to the run. The eval group carries no // dataset today, so it is supplied here. func buildRunDataSource( From d09111e06c5a6828352076b3063082734468c2e6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:19:22 +0300 Subject: [PATCH 029/114] fix(azure.ai.evaluations): rebase a supplied dataset path onto the deploy spec init --dataset ./tests/golden.jsonl wrote that path into evals/azure.yaml unchanged, but source: is resolved relative to the file it appears in, so the deploy looked for evals/tests/golden.jsonl and failed on a file the user had just pointed at. This is the spec's bring-your-own-data example exactly as written, so that example could never have worked. The path is now rebased onto the output directory, with forward slashes so the config reads the same on every platform, and absolute paths left alone. With this the documented examples all run end to end, which is what M1 exits on: bring-your-own-data through init, azd up and run; results show --failed-only -O writing its file; and the CI/CD sequence of dataset create, run start -o json and results export --format csv. --- .../azure.ai.evaluations/internal/cmd/init.go | 42 ++++++++++++++++++- .../internal/cmd/init_test.go | 15 ++++--- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 047257edd2e..2905b7e42ca 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -64,7 +64,7 @@ func newInitCommand() *cobra.Command { return err } - depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel) + depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel, outDir) if err := writeYAML(depPath, depCfg); err != nil { return err } @@ -130,10 +130,45 @@ func buildGenerateScaffold(target, rubricName, evalModel string) *project.Genera } } +// relativeToConfig rewrites a path given relative to the working directory so +// it resolves from the directory holding the deploy spec. +// +// `--dataset ./tests/golden.jsonl` means "relative to where I am", but the +// deploy spec's `source:` is resolved relative to that file, so writing the +// path through unchanged sends the deploy looking inside evals/. An absolute +// path is left alone, and forward slashes are kept so the config reads the same +// on every platform. +func relativeToConfig(path, outDir string) string { + if filepath.IsAbs(path) { + return path + } + + absPath, err := filepath.Abs(path) + if err != nil { + return path + } + absOut, err := filepath.Abs(outDir) + if err != nil { + return path + } + + rel, err := filepath.Rel(absOut, absPath) + if err != nil { + return path + } + + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, ".") { + rel = "./" + rel + } + return rel +} + func buildDeployScaffold( target, rubricName, dataset string, evaluators []string, evalModel string, + outDir string, ) *project.EvalConfig { cfg := &project.EvalConfig{} @@ -141,7 +176,10 @@ func buildDeployScaffold( datasetSource := "" if dataset != "" { if looksLikeLocalDataset(dataset) { - datasetSource = dataset + // --dataset is given relative to where the user is standing, but + // source: is resolved relative to the deploy spec, so the path has + // to be rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(dataset, outDir) datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) } else { // A bare name references an already-registered dataset. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 55820051efb..f0db8167284 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -19,7 +19,7 @@ func TestScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() depPath := filepath.Join(dir, "azure.yaml") - cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano") + cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano", project.DefaultEvalDir) require.NoError(t, writeYAML(depPath, cfg)) loaded, err := project.LoadEvalConfig(depPath) @@ -54,7 +54,7 @@ func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { cfg := buildDeployScaffold( "support-agent", "unused", "", - []string{"builtin.task_adherence", "my-custom"}, "", + []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir, ) require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") @@ -74,20 +74,23 @@ func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { // A bare name means an already-registered dataset; a path means a local file. func TestScaffold_DatasetReferenceForms(t *testing.T) { t.Run("local path becomes a source", func(t *testing.T) { - cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "") - require.Equal(t, "./tests/golden.jsonl", cfg.Datasets[0].Source) + // --dataset is relative to the working directory, but source: is + // resolved relative to the deploy spec, so it has to be rebased. + cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "", "evals") + require.Equal(t, "../tests/golden.jsonl", cfg.Datasets[0].Source, + "a dataset outside the eval dir must be reached with ..") require.Equal(t, "golden", cfg.Datasets[0].Name) }) t.Run("bare name references a registered dataset", func(t *testing.T) { - cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "") + cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "", project.DefaultEvalDir) require.Equal(t, "prod-sample", cfg.Datasets[0].Name) require.Empty(t, cfg.Datasets[0].Source, "a registered dataset must not get a local source") }) t.Run("no dataset flag scaffolds a local path", func(t *testing.T) { - cfg := buildDeployScaffold("support-agent", "r", "", nil, "") + cfg := buildDeployScaffold("support-agent", "r", "", nil, "", project.DefaultEvalDir) require.Contains(t, cfg.Datasets[0].Source, "support-agent-golden.jsonl") }) } From ce2ce3a8c2b6dd209cc7e7af8ebe861af2d05a23 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:29:24 +0300 Subject: [PATCH 030/114] feat(azure.ai.evaluations): say when a trace setting is being ignored agent.context.traces accepts source, window and sample, but the generation API takes a day count and nothing else, so source and sample were parsed and dropped without a word. An author who set sample: 500 believed they had narrowed the trace selection when nothing had changed. Both fields are documented in the spec, so this was reachable by following it. They are now reported as having no effect, naming each one, with the verb agreeing so one field reads "has" and two read "have". The warning goes to stdout rather than stderr because azd does not surface an extension's stderr -- written to stderr it was invisible in a real run even though the unit test passed -- and is suppressed under -o json so the output stays parseable. --- .../internal/cmd/generate.go | 44 ++++++++++++++++++ .../internal/cmd/trace_warning_test.go | 46 +++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 0a574635044..cdc411a2228 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -65,6 +65,11 @@ func newGenerateCommand() *cobra.Command { if err := cfg.Validate(); err != nil { return err } + // Written to stdout because azd does not surface an extension's + // stderr, and guarded so `-o json` stays parseable. + if !isJSON(cmd) { + warnIgnoredTraceFields(cfg, out) + } ec, err := newEvalContext(ctx, endpointFlg) if err != nil { @@ -149,6 +154,45 @@ func newGenerateCommand() *cobra.Command { return cmd } +// warnIgnoredTraceFields reports trace settings that are accepted but have no +// effect yet. +// +// The generation API takes a day window and nothing else, so `source` and +// `sample` are parsed and dropped. Silently discarding them is worse than not +// accepting them: the author believes they narrowed the trace selection when +// nothing changed. +func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { + traces := cfg.Agent.Context.Traces + if traces == nil { + return + } + + var ignored []string + if traces.Source != "" { + ignored = append(ignored, "source") + } + if traces.Sample > 0 { + ignored = append(ignored, "sample") + } + if len(ignored) == 0 { + return + } + + fields := make([]string, 0, len(ignored)) + for _, name := range ignored { + fields = append(fields, "agent.context.traces."+name) + } + + verb := "has" + if len(fields) > 1 { + verb = "have" + } + fmt.Fprintf(out, + "warning: %s %s no effect yet; trace seeding uses only `window`. "+ + "Trace selection lands with the trace scenarios.\n", + strings.Join(fields, " and "), verb) +} + // resolveInstruction returns the generation instruction, reading it from a // file when one is named. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go new file mode 100644 index 00000000000..6c952569aa4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +func warnFor(t *testing.T, traces *project.TraceSpec) string { + t.Helper() + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Traces = traces + + var buf bytes.Buffer + warnIgnoredTraceFields(cfg, &buf) + return buf.String() +} + +// source and sample are accepted by the config model but the generation API +// takes only a day window, so they are dropped. Dropping them silently lets an +// author believe they narrowed the trace selection when nothing changed. +func TestWarnsAboutTraceFieldsWithNoEffect(t *testing.T) { + out := warnFor(t, &project.TraceSpec{Source: "production", Window: "30d", Sample: 500}) + require.Contains(t, out, "agent.context.traces.source") + require.Contains(t, out, "agent.context.traces.sample") + require.Contains(t, out, "window") + require.Contains(t, out, "have no effect", "two fields take a plural verb") + + out = warnFor(t, &project.TraceSpec{Source: "production", Window: "30d"}) + require.Contains(t, out, "agent.context.traces.source") + require.NotContains(t, out, "sample") + require.Contains(t, out, "has no effect", "one field takes a singular verb") +} + +// The field that does work draws no warning, and neither does an absent block. +func TestNoWarningWhenOnlyWindowIsSet(t *testing.T) { + require.Empty(t, warnFor(t, &project.TraceSpec{Window: "30d"})) + require.Empty(t, warnFor(t, nil)) + require.Empty(t, warnFor(t, &project.TraceSpec{})) +} From b10f7060b6465eafa8c8a8448bb88dfbdbe37403 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:37:06 +0300 Subject: [PATCH 031/114] fix(azure.ai.evaluations): honour the config fields that were being dropped Swept every field the two config files declare against what the code reads. Three were parsed and ignored. options.max_samples did nothing, so a group that caps its sample count in config sent the whole dataset and only a flag on every invocation honoured the cap. It now resolves the same way the evaluation level already did: flag first, then the group's options. generate.dataset.strategy accepted from-traces, passed validation, and then generated synthetic rows anyway. Answering a different question than the one asked is worse than refusing, so it is now rejected with a pointer to agent.context.traces.window, which is how traces actually seed generation. An eval group's description was dropped because the create request has no field for it. It now rides in metadata, which the service stores and returns. It stays out of the group fingerprint deliberately: recreating an immutable group over a reworded description would cost the group id and break comparison against earlier runs, so an edit lands the next time the group is recreated for a reason that matters. --- .../internal/cmd/build.go | 5 ++ .../internal/cmd/description_test.go | 46 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 19 +++++++- .../internal/project/generate_config.go | 15 ++++-- .../internal/project/generate_config_test.go | 40 ++++++++++++++++ .../internal/project/service_target_eval.go | 8 +++- 6 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index b9c585c417c..803eed1c4e0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -237,6 +237,11 @@ func buildEvalGroupRequest( metadata["azd_agent"] = group.Target.Name } metadata["azd_eval_group"] = group.Name + // The create request has no description field, so the group's own + // description rides in metadata rather than being dropped. + if group.Description != "" { + metadata["azd_description"] = group.Description + } evalModel := "" level := "" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go new file mode 100644 index 00000000000..ea819af4ef9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// The create request has no description field, so a documented description +// would otherwise be parsed and dropped. +func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, + &project.Options{EvalModel: "m"}) + group.Description = "Quality gate for the support agent" + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Equal(t, "Quality gate for the support agent", req.Metadata["azd_description"]) +} + +// An absent description adds no metadata key rather than an empty one. +func TestBuildOmitsEmptyDescription(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.similarity": schema("builtin.similarity", + nil, []string{"query", "response"}, + []string{"deployment_name"}, []string{"deployment_name"}, "turn"), + } + group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, + &project.Options{EvalModel: "m"}) + + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.NotContains(t, req.Metadata, "azd_description") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index db25185ecde..7448c93329a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -99,7 +99,8 @@ func buildRunCommand(use, short string) *cobra.Command { if group == nil { dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) } else { - dataSource, err = buildRunDataSource(group, configPath, maxSamples) + dataSource, err = buildRunDataSource( + group, configPath, resolveMaxSamples(maxSamples, group)) } if err != nil { return err @@ -435,6 +436,22 @@ func resolveLevel(flag string, group *project.EvalGroup) string { return "" } +// resolveMaxSamples prefers the flag, then the group's options, matching how +// the evaluation level resolves. +// +// Without this, options.max_samples parsed and did nothing: a group that caps +// its sample count in config would send the whole dataset, and only a flag on +// every invocation would honour the cap. +func resolveMaxSamples(flag int, group *project.EvalGroup) int { + if flag > 0 { + return flag + } + if group != nil && group.Options != nil && group.Options.MaxSamples > 0 { + return group.Options.MaxSamples + } + return 0 +} + // pollRun waits for the run to reach a terminal state, reporting status changes. func (ec *evalContext) pollRun( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 4e1d647387d..22ec21ab3e0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -114,11 +114,20 @@ func (c *GenerateConfig) Validate() error { return fmt.Errorf("generate.dataset.name is required") } switch d.Strategy { - case "", StrategySynthetic, StrategyFromTraces: + case "", StrategySynthetic: + case StrategyFromTraces: + // Accepting this and generating synthetic rows anyway would hand back + // data that looks nothing like what was asked for. The generation API + // takes one dataset strategy today; traces seed generation through + // the agent's context instead. + return fmt.Errorf( + "generate.dataset.strategy %q is not supported yet; "+ + "use %q, and set agent.context.traces.window to seed generation from traces", + StrategyFromTraces, StrategySynthetic) default: return fmt.Errorf( - "generate.dataset.strategy %q is invalid; expected %q or %q", - d.Strategy, StrategySynthetic, StrategyFromTraces) + "generate.dataset.strategy %q is invalid; expected %q", + d.Strategy, StrategySynthetic) } if d.SampleSize != 0 && (d.SampleSize < MinSampleSize || d.SampleSize > MaxSampleSize) { return fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go new file mode 100644 index 00000000000..227674b0fc4 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func generateCfgWithStrategy(strategy string) *GenerateConfig { + cfg := &GenerateConfig{} + cfg.Agent.Name = "my-agent" + cfg.Generate.Rubric = &RubricSpec{Name: "r"} + cfg.Generate.Dataset = &DatasetSpec{Name: "d", Strategy: strategy} + return cfg +} + +// from-traces used to pass validation and then generate synthetic rows anyway, +// handing back data that looked nothing like what was asked for. Rejecting it +// is better than answering the wrong question. +func TestValidateRejectsUnsupportedDatasetStrategy(t *testing.T) { + err := generateCfgWithStrategy(StrategyFromTraces).Validate() + require.Error(t, err) + require.Contains(t, err.Error(), "not supported yet") + require.Contains(t, err.Error(), "agent.context.traces.window", + "the error should point at the way traces are actually used") +} + +func TestValidateAcceptsSupportedDatasetStrategies(t *testing.T) { + require.NoError(t, generateCfgWithStrategy("").Validate()) + require.NoError(t, generateCfgWithStrategy(StrategySynthetic).Validate()) +} + +func TestValidateRejectsUnknownDatasetStrategy(t *testing.T) { + err := generateCfgWithStrategy("made-up").Validate() + require.Error(t, err) + require.Contains(t, err.Error(), "invalid") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index b6dd930ced4..39e6f8ed9fa 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -321,8 +321,12 @@ func Fingerprint(path string) (string, error) { // evaluators are untouched. Without this a retargeted group keeps running // against the old definition. func FingerprintGroup(group EvalGroup) (string, error) { - // The id is server-assigned and the description is cosmetic; neither - // changes what the group evaluates. + // The id is server-assigned. The description is carried in the group's + // metadata, so editing it does change the request, but recreating an + // immutable group over a reworded description would cost the group id and + // break comparison against earlier runs. It is documentation, not + // evaluation semantics, so an edit lands the next time the group is + // recreated for a reason that matters. group.ID = "" group.Description = "" From 2a101c05fe2d1ca878a6bc25acbce981b87665fd Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:41:37 +0300 Subject: [PATCH 032/114] build(azure.ai.evaluations): ignore the artifacts the tooling produces ci-test.ps1 writes junitTestReport.xml for the pipeline to publish, and --debug writes a dated log, both into the extension directory. Neither was ignored, so a developer running the tests or debugging locally saw them offered for commit. Scoped to this extension rather than the root ignore file: the CLI's own test report has the same gap, but widening a repo-wide rule is not this change's to make. --- cli/azd/extensions/azure.ai.evaluations/.gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/.gitignore diff --git a/cli/azd/extensions/azure.ai.evaluations/.gitignore b/cli/azd/extensions/azure.ai.evaluations/.gitignore new file mode 100644 index 00000000000..0d5b6d76489 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/.gitignore @@ -0,0 +1,5 @@ +# Test report written by ci-test.ps1 for the pipeline to publish. +junitTestReport.xml + +# Debug log written when --debug or AZD_EXT_DEBUG is set. +azd-ai-eval-*.log From 7f1b067c2e1e82fc742b4da7f3c833f1e63e6ec5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:49:26 +0300 Subject: [PATCH 033/114] fix(azure.ai.evaluations): make the persistence warnings useful Two warnings about failing to persist a value into the azd environment went to stderr, which azd does not surface, so nobody ever saw them. The same mistake was already fixed for the trace warning; these were the remaining cases. They also fired for a situation that is not a problem. The atomic commands are meant to work standalone against the data plane, so running outside a project is ordinary, and reporting that there is nowhere to persist would be noise on every standalone invocation. An absent environment is now told apart from a write that genuinely failed, and only the latter is reported. What remains is written to stdout so it is visible, and suppressed under -o json so the output stays parseable. Verified both ways: creating a dataset outside any project prints only the result, and a deploy inside one still records its state. --- .../internal/cmd/context.go | 10 +++++- .../internal/cmd/dataset.go | 10 ++++-- .../internal/cmd/envwarn_test.go | 32 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 8 ++++- 4 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index ef7562644fa..7ec1da4237a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "errors" "fmt" "os" "strings" @@ -100,13 +101,20 @@ func lookupEndpointFromAzd(ctx context.Context, azdClient *azdext.AzdClient) (en return val.Value, envResp.Environment.Name } +// errNoAzdEnvironment reports that there is no azd environment to persist into. +// +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary rather than a problem worth reporting. +// A write that fails for any other reason still is. +var errNoAzdEnvironment = errors.New("no active azd environment") + // setEnvValue persists a value into the active azd environment. azd itself // writes none of these keys — the extension owns them. func (ec *evalContext) setEnvValue(ctx context.Context, key, value string) error { if ec.envName == "" { envResp, err := ec.azdClient.Environment().GetCurrent(ctx, &azdext.EmptyRequest{}) if err != nil || envResp == nil || envResp.Environment == nil { - return fmt.Errorf("no active azd environment to write %s into", key) + return fmt.Errorf("%w to write %s into", errNoAzdEnvironment, key) } ec.envName = envResp.Environment.Name } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index ebdc4439e83..5c18f453297 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "fmt" "os" "path/filepath" @@ -84,8 +85,13 @@ func newDatasetCreateCommand(update bool) *cobra.Command { } if err := ec.setEnvValue(ctx, envKeyDatasetVersion, ds.Version); err != nil { - // Persisting is a convenience; do not fail the command over it. - fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + // Persisting is a convenience, so this never fails the command. + // It goes to stdout because azd does not surface an extension's + // stderr, and is skipped outside a project, where having nowhere + // to persist is expected rather than notable. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprintf(cmd.OutOrStdout(), "warning: %v\n", err) + } } if isJSON(cmd) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go new file mode 100644 index 00000000000..8652dc6b958 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envwarn_test.go @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// The atomic commands are meant to work standalone against the data plane, so +// running outside a project is ordinary. Warning about nowhere to persist would +// be noise on every standalone invocation. +func TestNoAzdEnvironmentIsRecognisable(t *testing.T) { + err := fmt.Errorf("%w to write %s into", errNoAzdEnvironment, "EVAL_RUN_ID") + + require.ErrorIs(t, err, errNoAzdEnvironment, + "callers rely on telling this apart from a failed write") + require.Contains(t, err.Error(), "EVAL_RUN_ID", + "the key is still named when the message is shown") +} + +// A write that fails for any other reason stays reportable. +func TestOtherEnvironmentFailuresStayReportable(t *testing.T) { + err := fmt.Errorf("writing %s to the azd environment: %w", "EVAL_RUN_ID", errors.New("rpc failed")) + + require.NotErrorIs(t, err, errNoAzdEnvironment) + require.Contains(t, err.Error(), "rpc failed") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 7448c93329a..a90e4994745 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -7,6 +7,7 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -129,7 +130,12 @@ func buildRunCommand(use, short string) *cobra.Command { } if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), "warning: %v\n", err) + // Persisting the run id is a convenience for later commands. + // Reported on stdout because azd does not surface an + // extension's stderr, and skipped outside a project. + if !errors.Is(err, errNoAzdEnvironment) && !isJSON(cmd) { + fmt.Fprintf(out, "warning: %v\n", err) + } } if !wait { From 5853089bfec3e04ed774f16731b558078a8e8275 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 08:59:15 +0300 Subject: [PATCH 034/114] feat(azure.ai.evaluations): compare runs against a baseline M1 is complete and the comparison half of M2 was the next unblocked piece, so this adds `azd ai eval results compare`. Confirmed first that the APIs M2 needs are actually reachable in a live project rather than trusting the milestone note: schedules, evaluations/schedules and insights all answer. The spec described the request as {evalId, baselineRunId, treatmentRunIds}. Probing the live API found three things missing. displayName is required and is rejected before anything else is looked at. The comparison fields go inside a request wrapper. That wrapper is polymorphic and needs a type discriminator of EvaluationComparison, which the service itself named once asked. The synchronous POST /insights/sync returns a 500 for this shape, so the asynchronous form and a poll is the only usable path. With no flags it compares the two most recent completed runs, which is what asking whether a change helped usually means. --baseline and --treatment name runs explicitly, and --treatment repeats. Output carries the statistical method, per-metric baseline and treatment averages, a signed delta, the p-value, and the service's own effect classification. That last one matters: the probe returned TooFewSamples, and dropping it would make an inconclusive comparison read as a finding. --- .../internal/cmd/compare.go | 206 ++++++++++++++++++ .../internal/cmd/compare_test.go | 83 +++++++ .../internal/cmd/results.go | 2 +- .../internal/pkg/eval_api/insights.go | 121 ++++++++++ 4 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go new file mode 100644 index 00000000000..500f135b3fd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -0,0 +1,206 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// comparePollBudget bounds the wait for a comparison. The probe returned in +// about a second, so this is generous headroom rather than an expected wait. +const ( + comparePollInterval = 3 * time.Second + comparePollAttempts = 100 +) + +func newResultsCompareCommand() *cobra.Command { + var ( + baseline string + treatments []string + displayName string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "compare [eval-id]", + Short: "Compare runs of an eval group against a baseline.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args) + if err != nil { + return err + } + + baseline, treatments, err = ec.resolveComparisonRuns(ctx, evalID, baseline, treatments) + if err != nil { + return err + } + + if displayName == "" { + displayName = fmt.Sprintf("compare-%s", time.Now().UTC().Format("20060102-150405")) + } + + insight, err := ec.evalClient.CreateInsight(ctx, &eval_api.CreateInsightRequest{ + DisplayName: displayName, + Request: &eval_api.InsightRequest{ + Type: eval_api.InsightTypeEvaluationComparison, + EvalID: evalID, + BaselineRunID: baseline, + TreatmentRunIDs: treatments, + }, + }, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("starting the comparison: %w", err) + } + + if !isJSON(cmd) { + fmt.Fprintf(out, "Comparing %d run(s) against %s...\n", len(treatments), baseline) + } + + completed, err := ec.pollInsight(ctx, insight.ID) + if err != nil { + return err + } + if isJSON(cmd) { + return emitJSON(out, completed) + } + return renderComparison(out, completed) + }, + } + + cmd.Flags().StringVar(&baseline, "baseline", "", + "Run to compare against. Defaults to the second most recent completed run.") + cmd.Flags().StringArrayVar(&treatments, "treatment", nil, + "Run to measure, repeatable. Defaults to the most recent completed run.") + cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// resolveComparisonRuns fills in whichever runs were not named. +// +// Comparing the two most recent completed runs is what "did my change help?" +// means most of the time, so neither flag is required. +func (ec *evalContext) resolveComparisonRuns( + ctx context.Context, + evalID, baseline string, + treatments []string, +) (string, []string, error) { + if baseline != "" && len(treatments) > 0 { + return baseline, treatments, nil + } + + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) + if err != nil { + return "", nil, fmt.Errorf("listing runs of eval group %s: %w", evalID, err) + } + + completed := make([]string, 0, 2) + if list == nil { + return "", nil, fmt.Errorf("eval group %s has no runs", evalID) + } + for _, run := range list.Data { + if run.Status == "completed" { + completed = append(completed, run.ID) + } + } + + if len(treatments) == 0 { + if len(completed) == 0 { + return "", nil, fmt.Errorf( + "eval group %s has no completed runs to compare", evalID) + } + treatments = []string{completed[0]} + } + if baseline == "" { + if len(completed) < 2 { + return "", nil, fmt.Errorf( + "eval group %s has only one completed run, so there is nothing to compare it "+ + "against; run it again, or name a baseline with --baseline", + evalID) + } + baseline = completed[1] + } + return baseline, treatments, nil +} + +// pollInsight waits for the comparison to reach a terminal state. +func (ec *evalContext) pollInsight(ctx context.Context, insightID string) (*eval_api.Insight, error) { + for attempt := 0; attempt < comparePollAttempts; attempt++ { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(comparePollInterval): + } + + insight, err := ec.evalClient.GetInsight(ctx, insightID, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading comparison %s: %w", insightID, err) + } + if !insight.Terminal() { + continue + } + if !insight.Succeeded() { + return nil, fmt.Errorf("comparison %s finished with state %q", insightID, insight.State) + } + return insight, nil + } + return nil, fmt.Errorf("comparison %s did not finish in time", insightID) +} + +// renderComparison prints one row per criterion per treatment run. +func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_api.Insight) error { + if insight.Result == nil || len(insight.Result.Comparisons) == 0 { + fmt.Fprintln(w, "The comparison produced no metrics.") + return nil + } + + if insight.Result.Method != "" { + fmt.Fprintf(w, "Method: %s\n\n", insight.Result.Method) + } + + rows := [][]string{} + for _, c := range insight.Result.Comparisons { + baseAvg := "-" + if c.BaselineRunSummary != nil { + baseAvg = fmt.Sprintf("%.3f", c.BaselineRunSummary.Average) + } + for _, item := range c.CompareItems { + treatAvg := "-" + runID := "-" + if item.TreatmentRunSummary != nil { + treatAvg = fmt.Sprintf("%.3f", item.TreatmentRunSummary.Average) + runID = item.TreatmentRunSummary.RunID + } + rows = append(rows, []string{ + c.Metric, + runID, + baseAvg, + treatAvg, + fmt.Sprintf("%+.3f", item.DeltaEstimate), + fmt.Sprintf("%.3f", item.PValue), + item.TreatmentEffect, + }) + } + } + + return emitTable(w, + []string{"METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT"}, + rows) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go new file mode 100644 index 00000000000..5bbf0ad2370 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestInsightTerminalStates(t *testing.T) { + for _, state := range []string{"", "NotStarted", "Running", "InProgress", "Queued"} { + require.False(t, (&eval_api.Insight{State: state}).Terminal(), "%q is not terminal", state) + } + for _, state := range []string{"Succeeded", "Failed", "Cancelled"} { + require.True(t, (&eval_api.Insight{State: state}).Terminal(), "%q is terminal", state) + } + require.True(t, (&eval_api.Insight{State: "Succeeded"}).Succeeded()) + require.False(t, (&eval_api.Insight{State: "Failed"}).Succeeded()) +} + +// The rendered table is how a reader decides whether a change helped, so the +// delta carries its sign and the effect classification is not dropped. +func TestRenderComparisonShowsSignedDeltaAndEffect(t *testing.T) { + insight := &eval_api.Insight{ + State: "Succeeded", + Result: &eval_api.InsightResult{ + Method: "PairedTTest", + Comparisons: []eval_api.MetricComparison{{ + Metric: "task_adherence", + BaselineRunSummary: &eval_api.RunSummary{RunID: "base", Average: 0.75}, + CompareItems: []eval_api.CompareItem{{ + TreatmentRunSummary: &eval_api.RunSummary{RunID: "treat", Average: 0.5}, + DeltaEstimate: -0.25, + PValue: 0.39, + TreatmentEffect: "TooFewSamples", + }}, + }}, + }, + } + + var buf bytes.Buffer + require.NoError(t, renderComparison(&buf, insight)) + out := buf.String() + + require.Contains(t, out, "PairedTTest", "the statistical method qualifies the result") + require.Contains(t, out, "task_adherence") + require.Contains(t, out, "-0.250", "a regression must read as negative") + require.Contains(t, out, "0.390") + require.Contains(t, out, "TooFewSamples", + "an inconclusive result must not look like a finding") +} + +// A positive delta reads as an improvement rather than an unsigned number. +func TestRenderComparisonSignsImprovements(t *testing.T) { + insight := &eval_api.Insight{ + State: "Succeeded", + Result: &eval_api.InsightResult{ + Comparisons: []eval_api.MetricComparison{{ + Metric: "similarity", + BaselineRunSummary: &eval_api.RunSummary{Average: 0.5}, + CompareItems: []eval_api.CompareItem{{ + TreatmentRunSummary: &eval_api.RunSummary{RunID: "t", Average: 0.8}, + DeltaEstimate: 0.3, + }}, + }}, + }, + } + + var buf bytes.Buffer + require.NoError(t, renderComparison(&buf, insight)) + require.Contains(t, buf.String(), "+0.300") +} + +func TestRenderComparisonHandlesEmptyResult(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, renderComparison(&buf, &eval_api.Insight{State: "Succeeded"})) + require.Contains(t, buf.String(), "no metrics") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 3eb79a1bc22..985a4b7b461 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -21,7 +21,7 @@ func newResultsCommand() *cobra.Command { Use: "results", Short: "Inspect evaluation results.", } - cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) + cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand(), newResultsCompareCommand()) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go new file mode 100644 index 00000000000..8296fd21992 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +// InsightTypeEvaluationComparison compares evaluation runs. The service also +// defines EvaluationRunClusterInsight and AgentClusterInsight, which this +// extension does not use. +const InsightTypeEvaluationComparison = "EvaluationComparison" + +const pathInsights = "/insights" + +// InsightRequest is the polymorphic body the service dispatches on. `type` is +// the discriminator; without it the request is rejected because the underlying +// contract is an interface. +type InsightRequest struct { + Type string `json:"type"` + EvalID string `json:"evalId"` + BaselineRunID string `json:"baselineRunId"` + TreatmentRunIDs []string `json:"treatmentRunIds"` +} + +// CreateInsightRequest wraps the request. DisplayName is required; the service +// rejects a body without it before it looks at anything else. +type CreateInsightRequest struct { + DisplayName string `json:"displayName"` + Request *InsightRequest `json:"request"` +} + +// RunSummary is one run's aggregate for a single metric. +type RunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average float64 `json:"average"` + StandardDeviation float64 `json:"standardDeviation"` +} + +// CompareItem is one treatment run measured against the baseline. +type CompareItem struct { + TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` + DeltaEstimate float64 `json:"deltaEstimate"` + PValue float64 `json:"pValue"` + // TreatmentEffect classifies the result, e.g. TooFewSamples when the + // sample count cannot support a conclusion. + TreatmentEffect string `json:"treatmentEffect,omitempty"` +} + +// MetricComparison is the baseline and treatments for one testing criterion. +type MetricComparison struct { + TestingCriteria string `json:"testingCriteria"` + Metric string `json:"metric"` + Evaluator string `json:"evaluator"` + BaselineRunSummary *RunSummary `json:"baselineRunSummary,omitempty"` + CompareItems []CompareItem `json:"compareItems,omitempty"` +} + +// InsightResult carries the comparison once the insight succeeds. +type InsightResult struct { + Comparisons []MetricComparison `json:"comparisons,omitempty"` + // Method names the statistical test, e.g. PairedTTest. + Method string `json:"method,omitempty"` + Type string `json:"type,omitempty"` + Error any `json:"error,omitempty"` +} + +// Insight is the long-running operation the comparison runs as. +type Insight struct { + ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + State string `json:"state,omitempty"` + Request *InsightRequest `json:"request,omitempty"` + Result *InsightResult `json:"result,omitempty"` +} + +// Succeeded reports whether the insight finished with a result. +func (i *Insight) Succeeded() bool { + return i != nil && i.State == "Succeeded" +} + +// Terminal reports whether the insight has stopped changing. +func (i *Insight) Terminal() bool { + if i == nil { + return false + } + switch i.State { + case "", "NotStarted", "Running", "InProgress", "Queued": + return false + default: + return true + } +} + +// CreateInsight starts a comparison. +// +// The synchronous variant, POST /insights/sync, returns a 500 for this request +// shape, so the asynchronous form is the only usable one and the caller polls. +func (c *EvalClient) CreateInsight( + ctx context.Context, + request *CreateInsightRequest, + apiVersion string, +) (*Insight, error) { + return doRequestTyped[Insight]( + c, ctx, http.MethodPost, pathInsights, nil, request, apiVersion) +} + +// GetInsight reads a comparison's current state. +func (c *EvalClient) GetInsight( + ctx context.Context, + insightID string, + apiVersion string, +) (*Insight, error) { + path := fmt.Sprintf("%s/%s", pathInsights, url.PathEscape(insightID)) + return doRequestTyped[Insight](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} From 63a7a33266e357d41a8517297f828eeb4642459f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 09:24:06 +0300 Subject: [PATCH 035/114] fix(azure.ai.evaluations): finish the generate step of the documented flow M1 exits on the spec examples running end to end, and the first one could not: its generate step failed. Two separate causes, the second only reachable once the first was out of the way. Agent-seeded data generation fails server-side for every agent, which no user can act on. The same request carrying only the prompt succeeds and produces a usable dataset, so a failure now retries without the agent and says that it did. The rubric still uses the agent's context; only the dataset falls back. That exposed the real defect underneath. A dataset's URI points at either the blob or the container holding it, and nothing in the payload distinguishes them: isSingleFile is true either way. Uploads end in the file name, generated datasets end in the container, and downloading a container returns 409. The download path also sent no SAS token, so it could not have authenticated even against the right URL. It had never run successfully, because generation always failed before reaching it. Downloads now fetch a credential, use the blob directly when the URI names one, and otherwise list the container and take the JSONL inside. This retires an assumption recorded earlier as holding: that a dataset blob URI can be downloaded directly. It is true only for uploads. --- .../internal/cmd/generate.go | 41 +++++++++- .../internal/pkg/dataset_api/download_test.go | 47 ++++++++++++ .../internal/pkg/dataset_api/operations.go | 75 +++++++++++++++++++ .../internal/pkg/eval_api/generation.go | 26 +++++++ 4 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index cdc411a2228..eff801fc119 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -342,6 +342,27 @@ func (ec *evalContext) generateDataset( completed, err := ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, ec.evalClient.GetDataGenerationJob) + if err != nil && isAgentSeededGenerationFailure(err) { + // Agent-seeded generation fails server-side for every agent, while the + // same request carrying only the prompt succeeds. Failing the whole + // command would block the documented flow on a defect the user cannot + // do anything about, so retry without the agent and say so. + promptOnly := eval_api.WithoutAgentSource(sources) + if eval_api.HasPromptSource(promptOnly) { + fmt.Fprintf(out, + " warning: generating from agent %q failed in the service; "+ + "retrying from the instruction alone.\n", cfg.Agent.Name) + + req = eval_api.NewDataGenerationJobRequest( + spec.Name, model, spec.SampleSize, promptOnly) + job, err = ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) + if err != nil { + return nil, fmt.Errorf("submitting the data generation job: %w", err) + } + completed, err = ec.pollGeneration(ctx, job.ID, DataGenerationAPIVersion, + ec.evalClient.GetDataGenerationJob) + } + } if err != nil { return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, cfg.Agent.Name)) } @@ -351,11 +372,14 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("the data generation job returned no dataset reference") } - ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) - if err != nil { + // Confirm the version exists before reading it, so a missing dataset is + // reported as such rather than as a download failure. + if _, err := ec.datasetClient.GetDataset( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { return nil, fmt.Errorf("reading the generated dataset %q: %w", name, err) } - content, err := ec.datasetClient.DownloadDataset(ctx, ds.ResolvedBlobURI()) + content, err := ec.datasetClient.DownloadDatasetContent(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) } @@ -372,6 +396,17 @@ func (ec *evalContext) generateDataset( return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil } +// isAgentSeededGenerationFailure recognises the service-side failure that hits +// every agent, so it can be retried without the agent rather than surfaced. +func isAgentSeededGenerationFailure(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "DataGenerationJobSystemError") || + strings.Contains(text, "Something went wrong during data generation") +} + // explainDataGenerationFailure adds context to the service's opaque system // error. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go new file mode 100644 index 00000000000..9ac6f21e92b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/download_test.go @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// A dataset's URI points at either the blob or the container holding it, +// depending on how it was created, and nothing in the payload says which: +// isSingleFile is true either way. Uploaded datasets end in the file name; +// generated ones end in the container. Downloading a container returns 409. +func TestLooksLikeBlobURI(t *testing.T) { + uploaded := "https://acct.blob.core.windows.net:443/container-guid/azd-smoke-golden.jsonl" + generated := "https://acct.blob.core.windows.net/asayedahme-420d0b21-956c-513b-bb18-f60bfbf5e724" + + require.True(t, looksLikeBlobURI(uploaded), "an uploaded dataset names its file") + require.False(t, looksLikeBlobURI(generated), "a generated dataset names its container") +} + +// A SAS token on the URI must not change the answer. +func TestLooksLikeBlobURIIgnoresQuery(t *testing.T) { + require.True(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c/data.jsonl?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI( + "https://acct.blob.core.windows.net/c?sv=2021&sig=abc")) + require.False(t, looksLikeBlobURI("https://acct.blob.core.windows.net/c/")) +} + +// An evaluation dataset is JSONL, so that is preferred when a container holds +// more than one file. +func TestPickDatasetBlobPrefersJSONL(t *testing.T) { + require.Equal(t, "data.jsonl", + pickDatasetBlob([]string{"_meta.json", "data.jsonl", "readme.txt"})) + require.Equal(t, "data.JSONL", + pickDatasetBlob([]string{"data.JSONL"}), "the extension match is case-insensitive") +} + +// With nothing recognisable, any real file beats returning nothing. +func TestPickDatasetBlobFallsBackToAnyFile(t *testing.T) { + require.Equal(t, "data.csv", pickDatasetBlob([]string{"data.csv"})) + require.Empty(t, pickDatasetBlob([]string{"folder/"})) + require.Empty(t, pickDatasetBlob(nil)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 2589981ed15..be1e3e03e5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -14,6 +14,7 @@ import ( "log" "net/http" "net/url" + "path" "strings" "azureaieval/internal/version" @@ -276,6 +277,80 @@ func (c *DatasetClient) GetDatasetCredential( return doRequestTyped[DatasetCredential](c, ctx, http.MethodPost, path, nil, nil, apiVersion) } +// DownloadDatasetContent fetches a dataset version's content, whether its URI +// names a blob or a container. +// +// The two differ by origin, not by any field: a dataset uploaded through +// startPendingUpload gets a URI ending in the file name, while one produced by +// a generation job gets the container it was written into, with isSingleFile +// true either way. Downloading the container directly returns a 409, so the +// blob inside has to be found first. +// +// A credential is always fetched, because the URI on the dataset carries no +// SAS token and an unauthenticated read fails. +func (c *DatasetClient) DownloadDatasetContent( + ctx context.Context, + name string, + version string, + apiVersion string, +) ([]byte, error) { + cred, err := c.GetDatasetCredential(ctx, name, version, apiVersion) + if err != nil { + return nil, fmt.Errorf("reading download credentials for %q: %w", name, err) + } + + sasURI := cred.ResolvedDownloadURI() + if sasURI == "" { + return nil, fmt.Errorf("no download URI returned for dataset %q", name) + } + + // A URI whose last path segment carries a file extension is the blob + // itself; anything else is the container holding it. + if looksLikeBlobURI(sasURI) { + data, err := c.DownloadDataset(ctx, sasURI) + if err == nil { + return data, nil + } + log.Printf("[dataset_api] direct download failed (%v); treating the URI as a container", err) + } + + names, err := c.ListContainerBlobs(ctx, sasURI) + if err != nil { + return nil, fmt.Errorf("listing the content of dataset %q: %w", name, err) + } + blobName := pickDatasetBlob(names) + if blobName == "" { + return nil, fmt.Errorf("dataset %q holds no downloadable file", name) + } + return c.DownloadBlob(ctx, sasURI, blobName) +} + +// looksLikeBlobURI reports whether the URI's final segment names a file. +func looksLikeBlobURI(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + last := path.Base(strings.TrimSuffix(u.Path, "/")) + return path.Ext(last) != "" +} + +// pickDatasetBlob chooses the file to read from a container, preferring JSONL +// since that is what an evaluation dataset is. +func pickDatasetBlob(names []string) string { + for _, n := range names { + if strings.EqualFold(path.Ext(n), ".jsonl") { + return n + } + } + for _, n := range names { + if n != "" && !strings.HasSuffix(n, "/") { + return n + } + } + return "" +} + // DownloadDataset downloads dataset content from blob storage using a SAS-authenticated URL. // Returns the raw content as bytes. The downloadURL should be the full URL with SAS token // (e.g., from DatasetCredential.ResolvedDownloadURI()). diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go index 2b12968065a..26727677270 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -24,6 +24,32 @@ type TraceOptions struct { // A prompt source is included when instruction is non-empty, along with the // agent source. When traces is non-nil and Days > 0, a traces source is // appended with start_time computed from the current time. +// WithoutAgentSource returns the sources with the agent entry removed. +// +// Agent-seeded data generation currently fails server-side for every agent, +// while the same request carrying only the prompt succeeds, so this is what a +// retry falls back to. +func WithoutAgentSource(sources []GenerationSource) []GenerationSource { + kept := make([]GenerationSource, 0, len(sources)) + for _, s := range sources { + if s.Type == "agent" { + continue + } + kept = append(kept, s) + } + return kept +} + +// HasPromptSource reports whether anything remains to generate from. +func HasPromptSource(sources []GenerationSource) bool { + for _, s := range sources { + if s.Type == "prompt" && s.Prompt != "" { + return true + } + } + return false +} + func BuildGenerationSources(agentKind, agentName, version, instruction string, traces *TraceOptions) []GenerationSource { var sources []GenerationSource From b122b0f4ea5449ec37655db22ad7c15a18916bac Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 10:27:10 +0300 Subject: [PATCH 036/114] fix(azure.ai.evaluations): seed generation from the agent, not from a source that never works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-seeded data generation fails server-side for every agent. Following it through the stack, the request reaches AOAI unchanged: RAISvc forwards it and FineTuning rewrites the path, and neither resolves the agent. Nothing in this repo can fix it, and nothing the client sends changes the outcome — a nonexistent agent name fails exactly like a real one, so the agent is never looked up at all. But the contract says what that source was for: it "references an agent to fetch instructions / metadata from". That is a read this client can do itself. So `generate` now resolves the agent's context locally, preferring what the author supplied: an explicit instruction, else the file named by agent.context.instructions, else the agent's published instructions. That last step is what makes the scaffolded flow work. `init` writes agent.context.instructions pointing at a conventional path, and nothing read it — the field, and its `tools` sibling, were accepted and dropped. With no instruction anywhere, generation had nothing to fall back to when the agent source failed, so `init` followed by `generate` could not produce a dataset. It now generates one from the agent's own instructions, on topic and scored by a rubric drawn from the same source. `tools` is still unread, so it is called out rather than silently dropped, and init no longer scaffolds it — warning about a field the user never chose is just noise. Also stops relying on the service to reject a missing model. Generation is billed against a deployment, and without one the request failed halfway through the command with a service error naming nothing the caller controls. It now fails before any network call, naming both ways to supply it. --- .../internal/cmd/agent_context_test.go | 149 ++++++++++++++++++ .../internal/cmd/generate.go | 146 ++++++++++++++--- .../azure.ai.evaluations/internal/cmd/init.go | 5 +- .../internal/pkg/eval_api/models.go | 34 +++- .../internal/pkg/eval_api/operations.go | 14 ++ 5 files changed, 322 insertions(+), 26 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go new file mode 100644 index 00000000000..5fc740610e5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The generation spec names an instructions file relative to itself, not to the +// working directory, so `generate --config ` reads the same file the +// author sees next to the spec. +func TestAgentContextInstructions_ResolvesRelativeToTheSpec(t *testing.T) { + dir := t.TempDir() + specDir := filepath.Join(dir, "evals") + require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o755)) + + body := "Answer only from the product catalog." + require.NoError(t, os.WriteFile( + filepath.Join(specDir, "agent", "instructions.md"), []byte(" "+body+"\n"), 0o600)) + + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Instructions = "./agent/instructions.md" + + got, err := agentContextInstructions(cfg, filepath.Join(specDir, "eval_generate.yaml")) + require.NoError(t, err) + assert.Equal(t, body, got, "the file's contents should be used, trimmed") +} + +// `init` writes the instructions path before that file exists. Treating the +// gap as an error would break the flow init itself scaffolds. +func TestAgentContextInstructions_MissingFileIsNotAnError(t *testing.T) { + dir := t.TempDir() + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Instructions = "./agent/instructions.md" + + got, err := agentContextInstructions(cfg, filepath.Join(dir, "eval_generate.yaml")) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestAgentContextInstructions_UnsetIsEmpty(t *testing.T) { + got, err := agentContextInstructions(&project.GenerateConfig{}, "eval_generate.yaml") + require.NoError(t, err) + assert.Empty(t, got) +} + +// Only the newest version is read, and an agent with no published version must +// not panic the caller. +func TestAgentInstructions(t *testing.T) { + var agent eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{ + "name": "support", + "versions": { "latest": { "version": "2", "definition": { + "model": "gpt-5-mini", + "instructions": " You are a support assistant.\n" } } } + }`), &agent)) + assert.Equal(t, "You are a support assistant.", agent.Instructions()) + + var empty eval_api.Agent + require.NoError(t, json.Unmarshal([]byte(`{"name":"x","versions":{}}`), &empty)) + assert.Empty(t, empty.Instructions(), "an agent with no published version has no instructions") + + var nilAgent *eval_api.Agent + assert.Empty(t, nilAgent.Instructions()) +} + +// Dataset generation has no model of its own; it runs against the judge model +// the spec declares. +func TestGenerationModel(t *testing.T) { + cfg := &project.GenerateConfig{} + assert.Empty(t, generationModel(cfg), "no rubric means no model to borrow") + + cfg.Generate.Rubric = &project.RubricSpec{Model: "gpt-4.1-nano"} + assert.Equal(t, "gpt-4.1-nano", generationModel(cfg)) +} + +// `tools` is accepted and ignored, so it has to be called out — the same +// reasoning as the trace fields it now shares a warning with. +func TestWarnIgnoredFields_CoversTools(t *testing.T) { + cases := []struct { + name string + build func(*project.GenerateConfig) + want []string + quiet bool + }{ + { + name: "nothing set stays silent", + build: func(*project.GenerateConfig) {}, + quiet: true, + }, + { + name: "tools alone", + build: func(c *project.GenerateConfig) { c.Agent.Context.Tools = "./agent/tools.json" }, + want: []string{"agent.context.tools", "has no effect"}, + }, + { + name: "tools and a trace field agree in number", + build: func(c *project.GenerateConfig) { + c.Agent.Context.Tools = "./agent/tools.json" + c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} + }, + want: []string{"agent.context.traces.source", "agent.context.tools", "have no effect"}, + }, + { + name: "a window alone is honored, so no warning", + build: func(c *project.GenerateConfig) { + c.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} + }, + quiet: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := &project.GenerateConfig{} + tc.build(cfg) + + var buf bytes.Buffer + warnIgnoredTraceFields(cfg, &buf) + + if tc.quiet { + assert.Empty(t, buf.String()) + return + } + for _, want := range tc.want { + assert.Contains(t, buf.String(), want) + } + }) + } +} + +// init scaffolds only the context fields that are read. +func TestInitScaffold_OmitsToolsButKeepsInstructions(t *testing.T) { + cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") + assert.Equal(t, "./agent/instructions.md", cfg.Agent.Context.Instructions) + assert.Empty(t, cfg.Agent.Context.Tools, + "scaffolding a field nothing reads would warn on every default init") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index eff801fc119..a3508ade263 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -6,6 +6,7 @@ package cmd import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -71,12 +72,30 @@ func newGenerateCommand() *cobra.Command { warnIgnoredTraceFields(cfg, out) } + // Both jobs are billed against a model deployment. Checking before + // any network work keeps the failure at the flag the caller can act + // on, instead of a service rejection partway through the command. + generatingRubric := len(evaluators) == 0 && cfg.Generate.Rubric != nil + generatingDataset := datasetFlag == "" && cfg.Generate.Dataset != nil + if (generatingRubric || generatingDataset) && generationModel(cfg) == "" { + return fmt.Errorf( + "a model deployment is required to generate: pass --eval-model, " + + "or set generate.rubric.model in the generation spec") + } + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err } defer ec.Close() + instruction, err = ec.resolveGenerationInstruction( + ctx, cfg, instruction, configPath, out, isJSON(cmd), + ) + if err != nil { + return err + } + baseDir := filepath.Dir(deployPath) var datasetRefs, evaluatorRefs []project.ArtifactRef @@ -154,42 +173,39 @@ func newGenerateCommand() *cobra.Command { return cmd } -// warnIgnoredTraceFields reports trace settings that are accepted but have no -// effect yet. +// warnIgnoredTraceFields reports generation settings that are accepted but have +// no effect yet. // // The generation API takes a day window and nothing else, so `source` and // `sample` are parsed and dropped. Silently discarding them is worse than not // accepting them: the author believes they narrowed the trace selection when -// nothing changed. +// nothing changed. `agent.context.tools` is in the same position — nothing +// reads it, and only the instructions half of the agent's context is used. func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { - traces := cfg.Agent.Context.Traces - if traces == nil { - return - } + var fields []string - var ignored []string - if traces.Source != "" { - ignored = append(ignored, "source") + if traces := cfg.Agent.Context.Traces; traces != nil { + if traces.Source != "" { + fields = append(fields, "agent.context.traces.source") + } + if traces.Sample > 0 { + fields = append(fields, "agent.context.traces.sample") + } } - if traces.Sample > 0 { - ignored = append(ignored, "sample") + if cfg.Agent.Context.Tools != "" { + fields = append(fields, "agent.context.tools") } - if len(ignored) == 0 { + if len(fields) == 0 { return } - fields := make([]string, 0, len(ignored)) - for _, name := range ignored { - fields = append(fields, "agent.context.traces."+name) - } - verb := "has" if len(fields) > 1 { verb = "have" } fmt.Fprintf(out, - "warning: %s %s no effect yet; trace seeding uses only `window`. "+ - "Trace selection lands with the trace scenarios.\n", + "warning: %s %s no effect yet; generation is seeded from the agent's "+ + "instructions and, when a window is set, its traces.\n", strings.Join(fields, " and "), verb) } @@ -214,6 +230,91 @@ func resolveInstruction(inline, path string) (string, error) { return text, nil } +// generationModel returns the deployment both generation jobs run against. +// +// Dataset generation has no model of its own: the spec carries one judge model +// and both jobs use it. +func generationModel(cfg *project.GenerateConfig) string { + if cfg.Generate.Rubric == nil { + return "" + } + return cfg.Generate.Rubric.Model +} + +// agentContextInstructions reads the instructions named by +// `agent.context.instructions`, relative to the spec that declared them. +// +// A missing file is not an error. `init` writes the field pointing at a +// conventional path before that file exists, so treating its absence as a +// failure would break the flow it scaffolds. +func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (string, error) { + named := cfg.Agent.Context.Instructions + if named == "" { + return "", nil + } + + path := named + if !filepath.IsAbs(path) { + path = filepath.Join(filepath.Dir(configPath), filepath.FromSlash(named)) + } + raw, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("reading agent.context.instructions %q: %w", named, err) + } + return strings.TrimSpace(string(raw)), nil +} + +// resolveGenerationInstruction decides what generation is seeded from. +// +// The service accepts an agent source that is meant to pull the agent's own +// instructions, but it fails for every agent, so the agent's context is read +// here instead. In precedence order: what the caller passed, then the +// instructions file the spec names, then the agent's published instructions. +// +// The last step is what makes `generate` work with no authored input at all, +// which is the flow `init` sets up. +func (ec *evalContext) resolveGenerationInstruction( + ctx context.Context, + cfg *project.GenerateConfig, + explicit, configPath string, + out io.Writer, + quiet bool, +) (string, error) { + if explicit != "" { + return explicit, nil + } + + fromFile, err := agentContextInstructions(cfg, configPath) + if err != nil { + return "", err + } + if fromFile != "" { + return fromFile, nil + } + + if cfg.Agent.Name == "" { + return "", nil + } + agent, err := ec.evalClient.GetAgent(ctx, cfg.Agent.Name, ProjectEndpointAPIVersion) + if err != nil { + // Generation can still proceed from the agent source alone, so a + // failure to read the agent is reported without stopping. + if !quiet { + fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", + cfg.Agent.Name, err) + } + return "", nil + } + instructions := agent.Instructions() + if instructions != "" && !quiet { + fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", cfg.Agent.Name) + } + return instructions, nil +} + // resolveGenerateConfig loads the spec when present, then layers flags on top. // A missing file is not an error: flags alone are sufficient. func resolveGenerateConfig( @@ -325,10 +426,7 @@ func (ec *evalContext) generateDataset( sources := eval_api.BuildGenerationSources( "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), ) - model := "" - if cfg.Generate.Rubric != nil { - model = cfg.Generate.Rubric.Model - } + model := generationModel(cfg) req := eval_api.NewDataGenerationJobRequest(spec.Name, model, spec.SampleSize, sources) job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 2905b7e42ca..1f62f8be3d2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -110,8 +110,11 @@ func buildGenerateScaffold(target, rubricName, evalModel string) *project.Genera Agent: project.AgentSpec{ Name: target, Context: project.AgentContext{ + // Scaffolded even though the file does not exist yet: writing + // it overrides the agent's published instructions, which is the + // usual way to narrow what gets generated. `tools` is left out + // because nothing reads it yet. Instructions: "./agent/instructions.md", - Tools: "./agent/tools.json", }, }, Generate: project.GenerateSpec{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index cf51017746c..d5075eafbf1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -3,7 +3,10 @@ package eval_api -import "encoding/json" +import ( + "encoding/json" + "strings" +) // --------------------------------------------------------------------------- // Data Generation Jobs @@ -43,6 +46,35 @@ type GenerationSource struct { StartTime int64 `json:"start_time,omitempty"` } +// Agent is the part of a catalog agent that describes what it does. +// +// An agent is returned with its versions inlined rather than as a list, and +// only `latest` is populated on a plain read. +type Agent struct { + Name string `json:"name"` + Versions struct { + Latest *AgentVersion `json:"latest"` + } `json:"versions"` +} + +// AgentVersion is one published revision of an agent. +type AgentVersion struct { + Version string `json:"version"` + Definition struct { + Model string `json:"model"` + Instructions string `json:"instructions"` + } `json:"definition"` +} + +// Instructions returns the newest version's system prompt, or "" when the agent +// has no published version. +func (a *Agent) Instructions() string { + if a == nil || a.Versions.Latest == nil { + return "" + } + return strings.TrimSpace(a.Versions.Latest.Definition.Instructions) +} + // GenerationJob is the response for data and evaluator generation job operations. type GenerationJob struct { ID string `json:"id"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index d04a3104636..677e525f59f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -30,6 +30,7 @@ const ( pathEvaluators = "/evaluators" pathDatasets = "/datasets" pathOpenAIEvals = "/openai/v1/evals" + pathAgents = "/agents" ) // EvalClient provides methods for interacting with the Azure AI eval APIs. @@ -114,6 +115,19 @@ func (c *EvalClient) GetEvaluatorGenerationJob( return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) } +// GetAgent reads an agent from the project's catalog. +// +// Only the newest version is returned, which is the one generation is seeded +// from: the point is to describe what the agent does now. +func (c *EvalClient) GetAgent( + ctx context.Context, + name string, + apiVersion string, +) (*Agent, error) { + path := pathAgents + "/" + url.PathEscape(name) + return doRequestTyped[Agent](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + // CreateEvaluatorVersion creates a new version of a named evaluator. // The body should be the full evaluator JSON with the definition field updated. func (c *EvalClient) CreateEvaluatorVersion( From b9a30781f7ce329190fa8f272d7db4109d8f2530 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:20:42 +0300 Subject: [PATCH 037/114] feat(azure.ai.evaluations): run an eval group on a schedule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the scheduling half of M2. A schedule pairs a trigger with an eval group and the run to repeat, since the group holds only its testing criteria and the target and dataset travel with the run. `set` builds that run the same way `run` does, so a scheduled evaluation is the one already being run by hand. Triggers come from flags rather than raw JSON: --cron, or --every with the qualifier that period actually reads. Passing a qualifier from another period is rejected instead of dropped, because the service ignores what it does not recognise and the author would never learn the selection did not narrow. The route took finding. GET answers on both /schedules and /evaluations/schedules, but neither accepts a POST; creation is a PUT on a named resource, and everything else 404s. Three service behaviours shape the command, all measured rather than assumed: A project holds one schedule at a time. A second is refused with a 400 that carries no body at all, so the count is what explains it — the error now names the schedule already there and how to remove it. This also corrected an earlier reading of mine: a run of "required field" probes all failed for this reason, not the one I first recorded. A PUT over an existing schedule is accepted, echoes the new body, and changes nothing. Neither a new cron expression nor a new trigger type takes effect. So `set` refuses an existing name rather than reporting an edit that did not happen. Recreating under a name that was used before never leaves Creating and then cannot be deleted, so the refusal points at a different name rather than offering to replace in place. Deleting also needed care: a schedule mid-provision refuses, as 409 while busy or 404 because its trigger does not exist yet, so delete waits for it to settle. Also fixes a latent client bug this surfaced: 204 was not in the set of accepted statuses, so every successful delete was reported as a failure. --- .../azure.ai.evaluations/internal/cmd/root.go | 1 + .../internal/cmd/schedule.go | 595 ++++++++++++++++++ .../internal/cmd/schedule_test.go | 155 +++++ .../internal/pkg/eval_api/operations.go | 6 +- .../internal/pkg/eval_api/schedules.go | 175 ++++++ 5 files changed, 931 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 40724c5a0a7..118a831fdde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -51,6 +51,7 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), + newScheduleCommand(), newListenCommand(), ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go new file mode 100644 index 00000000000..e0afd5c468b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -0,0 +1,595 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// newScheduleCommand groups the recurring-evaluation commands. +func newScheduleCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "schedule", + Short: "Run an eval group on a schedule.", + } + cmd.AddCommand( + newScheduleSetCommand(), + newScheduleListCommand(), + newScheduleShowCommand(), + newScheduleDeleteCommand(), + ) + return cmd +} + +// newScheduleSetCommand creates the schedule that runs an eval group. +// +// It does not update. The service accepts a PUT over an existing schedule, +// echoes the new body and keeps the old trigger, so an in-place edit would +// report a change that did not happen. Recreating under the same name is not +// an escape either: the replacement never leaves Creating and cannot then be +// deleted. So an existing schedule is refused, and changing one means deleting +// it and creating another under a different name. +func newScheduleSetCommand() *cobra.Command { + var ( + configPath string + groupName string + evalID string + name string + description string + cron string + every string + interval int + atHours []int + onDays []string + onDaysOfMon []int + atTime string + timezone string + startTime string + endTime string + disabled bool + level string + maxSamples int + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "set [eval-id]", + Short: "Create the schedule that runs an eval group.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + if len(args) == 1 { + evalID = args[0] + } + + trigger, err := buildTrigger(triggerFlags{ + cron: cron, + every: every, + interval: interval, + atHours: atHours, + onDays: onDays, + onDaysOfMon: onDaysOfMon, + atTime: atTime, + timezone: timezone, + startTime: startTime, + endTime: endTime, + }) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Same resolution as `run`: the config names the group unless + // --eval-id bypasses it, and the run payload carries the target + // and dataset because the group holds neither. + var group *project.EvalGroup + var dataSource *eval_api.EvalRunDataSource + if evalID == "" { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + group, err = cfg.ResolveGroup(groupName) + if err != nil { + return err + } + if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { + return err + } + evalID, err = ec.resolveEvalGroupID( + ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) + if err != nil { + return err + } + dataSource, err = buildRunDataSource( + group, configPath, resolveMaxSamples(maxSamples, group)) + if err != nil { + return err + } + } else { + dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) + if err != nil { + return err + } + } + + if name == "" { + name = defaultScheduleName(group) + } + if description == "" { + description = fmt.Sprintf("Scheduled evaluation of %s.", evalID) + } + + // An existing schedule cannot be edited: the service takes the PUT + // and ignores it. Recreating under the same name is worse — the + // replacement sticks in Creating and cannot be deleted — so the + // only safe answer is a different name. + if existing, err := ec.evalClient.GetSchedule( + ctx, name, ProjectEndpointAPIVersion); err == nil && existing != nil && existing.ID != "" { + return fmt.Errorf( + "schedule %q already exists, and the service ignores edits to it. "+ + "Delete it with `azd ai eval schedule delete %s` and create the new "+ + "one under a different name; reusing this one leaves it stuck", + name, name) + } + + metadata := map[string]string{} + if lvl := resolveLevel(level, group); lvl != "" { + metadata["evaluation_level"] = lvl + } + + schedule := &eval_api.Schedule{ + DisplayName: name, + Description: description, + Enabled: !disabled, + Trigger: trigger, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: dataSource, + Metadata: metadata, + }, + }, + } + + saved, err := ec.evalClient.PutSchedule(ctx, name, schedule, ProjectEndpointAPIVersion) + if err != nil { + return explainScheduleFailure(ctx, ec, name, err) + } + + if isJSON(cmd) { + return emitJSON(out, saved) + } + state := "enabled" + if !saved.Enabled { + state = "disabled" + } + fmt.Fprintf(out, "Schedule %s (%s) runs %s on %s\n", + saved.ID, state, saved.Summary(), evalID) + return nil + }, + } + + cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, + "Path to the eval deployment config.") + cmd.Flags().StringVar(&groupName, "eval-group", "", "Which evalGroups entry to schedule.") + cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval group by id, ignoring config.") + cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the group name.") + cmd.Flags().StringVar(&description, "description", "", "Schedule description.") + cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) + cmd.Flags().StringVar(&every, "every", "", + "Recur hourly, daily, weekly or monthly.") + cmd.Flags().IntVar(&interval, "interval", 0, "Repeat every N periods of --every. Defaults to 1.") + cmd.Flags().IntSliceVar(&atHours, "at", nil, "Hours of the day for --every daily, 0-23.") + cmd.Flags().StringSliceVar(&onDays, "on", nil, "Days of the week for --every weekly, for example Monday.") + cmd.Flags().IntSliceVar(&onDaysOfMon, "on-day", nil, "Days of the month for --every monthly, 1-31.") + cmd.Flags().StringVar(&atTime, "at-time", "", "Run once at this RFC3339 time.") + cmd.Flags().StringVar(&timezone, "timezone", "", "Timezone for the trigger. Defaults to UTC.") + cmd.Flags().StringVar(&startTime, "start-time", "", "RFC3339 time before which the schedule does not fire.") + cmd.Flags().StringVar(&endTime, "end-time", "", "RFC3339 time after which the schedule stops firing.") + cmd.Flags().BoolVar(&disabled, "disabled", false, "Create the schedule without enabling it.") + cmd.Flags().StringVar(&level, "level", "", "Evaluation level for the scheduled runs.") + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap rows sent from a local dataset file.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + cmd.MarkFlagsMutuallyExclusive("cron", "every", "at-time") + + return cmd +} + +func newScheduleListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's schedules.", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing schedules: %w", err) + } + if isJSON(cmd) { + return emitJSON(out, list) + } + if list == nil || len(list.Value) == 0 { + fmt.Fprintln(out, "No schedules.") + return nil + } + + rows := make([][]string, 0, len(list.Value)) + for i := range list.Value { + s := &list.Value[i] + evalGroup := "" + if s.Task != nil { + evalGroup = s.Task.EvalID + } + rows = append(rows, []string{ + s.ID, + strconv.FormatBool(s.Enabled), + s.ProvisioningStatus, + s.Summary(), + evalGroup, + }) + } + return emitTable(out, + []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL GROUP"}, rows) + }, + } + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newScheduleShowCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "show [name]", + Short: "Show one schedule.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + if len(args) == 1 { + name = args[0] + } + if name == "" { + return requireFlag("name") + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("reading schedule %q: %w", name, err) + } + if isJSON(cmd) { + return emitJSON(out, s) + } + + fmt.Fprintf(out, "Schedule %s\n", s.ID) + fmt.Fprintf(out, " enabled: %t\n", s.Enabled) + fmt.Fprintf(out, " status: %s\n", s.ProvisioningStatus) + fmt.Fprintf(out, " trigger: %s\n", s.Summary()) + if s.Trigger != nil && s.Trigger.Timezone != "" { + fmt.Fprintf(out, " timezone: %s\n", s.Trigger.Timezone) + } + if s.Task != nil { + fmt.Fprintf(out, " group: %s\n", s.Task.EvalID) + } + if s.Description != "" { + fmt.Fprintf(out, " about: %s\n", s.Description) + } + return nil + }, + } + cmd.Flags().StringVar(&name, "name", "", "Schedule name.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newScheduleDeleteCommand() *cobra.Command { + var ( + name string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "delete [name]", + Short: "Delete a schedule.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + out := cmd.OutOrStdout() + + if len(args) == 1 { + name = args[0] + } + if name == "" { + return requireFlag("name") + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.deleteScheduleWhenSettled(ctx, name); err != nil { + return fmt.Errorf("deleting schedule %q: %w", name, err) + } + fmt.Fprintf(out, "Deleted schedule %s\n", name) + return nil + }, + } + cmd.Flags().StringVar(&name, "name", "", "Schedule name.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// triggerFlags carries the schedule's timing flags so buildTrigger can be +// tested without a command. +type triggerFlags struct { + cron string + every string + interval int + atHours []int + onDays []string + onDaysOfMon []int + atTime string + timezone string + startTime string + endTime string +} + +// buildTrigger turns the timing flags into the trigger the API expects. +func buildTrigger(f triggerFlags) (*eval_api.ScheduleTrigger, error) { + tz := f.timezone + if tz == "" { + tz = "UTC" + } + + switch { + case f.cron != "": + return &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerCron, + Expression: f.cron, + StartTime: f.startTime, + EndTime: f.endTime, + Timezone: tz, + }, nil + + case f.atTime != "": + if _, err := time.Parse(time.RFC3339, f.atTime); err != nil { + return nil, fmt.Errorf("--at-time %q is not an RFC3339 time", f.atTime) + } + return &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerOneTime, + ScheduledTime: f.atTime, + Timezone: tz, + }, nil + + case f.every != "": + pattern, err := buildRecurrence(f) + if err != nil { + return nil, err + } + interval := f.interval + if interval <= 0 { + interval = 1 + } + return &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerRecurrence, + Schedule: pattern, + Interval: interval, + StartTime: f.startTime, + EndTime: f.endTime, + Timezone: tz, + }, nil + } + + return nil, fmt.Errorf("a schedule needs a trigger: pass --cron, --every or --at-time") +} + +// buildRecurrence maps --every and its qualifiers onto a recurrence pattern. +// +// Each period reads only its own qualifier, so passing one that does not apply +// is rejected rather than dropped. +func buildRecurrence(f triggerFlags) (*eval_api.RecurrencePattern, error) { + period := strings.ToLower(strings.TrimSpace(f.every)) + + reject := func(flag, applies string) error { + return fmt.Errorf("--%s applies to --every %s, not %s", flag, applies, period) + } + + switch period { + case "hourly": + if len(f.atHours) > 0 { + return nil, reject("at", "daily") + } + if len(f.onDays) > 0 { + return nil, reject("on", "weekly") + } + if len(f.onDaysOfMon) > 0 { + return nil, reject("on-day", "monthly") + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceHourly}, nil + + case "daily": + if len(f.onDays) > 0 { + return nil, reject("on", "weekly") + } + if len(f.onDaysOfMon) > 0 { + return nil, reject("on-day", "monthly") + } + for _, h := range f.atHours { + if h < 0 || h > 23 { + return nil, fmt.Errorf("--at %d is not an hour of the day (0-23)", h) + } + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceDaily, Hours: f.atHours}, nil + + case "weekly": + if len(f.atHours) > 0 { + return nil, reject("at", "daily") + } + if len(f.onDaysOfMon) > 0 { + return nil, reject("on-day", "monthly") + } + days, err := normalizeDaysOfWeek(f.onDays) + if err != nil { + return nil, err + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly, DaysOfWeek: days}, nil + + case "monthly": + if len(f.atHours) > 0 { + return nil, reject("at", "daily") + } + if len(f.onDays) > 0 { + return nil, reject("on", "weekly") + } + for _, d := range f.onDaysOfMon { + if d < 1 || d > 31 { + return nil, fmt.Errorf("--on-day %d is not a day of the month (1-31)", d) + } + } + return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceMonthly, DaysOfMonth: f.onDaysOfMon}, nil + } + + return nil, fmt.Errorf( + "--every %q is not a recurrence: use hourly, daily, weekly or monthly", f.every) +} + +// normalizeDaysOfWeek accepts day names in any casing and returns the spelling +// the service expects. +func normalizeDaysOfWeek(days []string) ([]string, error) { + if len(days) == 0 { + return nil, nil + } + canonical := map[string]string{} + for d := time.Sunday; d <= time.Saturday; d++ { + canonical[strings.ToLower(d.String())] = d.String() + } + + out := make([]string, 0, len(days)) + for _, raw := range days { + name, ok := canonical[strings.ToLower(strings.TrimSpace(raw))] + if !ok { + return nil, fmt.Errorf("--on %q is not a day of the week", raw) + } + out = append(out, name) + } + return out, nil +} + +// defaultScheduleName derives a schedule name from the group being scheduled. +func defaultScheduleName(group *project.EvalGroup) string { + if group != nil && group.Name != "" { + return group.Name + } + return "eval-" + strconv.FormatInt(time.Now().UTC().Unix(), 10) +} + +// deleteScheduleWhenSettled removes a schedule, waiting out the window where +// the service is still provisioning it. +// +// A schedule that is mid-provision refuses the delete, and does it two ways: +// 409 while it is busy, or 404 because the trigger behind it does not exist +// yet. Either way the caller neither caused it nor can see it, so the wait +// happens here. +func (ec *evalContext) deleteScheduleWhenSettled(ctx context.Context, name string) error { + const attempts = 30 + + for i := 0; i < attempts; i++ { + s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + if err != nil || s == nil || s.ID == "" { + // Nothing to wait for: let the delete report what it finds. + break + } + if s.Settled() { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(4 * time.Second): + } + } + + for i := 0; ; i++ { + err := ec.evalClient.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion) + if err == nil || !eval_api.IsConflict(err) || i == attempts-1 { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(4 * time.Second): + } + } +} + +// explainScheduleFailure turns the service's bodiless rejection into the reason +// it actually happened. +// +// The project accepts one schedule at a time and refuses a second with a 400 +// carrying no message, so the count is what explains it. +func explainScheduleFailure( + ctx context.Context, + ec *evalContext, + name string, + cause error, +) error { + list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) + if listErr != nil || list == nil { + return fmt.Errorf("saving schedule %q: %w", name, cause) + } + + for i := range list.Value { + if other := list.Value[i].ID; other != "" && other != name { + return fmt.Errorf( + "saving schedule %q: the project already has a schedule, %q, and only one is "+ + "allowed at a time. Delete it first with "+ + "`azd ai eval schedule delete %s`", name, other, other) + } + } + return fmt.Errorf("saving schedule %q: %w", name, cause) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go new file mode 100644 index 00000000000..9f93207950b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildTrigger_Cron(t *testing.T) { + got, err := buildTrigger(triggerFlags{cron: "0 9 * * *"}) + require.NoError(t, err) + assert.Equal(t, eval_api.TriggerCron, got.Type) + assert.Equal(t, "0 9 * * *", got.Expression) + assert.Equal(t, "UTC", got.Timezone, "UTC unless the caller says otherwise") +} + +func TestBuildTrigger_OneTime(t *testing.T) { + got, err := buildTrigger(triggerFlags{atTime: "2026-08-01T09:00:00Z", timezone: "Europe/Dublin"}) + require.NoError(t, err) + assert.Equal(t, eval_api.TriggerOneTime, got.Type) + assert.Equal(t, "2026-08-01T09:00:00Z", got.ScheduledTime) + assert.Equal(t, "Europe/Dublin", got.Timezone) + + _, err = buildTrigger(triggerFlags{atTime: "next tuesday"}) + require.ErrorContains(t, err, "RFC3339") +} + +func TestBuildTrigger_Recurrence(t *testing.T) { + cases := []struct { + name string + flags triggerFlags + wantType string + assert func(*testing.T, *eval_api.RecurrencePattern) + }{ + { + name: "hourly", + flags: triggerFlags{every: "hourly", interval: 6}, + wantType: eval_api.RecurrenceHourly, + }, + { + name: "daily with hours", + flags: triggerFlags{every: "Daily", atHours: []int{9, 17}}, + wantType: eval_api.RecurrenceDaily, + assert: func(t *testing.T, p *eval_api.RecurrencePattern) { + assert.Equal(t, []int{9, 17}, p.Hours) + }, + }, + { + name: "weekly normalizes day casing", + flags: triggerFlags{every: "weekly", onDays: []string{"monday", "THURSDAY"}}, + wantType: eval_api.RecurrenceWeekly, + assert: func(t *testing.T, p *eval_api.RecurrencePattern) { + assert.Equal(t, []string{"Monday", "Thursday"}, p.DaysOfWeek) + }, + }, + { + name: "monthly", + flags: triggerFlags{every: "monthly", onDaysOfMon: []int{1, 15}}, + wantType: eval_api.RecurrenceMonthly, + assert: func(t *testing.T, p *eval_api.RecurrencePattern) { + assert.Equal(t, []int{1, 15}, p.DaysOfMonth) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := buildTrigger(tc.flags) + require.NoError(t, err) + assert.Equal(t, eval_api.TriggerRecurrence, got.Type) + require.NotNil(t, got.Schedule) + assert.Equal(t, tc.wantType, got.Schedule.Type) + if tc.assert != nil { + tc.assert(t, got.Schedule) + } + }) + } +} + +// An interval is always sent, so the service never has to infer one. +func TestBuildTrigger_IntervalDefaultsToOne(t *testing.T) { + got, err := buildTrigger(triggerFlags{every: "daily"}) + require.NoError(t, err) + assert.Equal(t, 1, got.Interval) + + got, err = buildTrigger(triggerFlags{every: "daily", interval: 3}) + require.NoError(t, err) + assert.Equal(t, 3, got.Interval) +} + +// Each period reads only its own qualifier. Accepting one that does not apply +// would drop it silently, which is the failure mode the trace fields already +// taught us to avoid. +func TestBuildTrigger_RejectsQualifiersFromAnotherPeriod(t *testing.T) { + cases := []struct { + name string + flags triggerFlags + want string + }{ + {"hours on weekly", triggerFlags{every: "weekly", atHours: []int{9}}, "--at applies to --every daily"}, + {"days on daily", triggerFlags{every: "daily", onDays: []string{"Monday"}}, "--on applies to --every weekly"}, + {"month days on hourly", triggerFlags{every: "hourly", onDaysOfMon: []int{1}}, "--on-day applies to --every monthly"}, + {"hours on monthly", triggerFlags{every: "monthly", atHours: []int{9}}, "--at applies to --every daily"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := buildTrigger(tc.flags) + require.ErrorContains(t, err, tc.want) + }) + } +} + +func TestBuildTrigger_RejectsOutOfRangeValues(t *testing.T) { + _, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{24}}) + require.ErrorContains(t, err, "hour of the day") + + _, err = buildTrigger(triggerFlags{every: "monthly", onDaysOfMon: []int{0}}) + require.ErrorContains(t, err, "day of the month") + + _, err = buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Caturday"}}) + require.ErrorContains(t, err, "not a day of the week") + + _, err = buildTrigger(triggerFlags{every: "fortnightly"}) + require.ErrorContains(t, err, "hourly, daily, weekly or monthly") +} + +func TestBuildTrigger_NeedsATrigger(t *testing.T) { + _, err := buildTrigger(triggerFlags{}) + require.ErrorContains(t, err, "--cron, --every or --at-time") +} + +func TestScheduleSummary(t *testing.T) { + cron := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerCron, Expression: "0 9 * * *"}} + assert.Equal(t, "cron 0 9 * * *", cron.Summary()) + + weekly := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerRecurrence, + Schedule: &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly}}} + assert.Equal(t, "every Weekly", weekly.Summary()) + + once := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ + Type: eval_api.TriggerOneTime, ScheduledTime: "2026-08-01T09:00:00Z"}} + assert.Equal(t, "once at 2026-08-01T09:00:00Z", once.Summary()) + + var nilSchedule *eval_api.Schedule + assert.Empty(t, nilSchedule.Summary()) + assert.Empty(t, (&eval_api.Schedule{}).Summary()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 677e525f59f..5b2a74923bf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -330,7 +330,11 @@ func (c *EvalClient) doRequest( log.Printf("[eval_api] response status: %d", resp.StatusCode) - if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + // 204 belongs here: a delete that removed the resource answers No Content, + // and treating that as a failure reports every successful delete as an + // error. doRequestTyped already tolerates the empty body. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { // Restore the body so runtime.NewResponseError can read it. resp.Body = io.NopCloser(bytes.NewReader(respBody)) return nil, runtime.NewResponseError(resp) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go new file mode 100644 index 00000000000..ed205bf1f79 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "errors" + "net/http" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +const pathSchedules = "/schedules" + +// Trigger types accepted by the schedules API. +const ( + TriggerCron = "Cron" + TriggerRecurrence = "Recurrence" + TriggerOneTime = "OneTime" +) + +// Recurrence patterns accepted under a Recurrence trigger. +const ( + RecurrenceHourly = "Hourly" + RecurrenceDaily = "Daily" + RecurrenceWeekly = "Weekly" + RecurrenceMonthly = "Monthly" +) + +// ScheduleTaskEvaluation is the only task type the service accepts today; an +// Insight task is rejected on validation. +const ScheduleTaskEvaluation = "Evaluation" + +// RecurrencePattern is the shape of a Recurrence trigger's repeat rule. Which +// fields apply depends on Type: Daily reads Hours, Weekly reads DaysOfWeek, +// Monthly reads DaysOfMonth, and Hourly reads neither. +type RecurrencePattern struct { + Type string `json:"type"` + Hours []int `json:"hours,omitempty"` + DaysOfWeek []string `json:"daysOfWeek,omitempty"` + DaysOfMonth []int `json:"daysOfMonth,omitempty"` +} + +// ScheduleTrigger says when the task runs. The discriminator is Type; the +// other fields are per-type and only one set is ever populated. +type ScheduleTrigger struct { + Type string `json:"type"` + + // Cron + Expression string `json:"expression,omitempty"` + + // Recurrence + Schedule *RecurrencePattern `json:"schedule,omitempty"` + Interval int `json:"interval,omitempty"` + + // OneTime + ScheduledTime string `json:"scheduledTime,omitempty"` + + // Cron and Recurrence + StartTime string `json:"startTime,omitempty"` + EndTime string `json:"endTime,omitempty"` + + Timezone string `json:"timezone,omitempty"` +} + +// ScheduleTask is what the trigger fires. An evaluation task needs both the +// group and the run to repeat: the group holds only its testing criteria, so +// the target and dataset travel with the run. +type ScheduleTask struct { + Type string `json:"type"` + EvalID string `json:"evalId,omitempty"` + EvalRun *CreateOpenAIEvalRunRequest `json:"evalRun,omitempty"` +} + +// Schedule is a named, project-scoped recurring evaluation. +type Schedule struct { + ID string `json:"id,omitempty"` + DisplayName string `json:"displayName,omitempty"` + Description string `json:"description,omitempty"` + Enabled bool `json:"enabled"` + ProvisioningStatus string `json:"provisioningStatus,omitempty"` + Trigger *ScheduleTrigger `json:"trigger,omitempty"` + Task *ScheduleTask `json:"task,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// Summary renders the trigger as a single line for listings. +func (s *Schedule) Summary() string { + if s == nil || s.Trigger == nil { + return "" + } + switch s.Trigger.Type { + case TriggerCron: + return "cron " + s.Trigger.Expression + case TriggerOneTime: + return "once at " + s.Trigger.ScheduledTime + case TriggerRecurrence: + if s.Trigger.Schedule == nil { + return "recurrence" + } + return "every " + s.Trigger.Schedule.Type + } + return s.Trigger.Type +} + +// ScheduleList is the response for ListSchedules. +type ScheduleList struct { + Value []Schedule `json:"value"` +} + +// Settled reports whether the schedule has finished provisioning. +// +// A schedule that is still being created refuses a delete, and does it two +// different ways: 409 while it is busy, or 404 because the trigger behind it +// does not exist yet. Waiting for it to settle avoids both. +func (s *Schedule) Settled() bool { + if s == nil { + return true + } + switch s.ProvisioningStatus { + case "Creating", "Updating", "Deleting": + return false + } + return true +} + +// PutSchedule creates or replaces a schedule. The route is keyed by name, and +// the same call updates an existing schedule in place. +func (c *EvalClient) PutSchedule( + ctx context.Context, + name string, + schedule *Schedule, + apiVersion string, +) (*Schedule, error) { + path := pathSchedules + "/" + url.PathEscape(name) + return doRequestTyped[Schedule](c, ctx, http.MethodPut, path, nil, schedule, apiVersion) +} + +// GetSchedule reads one schedule by name. +func (c *EvalClient) GetSchedule( + ctx context.Context, + name string, + apiVersion string, +) (*Schedule, error) { + path := pathSchedules + "/" + url.PathEscape(name) + return doRequestTyped[Schedule](c, ctx, http.MethodGet, path, nil, nil, apiVersion) +} + +// ListSchedules returns the project's schedules. +func (c *EvalClient) ListSchedules(ctx context.Context, apiVersion string) (*ScheduleList, error) { + return doRequestTyped[ScheduleList](c, ctx, http.MethodGet, pathSchedules, nil, nil, apiVersion) +} + +// DeleteSchedule removes a schedule by name. +func (c *EvalClient) DeleteSchedule(ctx context.Context, name string, apiVersion string) error { + path := pathSchedules + "/" + url.PathEscape(name) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + +// IsConflict reports whether the service refused because the resource is busy. +// +// A schedule that is still provisioning answers 409 to a delete, which is worth +// waiting out rather than reporting. +func IsConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} From 1c42efab828bc34048af4c2d790ae27b839f5653 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:34:50 +0300 Subject: [PATCH 038/114] feat(azure.ai.evaluations): evaluate an agent's traces, and say why a run failed Traces as a run data source were recorded as awaiting service support. They are not: azure_ai_traces is in the run data-source discriminator and the service accepts and executes it. The note was an assumption I had not re-tested, so `run --from-traces` now evaluates what the agent already did rather than asking it fresh questions from a dataset. The agent comes from the group's target, or from whatever the group ran against last when --eval-id bypasses the config, so the flag needs no argument. A window is only sent when asked for: the service defaults it, and an unset bound sent as an epoch would quietly mean 1970. Verified against the live service, which stored the payload and normalised 7d into lookback_hours 168 while honouring max_traces. The other half of this was finding the run had failed and not being told why. A failed run carries the reason, and it is usually the only actionable thing in the response, but it was dropped: the trace case explains that the agent emitted no GenAI content and what to do about it, and all the caller saw was the word "failed". Every failure path was losing this, not just traces. --- .../azure.ai.evaluations/internal/cmd/run.go | 63 +++++++++++++- .../internal/cmd/traces_test.go | 85 +++++++++++++++++++ .../internal/pkg/eval_api/models.go | 44 +++++++++- 3 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a90e4994745..ca2a16647f8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -51,6 +51,9 @@ func buildRunCommand(use, short string) *cobra.Command { runName string level string maxSamples int + fromTraces bool + traceWindow string + maxTraces int wait bool endpointFlg string ) @@ -97,9 +100,12 @@ func buildRunCommand(use, short string) *cobra.Command { // With --eval-id there is no config to read, so the pairing of // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource - if group == nil { + switch { + case fromTraces: + dataSource, err = buildTracesDataSource(ctx, ec, group, evalID, traceWindow, maxTraces) + case group == nil: dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) - } else { + default: dataSource, err = buildRunDataSource( group, configPath, resolveMaxSamples(maxSamples, group)) } @@ -170,6 +176,11 @@ func buildRunCommand(use, short string) *cobra.Command { "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from a local dataset file. Ignored for registered datasets.") + cmd.Flags().BoolVar(&fromTraces, "from-traces", false, + "Evaluate the agent's recorded traces instead of the dataset.") + cmd.Flags().StringVar(&traceWindow, "trace-window", "", + "How far back to read traces, for example 7d. Defaults to the service's window.") + cmd.Flags().IntVar(&maxTraces, "max-traces", 0, "Cap the traces evaluated.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool @@ -316,6 +327,48 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return list.Data[0].DataSource, nil } +// buildTracesDataSource evaluates what the agent has already done, rather than +// asking it fresh questions from a dataset. +// +// The service reads the traces from Application Insights, so the agent has to +// be emitting gen_ai.input.messages / gen_ai.output.messages for anything to be +// found; when it is not, the run fails with the service saying so. +func buildTracesDataSource( + ctx context.Context, + ec *evalContext, + group *project.EvalGroup, + evalID, window string, + maxTraces int, +) (*eval_api.EvalRunDataSource, error) { + agent := "" + switch { + case group != nil && group.Target != nil: + agent = group.Target.Name + default: + // With --eval-id there is no config, so the agent comes from whatever + // the group ran against last. + last, err := ec.reuseDataSourceFromLastRun(ctx, evalID) + if err != nil { + return nil, err + } + if last != nil && last.Target != nil { + agent = last.Target.Name + } + } + if agent == "" { + return nil, fmt.Errorf( + "--from-traces needs to know whose traces to read, and the eval group does not " + + "name an agent. Declare target.type: agent on the group") + } + + var start, end time.Time + if days := parseWindowDays(window); days > 0 { + end = time.Now().UTC() + start = end.AddDate(0, 0, -days) + } + return eval_api.NewTracesDataSource(agent, start, end, maxTraces), nil +} + // buildRunDataSource binds the dataset to the run. The eval group carries no // dataset today, so it is supplied here. func buildRunDataSource( @@ -492,6 +545,12 @@ func (ec *evalContext) pollRun( func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) + // A run that failed carries why, and it is usually the only actionable + // thing in the response — dropping it leaves the caller with just the word + // "failed". + if why := run.Failure(); why != "" { + fmt.Fprintf(out, " %s\n", why) + } if run.ReportURL != "" { fmt.Fprintf(out, "Report: %s\n", run.ReportURL) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go new file mode 100644 index 00000000000..dacbdd02104 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/json" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A window is only sent when one was asked for; the service defaults it +// otherwise, and an epoch-zero bound would silently mean 1970. +func TestNewTracesDataSource_OmitsAnUnsetWindow(t *testing.T) { + ds := eval_api.NewTracesDataSource("support-agent", time.Time{}, time.Time{}, 0) + assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) + assert.Equal(t, "support-agent", ds.AgentName) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.NotContains(t, body, "start_time") + assert.NotContains(t, body, "end_time") + assert.NotContains(t, body, "max_traces") + assert.NotContains(t, body, "input_messages", "traces carry no template") +} + +func TestNewTracesDataSource_SendsTheWindowItWasGiven(t *testing.T) { + end := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) + start := end.AddDate(0, 0, -7) + + ds := eval_api.NewTracesDataSource("support-agent", start, end, 25) + assert.Equal(t, start.Unix(), ds.StartTime) + assert.Equal(t, end.Unix(), ds.EndTime) + assert.Equal(t, 25, ds.MaxTraces) +} + +// The reason a run failed is the only actionable part of the response, so it +// has to survive into the output. +func TestRunFailureMessage(t *testing.T) { + var run eval_api.OpenAIEvalRun + require.NoError(t, json.Unmarshal([]byte(`{ + "id": "evalrun_x", "status": "failed", + "error": { "code": "UserError", "message": " No trace data found for agent_name 'a'. " } + }`), &run)) + assert.Equal(t, "No trace data found for agent_name 'a'.", run.Failure()) + + // The field is present and null-valued on success, so presence alone + // must not read as failure. + var ok eval_api.OpenAIEvalRun + require.NoError(t, json.Unmarshal([]byte(`{ + "id": "evalrun_y", "status": "completed", + "error": { "code": null, "message": null } + }`), &ok)) + assert.Empty(t, ok.Failure()) + + var absent eval_api.OpenAIEvalRun + require.NoError(t, json.Unmarshal([]byte(`{"id":"evalrun_z","status":"completed"}`), &absent)) + assert.Empty(t, absent.Failure()) + + var nilRun *eval_api.OpenAIEvalRun + assert.Empty(t, nilRun.Failure()) +} + +func TestRenderRun_ShowsTheFailureReason(t *testing.T) { + var buf bytes.Buffer + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_x", + Status: "failed", + Error: &eval_api.JobError{Code: "UserError", Message: "No trace data found."}, + } + require.NoError(t, renderRun(&buf, run)) + assert.Contains(t, buf.String(), "failed") + assert.Contains(t, buf.String(), "No trace data found.") + + var clean bytes.Buffer + require.NoError(t, renderRun(&clean, &eval_api.OpenAIEvalRun{ID: "evalrun_y", Status: "completed"})) + assert.NotContains(t, clean.String(), " \n", "a successful run gains no blank reason line") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index d5075eafbf1..92b7f2817df 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -6,6 +6,7 @@ package eval_api import ( "encoding/json" "strings" + "time" ) // --------------------------------------------------------------------------- @@ -315,6 +316,11 @@ type EvalRunDataSourceType string const ( // EvalRunDataSourceTypeAgentTarget is the data source type for agent target completions. EvalRunDataSourceTypeAgentTarget EvalRunDataSourceType = "azure_ai_target_completions" + + // EvalRunDataSourceTypeTraces evaluates an agent's recorded traces instead of + // a dataset. The service reads them from Application Insights, so the agent + // must be emitting gen_ai.input.messages / gen_ai.output.messages. + EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" ) // EvalRunDataContentType defines the source type for eval run data content. @@ -331,6 +337,12 @@ type EvalRunDataSource struct { InputMessages *EvalRunInputMessages `json:"input_messages,omitempty"` Source *EvalRunDataContent `json:"source,omitempty"` Target *EvalRunTarget `json:"target,omitempty"` + + // Traces only. The window defaults to the last seven days when left unset. + AgentName string `json:"agent_name,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` } // EvalRunInputMessages describes how input messages are constructed from dataset items. @@ -385,6 +397,25 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } } +// NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. +// +// The service reads them from Application Insights and defaults to the last +// seven days, so a zero window is left off rather than sent as an epoch. +func NewTracesDataSource(agentName string, start, end time.Time, maxTraces int) *EvalRunDataSource { + ds := &EvalRunDataSource{ + Type: EvalRunDataSourceTypeTraces, + AgentName: agentName, + MaxTraces: maxTraces, + } + if !start.IsZero() { + ds.StartTime = start.Unix() + } + if !end.IsZero() { + ds.EndTime = end.Unix() + } + return ds +} + // SetFileContent sets the data source to use inline file content. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ @@ -417,7 +448,18 @@ type OpenAIEvalRun struct { // Result summary ResultCounts *EvalRunResultCounts `json:"result_counts,omitempty"` PerTestingCriteria []EvalRunCriteriaResult `json:"per_testing_criteria_results,omitempty"` - Error any `json:"error,omitempty"` + Error *JobError `json:"error,omitempty"` +} + +// Failure returns why the run failed, or "" when it did not. +// +// The field is always present and its members are null on success, so its +// presence says nothing on its own. +func (r *OpenAIEvalRun) Failure() string { + if r == nil || r.Error == nil { + return "" + } + return strings.TrimSpace(r.Error.Message) } // EvalRunResultCounts holds pass/fail/error/skip counts for a run. From 03c7026d24d5041337e994f1d21b27435ec0fe21 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:40:57 +0300 Subject: [PATCH 039/114] fix(azure.ai.evaluations): send the trace window in the field the service reads --trace-window was being dropped. The traces data source has no start bound: its window is lookback_hours, and a start_time is accepted and discarded, leaving the default seven days in place. It looked like it worked because the value I first tested with, 7d, is the default. Asking for 30d stored lookback_hours 168 and queried a week, which is the failure mode this extension has been fixing everywhere else: a field accepted, ignored, and never reported. Now sends lookback_hours, verified against the service, which stored 720 and queried 2026-06-28 to 2026-07-28. Found by reading the service's own contract rather than the response echo. The echo was no help: it reflected a well-formed request whose window had already been replaced by the default. --- .../azure.ai.evaluations/internal/cmd/run.go | 7 ++--- .../internal/cmd/traces_test.go | 26 +++++++++------- .../internal/pkg/eval_api/models.go | 30 ++++++++++--------- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index ca2a16647f8..436b94ae456 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -361,12 +361,11 @@ func buildTracesDataSource( "name an agent. Declare target.type: agent on the group") } - var start, end time.Time + var lookbackHours int if days := parseWindowDays(window); days > 0 { - end = time.Now().UTC() - start = end.AddDate(0, 0, -days) + lookbackHours = days * 24 } - return eval_api.NewTracesDataSource(agent, start, end, maxTraces), nil + return eval_api.NewTracesDataSource(agent, lookbackHours, time.Time{}, maxTraces), nil } // buildRunDataSource binds the dataset to the run. The eval group carries no diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go index dacbdd02104..75a175d6d3f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go @@ -16,29 +16,35 @@ import ( ) // A window is only sent when one was asked for; the service defaults it -// otherwise, and an epoch-zero bound would silently mean 1970. +// otherwise. func TestNewTracesDataSource_OmitsAnUnsetWindow(t *testing.T) { - ds := eval_api.NewTracesDataSource("support-agent", time.Time{}, time.Time{}, 0) + ds := eval_api.NewTracesDataSource("support-agent", 0, time.Time{}, 0) assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) assert.Equal(t, "support-agent", ds.AgentName) raw, err := json.Marshal(ds) require.NoError(t, err) body := string(raw) - assert.NotContains(t, body, "start_time") + assert.NotContains(t, body, "lookback_hours") assert.NotContains(t, body, "end_time") assert.NotContains(t, body, "max_traces") assert.NotContains(t, body, "input_messages", "traces carry no template") } -func TestNewTracesDataSource_SendsTheWindowItWasGiven(t *testing.T) { - end := time.Date(2026, 7, 28, 12, 0, 0, 0, time.UTC) - start := end.AddDate(0, 0, -7) - - ds := eval_api.NewTracesDataSource("support-agent", start, end, 25) - assert.Equal(t, start.Unix(), ds.StartTime) - assert.Equal(t, end.Unix(), ds.EndTime) +// The service reads `lookback_hours` and has no start bound. Sending a +// start_time is accepted and dropped, which silently leaves the default seven +// days in place, so the window has to travel as hours. +func TestNewTracesDataSource_SendsAWindowTheServiceReads(t *testing.T) { + ds := eval_api.NewTracesDataSource("support-agent", 30*24, time.Time{}, 25) + assert.Equal(t, 720, ds.LookbackHours) assert.Equal(t, 25, ds.MaxTraces) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.Contains(t, body, `"lookback_hours":720`) + assert.NotContains(t, body, "start_time", + "the service drops start_time and falls back to its default window") } // The reason a run failed is the only actionable part of the response, so it diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 92b7f2817df..6d2f4c2c1c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -338,11 +338,13 @@ type EvalRunDataSource struct { Source *EvalRunDataContent `json:"source,omitempty"` Target *EvalRunTarget `json:"target,omitempty"` - // Traces only. The window defaults to the last seven days when left unset. - AgentName string `json:"agent_name,omitempty"` - StartTime int64 `json:"start_time,omitempty"` - EndTime int64 `json:"end_time,omitempty"` - MaxTraces int `json:"max_traces,omitempty"` + // Traces only. The window is expressed as a lookback in hours, not as a + // start bound: the service has no start_time on this data source and + // silently falls back to its default when one is sent. + AgentName string `json:"agent_name,omitempty"` + LookbackHours int `json:"lookback_hours,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + MaxTraces int `json:"max_traces,omitempty"` } // EvalRunInputMessages describes how input messages are constructed from dataset items. @@ -399,16 +401,16 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa // NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. // -// The service reads them from Application Insights and defaults to the last -// seven days, so a zero window is left off rather than sent as an epoch. -func NewTracesDataSource(agentName string, start, end time.Time, maxTraces int) *EvalRunDataSource { +// The window is a lookback in hours. The service's own field is +// `lookback_hours` and it has no start bound: a `start_time` is accepted and +// dropped, leaving the default seven days in place, so the conversion happens +// here rather than being left to look like it worked. +func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, maxTraces int) *EvalRunDataSource { ds := &EvalRunDataSource{ - Type: EvalRunDataSourceTypeTraces, - AgentName: agentName, - MaxTraces: maxTraces, - } - if !start.IsZero() { - ds.StartTime = start.Unix() + Type: EvalRunDataSourceTypeTraces, + AgentName: agentName, + LookbackHours: lookbackHours, + MaxTraces: maxTraces, } if !end.IsZero() { ds.EndTime = end.Unix() From 7905c97ff2f02666a433a75289264bebb4079285 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 11:47:59 +0300 Subject: [PATCH 040/114] feat(azure.ai.evaluations): evaluate stored responses by id The second M4 scenario listed as awaiting service support and reachable today. `run --response-id` evaluates responses the project already holds, so a run that already happened can be scored without replaying it. The shape is not the obvious one. There is no list of ids on the data source: the ids travel as ordinary JSONL rows and a data_mapping points the service at the field holding each one, which is what lets it retrieve the chat history behind the response. Every guess at a plainer shape was rejected with the same unhelpful "Item generation source content is empty", so this came from reading the service contract instead. Verified live: three stored responses evaluated, three passed, and the stored payload matched what was sent field for field. --- .../azure.ai.evaluations/internal/cmd/run.go | 9 ++++ .../internal/cmd/traces_test.go | 28 +++++++++++++ .../internal/pkg/eval_api/models.go | 41 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 436b94ae456..355bc06b678 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -54,6 +54,8 @@ func buildRunCommand(use, short string) *cobra.Command { fromTraces bool traceWindow string maxTraces int + responseIDs []string + maxTurns int wait bool endpointFlg string ) @@ -101,6 +103,8 @@ func buildRunCommand(use, short string) *cobra.Command { // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource switch { + case len(responseIDs) > 0: + dataSource = eval_api.NewResponsesDataSource(responseIDs, maxTurns) case fromTraces: dataSource, err = buildTracesDataSource(ctx, ec, group, evalID, traceWindow, maxTraces) case group == nil: @@ -181,6 +185,11 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().StringVar(&traceWindow, "trace-window", "", "How far back to read traces, for example 7d. Defaults to the service's window.") cmd.Flags().IntVar(&maxTraces, "max-traces", 0, "Cap the traces evaluated.") + cmd.Flags().StringSliceVar(&responseIDs, "response-id", nil, + "Evaluate stored responses by id instead of the dataset; repeatable.") + cmd.Flags().IntVar(&maxTurns, "max-turns", 0, + "Turns of chat history to pull back per response. Defaults to the service's limit.") + cmd.MarkFlagsMutuallyExclusive("from-traces", "response-id") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go index 75a175d6d3f..078a62435b9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go @@ -74,6 +74,34 @@ func TestRunFailureMessage(t *testing.T) { assert.Empty(t, nilRun.Failure()) } +// The ids travel as ordinary JSONL rows with a mapping pointing at the field +// that holds each one; that is how the service finds the chat history. +func TestNewResponsesDataSource(t *testing.T) { + ds := eval_api.NewResponsesDataSource([]string{"resp_a", "resp_b"}, 10) + assert.Equal(t, eval_api.EvalRunDataSourceTypeResponses, ds.Type) + require.NotNil(t, ds.ItemGenerationParams) + assert.Equal(t, "response_retrieval", ds.ItemGenerationParams.Type) + assert.Equal(t, 10, ds.ItemGenerationParams.MaxNumTurns) + assert.Equal(t, + map[string]string{"response_id": "{{item.response_id}}"}, + ds.ItemGenerationParams.DataMapping) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.Contains(t, body, `"response_id":"resp_a"`) + assert.Contains(t, body, `"response_id":"resp_b"`) + assert.NotContains(t, body, "agent_name", "responses carry no agent") +} + +// An unset turn limit is left to the service rather than sent as zero. +func TestNewResponsesDataSource_OmitsAnUnsetTurnLimit(t *testing.T) { + ds := eval_api.NewResponsesDataSource([]string{"resp_a"}, 0) + raw, err := json.Marshal(ds) + require.NoError(t, err) + assert.NotContains(t, string(raw), "max_num_turns") +} + func TestRenderRun_ShowsTheFailureReason(t *testing.T) { var buf bytes.Buffer run := &eval_api.OpenAIEvalRun{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 6d2f4c2c1c7..af5f880e268 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -321,6 +321,10 @@ const ( // a dataset. The service reads them from Application Insights, so the agent // must be emitting gen_ai.input.messages / gen_ai.output.messages. EvalRunDataSourceTypeTraces EvalRunDataSourceType = "azure_ai_traces" + + // EvalRunDataSourceTypeResponses evaluates responses the project already + // stored, addressed by id. + EvalRunDataSourceTypeResponses EvalRunDataSourceType = "azure_ai_responses" ) // EvalRunDataContentType defines the source type for eval run data content. @@ -345,6 +349,18 @@ type EvalRunDataSource struct { LookbackHours int `json:"lookback_hours,omitempty"` EndTime int64 `json:"end_time,omitempty"` MaxTraces int `json:"max_traces,omitempty"` + + // Responses only. + ItemGenerationParams *ItemGenerationParams `json:"item_generation_params,omitempty"` +} + +// ItemGenerationParams says how the service should turn a source into the items +// it evaluates. +type ItemGenerationParams struct { + Type string `json:"type"` + MaxNumTurns int `json:"max_num_turns,omitempty"` + DataMapping map[string]string `json:"data_mapping,omitempty"` + Source *EvalRunDataContent `json:"source,omitempty"` } // EvalRunInputMessages describes how input messages are constructed from dataset items. @@ -418,6 +434,31 @@ func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, max return ds } +// NewResponsesDataSource evaluates responses the project already stored. +// +// The ids travel as ordinary JSONL rows and a data_mapping points the service +// at the field holding each one, which is how it retrieves the chat history +// behind the response. +func NewResponsesDataSource(responseIDs []string, maxTurns int) *EvalRunDataSource { + rows := make([]map[string]any, 0, len(responseIDs)) + for _, id := range responseIDs { + rows = append(rows, map[string]any{"item": map[string]any{"response_id": id}}) + } + + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeResponses, + ItemGenerationParams: &ItemGenerationParams{ + Type: "response_retrieval", + MaxNumTurns: maxTurns, + DataMapping: map[string]string{"response_id": "{{item.response_id}}"}, + Source: &EvalRunDataContent{ + Type: EvalRunDataContentTypeFileContent, + Content: rows, + }, + }, + } +} + // SetFileContent sets the data source to use inline file content. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ From 1d195f56a82fb40a93ce177ec2565b43d0edef82 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:01:03 +0300 Subject: [PATCH 041/114] feat(azure.ai.evaluations): evaluate a model deployment directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit target.type: model evaluates a deployment with no agent in front of it. The config already rejected it by name, and the test asserting that used "model" as its example of an unsupported type, so the gap was recorded twice as deliberate. The service supports it. What makes it more than a new enum value is that the criteria have to change with it. An agent returns output_items and tool calls; a model answers as plain text under output_text. Binding the agent shape to a model run does not fail at validation, it fails per row at execution with "Missing inputs for line 1: data.sample.tool_definitions, data.sample.output_items" — the group looks well formed and every row errors. So the sample bindings are chosen from the target kind rather than from whether a target exists at all, and a model target binds response alone. Verified end to end: a model-target group deployed with response bound to {{sample.output_text}}, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency. --- .../internal/cmd/build.go | 31 +++++- .../internal/cmd/model_target_test.go | 98 +++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 9 +- .../internal/pkg/eval_api/models.go | 36 ++++++- .../internal/project/eval_config.go | 12 ++- .../internal/project/eval_config_test.go | 2 +- 6 files changed, 175 insertions(+), 13 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 803eed1c4e0..2fe71d974a5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -35,6 +35,25 @@ var sampleBindings = map[string]string{ "tool_definitions": "{{sample.tool_definitions}}", } +// modelSampleBindings are what a model target produces. A model answers as +// plain text and calls no tools, so binding an agent's richer output would +// leave the evaluator waiting on fields the run never produces. +var modelSampleBindings = map[string]string{ + "response": "{{sample.output_text}}", +} + +// sampleBindingsFor returns the run-time bindings a target of this kind can +// satisfy. An empty target kind means nothing is invoked, so nothing is bound. +func sampleBindingsFor(targetType string) map[string]string { + switch targetType { + case project.TargetTypeAgent: + return sampleBindings + case project.TargetTypeModel: + return modelSampleBindings + } + return nil +} + // legacyInputs is the mapping used when the service publishes no schema for an // evaluator, which is the case for freshly uploaded custom evaluators. It // matches the agent-target shape. @@ -116,7 +135,7 @@ func selectLevelFields(accepted, required []string, level string) []string { func planCriterion( ref evalcore.EvaluatorRef, schema *eval_api.EvaluatorSummary, - hasTarget bool, + targetBindings map[string]string, datasetColumns map[string]bool, evalModel string, level string, @@ -138,7 +157,7 @@ func planCriterion( } for _, field := range accepted { - if binding, ok := sampleBindings[field]; ok && hasTarget { + if binding, ok := targetBindings[field]; ok { plan.dataMapping[field] = binding continue } @@ -233,9 +252,15 @@ func buildEvalGroupRequest( ) (*eval_api.CreateOpenAIEvalRequest, error) { metadata := map[string]string{} hasTarget := group.Target != nil && group.Target.Name != "" + targetType := "" if hasTarget { metadata["azd_agent"] = group.Target.Name + targetType = group.Target.Type + if targetType == "" { + targetType = project.TargetTypeAgent + } } + targetBindings := sampleBindingsFor(targetType) metadata["azd_eval_group"] = group.Name // The create request has no description field, so the group's own // description rides in metadata rather than being dropped. @@ -263,7 +288,7 @@ func buildEvalGroupRequest( schema = &eval_api.EvaluatorSummary{Name: ref.Name} } - plan, err := planCriterion(ref, schema, hasTarget, datasetColumns, evalModel, level) + plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, evalModel, level) if err != nil { return nil, err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go new file mode 100644 index 00000000000..2a458a5778c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A model answers as plain text and calls no tools. Binding an agent's richer +// output would leave the evaluator waiting on fields the run never produces, +// which the service reports as a missing input rather than a mapping mistake. +func TestSampleBindingsFor(t *testing.T) { + agent := sampleBindingsFor(project.TargetTypeAgent) + assert.Equal(t, "{{sample.output_items}}", agent["response"]) + assert.Contains(t, agent, "tool_calls") + assert.Contains(t, agent, "tool_definitions") + + model := sampleBindingsFor(project.TargetTypeModel) + assert.Equal(t, "{{sample.output_text}}", model["response"]) + assert.NotContains(t, model, "tool_calls", "a model calls no tools") + assert.NotContains(t, model, "tool_definitions") + + assert.Nil(t, sampleBindingsFor(""), "with nothing invoked, nothing is bound") +} + +// The criteria a group sends depend on what it targets. +func TestBuildEvalGroupRequest_BindsByTargetKind(t *testing.T) { + schemas := map[string]*eval_api.EvaluatorSummary{ + "builtin.coherence": { + Name: "builtin.coherence", + Definition: &eval_api.EvaluatorContract{ + DataSchema: &eval_api.JSONSchema{ + Required: []string{"query", "response"}, + Properties: map[string]any{ + "query": map[string]any{"type": "string"}, + "response": map[string]any{"type": "string"}, + }, + }, + }, + }, + } + + for _, tc := range []struct { + targetType string + want string + }{ + {project.TargetTypeAgent, "{{sample.output_items}}"}, + {project.TargetTypeModel, "{{sample.output_text}}"}, + } { + t.Run(tc.targetType, func(t *testing.T) { + group := &project.EvalGroup{ + Name: "quality", + Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.coherence"}}, + Target: &project.Target{Type: tc.targetType, Name: "thing"}, + } + req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + require.NoError(t, err) + require.Len(t, req.TestingCriteria, 1) + assert.Equal(t, tc.want, req.TestingCriteria[0].DataMapping["response"]) + assert.Equal(t, "{{item.query}}", req.TestingCriteria[0].DataMapping["query"]) + }) + } +} + +// The target the run posts has to match what the group's criteria expect. +func TestNewModelTargetDataSource(t *testing.T) { + ds := eval_api.NewModelTargetDataSource("gpt-4.1-nano") + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_model", ds.Target.Type) + assert.Equal(t, "gpt-4.1-nano", ds.Target.Model) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + body := string(raw) + assert.Contains(t, body, `"model":"gpt-4.1-nano"`) + assert.NotContains(t, body, `"name"`, "a model target is addressed by deployment, not name") + assert.NotContains(t, body, "tool_descriptions", "a model calls no tools") +} + +func TestNewAgentTargetDataSource_StillSendsAgentFields(t *testing.T) { + ds := eval_api.NewAgentTargetDataSource("support-agent", nil) + require.NotNil(t, ds.Target) + assert.Equal(t, "azure_ai_agent", ds.Target.Type) + assert.Equal(t, "support-agent", ds.Target.Name) + + raw, err := json.Marshal(ds) + require.NoError(t, err) + assert.NotContains(t, string(raw), `"model"`) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 355bc06b678..1d3d20b53cb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -386,10 +386,15 @@ func buildRunDataSource( ) (*eval_api.EvalRunDataSource, error) { if group == nil || group.Target == nil { return nil, fmt.Errorf( - "the eval group must declare target.type: agent so the run knows what to invoke") + "the eval group must declare a target so the run knows what to invoke") } - ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + var ds *eval_api.EvalRunDataSource + if group.Target.Type == project.TargetTypeModel { + ds = eval_api.NewModelTargetDataSource(group.Target.Name) + } else { + ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + } if group.Dataset == "" { return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index af5f880e268..8e1ba82f1bf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -376,12 +376,14 @@ type EvalRunMessageTemplate struct { Type string `json:"type"` } -// EvalRunTarget describes the agent target for completions. +// EvalRunTarget describes what the run invokes: an agent by name, or a model +// deployment directly. Only the fields belonging to Type are sent. type EvalRunTarget struct { Type string `json:"type"` - Name string `json:"name"` - Version *string `json:"version"` - ToolDescriptions []string `json:"tool_descriptions"` + Name string `json:"name,omitempty"` + Version *string `json:"version,omitempty"` + ToolDescriptions []string `json:"tool_descriptions,omitempty"` + Model string `json:"model,omitempty"` } // EvalRunDataContent holds the source reference within an EvalRunDataSource. @@ -434,6 +436,32 @@ func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, max return ds } +// NewModelTargetDataSource sends the dataset's questions straight to a model +// deployment, with no agent in front of it. +// +// The model answers as plain text, so a group evaluating one has to bind its +// response to {{sample.output_text}} rather than the richer output an agent +// produces. +func NewModelTargetDataSource(model string) *EvalRunDataSource { + return &EvalRunDataSource{ + Type: EvalRunDataSourceTypeAgentTarget, + InputMessages: &EvalRunInputMessages{ + Type: "template", + Template: []EvalRunMessageTemplate{ + { + Role: "user", + Content: "{{item.query}}", + Type: "message", + }, + }, + }, + Target: &EvalRunTarget{ + Type: "azure_ai_model", + Model: model, + }, + } +} + // NewResponsesDataSource evaluates responses the project already stored. // // The ids travel as ordinary JSONL rows and a data_mapping points the service diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 37a0777448a..13b1eba685f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -58,6 +58,11 @@ type Target struct { const TargetTypeAgent = "agent" +// TargetTypeModel evaluates a model deployment directly, with no agent in +// front of it. A model answers as plain text, so a group targeting one binds +// its response differently from a group targeting an agent. +const TargetTypeModel = "model" + // Options are run settings carried on the group. type Options struct { EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` @@ -145,10 +150,11 @@ func (c *EvalConfig) Validate() error { i, g.Name, ref.Name, evalcore.BuiltinPrefix) } } - if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { + if g.Target != nil && g.Target.Type != "" && + g.Target.Type != TargetTypeAgent && g.Target.Type != TargetTypeModel { return fmt.Errorf( - "evalGroups[%d] (%s): target.type %q is not supported; only %q is available today", - i, g.Name, g.Target.Type, TargetTypeAgent) + "evalGroups[%d] (%s): target.type %q is not supported; use %q or %q", + i, g.Name, g.Target.Type, TargetTypeAgent, TargetTypeModel) } if g.Options != nil { switch g.Options.EvaluationLevel { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 172154ce714..177638d5507 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -148,7 +148,7 @@ func TestValidate_Rejects(t *testing.T) { }, { name: "unsupported target type", - body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: model\n", + body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", wantErr: "is not supported", }, { From fbf532fdc2bdccab20d25c10161aa69f0ba547f6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:16:31 +0300 Subject: [PATCH 042/114] fix(azure.ai.evaluations): run the datasets that are already registered A group whose dataset has no local source could not run at all. The run sent the dataset name as a file_id, and a file_id means an uploaded file, so the service answered "invalid data source file ids". Every test until now used a local source, which is why the path stayed unexercised. Registered datasets are now fetched and their rows sent inline, the same as a local file. That also gives --max-samples the meaning it always claimed: it was documented as ignored for registered datasets, because a file reference carries no row limit and the service cannot narrow one. Fetching the rows is what makes narrowing possible, so the flag now behaves the same either way and the caveat is gone from its help. Found while implementing the last M4 scenario, subsetting a registered dataset. The subsetting worked on the first try; the run underneath it did not, which is the part worth having found. Verified live against a published dataset: the whole set scores 2 passed / 1 failed, and --max-samples 2 scores 2 rows. --- .../internal/cmd/dataset_rows_test.go | 53 +++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 88 +++++++++++++++++-- .../internal/cmd/schedule.go | 4 +- 3 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go new file mode 100644 index 00000000000..dd25bcf022b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_rows_test.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadJSONLBytes(t *testing.T) { + content := []byte( + "{\"query\":\"a\"}\n" + + "\n" + // blank lines are skipped, not treated as rows + "{\"query\":\"b\"}\n" + + " {\"query\":\"c\"} \n") + + all, err := readJSONLBytes(content, 0) + require.NoError(t, err) + require.Len(t, all, 3) + assert.Equal(t, "a", all[0]["query"]) + assert.Equal(t, "c", all[2]["query"], "surrounding whitespace is not part of the row") +} + +// The limit is what makes --max-samples mean the same thing for a published +// dataset as for a local file. +func TestReadJSONLBytes_StopsAtTheLimit(t *testing.T) { + content := []byte("{\"n\":1}\n{\"n\":2}\n{\"n\":3}\n") + + two, err := readJSONLBytes(content, 2) + require.NoError(t, err) + require.Len(t, two, 2) + assert.EqualValues(t, 1, two[0]["n"]) + assert.EqualValues(t, 2, two[1]["n"]) + + // A limit larger than the file is not an error. + more, err := readJSONLBytes(content, 99) + require.NoError(t, err) + assert.Len(t, more, 3) +} + +func TestReadJSONLBytes_ReportsTheOffendingLine(t *testing.T) { + _, err := readJSONLBytes([]byte("{\"n\":1}\nnot json\n"), 0) + require.ErrorContains(t, err, "line 2") +} + +func TestReadJSONLBytes_EmptyIsNotAnError(t *testing.T) { + items, err := readJSONLBytes([]byte("\n\n"), 0) + require.NoError(t, err) + assert.Empty(t, items, "the caller decides whether no rows is a problem") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 1d3d20b53cb..39f98b811d1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -5,15 +5,18 @@ package cmd import ( "bufio" + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "strings" "time" + "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" @@ -110,8 +113,8 @@ func buildRunCommand(use, short string) *cobra.Command { case group == nil: dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) default: - dataSource, err = buildRunDataSource( - group, configPath, resolveMaxSamples(maxSamples, group)) + dataSource, err = ec.buildRunDataSource( + ctx, group, configPath, resolveMaxSamples(maxSamples, group)) } if err != nil { return err @@ -179,7 +182,7 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().StringVar(&level, "level", "", "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - "Cap the rows sent from a local dataset file. Ignored for registered datasets.") + "Cap the rows sent from the dataset.") cmd.Flags().BoolVar(&fromTraces, "from-traces", false, "Evaluate the agent's recorded traces instead of the dataset.") cmd.Flags().StringVar(&traceWindow, "trace-window", "", @@ -379,7 +382,8 @@ func buildTracesDataSource( // buildRunDataSource binds the dataset to the run. The eval group carries no // dataset today, so it is supplied here. -func buildRunDataSource( +func (ec *evalContext) buildRunDataSource( + ctx context.Context, group *project.EvalGroup, configPath string, maxSamples int, @@ -400,10 +404,17 @@ func buildRunDataSource( return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) } - // A local source is sent inline; anything else is a registered dataset. + // A local source is read from disk; anything else is already registered and + // has to be fetched. Either way the rows are sent inline, because a run's + // file_id means an uploaded file and a dataset name is not one: sending the + // name is rejected with "invalid data source file ids". localPath := localDatasetPath(configPath, group) if localPath == "" { - ds.SetFileID(group.Dataset) + items, err := ec.readRegisteredDataset(ctx, group.Dataset, maxSamples) + if err != nil { + return nil, err + } + ds.SetFileContent(items) return ds, nil } @@ -418,6 +429,49 @@ func buildRunDataSource( return ds, nil } +// readRegisteredDataset fetches a published dataset's rows, optionally keeping +// only the first n. +// +// The rows have to be fetched because a run cannot reference a dataset by +// name: `file_id` means an uploaded file, and passing a dataset name there is +// rejected. Fetching also makes --max-samples mean the same thing whether the +// dataset is local or published, which a file reference could not — that +// source carries no row limit. +func (ec *evalContext) readRegisteredDataset( + ctx context.Context, + name string, + maxSamples int, +) ([]map[string]any, error) { + version := ec.getEnvValue(ctx, envKeyDatasetVersion) + if version == "" { + versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", name, err) + } + if versions != nil { + version = dataset_api.LatestVersion(versions.Value) + } + } + if version == "" { + return nil, fmt.Errorf("dataset %q has no versions to read", name) + } + + content, err := ec.datasetClient.DownloadDatasetContent( + ctx, name, version, ProjectEndpointAPIVersion) + if err != nil { + return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + } + + items, err := readJSONLBytes(content, maxSamples) + if err != nil { + return nil, fmt.Errorf("reading dataset %q version %s: %w", name, version, err) + } + if len(items) == 0 { + return nil, fmt.Errorf("dataset %q version %s has no rows", name, version) + } + return items, nil +} + // datasetColumns reports the columns a group's dataset provides, so criteria // bind only to fields that exist and a missing required field is caught // locally rather than as a service rejection. @@ -472,8 +526,24 @@ func readJSONL(path string, limit int) ([]map[string]any, error) { } defer f.Close() + items, err := scanJSONL(f, limit) + if err != nil { + return nil, fmt.Errorf("reading dataset %q: %w", path, err) + } + return items, nil +} + +// readJSONLBytes parses JSONL already in memory, which is how a registered +// dataset arrives. +func readJSONLBytes(content []byte, limit int) ([]map[string]any, error) { + return scanJSONL(bytes.NewReader(content), limit) +} + +// scanJSONL reads rows until the limit is reached, so a subset costs only the +// rows it needs to parse. +func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { var items []map[string]any - scanner := bufio.NewScanner(f) + scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) line := 0 for scanner.Scan() { @@ -484,7 +554,7 @@ func readJSONL(path string, limit int) ([]map[string]any, error) { } var row map[string]any if err := json.Unmarshal([]byte(text), &row); err != nil { - return nil, fmt.Errorf("%s line %d is not valid JSON: %w", path, line, err) + return nil, fmt.Errorf("line %d is not valid JSON: %w", line, err) } items = append(items, row) if limit > 0 && len(items) >= limit { @@ -492,7 +562,7 @@ func readJSONL(path string, limit int) ([]map[string]any, error) { } } if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("reading dataset %q: %w", path, err) + return nil, err } return items, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index e0afd5c468b..96f5429e060 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -121,8 +121,8 @@ func newScheduleSetCommand() *cobra.Command { if err != nil { return err } - dataSource, err = buildRunDataSource( - group, configPath, resolveMaxSamples(maxSamples, group)) + dataSource, err = ec.buildRunDataSource( + ctx, group, configPath, resolveMaxSamples(maxSamples, group)) if err != nil { return err } From 09b7e5f965cfae62887d29abc8eb3454eae9d7ec Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:27:05 +0300 Subject: [PATCH 043/114] fix(azure.ai.evaluations): stop a second eval group from taking the first one's id A config with two eval groups aliased them. Resolved ids were kept under one shared EVAL_GROUP_ID, so on the second deploy the first group read the id the second had left there, confirmed that group existed, and adopted it. Both declarations then pointed at the same group, and running one scored the other's criteria against its own dataset. Nothing failed; the results were just wrong. The first deploy hid it, because the env is not readable back within the same deploy, so both groups were created correctly and only the cache was left crossed. It needed a second deploy to surface, which is why the single-group testing everything else used never reached it. Ids are now keyed by declaration name, as the fingerprints already were. EVAL_GROUP_ID is still written as the last-deployed group, which is what the commands fall back to when a config names only one. Same fix for the dataset version read while narrowing a registered dataset: it was reading the shared EVAL_DATASET_VERSION, which with two datasets is whichever was published last. Verified live: two groups keep distinct ids across repeated deploys, and each runs its own criteria. --- .../internal/cmd/envkeys_test.go | 37 +++++++++++++++++++ .../internal/cmd/reconciler.go | 15 +++++++- .../azure.ai.evaluations/internal/cmd/run.go | 7 ++-- 3 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go new file mode 100644 index 00000000000..be8090b9d06 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Ids are per declaration. A shared key works only while a config has one +// group: with two, the second deploy finds the first's id cached, confirms it +// exists, and hands it back for the wrong group — so group A silently scores +// group B's criteria. +func TestIDKey_IsPerName(t *testing.T) { + a := idKey("evalgroup", "quality-a") + b := idKey("evalgroup", "quality-b") + + assert.NotEqual(t, a, b, "two groups must not share an id key") + assert.Contains(t, a, "QUALITY_A") + assert.True(t, len(a) > 3 && a[len(a)-3:] == "_ID") +} + +// Names that are not valid env identifiers still have to produce distinct, +// stable keys. +func TestIDKey_NormalizesNames(t *testing.T) { + assert.Equal(t, idKey("evalgroup", "my group"), idKey("evalgroup", "my-group"), + "characters that cannot appear in an env name normalize the same way") + assert.NotEqual(t, idKey("evalgroup", "a"), idKey("dataset", "a"), + "the kind keeps different resources apart") +} + +// The id and version keys for the same declaration must not collide. +func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { + assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index a00ce08711b..b645c8ce6ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -220,13 +220,14 @@ func (r *evalReconciler) EnsureEvalGroup( recreate = true } - cached := r.ec.getEnvValue(ctx, envKeyEvalGroupID) + cached := r.ec.getEnvValue(ctx, idKey("evalgroup", group.Name)) if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { // Record the digest on reuse as well, otherwise a group deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, cached) return cached, nil } } @@ -244,6 +245,9 @@ func (r *evalReconciler) EnsureEvalGroup( return "", err } _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID) + // EVAL_GROUP_ID stays the last-deployed group, which is what the commands + // fall back to when a config names only one. _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) return created.ID, nil } @@ -314,3 +318,12 @@ func versionFromRaw(raw []byte, fallback string) string { func versionKey(kind, name string) string { return project.FingerprintKey(kind, name) + "_VERSION" } + +// idKey names the env entry holding a resolved id. +// +// Ids are per declaration. A single shared key works only while a config has +// one group: with two, the second deploy finds the first's id cached, confirms +// it exists, and hands it back for the wrong group. +func idKey(kind, name string) string { + return project.FingerprintKey(kind, name) + "_ID" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 39f98b811d1..e0d187856b5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -223,7 +223,7 @@ func (ec *evalContext) resolveEvalGroupID( return group.ID, nil } - if cached := ec.getEnvValue(ctx, envKeyEvalGroupID); cached != "" { + if cached := ec.getEnvValue(ctx, idKey("evalgroup", group.Name)); cached != "" { // Confirm it still exists; a deleted group should fall through to create. if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { return cached, nil @@ -258,9 +258,10 @@ func (ec *evalContext) resolveEvalGroupID( if err != nil { return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) } - if err := ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID); err != nil { + if err := ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID); err != nil { fmt.Fprintf(out, "warning: %v\n", err) } + _ = ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) return created.ID, nil } @@ -442,7 +443,7 @@ func (ec *evalContext) readRegisteredDataset( name string, maxSamples int, ) ([]map[string]any, error) { - version := ec.getEnvValue(ctx, envKeyDatasetVersion) + version := ec.getEnvValue(ctx, versionKey("dataset", name)) if version == "" { versions, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) if err != nil { From d961d3bb0773f19a7912b8ba55c4000d38e633b5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:37:45 +0300 Subject: [PATCH 044/114] feat(azure.ai.evaluations): reach a group by name, and remember its run separately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finishes what the group-id fix started. Two more places assumed one group. Every command taking an eval-id could only be pointed at a group by its service id. With one group the cached id made that invisible; with two there was no way to ask for the other except by looking its id up. They now accept --eval-group, naming the group the way the config does. That exposed the next layer: the remembered run was also a single key, so it belonged to whichever group ran last. Asking group A for its latest results fetched group B's run id inside group A and returned 404 — the group resolved correctly and the run did not. Runs are now remembered per group. A remembered run that no longer resolves also stops being fatal. It is a convenience, so a stale one falls through to the group's current latest rather than failing; an explicit --run-id still reports what went wrong. Verified live with two groups: each resolves to its own run and its own criteria, and an undeployed name is refused by name. --- .../internal/cmd/compare.go | 4 +- .../internal/cmd/results.go | 59 +++++++++++++++---- .../azure.ai.evaluations/internal/cmd/run.go | 4 ++ .../internal/cmd/run_ops.go | 16 +++-- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index 500f135b3fd..815f4ab47c6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -26,6 +26,7 @@ func newResultsCompareCommand() *cobra.Command { treatments []string displayName string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -42,7 +43,7 @@ func newResultsCompareCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -89,6 +90,7 @@ func newResultsCompareCommand() *cobra.Command { cmd.Flags().StringArrayVar(&treatments, "treatment", nil, "Run to measure, repeatable. Defaults to the most recent completed run.") cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 985a4b7b461..3a2f4d87f5a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -31,6 +31,7 @@ func newResultsShowCommand() *cobra.Command { failedOnly bool outFile string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -45,7 +46,7 @@ func newResultsShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -73,6 +74,7 @@ func newResultsShowCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -83,6 +85,7 @@ func newResultsExportCommand() *cobra.Command { format string outFile string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -102,7 +105,7 @@ func newResultsExportCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -132,24 +135,53 @@ func newResultsExportCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// resolveEvalID takes the eval group id from the argument, falling back to the -// id cached in the azd environment. -func resolveEvalID(cmd *cobra.Command, ec *evalContext, args []string) (string, error) { +// resolveEvalID takes the eval group id from the argument, from a group named +// with --eval-group, or from the id cached in the azd environment. +// +// The cached id is the last group deployed, which is unambiguous only while a +// config declares one. --eval-group is how the others are reached without +// having to know their service ids. +func resolveEvalID( + cmd *cobra.Command, + ec *evalContext, + args []string, + groupName string, +) (string, error) { if len(args) > 0 && args[0] != "" { return args[0], nil } + + if groupName != "" { + if id := ec.getEnvValue(cmd.Context(), idKey("evalgroup", groupName)); id != "" { + return id, nil + } + return "", fmt.Errorf( + "eval group %q has no id recorded in this environment; deploy it first, "+ + "or pass its id directly", groupName) + } + if cached := ec.getEnvValue(cmd.Context(), envKeyEvalGroupID); cached != "" { return cached, nil } return "", fmt.Errorf( - "no eval group id given; pass it as an argument or set %s in the azd environment", + "no eval group id given; pass it as an argument, name one with --eval-group, "+ + "or set %s in the azd environment", envKeyEvalGroupID) } +// addEvalGroupFlag registers the flag that names a group from the config, so +// every command taking an eval-id can reach a group by the name its author +// used. +func addEvalGroupFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval-group", "", + "Name a group from the config instead of passing its id.") +} + // latestOrNamedRun returns the named run, or the most recent one for the group. func (ec *evalContext) latestOrNamedRun( cmd *cobra.Command, @@ -157,17 +189,22 @@ func (ec *evalContext) latestOrNamedRun( ) (*eval_api.OpenAIEvalRun, error) { ctx := cmd.Context() + // The remembered run is per group. A single shared one belongs to whichever + // group ran last, and asking another group for it returns 404 rather than + // that group's own latest run. if runID == "" { - if cached := ec.getEnvValue(ctx, envKeyEvalRunID); cached != "" { - runID = cached - } + runID = ec.getEnvValue(ctx, idKey("evalrun", evalID)) } if runID != "" { run, err := ec.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) - if err != nil { + if err == nil { + return run, nil + } + // A remembered run that no longer resolves is not worth failing on: + // fall through to whatever the group has now. + if cmd.Flag("run-id") != nil && cmd.Flag("run-id").Changed { return nil, fmt.Errorf("reading run %s: %w", runID, err) } - return run, nil } list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e0d187856b5..31299be89de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -142,6 +142,10 @@ func buildRunCommand(use, short string) *cobra.Command { return fmt.Errorf("starting the evaluation run: %w", err) } + // Remembered per group as well as globally: a single shared key + // belongs to whichever group ran last, so another group asking for + // "the last run" would be handed one that is not its own. + _ = ec.setEnvValue(ctx, idKey("evalrun", evalID), run.ID) if err := ec.setEnvValue(ctx, envKeyEvalRunID, run.ID); err != nil { // Persisting the run id is a convenience for later commands. // Reported on stdout because azd does not surface an diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index c0eee5226cc..196a989a3bc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -25,7 +25,10 @@ func addRunSubcommands(cmd *cobra.Command) { } func newRunListCommand() *cobra.Command { - var endpointFlg string + var ( + endpointFlg string + groupName string + ) cmd := &cobra.Command{ Use: "list [eval-id]", @@ -39,7 +42,7 @@ func newRunListCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -64,6 +67,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -72,6 +76,7 @@ func newRunShowCommand() *cobra.Command { var ( runID string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -86,7 +91,7 @@ func newRunShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -113,6 +118,7 @@ func newRunShowCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -121,6 +127,7 @@ func newRunCancelCommand() *cobra.Command { var ( runID string endpointFlg string + groupName string ) cmd := &cobra.Command{ @@ -135,7 +142,7 @@ func newRunCancelCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args) + evalID, err := resolveEvalID(cmd, ec, args, groupName) if err != nil { return err } @@ -167,6 +174,7 @@ func newRunCancelCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") + addEvalGroupFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } From ca09620770a72d1bf74edbf7814f5e73fc61f3b0 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:46:56 +0300 Subject: [PATCH 045/114] fix(azure.ai.evaluations): publish the dataset version the config declares `version:` on a dataset was being passed to the upload helper whose argument is the version to count from, not the one to write. So a config declaring version "1.0" published 2.0, and every later edit climbed again. The author never got the version they asked for and nothing said so. The field also meant two different things depending on the branch taken: unchanged content resolved to the declared version, changed content published one above it. A declared version is now the version published. If it already exists and the local file differs, that is the author's decision to make, so the deploy stops and says to raise the pin or drop it. Without a pin nothing changes: each deploy still takes the next version. Verified live: version "1.0" on a new dataset publishes 1.0, and editing the file while still pinned to 1.0 fails with that instruction instead of quietly publishing 2.0. --- .../internal/cmd/reconciler.go | 22 +++++++++++++++ .../internal/pkg/dataset_api/operations.go | 22 ++++++++++++--- .../internal/pkg/dataset_api/version_test.go | 27 +++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index b645c8ce6ee..4b2c52401c8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -94,6 +94,28 @@ func (r *evalReconciler) EnsureDataset( dir = filepath.Dir(localPath) } + // A declared version is the version to publish, not one to count from. + // Reaching here means the content differs from what that version holds, so + // republishing over it would change a version the author pinned. + if decl.Version != "" { + ds, err := r.ec.datasetClient.UploadVersion( + ctx, decl.Name, decl.Version, dir, ProjectEndpointAPIVersion, + ) + if err != nil { + if dataset_api.IsVersionConflict(err) { + return "", false, fmt.Errorf( + "dataset %q version %s already exists and the local file differs from it. "+ + "Raise `version:` to publish the change, or drop it to let each "+ + "deploy take the next version", + decl.Name, decl.Version) + } + return "", false, err + } + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("dataset", decl.Name), ds.Version) + return ds.Version, true, nil + } + // UploadNextVersion discovers the currently registered version when none is // declared, so the upload does not restart at 1.0 and collide. ds, err := r.ec.datasetClient.UploadNextVersion( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index be1e3e03e5f..c285c19a450 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -107,9 +107,8 @@ func (c *DatasetClient) UploadNextVersion( if currentVersion == "" { currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) } - ds, err := c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) - if err == nil || !isVersionConflict(err) { + if err == nil || !IsVersionConflict(err) { return ds, err } @@ -136,7 +135,7 @@ func (c *DatasetClient) latestRegisteredVersion( // isVersionConflict reports whether the service refused the upload because the // target version already exists. -func isVersionConflict(err error) bool { +func IsVersionConflict(err error) bool { var respErr *azcore.ResponseError if !errors.As(err, &respErr) { return false @@ -156,13 +155,28 @@ func (c *DatasetClient) UploadNewVersion( currentVersion string, localDir string, apiVersion string, +) (*Dataset, error) { + return c.UploadVersion(ctx, name, NextVersion(currentVersion), localDir, apiVersion) +} + +// UploadVersion publishes the dataset at exactly this version. +// +// Separate from UploadNewVersion because its parameter is the version to +// count from, not the one to write: passing "1.0" there publishes 2.0. An +// author who declares a version means that version. +func (c *DatasetClient) UploadVersion( + ctx context.Context, + name string, + version string, + localDir string, + apiVersion string, ) (*Dataset, error) { content, err := ReadFirstJSONLFile(localDir) if err != nil { return nil, fmt.Errorf("reading dataset from %s: %w", localDir, err) } - newVersion := NextVersion(currentVersion) + newVersion := version // Step 1: Start pending upload to get a SAS URI. pending, err := c.StartPendingUpload(ctx, name, newVersion, apiVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go index de822def3cd..052a63ae504 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/version_test.go @@ -39,3 +39,30 @@ func TestVersionGreaterIgnoresUnorderable(t *testing.T) { require.False(t, VersionGreater("", "1.0")) require.False(t, VersionGreater("1.0", "")) } + +// The two upload entry points read their version argument differently, and the +// difference is the whole point: UploadNewVersion counts from it, UploadVersion +// writes it. Passing "1.0" to the counting one publishes 2.0, which is not what +// an author who wrote version: "1.0" asked for. +func TestNextVersionCountsFromTheArgument(t *testing.T) { + if got := NextVersion("1.0"); got != "2.0" { + t.Fatalf("NextVersion(1.0) = %q, want 2.0", got) + } + if got := NextVersion("1"); got != "2.0" { + t.Fatalf("NextVersion(1) = %q, want 2.0", got) + } + // An unknown current version starts the sequence rather than guessing. + if got := NextVersion(""); got != "1.0" { + t.Fatalf("NextVersion(empty) = %q, want 1.0", got) + } +} + +func TestLatestVersionOrdersNumerically(t *testing.T) { + got := LatestVersion([]Dataset{{Version: "1.0"}, {Version: "10.0"}, {Version: "2.0"}}) + if got != "10.0" { + t.Fatalf("LatestVersion = %q, want 10.0 (numeric, not lexical)", got) + } + if LatestVersion(nil) != "" { + t.Fatal("LatestVersion(nil) should be empty") + } +} From 131f172043a4053d8eb43111ce239c4affaddbec Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 12:55:38 +0300 Subject: [PATCH 046/114] fix(azure.ai.evaluations): refuse an evaluator version the service will not honour An evaluator declaring both a source and a version published at whatever version came next and the group bound that, so the pin described a version nothing used. A config asking for version 7 deployed version 1 and said nothing. Unlike a dataset, the version here is not ours to choose: the service assigns it on publish. So there is nothing to honour, and the field is refused alongside a source rather than accepted and ignored. The message names both ways out, since either is a coherent thing to have meant: drop version to publish the file, or drop source to reference a version already published. Both remaining forms verified live: source alone publishes and then reports unchanged, version alone references what is on the project. --- .../internal/project/eval_config.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 13b1eba685f..2a12cb6bfab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -118,6 +118,17 @@ func (c *EvalConfig) Validate() error { if evaluators[e.Name] { return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) } + // The service assigns an evaluator's version on publish, so a declared + // one cannot be honoured alongside a source: the upload lands on + // whatever comes next and the group binds that, leaving the pin + // describing a version nothing uses. + if e.Source != "" && e.Version != "" { + return fmt.Errorf( + "evaluators[%d] (%s): `version` cannot be set with `source`, because the "+ + "service assigns the version when it publishes. Drop `version` to "+ + "publish this file, or drop `source` to reference a version already "+ + "on the project", i, e.Name) + } evaluators[e.Name] = true } From 1c42eb2032d55e50b2566806815b41c9e8824e68 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:08:13 +0300 Subject: [PATCH 047/114] fix(azure.ai.evaluations): read the built-in evaluators' published schemas Criteria are meant to be shaped from each evaluator's own contract. For built-ins they never were: the schemas were fetched with an unfiltered list, which returns only the project's own evaluators. Built-ins have to be asked for by type, so every one of them fell back to the legacy field list instead. Nothing looked wrong because that list is query, response, tool_calls, tool_definitions, which is exactly what the common evaluators want. It fails where an evaluator wants anything else. builtin.task_completion at conversation level wants messages, so its criterion was published with an empty data_mapping and no evaluation_level: a group that could not score a single row, and no error anywhere. Also drops the requirement that a group declare a target. A dataset holding both sides of an exchange has nothing to invoke, and the service runs it happily; refusing it was ours alone. Verified live: the same group now publishes messages bound to {{item.messages}} with evaluation_level conversation, and runs to 1 passed / 1 failed / 0 errored. Note for anyone reading the earlier verification: groups built from these schemas were confirmed accepted by the service, and that is still true. The service accepts an empty mapping. Accepted was never the same as correct. --- .../internal/cmd/build.go | 24 ++++++++++++++++--- .../azure.ai.evaluations/internal/cmd/run.go | 15 ++++++++---- .../internal/pkg/eval_api/models.go | 11 +++++++++ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 2fe71d974a5..c92ff0bccf3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -18,12 +18,30 @@ import ( // // A failure is deliberately not fatal: without schemas the builder falls back // to the agent-target shape, which is what it always used to send. +// evaluatorSchemas indexes the published contract of every evaluator a group +// can reference. +// +// Built-ins have to be asked for separately. An unfiltered list returns only +// the project's own evaluators, so relying on it leaves every built-in without +// a schema and falling back to legacyInputs — which happens to match +// query/response and so looks right for the common evaluators while quietly +// dropping the fields anything else needs. func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_api.EvaluatorSummary { - list, err := ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) - if err != nil { + index := map[string]*eval_api.EvaluatorSummary{} + + for _, filter := range []string{"", eval_api.EvaluatorTypeBuiltin} { + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) + if err != nil { + continue + } + for name, summary := range list.ByName() { + index[name] = summary + } + } + if len(index) == 0 { return nil } - return list.ByName() + return index } // sampleBindings are the fields an agent target produces at run time. Anything diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 31299be89de..d7a3bb0e3eb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -393,15 +393,20 @@ func (ec *evalContext) buildRunDataSource( configPath string, maxSamples int, ) (*eval_api.EvalRunDataSource, error) { - if group == nil || group.Target == nil { - return nil, fmt.Errorf( - "the eval group must declare a target so the run knows what to invoke") + if group == nil { + return nil, fmt.Errorf("no eval group to run") } + // A group with no target scores a dataset that already holds the exchange, + // so there is nothing to invoke. That is how recorded conversations are + // evaluated. var ds *eval_api.EvalRunDataSource - if group.Target.Type == project.TargetTypeModel { + switch { + case group.Target == nil || group.Target.Name == "": + ds = eval_api.NewDatasetOnlyDataSource() + case group.Target.Type == project.TargetTypeModel: ds = eval_api.NewModelTargetDataSource(group.Target.Name) - } else { + default: ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 8e1ba82f1bf..506fefd9578 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -325,6 +325,9 @@ const ( // EvalRunDataSourceTypeResponses evaluates responses the project already // stored, addressed by id. EvalRunDataSourceTypeResponses EvalRunDataSourceType = "azure_ai_responses" + + // EvalRunDataSourceTypeJSONL scores the rows as they are, invoking nothing. + EvalRunDataSourceTypeJSONL EvalRunDataSourceType = "jsonl" ) // EvalRunDataContentType defines the source type for eval run data content. @@ -436,6 +439,14 @@ func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, max return ds } +// NewDatasetOnlyDataSource scores the dataset as it stands, invoking nothing. +// +// Used when a group declares no target: the rows already hold both sides of +// the exchange, which is how a recorded conversation is evaluated. +func NewDatasetOnlyDataSource() *EvalRunDataSource { + return &EvalRunDataSource{Type: EvalRunDataSourceTypeJSONL} +} + // NewModelTargetDataSource sends the dataset's questions straight to a model // deployment, with no agent in front of it. // From ed62aeed1f824778f3ade07d90ba5af5f2012df7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:15:02 +0300 Subject: [PATCH 048/114] test(azure.ai.evaluations): cover the lookup that supplies evaluator schemas The built-in schemas were never reaching the builder in the shipping path, and the test that walks every built-in could not see it: it fetched the schemas itself with the Builtin filter and handed them in. It was building the input the product was failing to build, so the builder was verified against every published contract while the code that supplies them resolved none. That test now takes its schemas from the production lookup, so a break in supply breaks the test. Two more cover the lookup directly: one asserts built-ins resolve at all and reports how many the unfiltered listing returns, the other that a conversation-level evaluator binds its conversation field rather than producing the empty mapping that scores nothing. Checked by reverting the fix: the new test fails with "the unfiltered listing returns 0 of them", and all ten built-ins pass again with it restored. --- .../internal/cmd/build_live_test.go | 11 ++- .../internal/cmd/schemas_live_test.go | 96 +++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 0eefc25a34c..124b56c0249 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -64,11 +64,20 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { listed, err := client.ListEvaluators(ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) require.NoError(t, err) require.NotEmpty(t, listed.Value) - schemas := listed.ByName() + + // Deliberately the production lookup rather than the listing above. Taking + // the schemas straight from a filtered list is what let this test pass + // while the shipping path resolved none of them: it built the input the + // product was failing to build. + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) for _, summary := range listed.Value { summary := summary t.Run(summary.Name, func(t *testing.T) { + require.NotNil(t, schemas[summary.Name], + "the shipping lookup did not resolve %s", summary.Name) // Give the builder a dataset carrying every column the evaluator // accepts, so a rejection means the request shape is wrong rather // than the data being genuinely absent. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go new file mode 100644 index 00000000000..dd852e0a778 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cmd + +import ( + "context" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorSchemasIncludesBuiltins covers the function that supplies +// the schemas, rather than the builder that consumes them. +// +// The builder was already tested against every built-in, but the test fetched +// them itself with the Builtin filter. Production did not: it listed +// unfiltered, which returns only the project's own evaluators, so every +// built-in reached the builder with no schema at all. The builder was correct +// and the criteria were still wrong, and no test could see it because each one +// constructed the input production was failing to construct. +func TestLiveEvaluatorSchemasIncludesBuiltins(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + // The listing production used to rely on, to show what it omits. + unfiltered, err := client.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + require.NoError(t, err) + builtinsInUnfiltered := 0 + for _, e := range unfiltered.Value { + if eval_api.IsBuiltinEvaluator(e.Name) { + builtinsInUnfiltered++ + } + } + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas, "no evaluator schemas were resolved at all") + + builtins := 0 + for name, summary := range schemas { + if !eval_api.IsBuiltinEvaluator(name) { + continue + } + builtins++ + assert.NotNil(t, summary.DataSchema(), + "%s resolved without the contract the criteria are shaped from", name) + } + + require.NotZero(t, builtins, + "built-ins must be resolvable; the unfiltered listing returns %d of them, "+ + "so they have to be asked for by type", builtinsInUnfiltered) +} + +// The fields an evaluator declares are the ones its criterion has to bind, so +// a conversation-level evaluator must resolve to its conversation field. +func TestLiveConversationEvaluatorBindsMessages(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas) + + var name string + for n, summary := range schemas { + if eval_api.IsBuiltinEvaluator(n) && summary.SupportsLevel("conversation") { + if ds := summary.DataSchema(); ds != nil && ds.Accepts(conversationField) { + name = n + break + } + } + } + if name == "" { + t.Skip("no built-in advertises a conversation contract on this project") + } + + plan, err := planCriterion( + evalcore.EvaluatorRef{Name: name}, + schemas[name], + nil, // no target: the dataset holds both sides of the exchange + map[string]bool{conversationField: true}, + judge, + "conversation", + ) + require.NoError(t, err) + assert.Equal(t, "{{item."+conversationField+"}}", plan.dataMapping[conversationField], + "%s must bind its conversation field", name) + assert.NotEmpty(t, plan.dataMapping, "an empty mapping scores nothing") +} From 36b51c9a8b994b9dd21b8981bc3469b90347e48b Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:21:12 +0300 Subject: [PATCH 049/114] test(azure.ai.evaluations): pin the decisions that fail quietly Coverage across the decision-making code was thin in exactly the places this extension has been getting wrong: precedence rules and URI resolution, where a wrong answer produces no error at all. Precedence: which of a flag and a config option wins, for the sample cap and the evaluation level. options.max_samples was already parsed and dropped once, and nothing failed when it was; a test would have said so. Level filtering: that a conversation evaluator is sent messages and a turn one query and response, in both directions, and that a required field survives the filter so a real conflict still surfaces as a missing field rather than being reshaped away. URI resolution: the service spells these fields several ways and a URI read from the wrong spelling comes back empty rather than wrong, which is how the dataset URI went unbound the first time. Also that an upload's SAS-bearing URI and its plain finalize URI are read from their own places, since confusing them fails at different stages. --- .../internal/cmd/resolution_test.go | 74 ++++++++++++++++++ .../internal/pkg/dataset_api/uri_test.go | 75 +++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go new file mode 100644 index 00000000000..9bd441b0f4b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" +) + +// Precedence decides behaviour without announcing it, so a wrong answer here +// is silent. options.max_samples was parsed and dropped once already, which is +// what these lock down. +func TestResolveMaxSamples_Precedence(t *testing.T) { + withOptions := &project.EvalGroup{Options: &project.Options{MaxSamples: 25}} + + assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") + assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") + assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{}), "neither means no cap") + assert.Equal(t, 0, resolveMaxSamples(0, nil)) + assert.Equal(t, 7, resolveMaxSamples(7, nil), "a flag stands on its own") + + // Zero in config is absent, not a cap of zero: a cap of zero would send + // nothing at all. + assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{Options: &project.Options{MaxSamples: 0}})) +} + +func TestResolveLevel_Precedence(t *testing.T) { + withOptions := &project.EvalGroup{ + Options: &project.Options{EvaluationLevel: project.EvaluationLevelConversation}, + } + + assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), + "the flag wins over the config") + assert.Equal(t, project.EvaluationLevelConversation, resolveLevel("", withOptions)) + assert.Empty(t, resolveLevel("", &project.EvalGroup{}), "unset defers to the service default") + assert.Empty(t, resolveLevel("", nil)) +} + +// A group's target decides which run-time fields its criteria can bind. Getting +// this wrong passes validation and then errors on every row. +func TestSampleBindingsFor_UnknownTargetBindsNothing(t *testing.T) { + assert.Nil(t, sampleBindingsFor("prompt"), + "an unrecognised target must bind nothing rather than guess at agent fields") +} + +// The level filter is what keeps a conversation evaluator from being sent turn +// fields and the reverse. Both directions matter. +func TestSelectLevelFields_KeepsOnlyTheLevelsShape(t *testing.T) { + accepted := []string{"query", "response", "messages", "tool_definitions"} + + conv := selectLevelFields(accepted, nil, project.EvaluationLevelConversation) + assert.Contains(t, conv, "messages") + assert.NotContains(t, conv, "query") + assert.NotContains(t, conv, "response") + assert.Contains(t, conv, "tool_definitions", "fields outside the split are untouched") + + turn := selectLevelFields(accepted, nil, project.EvaluationLevelTurn) + assert.Contains(t, turn, "query") + assert.Contains(t, turn, "response") + assert.NotContains(t, turn, "messages") + + // An evaluator offering only one shape is left alone, whatever the level. + only := []string{"query", "response"} + assert.Equal(t, only, selectLevelFields(only, nil, project.EvaluationLevelConversation)) + + // A required field is never dropped: a genuine conflict has to surface as a + // missing-field error rather than being reshaped away. + kept := selectLevelFields(accepted, []string{"query"}, project.EvaluationLevelConversation) + assert.Contains(t, kept, "query", "a required field survives the level filter") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go new file mode 100644 index 00000000000..e6cc46c2e3d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/uri_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The service spells these fields inconsistently, and a URI read from the +// wrong spelling comes back empty rather than wrong — which is how the dataset +// URI went unbound the first time. +func TestDatasetResolvedBlobURI_AcceptsEitherSpelling(t *testing.T) { + cases := map[string]string{ + `{"dataUri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"data_uri":"https://x/y.jsonl"}`: "https://x/y.jsonl", + `{"blobUri":"https://x/b.jsonl"}`: "https://x/b.jsonl", + `{"contentUri":"https://x/c.jsonl"}`: "https://x/c.jsonl", + } + for body, want := range cases { + var ds Dataset + require.NoError(t, json.Unmarshal([]byte(body), &ds), body) + assert.Equal(t, want, ds.ResolvedBlobURI(), body) + } + + var none Dataset + require.NoError(t, json.Unmarshal([]byte(`{"name":"x"}`), &none)) + assert.Empty(t, none.ResolvedBlobURI(), + "no URI means the caller has to fetch a credential, not that the dataset is unreadable") +} + +// An upload needs the SAS-bearing URI to write to and the plain one to +// finalize with. Confusing them fails at different stages, so both are read +// from their own place. +func TestPendingUploadURIs(t *testing.T) { + var p PendingUploadResponse + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReference": { + "blobUri": "https://acct.blob.core.windows.net/container", + "credential": { "sasUri": "https://acct.blob.core.windows.net/container?sig=abc" } + } + }`), &p)) + + assert.Equal(t, "https://acct.blob.core.windows.net/container?sig=abc", p.ResolvedUploadURI(), + "the upload target carries the SAS") + assert.Equal(t, "https://acct.blob.core.windows.net/container", p.ResolvedBlobURI(), + "the finalize URI does not") + + var empty PendingUploadResponse + assert.Empty(t, empty.ResolvedUploadURI()) + assert.Empty(t, empty.ResolvedBlobURI()) +} + +// Credentials arrive in two shapes and the consumption one takes precedence, +// because that is the one scoped for reading. +func TestCredentialResolvedDownloadURI(t *testing.T) { + var c DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{ + "blobReferenceForConsumption": { "credential": { "sasUri": "https://acct/read?sig=r" } }, + "blobReference": { "credential": { "sasUri": "https://acct/write?sig=w" } } + }`), &c)) + assert.Equal(t, "https://acct/read?sig=r", c.ResolvedDownloadURI()) + + var legacy DatasetCredential + require.NoError(t, json.Unmarshal([]byte(`{"sas_uri":"https://acct/legacy?sig=l"}`), &legacy)) + assert.Equal(t, "https://acct/legacy?sig=l", legacy.ResolvedDownloadURI(), + "the flat spelling is still honoured") + + var none DatasetCredential + assert.Empty(t, none.ResolvedDownloadURI()) +} From b28dfaa7f142d73864ac8f84a86eda2f50b286ac Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:51:43 +0300 Subject: [PATCH 050/114] Honour a hand-set EVAL_GROUP_ID again for single-group configs Giving each eval group its own env entry stopped a second group from adopting the first one's id, but it also removed the only read of EVAL_GROUP_ID on the run path. Setting that key by hand is the documented way to point a config at a group created in the portal or by another tool, so the documented behaviour quietly stopped working: the override was ignored and a new group created. The shared key is read again, but only when the config declares a single group. With more than one there is no way to tell which group it refers to, which is what caused the aliasing in the first place. --- .../internal/cmd/envkeys_test.go | 17 ++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 27 +++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index be8090b9d06..dbda6180b23 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -35,3 +35,20 @@ func TestIDKey_NormalizesNames(t *testing.T) { func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) } + +// Setting EVAL_GROUP_ID by hand is the documented way to point a config at a +// group that already exists. It is also the key the extension writes itself, +// which is what let a second group adopt the first one's id — so it stays +// readable only where it cannot be ambiguous. Fixing the aliasing dropped this +// fallback entirely once, silently breaking the documented behaviour. +func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { + sole := groupIDKeys("quality", true) + assert.Equal(t, idKey("evalgroup", "quality"), sole[0], + "a group's own entry is preferred over the shared one") + assert.Contains(t, sole, envKeyEvalGroupID, + "a single-group config honours an id set by hand") + + assert.Equal(t, []string{idKey("evalgroup", "quality")}, groupIDKeys("quality", false), + "with several groups the shared entry cannot say which group it means") +} + diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index d7a3bb0e3eb..3f5d81ebb0f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -96,7 +96,8 @@ func buildRunCommand(use, short string) *cobra.Command { } evalID, err = ec.resolveEvalGroupID( - ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) + ctx, group, configPath, resolveLevel(level, group), + len(cfg.EvalGroups) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -220,6 +221,7 @@ func (ec *evalContext) resolveEvalGroupID( group *project.EvalGroup, configPath string, level string, + soleGroup bool, out interface{ Write([]byte) (int, error) }, jsonMode bool, ) (string, error) { @@ -227,7 +229,11 @@ func (ec *evalContext) resolveEvalGroupID( return group.ID, nil } - if cached := ec.getEnvValue(ctx, idKey("evalgroup", group.Name)); cached != "" { + for _, key := range groupIDKeys(group.Name, soleGroup) { + cached := ec.getEnvValue(ctx, key) + if cached == "" { + continue + } // Confirm it still exists; a deleted group should fall through to create. if _, err := ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { return cached, nil @@ -269,6 +275,23 @@ func (ec *evalContext) resolveEvalGroupID( return created.ID, nil } +// groupIDKeys lists the env entries that may hold this group's id, most +// specific first. +// +// The per-name entry is what the extension writes. EVAL_GROUP_ID is also the +// documented way to point a config at a group that already exists, created in +// the portal or by another tool, so it stays readable — but only when the +// config declares a single group. With more than one there is no way to tell +// which group a shared entry refers to, and reading it anyway is what let a +// second group adopt the first one's id. +func groupIDKeys(name string, soleGroup bool) []string { + keys := []string{idKey("evalgroup", name)} + if soleGroup { + keys = append(keys, envKeyEvalGroupID) + } + return keys +} + // checkDatasetRegistered fails when the group's local dataset has edits that // were never deployed. // From 69a6ec8527a4b0a5e5be1910f90acb92e0f6550c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 13:51:53 +0300 Subject: [PATCH 051/114] Emit a JSON array from every list command The list commands passed the service's own envelope straight through, and the two services behind them do not agree on one: runs came back as {"data": [..]} and datasets, evaluators and schedules as {"value": [..]}. A script had to know which API happened to back each command to read its output, and the envelopes also carry paging fields the extension never follows, implying there is more to fetch when there is not. Each list now emits a bare array. A nil slice is normalized to [] so that an empty listing is still something a caller can iterate rather than null. --- .../internal/cmd/dataset.go | 2 +- .../internal/cmd/evaluator.go | 2 +- .../internal/cmd/output.go | 18 +++++++++++ .../internal/cmd/output_test.go | 31 +++++++++++++++++++ .../internal/cmd/run_ops.go | 6 +++- .../internal/cmd/schedule.go | 9 ++++-- 6 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 5c18f453297..3e4513b07c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -138,7 +138,7 @@ func newDatasetListCommand() *cobra.Command { } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), list) + return emitJSONList(cmd.OutOrStdout(), list.Value) } rows := make([][]string, 0, len(list.Value)) for _, d := range list.Value { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index ca1d597eaa3..4ec85fca946 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -227,7 +227,7 @@ func newEvaluatorBuiltinsCommand() *cobra.Command { func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) error { if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), list) + return emitJSONList(cmd.OutOrStdout(), list.Value) } if len(list.Value) == 0 { fmt.Fprintln(cmd.OutOrStdout(), "No evaluators found.") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index ee6a1928fdb..319ed2565f0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -39,6 +39,24 @@ func emitJSON(w io.Writer, v any) error { return enc.Encode(v) } +// emitJSONList writes items as a JSON array. +// +// List commands emit a bare array rather than the envelope the service replied +// with. The envelopes disagree with each other — the OpenAI-shaped APIs wrap +// results in `data`, the ARM-shaped ones in `value` — so passing them through +// would make a caller's parsing depend on which service happens to back a given +// command. They also carry paging fields that this extension does not follow, +// which would suggest there is more to fetch when there is not. +// +// A nil slice encodes as `null`, so it is normalized to an empty array: a +// caller iterating the result should see no elements, not a type error. +func emitJSONList[T any](w io.Writer, items []T) error { + if items == nil { + items = []T{} + } + return emitJSON(w, items) +} + // emitTable writes a simple aligned table. Rows must match the header width. func emitTable(w io.Writer, headers []string, rows [][]string) error { tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go new file mode 100644 index 00000000000..2914e5c3fbd --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output_test.go @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The list commands are backed by two different services whose envelopes +// disagree — `data` on one side, `value` on the other. Emitting whichever one +// came back would make a caller's parsing depend on that accident, so every +// list emits a bare array instead. +func TestEmitJSONList_EmitsAnArrayNotAnEnvelope(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, emitJSONList(&buf, []string{"a", "b"})) + assert.Equal(t, "[\n \"a\",\n \"b\"\n]\n", buf.String()) +} + +// A nil slice marshals to `null`, which a caller iterating the output cannot +// range over. An empty listing has to come back as an empty array. +func TestEmitJSONList_NilBecomesEmptyArray(t *testing.T) { + var buf bytes.Buffer + var none []string + require.NoError(t, emitJSONList(&buf, none)) + assert.Equal(t, "[]\n", buf.String()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 196a989a3bc..509f7f48026 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -52,7 +52,11 @@ func newRunListCommand() *cobra.Command { return fmt.Errorf("listing runs for %q: %w", evalID, err) } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), list) + var runs []eval_api.OpenAIEvalRun + if list != nil { + runs = list.Data + } + return emitJSONList(cmd.OutOrStdout(), runs) } if list == nil || len(list.Data) == 0 { fmt.Fprintf(cmd.OutOrStdout(), "Eval group %s has no runs yet.\n", evalID) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index 96f5429e060..ae86cb60dc8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -117,7 +117,8 @@ func newScheduleSetCommand() *cobra.Command { return err } evalID, err = ec.resolveEvalGroupID( - ctx, group, configPath, resolveLevel(level, group), out, isJSON(cmd)) + ctx, group, configPath, resolveLevel(level, group), + len(cfg.EvalGroups) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -239,7 +240,11 @@ func newScheduleListCommand() *cobra.Command { return fmt.Errorf("listing schedules: %w", err) } if isJSON(cmd) { - return emitJSON(out, list) + var schedules []eval_api.Schedule + if list != nil { + schedules = list.Value + } + return emitJSONList(out, schedules) } if list == nil || len(list.Value) == 0 { fmt.Fprintln(out, "No schedules.") From ab06f367184f6abf96260c6a0491a0859ebe9f7f Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 14:12:13 +0300 Subject: [PATCH 052/114] Accept --eval-id on every command that takes an eval group `run start --eval-id ` is the form the CI example uses, but the sibling commands accepted the id only as a positional argument, so `run list --eval-id` failed with "unknown flag". Writing the verification script for the spec examples is what surfaced it: the flag learned in one command does not work in the next one a script reaches for. --eval-id now sits alongside --eval-group on run list/show/cancel and results show/export/compare, resolved with the same precedence as before. The positional argument keeps working and still wins. --- .../internal/cmd/compare.go | 2 +- .../internal/cmd/results.go | 19 ++++++++++++--- .../internal/cmd/run_ops.go | 6 ++--- .../internal/cmd/run_ops_test.go | 24 +++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index 815f4ab47c6..05775d3a1e4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -90,7 +90,7 @@ func newResultsCompareCommand() *cobra.Command { cmd.Flags().StringArrayVar(&treatments, "treatment", nil, "Run to measure, repeatable. Defaults to the most recent completed run.") cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 3a2f4d87f5a..1fc0ee4157b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -74,7 +74,7 @@ func newResultsShowCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -135,7 +135,7 @@ func newResultsExportCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -156,6 +156,10 @@ func resolveEvalID( return args[0], nil } + if flag, err := cmd.Flags().GetString("eval-id"); err == nil && flag != "" { + return flag, nil + } + if groupName != "" { if id := ec.getEnvValue(cmd.Context(), idKey("evalgroup", groupName)); id != "" { return id, nil @@ -177,9 +181,18 @@ func resolveEvalID( // addEvalGroupFlag registers the flag that names a group from the config, so // every command taking an eval-id can reach a group by the name its author // used. -func addEvalGroupFlag(cmd *cobra.Command, target *string) { +// addEvalGroupFlags registers the two ways to say which group a command acts +// on: --eval-group names one from the config, --eval-id gives its service id. +// +// The id is also accepted as a positional argument. The flag exists because +// `run start --eval-id` already spells it that way, and a script that learned +// it there should not have to find out that the sibling commands take only a +// positional. +func addEvalGroupFlags(cmd *cobra.Command, target *string) { cmd.Flags().StringVar(target, "eval-group", "", "Name a group from the config instead of passing its id.") + cmd.Flags().String("eval-id", "", + "Id of the eval group. Same as passing the id as an argument.") } // latestOrNamedRun returns the named run, or the most recent one for the group. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 509f7f48026..1ff2ee7cebb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -71,7 +71,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -122,7 +122,7 @@ func newRunShowCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -178,7 +178,7 @@ func newRunCancelCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") - addEvalGroupFlag(cmd, &groupName) + addEvalGroupFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 63ef3344e2e..edc4d99ccef 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -62,6 +62,30 @@ func TestRunStartMirrorsCompositeFlags(t *testing.T) { } } +// Every command that acts on an eval group takes the id the same two ways. +// `run start --eval-id` is the form the CI example uses, and `run list` used to +// reject that flag and accept only a positional, so a script that worked for +// one sibling failed on the next. +func TestEvalGroupCommandsAcceptIDAsAFlag(t *testing.T) { + subs := map[string]*cobra.Command{} + for _, sub := range newRunCommand().Commands() { + subs["run "+sub.Name()] = sub + } + for _, sub := range newResultsCommand().Commands() { + subs["results "+sub.Name()] = sub + } + + for _, name := range []string{ + "run list", "run show", "run cancel", + "results show", "results export", "results compare", + } { + cmd := subs[name] + require.NotNil(t, cmd, "%s should exist", name) + require.NotNil(t, cmd.Flags().Lookup("eval-id"), "%s should accept --eval-id", name) + require.NotNil(t, cmd.Flags().Lookup("eval-group"), "%s should accept --eval-group", name) + } +} + // --no-wait is documented in the spec, and cobra does not derive it from the // --wait bool. func TestRunCommandAcceptsNoWait(t *testing.T) { From 3a2fb1eae23085569a6755bf3dcd0a098f0c2683 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 14:20:22 +0300 Subject: [PATCH 053/114] Survive the quoted NaN the service sends for an undefined statistic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run with a single sample has no standard deviation, and the service reports it as the string "NaN" because JSON has no NaN literal. The model declared float64, so decoding failed and the whole comparison was thrown away with "cannot unmarshal string into Go struct field ... of type float64" — including the TooFewSamples verdict that exists to explain that exact situation. Comparing a one-sample gate, which is the cheapest thing a pipeline does, could not work. The statistics are now a LenientFloat that decodes a number or any of the quoted forms, and marshals non-finite values as null, because encoding/json refuses NaN outright and would have broken -o json instead. The table prints an undefined statistic as a dash rather than the literal NaN. The earlier live check passed only because every run it compared had enough samples for the deviation to be a real number. --- .../internal/cmd/compare.go | 19 ++- .../internal/cmd/envkeys_test.go | 1 - .../internal/pkg/eval_api/insights.go | 67 +++++++++-- .../internal/pkg/eval_api/insights_test.go | 108 ++++++++++++++++++ 4 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index 05775d3a1e4..f90ecfb5ef8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -167,6 +167,17 @@ func (ec *evalContext) pollInsight(ctx context.Context, insightID string) (*eval } // renderComparison prints one row per criterion per treatment run. +// formatStat renders a statistic, showing an undefined one as a dash. A +// standard deviation over a single sample has no value, and printing the +// literal "NaN" in a results table reads like a failure rather than the +// arithmetic it is. +func formatStat(verb string, v eval_api.LenientFloat) string { + if !v.Defined() { + return "-" + } + return fmt.Sprintf(verb, float64(v)) +} + func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_api.Insight) error { if insight.Result == nil || len(insight.Result.Comparisons) == 0 { fmt.Fprintln(w, "The comparison produced no metrics.") @@ -181,13 +192,13 @@ func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_a for _, c := range insight.Result.Comparisons { baseAvg := "-" if c.BaselineRunSummary != nil { - baseAvg = fmt.Sprintf("%.3f", c.BaselineRunSummary.Average) + baseAvg = formatStat("%.3f", c.BaselineRunSummary.Average) } for _, item := range c.CompareItems { treatAvg := "-" runID := "-" if item.TreatmentRunSummary != nil { - treatAvg = fmt.Sprintf("%.3f", item.TreatmentRunSummary.Average) + treatAvg = formatStat("%.3f", item.TreatmentRunSummary.Average) runID = item.TreatmentRunSummary.RunID } rows = append(rows, []string{ @@ -195,8 +206,8 @@ func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_a runID, baseAvg, treatAvg, - fmt.Sprintf("%+.3f", item.DeltaEstimate), - fmt.Sprintf("%.3f", item.PValue), + formatStat("%+.3f", item.DeltaEstimate), + formatStat("%.3f", item.PValue), item.TreatmentEffect, }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index dbda6180b23..b78bd8debf9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -51,4 +51,3 @@ func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { assert.Equal(t, []string{idKey("evalgroup", "quality")}, groupIDKeys("quality", false), "with several groups the shared entry cannot say which group it means") } - diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go index 8296fd21992..0335f951d62 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights.go @@ -5,9 +5,13 @@ package eval_api import ( "context" + "encoding/json" "fmt" + "math" "net/http" "net/url" + "strconv" + "strings" ) // InsightTypeEvaluationComparison compares evaluation runs. The service also @@ -34,19 +38,68 @@ type CreateInsightRequest struct { Request *InsightRequest `json:"request"` } +// LenientFloat is a float64 that also decodes the quoted forms the service +// uses for values JSON cannot express. +// +// A run with a single sample has an undefined standard deviation, and the +// service sends it as the string "NaN" because JSON has no NaN literal. +// Decoding that into a plain float64 fails the entire comparison — including +// the TooFewSamples verdict that exists to explain exactly this case — so a +// one-sample gate reported a parse error instead of its result. +type LenientFloat float64 + +func (f *LenientFloat) UnmarshalJSON(data []byte) error { + s := strings.TrimSpace(string(data)) + if s == "null" { + *f = LenientFloat(math.NaN()) + return nil + } + // "NaN", "Infinity", "-Infinity" and ordinary numbers arrive quoted; + // ParseFloat accepts all of them once the quotes are gone. + if unquoted, err := strconv.Unquote(s); err == nil { + s = strings.TrimSpace(unquoted) + if s == "" { + *f = LenientFloat(math.NaN()) + return nil + } + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return fmt.Errorf("parsing number %s: %w", data, err) + } + *f = LenientFloat(v) + return nil +} + +// MarshalJSON writes non-finite values as null. encoding/json refuses to +// marshal NaN or ±Inf at all, which would turn `-o json` into an error the +// moment a comparison contained one; null is valid JSON and reads as the +// "undefined" that a one-sample standard deviation actually is. +func (f LenientFloat) MarshalJSON() ([]byte, error) { + if math.IsNaN(float64(f)) || math.IsInf(float64(f), 0) { + return []byte("null"), nil + } + return json.Marshal(float64(f)) +} + +// Defined reports whether the value is a real number that can be shown. +func (f LenientFloat) Defined() bool { + return !math.IsNaN(float64(f)) && !math.IsInf(float64(f), 0) +} + // RunSummary is one run's aggregate for a single metric. type RunSummary struct { - RunID string `json:"runId"` - SampleCount int `json:"sampleCount"` - Average float64 `json:"average"` - StandardDeviation float64 `json:"standardDeviation"` + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average LenientFloat `json:"average"` + StandardDeviation LenientFloat `json:"standardDeviation"` } // CompareItem is one treatment run measured against the baseline. type CompareItem struct { - TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` - DeltaEstimate float64 `json:"deltaEstimate"` - PValue float64 `json:"pValue"` + TreatmentRunSummary *RunSummary `json:"treatmentRunSummary,omitempty"` + DeltaEstimate LenientFloat `json:"deltaEstimate"` + PValue LenientFloat `json:"pValue"` // TreatmentEffect classifies the result, e.g. TooFewSamples when the // sample count cannot support a conclusion. TreatmentEffect string `json:"treatmentEffect,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go new file mode 100644 index 00000000000..b8db134dc79 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/insights_test.go @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exact body the service returned for a comparison of two single-sample +// runs. `standardDeviation` is the string "NaN" because JSON has no NaN +// literal, and decoding it into a float64 failed the whole comparison — losing +// the TooFewSamples verdict that explains the very situation that produced it. +const oneSampleComparison = `{ + "comparisons": [ + { + "testingCriteria": "task_adherence", + "metric": "task_adherence", + "evaluator": "builtin.task_adherence", + "baselineRunSummary": { + "runId": "evalrun_base", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "compareItems": [ + { + "treatmentRunSummary": { + "runId": "evalrun_treat", + "sampleCount": 1, + "average": 1.0, + "standardDeviation": "NaN" + }, + "deltaEstimate": 0.0, + "pValue": 1.0, + "treatmentEffect": "TooFewSamples" + } + ] + } + ], + "method": "TTest", + "type": "EvaluationComparison" +}` + +func TestInsightResult_DecodesQuotedNaN(t *testing.T) { + var got InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &got)) + + require.Len(t, got.Comparisons, 1) + c := got.Comparisons[0] + require.NotNil(t, c.BaselineRunSummary) + + assert.Equal(t, 1.0, float64(c.BaselineRunSummary.Average)) + assert.False(t, c.BaselineRunSummary.StandardDeviation.Defined(), + "a single sample has no standard deviation") + + require.Len(t, c.CompareItems, 1) + assert.Equal(t, "TooFewSamples", c.CompareItems[0].TreatmentEffect, + "the verdict survives, which is the whole point of not failing the parse") + assert.Equal(t, 1.0, float64(c.CompareItems[0].PValue)) +} + +func TestLenientFloat_AcceptsBothShapes(t *testing.T) { + cases := map[string]func(LenientFloat) bool{ + `0.75`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"0.75"`: func(f LenientFloat) bool { return float64(f) == 0.75 }, + `"NaN"`: func(f LenientFloat) bool { return !f.Defined() }, + `"Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `"-Infinity"`: func(f LenientFloat) bool { return !f.Defined() }, + `null`: func(f LenientFloat) bool { return !f.Defined() }, + `""`: func(f LenientFloat) bool { return !f.Defined() }, + } + + for raw, ok := range cases { + var f LenientFloat + require.NoError(t, json.Unmarshal([]byte(raw), &f), "decoding %s", raw) + assert.True(t, ok(f), "unexpected value decoding %s", raw) + } + + var f LenientFloat + assert.Error(t, json.Unmarshal([]byte(`"not a number"`), &f), + "genuine garbage must still be reported") +} + +// encoding/json refuses to marshal NaN, so `-o json` would fail on any +// comparison holding one unless it is written as null. +func TestLenientFloat_MarshalsNonFiniteAsNull(t *testing.T) { + b, err := json.Marshal(LenientFloat(math.NaN())) + require.NoError(t, err) + assert.Equal(t, "null", string(b)) + + b, err = json.Marshal(LenientFloat(0.5)) + require.NoError(t, err) + assert.Equal(t, "0.5", string(b)) + + // The whole result has to survive a round trip, since that is what + // `results compare -o json` emits. + var res InsightResult + require.NoError(t, json.Unmarshal([]byte(oneSampleComparison), &res)) + out, err := json.Marshal(res) + require.NoError(t, err, "a comparison containing NaN must still emit JSON") + assert.Contains(t, string(out), `"standardDeviation":null`) +} From fe590edb259e650a8e15d80db72477b1c4d603ea Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 14:49:21 +0300 Subject: [PATCH 054/114] Explain the two schedule failures the E2E scenarios turned up A schedule repeats the eval group's most recent run, so scheduling a group whose last run came from --from-traces makes it a trace evaluation, and the service then allows only an hourly trigger. Asking for a daily one produced a raw 400 saying "Scheduled trace evaluations only support hourly recurrence triggers", which is baffling when a trigger was the only thing asked for and traces were never mentioned. Confirmed by experiment: daily was accepted after an agent run, refused after a traces run on the same group, and hourly accepted for that same traces run. The error now names the cause and both ways out. Deleting a schedule that does not exist printed 2364 characters of service error document wrapping an inner 404 from the trigger service. A missing name is the ordinary typo, so it now says so in one line and points at schedule list. The live run test also asserted only that the run reached a terminal state. A run whose every sample errors still reports "completed", so it now requires no errored samples and at least one scored one - otherwise a broken target or evaluator would leave the suite green. --- .../internal/cmd/schedule.go | 29 +++++++++++++++++++ .../internal/cmd/schedule_test.go | 13 +++++++++ .../internal/pkg/eval_api/schedules.go | 9 ++++++ .../tests/live/live_test.go | 14 +++++++++ 4 files changed, 65 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index ae86cb60dc8..b2972b29ac9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -358,6 +358,15 @@ func newScheduleDeleteCommand() *cobra.Command { defer ec.Close() if err := ec.deleteScheduleWhenSettled(ctx, name); err != nil { + // A name that was never there is the common typo, and the + // service answers it with a full error document wrapping an + // inner 404 from the trigger service. Saying so in one line is + // more use than reproducing that. + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no schedule named %q in this project; "+ + "`azd ai eval schedule list` shows the ones that exist", name) + } return fmt.Errorf("deleting schedule %q: %w", name, err) } fmt.Fprintf(out, "Deleted schedule %s\n", name) @@ -583,6 +592,19 @@ func explainScheduleFailure( name string, cause error, ) error { + // A schedule repeats the group's most recent run, so scheduling a group + // whose last run came from --from-traces creates a trace evaluation, and + // the service allows only an hourly trigger for those. The message it + // returns says so without saying why it thinks the schedule is one, which + // is bewildering when the trigger was the only thing asked for. + if isTracesHourlyOnly(cause) { + return fmt.Errorf( + "saving schedule %q: this group's most recent run read from traces, and a schedule "+ + "repeats that run, so the service treats it as a scheduled trace evaluation "+ + "and allows only `--every hourly`. Use `--every hourly`, or run the group "+ + "once against its dataset first so the schedule repeats that instead", name) + } + list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) if listErr != nil || list == nil { return fmt.Errorf("saving schedule %q: %w", name, cause) @@ -598,3 +620,10 @@ func explainScheduleFailure( } return fmt.Errorf("saving schedule %q: %w", name, cause) } + +// isTracesHourlyOnly matches the service's refusal of a non-hourly trigger on a +// schedule it considers a trace evaluation. +func isTracesHourlyOnly(err error) bool { + return err != nil && + strings.Contains(err.Error(), "trace evaluations only support hourly") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go index 9f93207950b..f518397d843 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "testing" "azureaieval/internal/pkg/eval_api" @@ -20,6 +21,18 @@ func TestBuildTrigger_Cron(t *testing.T) { assert.Equal(t, "UTC", got.Timezone, "UTC unless the caller says otherwise") } +// A schedule repeats the group's most recent run. Scheduling a group whose last +// run came from --from-traces therefore creates a trace evaluation, and the +// service allows only an hourly trigger for those. Confirmed live: a daily +// trigger was accepted after an agent run, refused after a traces run on the +// same group, and hourly was accepted for that same traces run. +func TestIsTracesHourlyOnly(t *testing.T) { + assert.True(t, isTracesHourlyOnly( + errors.New(`{"message": "Scheduled trace evaluations only support hourly recurrence triggers. is invalid"}`))) + assert.False(t, isTracesHourlyOnly(errors.New("some other 400"))) + assert.False(t, isTracesHourlyOnly(nil)) +} + func TestBuildTrigger_OneTime(t *testing.T) { got, err := buildTrigger(triggerFlags{atTime: "2026-08-01T09:00:00Z", timezone: "Europe/Dublin"}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go index ed205bf1f79..a6ab65746a4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go @@ -173,3 +173,12 @@ func IsConflict(err error) bool { } return respErr.StatusCode == http.StatusConflict } + +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 9a4a209bffb..2ecceff6f7e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -320,6 +320,20 @@ func TestLiveRun(t *testing.T) { } body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") t.Logf("per-criteria results: %s", string(body)) + + // Reaching a terminal state is not the same as having evaluated + // anything. A run whose every sample errors still reports + // "completed", so asserting only on the status would let the target + // or the evaluator break without the test noticing. + require.Equal(t, "completed", strings.ToLower(current.Status), + "the run must complete rather than fail or cancel") + require.NotNil(t, current.ResultCounts, "a completed run must report counts") + require.Zero(t, current.ResultCounts.Errored, + "an errored sample means the target or the evaluator did not run") + require.Positive(t, + current.ResultCounts.Passed+current.ResultCounts.Failed, + "the run must score at least one sample; a pass or a fail are both fine, "+ + "but scoring nothing means the data never reached the evaluator") return } if time.Now().After(deadline) { From 42f8fe120a5816591a1c39b07bd1260ac458b044 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 18:06:55 +0300 Subject: [PATCH 055/114] Declare the eval service in the root azure.yaml instead of describing it init scaffolded evals/azure.yaml and then printed a services block for the reader to paste into the root azure.yaml. azd acts on nothing until that reference exists, so the documented flow stopped between init and azd up, and every scenario run so far had passed only because the harness wrote that file itself before calling init. The reference is mechanical, so init writes it: creating the project file when the folder has none, and otherwise adding the service through the YAML node tree so the project keeps its name, comments, key order and other services. A project already declaring an azure.ai.eval service is left alone - matched on the host rather than the service name, so running init twice cannot deploy the same evals twice. Anything that cannot be edited safely still prints the block. The next steps now suggest azd deploy rather than azd up when the project has no infra to provision, since azd up stops at a missing infra/main.bicep without saying that provisioning is what it wanted. --- .../azure.ai.evaluations/internal/cmd/init.go | 184 +++++++++++++++++- .../internal/cmd/init_wiring_test.go | 126 ++++++++++++ 2 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 1f62f8be3d2..f4de5319b79 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "fmt" "os" "path/filepath" @@ -69,27 +70,59 @@ func newInitCommand() *cobra.Command { return err } + // Scaffolding a config azd cannot see is half a step: the eval + // service has to be referenced from the root config before any of + // `azd up`, `azd deploy` or `azd ai eval run` will act on it. + // Printing the block and leaving the edit to the reader was enough + // to make the documented flow stop working between `init` and + // `azd up`. + rootWiring, err := ensureRootEvalService(rootConfigName, depPath) + if err != nil { + return err + } + if isJSON(cmd) { return emitJSON(out, map[string]any{ "generateConfig": genPath, "deployConfig": depPath, "datasetsDir": filepath.Join(outDir, project.DefaultDatasetsDir), "evaluatorsDir": filepath.Join(outDir, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, }) } fmt.Fprintf(out, "Wrote %s\n", genPath) fmt.Fprintf(out, "Wrote %s\n", depPath) + fmt.Fprintln(out, "\nNext:") - fmt.Fprintf(out, " 1. Reference %s from your root azure.yaml:\n", depPath) - fmt.Fprintln(out, " services:") - fmt.Fprintln(out, " evals:") - fmt.Fprintln(out, " host: azure.ai.eval") - fmt.Fprintln(out, " uses: [ai-project]") - fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) - fmt.Fprintln(out, " 2. azd ai eval generate (or supply your own dataset)") - fmt.Fprintln(out, " 3. azd up") - fmt.Fprintln(out, " 4. azd ai eval run") + step := 1 + if rootWiring == wiringManual { + // Only reached when the root config could not be read or has a + // shape this cannot safely edit, so the wiring is the caller's + // to do. + fmt.Fprintf(out, " %d. Reference %s from your root azure.yaml:\n", step, depPath) + fmt.Fprintln(out, " services:") + fmt.Fprintln(out, " evals:") + fmt.Fprintln(out, " host: azure.ai.eval") + fmt.Fprintln(out, " uses: [ai-project]") + fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) + step++ + } else { + fmt.Fprintf(out, " (%s references %s)\n", rootConfigName, depPath) + } + fmt.Fprintf(out, " %d. azd ai eval generate (or supply your own dataset)\n", step) + step++ + // azd up provisions before it deploys, which needs a bicep template. + // An eval-only project has none, and the failure names a missing + // infra/main.bicep rather than the reason, so it is only suggested + // where it can work. + if hasInfra() { + fmt.Fprintf(out, " %d. azd up\n", step) + } else { + fmt.Fprintf(out, " %d. azd deploy evals (azd up once the project has infra to provision)\n", step) + } + step++ + fmt.Fprintf(out, " %d. azd ai eval run\n", step) return nil }, } @@ -105,6 +138,139 @@ func newInitCommand() *cobra.Command { return cmd } +// rootConfigName is azd's project file, which the eval service is declared in. +const rootConfigName = "azure.yaml" + +// How the root config ended up referencing the eval service. +const ( + wiringCreated = "created" // there was no root config, so one was written + wiringAdded = "added" // the service was added to an existing config + wiringPresent = "present" // an eval service was already declared + wiringManual = "manual" // the caller has to do it; the block is printed +) + +// ensureRootEvalService declares the eval service in azd's project file. +// +// A config azd cannot see does nothing, and the reference is mechanical, so it +// is written rather than described. An existing project file is edited in place +// through the YAML node tree, which keeps its comments and key order; anything +// that cannot be edited safely falls back to printing the block. +func ensureRootEvalService(rootPath, depPath string) (string, error) { + ref := "./" + filepath.ToSlash(depPath) + + raw, err := os.ReadFile(rootPath) + if errors.Is(err, os.ErrNotExist) { + name := filepath.Base(mustAbs(filepath.Dir(rootPath))) + body := fmt.Sprintf(""+ + "name: %s\n"+ + "services:\n"+ + " evals:\n"+ + " host: azure.ai.eval\n"+ + " $ref: %s\n", name, ref) + if err := os.WriteFile(rootPath, []byte(body), 0o600); err != nil { + return "", fmt.Errorf("writing %s: %w", rootPath, err) + } + return wiringCreated, nil + } + if err != nil { + return wiringManual, nil + } + + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.Content) == 0 { + return wiringManual, nil + } + root := doc.Content[0] + if root.Kind != yaml.MappingNode { + return wiringManual, nil + } + + services := mappingValue(root, "services") + if services != nil && services.Kind == yaml.MappingNode { + // A service already pointing at an eval config is left alone, whatever + // it is called: adding a second would deploy the same evals twice. + for i := 0; i+1 < len(services.Content); i += 2 { + if host := mappingValue(services.Content[i+1], "host"); host != nil && + host.Value == project.EvalHost { + return wiringPresent, nil + } + } + } + if services == nil { + root.Content = append(root.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "services"}, + &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}) + services = root.Content[len(root.Content)-1] + } + if services.Kind != yaml.MappingNode { + return wiringManual, nil + } + + entry := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + entry.Content = append(entry.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "host"}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: project.EvalHost}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "$ref"}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: ref}) + services.Content = append(services.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: uniqueServiceName(services)}, + entry) + + out, err := yaml.Marshal(&doc) + if err != nil { + return wiringManual, nil + } + if err := os.WriteFile(rootPath, out, 0o600); err != nil { + return "", fmt.Errorf("updating %s: %w", rootPath, err) + } + return wiringAdded, nil +} + +// mappingValue returns the value node for key, or nil. +func mappingValue(m *yaml.Node, key string) *yaml.Node { + if m == nil || m.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1] + } + } + return nil +} + +// uniqueServiceName avoids colliding with a service the project already has. +func uniqueServiceName(services *yaml.Node) string { + taken := map[string]bool{} + for i := 0; i+1 < len(services.Content); i += 2 { + taken[services.Content[i].Value] = true + } + if !taken["evals"] { + return "evals" + } + for i := 2; ; i++ { + candidate := fmt.Sprintf("evals%d", i) + if !taken[candidate] { + return candidate + } + } +} + +func mustAbs(p string) string { + abs, err := filepath.Abs(p) + if err != nil { + return p + } + return abs +} + +// hasInfra reports whether azd has a template to provision, which decides +// whether `azd up` can work here. +func hasInfra() bool { + _, err := os.Stat(filepath.Join("infra", "main.bicep")) + return err == nil +} + func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { return &project.GenerateConfig{ Agent: project.AgentSpec{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go new file mode 100644 index 00000000000..86288bdf578 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// init scaffolds a config that azd only acts on once the root project file +// references it. Printing the block and leaving the edit to the reader is what +// made the documented flow stop between `init` and `azd up`. +func TestEnsureRootEvalService_CreatesTheProjectFileWhenAbsent(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringCreated, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "host: azure.ai.eval") + assert.Contains(t, string(body), "$ref: ./evals/azure.yaml") + assert.Contains(t, string(body), "name: "+filepath.Base(dir), + "azd needs a project name, taken from the directory") +} + +// An existing project file belongs to the caller, so the service is added +// without disturbing what is already declared. +func TestEnsureRootEvalService_AddsToAnExistingProject(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + existing := "name: my-app\n" + + "services:\n" + + " api:\n" + + " host: containerapp\n" + + " language: python\n" + require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringAdded, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "host: azure.ai.eval") + assert.Contains(t, string(body), "host: containerapp", "the existing service survives") + assert.Contains(t, string(body), "language: python") + assert.Contains(t, string(body), "name: my-app", "the project keeps its name") +} + +// Running init twice must not declare the evals twice, which would deploy them +// twice. The name is not what identifies it — the host is. +func TestEnsureRootEvalService_LeavesAnExistingEvalServiceAlone(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + existing := "name: my-app\n" + + "services:\n" + + " quality:\n" + + " host: azure.ai.eval\n" + + " $ref: ./evals/azure.yaml\n" + require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringPresent, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Equal(t, existing, string(body), "an already-wired project is untouched") +} + +// A project declaring no services at all still needs the key adding. +func TestEnsureRootEvalService_AddsTheServicesKeyWhenMissing(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + require.NoError(t, os.WriteFile(rootConfigName, []byte("name: my-app\n"), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringAdded, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "services:") + assert.Contains(t, string(body), "host: azure.ai.eval") +} + +// A file that is not a YAML mapping is someone else's to fix; the block is +// printed instead of guessing at an edit. +func TestEnsureRootEvalService_FallsBackWhenTheProjectFileIsNotAMapping(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + require.NoError(t, os.WriteFile(rootConfigName, []byte("- not\n- a mapping\n"), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringManual, action) +} + +// A service called "evals" already existing for something else must not be +// overwritten. +func TestEnsureRootEvalService_DoesNotClobberAnUnrelatedEvalsService(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + existing := "name: my-app\n" + + "services:\n" + + " evals:\n" + + " host: containerapp\n" + require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) + + action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) + require.NoError(t, err) + assert.Equal(t, wiringAdded, action) + + body, err := os.ReadFile(rootConfigName) + require.NoError(t, err) + assert.Contains(t, string(body), "host: containerapp", "the unrelated service is intact") + assert.Contains(t, string(body), "evals2:", "the eval service takes a free name") +} From 2713a8e3e74171ee4202f930af1e51d4fb8dda1d Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 20:01:12 +0300 Subject: [PATCH 056/114] Add the eval service through azd, and catch two things the corner cases found init hand-edited the root azure.yaml through the YAML node tree. No other extension does that - the agents extension adds its service with azd's own Project().AddService, and the only azure.yaml any extension writes is a staged copy in a temp directory. So the eval service goes through AddService too, and azd owns the edit. A folder with no project is no longer given one: evals attach to a project, so init says to run azd init. That also removes the azd up divergence. init suggested azd deploy when the project had no infra, which quietly moved people off the flow the spec describes; it says azd up again, and a project without infra fails in provisioning where the reason belongs. Two defects the corner cases turned up: A malformed dataset row was published without complaint. {not json at all} became version 1.0 with an eval group bound to it, and would have failed much later on a row nobody had looked at. Every row is now checked before upload and the offending line is named. The scan buffer is 8MB because a conversation-level row runs well past bufio's 64KB default. Asking for runs of an eval group that does not exist printed 1682 characters of HTTP error. A mistyped or undeployed id is ordinary, so it now says so in one line. --- .../azure.ai.evaluations/internal/cmd/init.go | 191 ++++++------------ .../internal/cmd/init_wiring_test.go | 126 ++---------- .../internal/cmd/jsonl_validation_test.go | 75 +++++++ .../internal/cmd/reconciler.go | 53 +++++ .../internal/cmd/results.go | 5 + .../internal/cmd/run_ops.go | 5 + 6 files changed, 214 insertions(+), 241 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index f4de5319b79..be2d29941e2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -4,7 +4,7 @@ package cmd import ( - "errors" + "context" "fmt" "os" "path/filepath" @@ -13,8 +13,10 @@ import ( "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/spf13/cobra" "go.yaml.in/yaml/v3" + "google.golang.org/protobuf/types/known/structpb" ) // newInitCommand scaffolds the eval configuration. It makes no service calls at @@ -76,7 +78,7 @@ func newInitCommand() *cobra.Command { // Printing the block and leaving the edit to the reader was enough // to make the documented flow stop working between `init` and // `azd up`. - rootWiring, err := ensureRootEvalService(rootConfigName, depPath) + rootWiring, err := ensureRootEvalService(cmd.Context(), depPath) if err != nil { return err } @@ -93,36 +95,17 @@ func newInitCommand() *cobra.Command { fmt.Fprintf(out, "Wrote %s\n", genPath) fmt.Fprintf(out, "Wrote %s\n", depPath) + switch rootWiring { + case wiringAdded: + fmt.Fprintf(out, "Added the evals service to %s\n", rootConfigName) + case wiringPresent: + fmt.Fprintf(out, "%s already declares an eval service\n", rootConfigName) + } fmt.Fprintln(out, "\nNext:") - step := 1 - if rootWiring == wiringManual { - // Only reached when the root config could not be read or has a - // shape this cannot safely edit, so the wiring is the caller's - // to do. - fmt.Fprintf(out, " %d. Reference %s from your root azure.yaml:\n", step, depPath) - fmt.Fprintln(out, " services:") - fmt.Fprintln(out, " evals:") - fmt.Fprintln(out, " host: azure.ai.eval") - fmt.Fprintln(out, " uses: [ai-project]") - fmt.Fprintf(out, " $ref: ./%s\n", filepath.ToSlash(depPath)) - step++ - } else { - fmt.Fprintf(out, " (%s references %s)\n", rootConfigName, depPath) - } - fmt.Fprintf(out, " %d. azd ai eval generate (or supply your own dataset)\n", step) - step++ - // azd up provisions before it deploys, which needs a bicep template. - // An eval-only project has none, and the failure names a missing - // infra/main.bicep rather than the reason, so it is only suggested - // where it can work. - if hasInfra() { - fmt.Fprintf(out, " %d. azd up\n", step) - } else { - fmt.Fprintf(out, " %d. azd deploy evals (azd up once the project has infra to provision)\n", step) - } - step++ - fmt.Fprintf(out, " %d. azd ai eval run\n", step) + fmt.Fprintln(out, " 1. azd ai eval generate (or supply your own dataset)") + fmt.Fprintln(out, " 2. azd up") + fmt.Fprintln(out, " 3. azd ai eval run") return nil }, } @@ -143,134 +126,80 @@ const rootConfigName = "azure.yaml" // How the root config ended up referencing the eval service. const ( - wiringCreated = "created" // there was no root config, so one was written - wiringAdded = "added" // the service was added to an existing config + wiringAdded = "added" // the service was added to the project wiringPresent = "present" // an eval service was already declared - wiringManual = "manual" // the caller has to do it; the block is printed ) // ensureRootEvalService declares the eval service in azd's project file. // -// A config azd cannot see does nothing, and the reference is mechanical, so it -// is written rather than described. An existing project file is edited in place -// through the YAML node tree, which keeps its comments and key order; anything -// that cannot be edited safely falls back to printing the block. -func ensureRootEvalService(rootPath, depPath string) (string, error) { - ref := "./" + filepath.ToSlash(depPath) - - raw, err := os.ReadFile(rootPath) - if errors.Is(err, os.ErrNotExist) { - name := filepath.Base(mustAbs(filepath.Dir(rootPath))) - body := fmt.Sprintf(""+ - "name: %s\n"+ - "services:\n"+ - " evals:\n"+ - " host: azure.ai.eval\n"+ - " $ref: %s\n", name, ref) - if err := os.WriteFile(rootPath, []byte(body), 0o600); err != nil { - return "", fmt.Errorf("writing %s: %w", rootPath, err) - } - return wiringCreated, nil - } +// azd acts on nothing until the service exists, so the reference is made rather +// than described. It goes through azd's own Project().AddService, the same call +// the agents extension uses, so azd owns the edit and the project file keeps +// whatever shape azd gives it. +// +// The eval config itself stays in evals/azure.yaml and is referenced with +// `$ref`. azd carries unknown keys through AdditionalProperties untouched, +// which is how the extension gets it back at deploy time. +func ensureRootEvalService(ctx context.Context, depPath string) (string, error) { + azdClient, err := azdext.NewAzdClient() if err != nil { - return wiringManual, nil - } - - var doc yaml.Node - if err := yaml.Unmarshal(raw, &doc); err != nil || len(doc.Content) == 0 { - return wiringManual, nil + return "", fmt.Errorf("connecting to azd: %w", err) } - root := doc.Content[0] - if root.Kind != yaml.MappingNode { - return wiringManual, nil + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + // Evals attach to a project; they do not create one. Saying which + // command makes one is more use than a gRPC error. + return "", fmt.Errorf( + "no azd project found in this directory. Run `azd init` first, "+ + "or run this from the root of an existing one; the eval service is "+ + "added to its %s", rootConfigName) } - services := mappingValue(root, "services") - if services != nil && services.Kind == yaml.MappingNode { - // A service already pointing at an eval config is left alone, whatever - // it is called: adding a second would deploy the same evals twice. - for i := 0; i+1 < len(services.Content); i += 2 { - if host := mappingValue(services.Content[i+1], "host"); host != nil && - host.Value == project.EvalHost { - return wiringPresent, nil - } + // A service already pointing at an eval config is left alone, whatever it + // is called: a second one would deploy the same evals twice. + for _, svc := range resp.GetProject().GetServices() { + if svc.GetHost() == project.EvalHost { + return wiringPresent, nil } } - if services == nil { - root.Content = append(root.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "services"}, - &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}) - services = root.Content[len(root.Content)-1] - } - if services.Kind != yaml.MappingNode { - return wiringManual, nil - } - entry := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - entry.Content = append(entry.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "host"}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: project.EvalHost}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "$ref"}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: ref}) - services.Content = append(services.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: uniqueServiceName(services)}, - entry) - - out, err := yaml.Marshal(&doc) + props, err := structpb.NewStruct(map[string]any{ + "$ref": "./" + filepath.ToSlash(depPath), + }) if err != nil { - return wiringManual, nil + return "", fmt.Errorf("building the eval service entry: %w", err) } - if err := os.WriteFile(rootPath, out, 0o600); err != nil { - return "", fmt.Errorf("updating %s: %w", rootPath, err) - } - return wiringAdded, nil -} -// mappingValue returns the value node for key, or nil. -func mappingValue(m *yaml.Node, key string) *yaml.Node { - if m == nil || m.Kind != yaml.MappingNode { - return nil - } - for i := 0; i+1 < len(m.Content); i += 2 { - if m.Content[i].Value == key { - return m.Content[i+1] - } + _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ + Service: &azdext.ServiceConfig{ + Name: evalServiceName(resp.GetProject()), + Host: project.EvalHost, + AdditionalProperties: props, + }, + }) + if err != nil { + return "", fmt.Errorf("adding the eval service to %s: %w", rootConfigName, err) } - return nil + return wiringAdded, nil } -// uniqueServiceName avoids colliding with a service the project already has. -func uniqueServiceName(services *yaml.Node) string { - taken := map[string]bool{} - for i := 0; i+1 < len(services.Content); i += 2 { - taken[services.Content[i].Value] = true - } - if !taken["evals"] { +// evalServiceName avoids colliding with a service the project already has. +// azd keys services by name, so the map key is the name to avoid. +func evalServiceName(proj *azdext.ProjectConfig) string { + taken := proj.GetServices() + if _, exists := taken["evals"]; !exists { return "evals" } for i := 2; ; i++ { candidate := fmt.Sprintf("evals%d", i) - if !taken[candidate] { + if _, exists := taken[candidate]; !exists { return candidate } } } -func mustAbs(p string) string { - abs, err := filepath.Abs(p) - if err != nil { - return p - } - return abs -} - -// hasInfra reports whether azd has a template to provision, which decides -// whether `azd up` can work here. -func hasInfra() bool { - _, err := os.Stat(filepath.Join("infra", "main.bicep")) - return err == nil -} - func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { return &project.GenerateConfig{ Agent: project.AgentSpec{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go index 86288bdf578..c5371c23db2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -4,123 +4,29 @@ package cmd import ( - "os" - "path/filepath" "testing" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -// init scaffolds a config that azd only acts on once the root project file -// references it. Printing the block and leaving the edit to the reader is what -// made the documented flow stop between `init` and `azd up`. -func TestEnsureRootEvalService_CreatesTheProjectFileWhenAbsent(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringCreated, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "host: azure.ai.eval") - assert.Contains(t, string(body), "$ref: ./evals/azure.yaml") - assert.Contains(t, string(body), "name: "+filepath.Base(dir), - "azd needs a project name, taken from the directory") +func projectWith(names ...string) *azdext.ProjectConfig { + proj := &azdext.ProjectConfig{Services: map[string]*azdext.ServiceConfig{}} + for _, n := range names { + proj.Services[n] = &azdext.ServiceConfig{Name: n} + } + return proj } -// An existing project file belongs to the caller, so the service is added -// without disturbing what is already declared. -func TestEnsureRootEvalService_AddsToAnExistingProject(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - existing := "name: my-app\n" + - "services:\n" + - " api:\n" + - " host: containerapp\n" + - " language: python\n" - require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringAdded, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "host: azure.ai.eval") - assert.Contains(t, string(body), "host: containerapp", "the existing service survives") - assert.Contains(t, string(body), "language: python") - assert.Contains(t, string(body), "name: my-app", "the project keeps its name") +// The eval service is added through azd's own AddService, so it has to pick a +// name azd will accept. Reusing one already in the project would overwrite +// somebody else's service. +func TestEvalServiceName_PrefersEvals(t *testing.T) { + assert.Equal(t, "evals", evalServiceName(projectWith())) + assert.Equal(t, "evals", evalServiceName(projectWith("api", "web"))) } -// Running init twice must not declare the evals twice, which would deploy them -// twice. The name is not what identifies it — the host is. -func TestEnsureRootEvalService_LeavesAnExistingEvalServiceAlone(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - existing := "name: my-app\n" + - "services:\n" + - " quality:\n" + - " host: azure.ai.eval\n" + - " $ref: ./evals/azure.yaml\n" - require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringPresent, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Equal(t, existing, string(body), "an already-wired project is untouched") -} - -// A project declaring no services at all still needs the key adding. -func TestEnsureRootEvalService_AddsTheServicesKeyWhenMissing(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - require.NoError(t, os.WriteFile(rootConfigName, []byte("name: my-app\n"), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringAdded, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "services:") - assert.Contains(t, string(body), "host: azure.ai.eval") -} - -// A file that is not a YAML mapping is someone else's to fix; the block is -// printed instead of guessing at an edit. -func TestEnsureRootEvalService_FallsBackWhenTheProjectFileIsNotAMapping(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - require.NoError(t, os.WriteFile(rootConfigName, []byte("- not\n- a mapping\n"), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringManual, action) -} - -// A service called "evals" already existing for something else must not be -// overwritten. -func TestEnsureRootEvalService_DoesNotClobberAnUnrelatedEvalsService(t *testing.T) { - dir := t.TempDir() - t.Chdir(dir) - existing := "name: my-app\n" + - "services:\n" + - " evals:\n" + - " host: containerapp\n" - require.NoError(t, os.WriteFile(rootConfigName, []byte(existing), 0o600)) - - action, err := ensureRootEvalService(rootConfigName, filepath.Join("evals", "azure.yaml")) - require.NoError(t, err) - assert.Equal(t, wiringAdded, action) - - body, err := os.ReadFile(rootConfigName) - require.NoError(t, err) - assert.Contains(t, string(body), "host: containerapp", "the unrelated service is intact") - assert.Contains(t, string(body), "evals2:", "the eval service takes a free name") +func TestEvalServiceName_StepsAsideForAnExistingName(t *testing.T) { + assert.Equal(t, "evals2", evalServiceName(projectWith("evals"))) + assert.Equal(t, "evals3", evalServiceName(projectWith("evals", "evals2"))) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go new file mode 100644 index 00000000000..e614af18378 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeJSONL(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "d.jsonl") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// The service accepts whatever bytes it is given, so a malformed row becomes a +// published version with an eval group bound to it, and only fails much later +// on a row nobody has looked at. A live deploy published `{not json at all}` +// as version 1.0 before this existed. +func TestValidateJSONL_RejectsAMalformedRowByLine(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"fine\"}\n{not json at all}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "one JSON object") +} + +func TestValidateJSONL_AcceptsWellFormedRows(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n{\"query\":\"b\"}\n"))) +} + +// Trailing and interior blank lines are formatting, not rows. +func TestValidateJSONL_IgnoresBlankLines(t *testing.T) { + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\"a\"}\n\n{\"query\":\"b\"}\n\n"))) +} + +// A file with nothing in it publishes a version that can never score anything. +func TestValidateJSONL_RejectsAFileWithNoRows(t *testing.T) { + err := validateJSONL(writeJSONL(t, "\n\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "no rows") +} + +// A JSON array is the shape people reach for when they mean JSONL. +func TestValidateJSONL_RejectsAJSONArray(t *testing.T) { + err := validateJSONL(writeJSONL(t, "[{\"query\":\"a\"},{\"query\":\"b\"}]\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 1") +} + +// An empty object parses but evaluates to nothing. +func TestValidateJSONL_RejectsAnEmptyObject(t *testing.T) { + err := validateJSONL(writeJSONL(t, "{\"query\":\"a\"}\n{}\n")) + require.Error(t, err) + assert.Contains(t, err.Error(), "line 2") + assert.Contains(t, err.Error(), "empty object") +} + +// A conversation-level row holds a whole transcript and runs past bufio's +// default 64KB line limit, which would otherwise be reported as invalid JSON. +func TestValidateJSONL_AcceptsAVeryLongRow(t *testing.T) { + long := make([]byte, 200*1024) + for i := range long { + long[i] = 'x' + } + assert.NoError(t, validateJSONL(writeJSONL(t, + "{\"query\":\""+string(long)+"\"}\n"))) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 4b2c52401c8..a9d6737924e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -4,12 +4,14 @@ package cmd import ( + "bufio" "context" "encoding/json" "fmt" "os" "path/filepath" "reflect" + "strings" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/project" @@ -66,6 +68,13 @@ func (r *evalReconciler) EnsureDataset( return "", false, fmt.Errorf("dataset source %q: %w", localPath, err) } + // A malformed row is only noticed once the service tries to evaluate it, + // by which point a version has been published and the eval group points at + // it. Reading the file here costs nothing and names the offending line. + if err := validateJSONL(localPath); err != nil { + return "", false, fmt.Errorf("dataset %q: %w", decl.Name, err) + } + digest, err := project.Fingerprint(localPath) if err != nil { return "", false, err @@ -140,6 +149,50 @@ func (r *evalReconciler) EnsureDataset( // pinning the eval group to the older version would quietly evaluate against // stale data. Publishing is not destructive — versions are immutable — so the // remedy is to sync, not to overwrite. +// validateJSONL checks that every row is a JSON object before the file is +// published. +// +// The service accepts the upload whatever the bytes are, so a typo becomes a +// registered version, an eval group bound to it, and a run that fails on a row +// nobody has looked at. Blank lines are skipped: they are not rows. +func validateJSONL(path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + // A row carrying a whole conversation runs well past the 64KB default. + scanner.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + rows := 0 + for line := 1; scanner.Scan(); line++ { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(text), &row); err != nil { + return fmt.Errorf( + "%s line %d is not valid JSON: %w. Every line must be one JSON object", + path, line, err) + } + if len(row) == 0 { + return fmt.Errorf( + "%s line %d is an empty object, which evaluates to nothing", path, line) + } + rows++ + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("reading %s: %w", path, err) + } + if rows == 0 { + return fmt.Errorf("%s has no rows to evaluate", path) + } + return nil +} + func (r *evalReconciler) checkDatasetDrift( ctx context.Context, name, recorded string, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 1fc0ee4157b..0cc74767f1f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -222,6 +222,11 @@ func (ec *evalContext) latestOrNamedRun( list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { + if eval_api.IsNotFound(err) { + return nil, fmt.Errorf( + "no eval group %q in this project; "+ + "`azd up` creates the ones your config declares", evalID) + } return nil, fmt.Errorf("listing runs for eval group %s: %w", evalID, err) } if len(list.Data) == 0 { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 1ff2ee7cebb..e815d4866bd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -49,6 +49,11 @@ func newRunListCommand() *cobra.Command { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no eval group %q in this project; "+ + "`azd up` creates the ones your config declares", evalID) + } return fmt.Errorf("listing runs for %q: %w", evalID, err) } if isJSON(cmd) { From f3a8ace964e5e8966228157233a559aded355512 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 20:52:52 +0300 Subject: [PATCH 057/114] Order the eval service after the Foundry project when the repo declares one The service entry named no dependencies, so azd had nothing to order it against. The agents extension wires uses: to the ai-project service it created, and the eval service wants the same: it evaluates against that project, so the project should be provisioned first. It is conditional for the reason agents makes it conditional - naming a service the project does not declare is a broken reference, and an eval config can sit in a repo that reaches an existing Foundry project by endpoint rather than declaring one. --- .../azure.ai.evaluations/internal/cmd/init.go | 21 +++++++++++++++++++ .../internal/cmd/init_wiring_test.go | 16 ++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index be2d29941e2..b7804ae91a6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -124,6 +124,10 @@ func newInitCommand() *cobra.Command { // rootConfigName is azd's project file, which the eval service is declared in. const rootConfigName = "azure.yaml" +// aiProjectHost is the Foundry project service other extensions declare. The +// eval service uses it for ordering when the repo has one. +const aiProjectHost = "azure.ai.project" + // How the root config ended up referencing the eval service. const ( wiringAdded = "added" // the service was added to the project @@ -176,6 +180,7 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) Service: &azdext.ServiceConfig{ Name: evalServiceName(resp.GetProject()), Host: project.EvalHost, + Uses: projectServiceUses(resp.GetProject()), AdditionalProperties: props, }, }) @@ -185,6 +190,22 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) return wiringAdded, nil } +// projectServiceUses points the eval service at the Foundry project service +// when the repo declares one, so azd provisions it first. +// +// It is conditional for the same reason the agents extension makes it +// conditional: naming a service the project does not declare is a broken +// reference, and an eval config can perfectly well sit in a repo that reaches +// an existing Foundry project by endpoint instead. +func projectServiceUses(proj *azdext.ProjectConfig) []string { + for name, svc := range proj.GetServices() { + if svc.GetHost() == aiProjectHost { + return []string{name} + } + } + return nil +} + // evalServiceName avoids colliding with a service the project already has. // azd keys services by name, so the map key is the name to avoid. func evalServiceName(proj *azdext.ProjectConfig) string { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go index c5371c23db2..18b37e8e969 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -30,3 +30,19 @@ func TestEvalServiceName_StepsAsideForAnExistingName(t *testing.T) { assert.Equal(t, "evals2", evalServiceName(projectWith("evals"))) assert.Equal(t, "evals3", evalServiceName(projectWith("evals", "evals2"))) } + +// The agents extension wires uses: only to services the project actually +// declares. Naming one it does not have is a broken reference, and an eval +// config can sit in a repo that reaches an existing Foundry project by +// endpoint instead of declaring one. +func TestProjectServiceUses_OnlyWhenTheProjectDeclaresOne(t *testing.T) { + assert.Nil(t, projectServiceUses(projectWith("api", "web")), + "no Foundry project service means no uses entry") + + withProject := projectWith("api") + withProject.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", Host: aiProjectHost, + } + assert.Equal(t, []string{"ai-project"}, projectServiceUses(withProject), + "the eval service should be ordered after the project it evaluates against") +} From cfe114072c21bbf6b07df3a211a4f304b14e3ad8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 22:14:15 +0300 Subject: [PATCH 058/114] Address review: drop "eval group", split the model flags, show output items Renames, all user-facing, from the spec review: - "eval group" is not a service concept. The object is an eval, so evalGroups: becomes evals:, --eval-group becomes --eval, EVAL_GROUP_ID becomes EVAL_ID, and the prose follows. Breaking, and cheapest now. - --eval-model meant the judge in run and the generator in generate, which is two different things wearing one name. Split into --judge-model and --generation-model. - --gen-instruction is the agent's instruction, so --agent-instruction. - evaluator upload becomes evaluator create, matching dataset create. update is gone: both published a new immutable version, so it was a synonym. - evaluator builtins folds into evaluator list --builtin. results show only ever fetched the run, which carries totals and a per-criterion breakdown - so it could say how many rows failed but never which or why, and --failed-only filtered criteria rather than rows. It now reads the run's output_items, which carry the evaluated row, each evaluator's score and verdict, and the judge's reason. --failed-only filters rows, and -o json carries the whole thing. run delete is added; the API supported it all along. --- .../internal/cmd/apiversions.go | 2 +- .../internal/cmd/build.go | 8 +- .../internal/cmd/build_live_test.go | 10 +- .../internal/cmd/build_test.go | 30 ++-- .../internal/cmd/compare.go | 12 +- .../internal/cmd/context.go | 4 +- .../internal/cmd/description_test.go | 4 +- .../internal/cmd/envkeys_test.go | 18 +- .../internal/cmd/evaluator.go | 62 +++---- .../internal/cmd/generate.go | 16 +- .../azure.ai.evaluations/internal/cmd/init.go | 6 +- .../internal/cmd/init_test.go | 6 +- .../internal/cmd/instruction_test.go | 2 +- .../internal/cmd/jsonl_validation_test.go | 2 +- .../internal/cmd/model_target_test.go | 6 +- .../internal/cmd/reconciler.go | 26 +-- .../internal/cmd/resolution_test.go | 10 +- .../internal/cmd/results.go | 156 +++++++++++++----- .../azure.ai.evaluations/internal/cmd/run.go | 76 ++++----- .../internal/cmd/run_ops.go | 68 +++++++- .../internal/cmd/run_ops_test.go | 8 +- .../internal/cmd/schedule.go | 18 +- .../internal/pkg/eval_api/evaluators.go | 34 ++++ .../internal/pkg/eval_api/models.go | 66 ++++++++ .../internal/pkg/evalcore/evaluator.go | 2 +- .../internal/project/eval_config.go | 54 +++--- .../internal/project/eval_config_test.go | 20 +-- .../internal/project/merge_test.go | 8 +- .../internal/project/service_target_eval.go | 28 ++-- .../project/service_target_eval_test.go | 22 +-- 30 files changed, 494 insertions(+), 290 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go index 89b1d125f34..0c3ddb78734 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/apiversions.go @@ -12,6 +12,6 @@ const ( // DataGenerationAPIVersion covers dataset generation jobs. DataGenerationAPIVersion = "v1" - // OpenAI-compatible eval group and run calls send no api-version, so there + // OpenAI-compatible eval and run calls send no api-version, so there // is deliberately no constant for them. ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index c92ff0bccf3..6ca6d2fcf61 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -246,7 +246,7 @@ func planCriterion( } if len(missingInit) > 0 { return nil, fmt.Errorf( - "evaluator %q requires %s; set the judge model on the eval group", + "evaluator %q requires %s; set the judge model on the eval", ref.Name, quoteList(missingInit), ) } @@ -255,7 +255,7 @@ func planCriterion( return plan, nil } -// buildEvalGroupRequest converts an eval group declaration into the create +// buildEvalRequest converts an eval declaration into the create // request. Each evaluator becomes a testing criterion bound to its own // contract, and the item schema declares every dataset column those bindings // reference. @@ -263,8 +263,8 @@ func planCriterion( // schemas may be nil or partial; an evaluator with no published contract falls // back to the agent-target shape. datasetColumns may be nil, meaning the // columns are unknown and every accepted field is assumed present. -func buildEvalGroupRequest( - group *project.EvalGroup, +func buildEvalRequest( + group *project.Eval, schemas map[string]*eval_api.EvaluatorSummary, datasetColumns map[string]bool, ) (*eval_api.CreateOpenAIEvalRequest, error) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 124b56c0249..b7c9c32500c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -3,7 +3,7 @@ //go:build live -// This file proves the request buildEvalGroupRequest produces is accepted by +// This file proves the request buildEvalRequest produces is accepted by // the real service. It lives in the cmd package on purpose: the tests under // tests/live can only hand-roll a request, which validates the API but not the // code that ships. @@ -93,7 +93,7 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { level = summary.SupportedEvaluationLevels[0] } - group := &project.EvalGroup{ + group := &project.Eval{ Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), Dataset: "inline", Target: &project.Target{Type: "agent", Name: "probe-agent"}, @@ -101,7 +101,7 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, } - req, err := buildEvalGroupRequest(group, schemas, columns) + req, err := buildEvalRequest(group, schemas, columns) require.NoError(t, err, "the builder must satisfy every published contract") created, err := client.CreateOpenAIEval(ctx, req) @@ -135,7 +135,7 @@ func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { require.NotEmpty(t, target.DataSchema().Required, "this test relies on ifeval declaring required inputs") - group := &project.EvalGroup{ + group := &project.Eval{ Name: "azd-live-negative", Dataset: "inline", Target: &project.Target{Type: "agent", Name: "probe-agent"}, @@ -144,7 +144,7 @@ func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { } // A dataset with only `query` cannot satisfy ifeval. - _, err = buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err = buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "instruction_id_list") t.Logf("pre-flight error: %v", err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index 89785096c8f..f6c80e602a8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -35,8 +35,8 @@ func schema(name string, dataRequired, dataProps, initRequired, initProps []stri } } -func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.EvalGroup { - return &project.EvalGroup{ +func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.Eval { + return &project.Eval{ Name: "g", Dataset: "d", Target: &project.Target{Type: "agent", Name: "my-agent"}, @@ -59,7 +59,7 @@ func TestBuildBindsAgentFieldsFromSample(t *testing.T) { &project.Options{EvalModel: "gpt-4.1-nano"}, ) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Len(t, req.TestingCriteria, 1) @@ -82,7 +82,7 @@ func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { } group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) - _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "instruction_id_list") require.Contains(t, err.Error(), "instruction_kwargs") @@ -98,7 +98,7 @@ func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { } group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + req, err := buildEvalRequest(group, schemas, map[string]bool{ "instruction_id_list": true, "instruction_kwargs": true, }) @@ -131,7 +131,7 @@ func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { {Name: "builtin.similarity", Threshold: &threshold}, }, &project.Options{EvalModel: "gpt-4.1-nano"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "ground_truth": true, }) require.NoError(t, err) @@ -163,7 +163,7 @@ func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { {Name: "builtin.similarity"}, }, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Equal(t, "turn", req.TestingCriteria[0].InitializationParameters["evaluation_level"]) @@ -181,7 +181,7 @@ func TestBuildRejectsUnsupportedLevel(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) - _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "conversation") require.Contains(t, err.Error(), "turn") @@ -196,7 +196,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { } group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) - _, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) require.Contains(t, err.Error(), "deployment_name") } @@ -207,7 +207,7 @@ func TestBuildFallsBackWithoutSchema(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "my-custom-evaluator"}}, &project.Options{EvalModel: "m"}) - req, err := buildEvalGroupRequest(group, nil, nil) + req, err := buildEvalRequest(group, nil, nil) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping @@ -232,7 +232,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { // Turn level keeps query/response and drops messages. turn := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) - req, err := buildEvalGroupRequest(turn, schemas, columns) + req, err := buildEvalRequest(turn, schemas, columns) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping require.Contains(t, mapping, "query") @@ -241,7 +241,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { // Conversation level keeps messages and drops query/response. conv := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) - req, err = buildEvalGroupRequest(conv, schemas, columns) + req, err = buildEvalRequest(conv, schemas, columns) require.NoError(t, err) mapping = req.TestingCriteria[0].DataMapping require.Contains(t, mapping, "messages") @@ -251,7 +251,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { // An unset level behaves as turn, matching the service default. dflt := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, &project.Options{EvalModel: "m"}) - req, err = buildEvalGroupRequest(dflt, schemas, columns) + req, err = buildEvalRequest(dflt, schemas, columns) require.NoError(t, err) require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") } @@ -273,7 +273,7 @@ func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { {Name: "my-rubric"}, }, &project.Options{EvalModel: "gpt-4.1-nano"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) builtin := req.TestingCriteria[0].InitializationParameters @@ -297,7 +297,7 @@ func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) group.Target = nil - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{ + req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "response": true, "ground_truth": true, }) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go index f90ecfb5ef8..3859dc3664f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go @@ -31,7 +31,7 @@ func newResultsCompareCommand() *cobra.Command { cmd := &cobra.Command{ Use: "compare [eval-id]", - Short: "Compare runs of an eval group against a baseline.", + Short: "Compare runs of an eval against a baseline.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -90,7 +90,7 @@ func newResultsCompareCommand() *cobra.Command { cmd.Flags().StringArrayVar(&treatments, "treatment", nil, "Run to measure, repeatable. Defaults to the most recent completed run.") cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -110,12 +110,12 @@ func (ec *evalContext) resolveComparisonRuns( list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) if err != nil { - return "", nil, fmt.Errorf("listing runs of eval group %s: %w", evalID, err) + return "", nil, fmt.Errorf("listing runs of eval %s: %w", evalID, err) } completed := make([]string, 0, 2) if list == nil { - return "", nil, fmt.Errorf("eval group %s has no runs", evalID) + return "", nil, fmt.Errorf("eval %s has no runs", evalID) } for _, run := range list.Data { if run.Status == "completed" { @@ -126,14 +126,14 @@ func (ec *evalContext) resolveComparisonRuns( if len(treatments) == 0 { if len(completed) == 0 { return "", nil, fmt.Errorf( - "eval group %s has no completed runs to compare", evalID) + "eval %s has no completed runs to compare", evalID) } treatments = []string{completed[0]} } if baseline == "" { if len(completed) < 2 { return "", nil, fmt.Errorf( - "eval group %s has only one completed run, so there is nothing to compare it "+ + "eval %s has only one completed run, so there is nothing to compare it "+ "against; run it again, or name a baseline with --baseline", evalID) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 7ec1da4237a..0b987db0315 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -48,7 +48,7 @@ func newEvalContext(ctx context.Context, endpointFlag string) (*evalContext, err ec := &evalContext{azdClient: azdClient} // The environment name is resolved regardless of where the endpoint comes - // from: it is what the cached eval group and run ids are read from and + // from: it is what the cached eval and run ids are read from and // written to. Deriving it only when the endpoint came from azd meant // --project-endpoint silently disabled that cache. azdEndpoint, envName := lookupEndpointFromAzd(ctx, azdClient) @@ -153,7 +153,7 @@ func (ec *evalContext) Close() { // azd environment keys written by this extension. const ( - envKeyEvalGroupID = "EVAL_GROUP_ID" + envKeyEvalID = "EVAL_ID" envKeyEvalRunID = "EVAL_RUN_ID" envKeyDatasetVersion = "EVAL_DATASET_VERSION" envKeyFingerprintPrefix = "EVAL_FINGERPRINT_" diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go index ea819af4ef9..6d14b672856 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -25,7 +25,7 @@ func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { &project.Options{EvalModel: "m"}) group.Description = "Quality gate for the support agent" - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Equal(t, "Quality gate for the support agent", req.Metadata["azd_description"]) } @@ -40,7 +40,7 @@ func TestBuildOmitsEmptyDescription(t *testing.T) { group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, &project.Options{EvalModel: "m"}) - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.NotContains(t, req.Metadata, "azd_description") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index b78bd8debf9..e7c404211c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -14,8 +14,8 @@ import ( // exists, and hands it back for the wrong group — so group A silently scores // group B's criteria. func TestIDKey_IsPerName(t *testing.T) { - a := idKey("evalgroup", "quality-a") - b := idKey("evalgroup", "quality-b") + a := idKey("eval", "quality-a") + b := idKey("eval", "quality-b") assert.NotEqual(t, a, b, "two groups must not share an id key") assert.Contains(t, a, "QUALITY_A") @@ -25,9 +25,9 @@ func TestIDKey_IsPerName(t *testing.T) { // Names that are not valid env identifiers still have to produce distinct, // stable keys. func TestIDKey_NormalizesNames(t *testing.T) { - assert.Equal(t, idKey("evalgroup", "my group"), idKey("evalgroup", "my-group"), + assert.Equal(t, idKey("eval", "my group"), idKey("eval", "my-group"), "characters that cannot appear in an env name normalize the same way") - assert.NotEqual(t, idKey("evalgroup", "a"), idKey("dataset", "a"), + assert.NotEqual(t, idKey("eval", "a"), idKey("dataset", "a"), "the kind keeps different resources apart") } @@ -36,18 +36,18 @@ func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) } -// Setting EVAL_GROUP_ID by hand is the documented way to point a config at a +// Setting EVAL_ID by hand is the documented way to point a config at a // group that already exists. It is also the key the extension writes itself, // which is what let a second group adopt the first one's id — so it stays // readable only where it cannot be ambiguous. Fixing the aliasing dropped this // fallback entirely once, silently breaking the documented behaviour. func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { - sole := groupIDKeys("quality", true) - assert.Equal(t, idKey("evalgroup", "quality"), sole[0], + sole := evalIDKeys("quality", true) + assert.Equal(t, idKey("eval", "quality"), sole[0], "a group's own entry is preferred over the shared one") - assert.Contains(t, sole, envKeyEvalGroupID, + assert.Contains(t, sole, envKeyEvalID, "a single-group config honours an id set by hand") - assert.Equal(t, []string{idKey("evalgroup", "quality")}, groupIDKeys("quality", false), + assert.Equal(t, []string{idKey("eval", "quality")}, evalIDKeys("quality", false), "with several groups the shared entry cannot say which group it means") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 4ec85fca946..100e8e7b10a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -19,32 +19,29 @@ func newEvaluatorCommand() *cobra.Command { Short: "Manage custom evaluators.", } cmd.AddCommand( - newEvaluatorUploadCommand(false), - newEvaluatorUploadCommand(true), + newEvaluatorCreateCommand(), newEvaluatorListCommand(), newEvaluatorShowCommand(), - newEvaluatorBuiltinsCommand(), newEvaluatorDeleteCommand(), ) return cmd } -// newEvaluatorUploadCommand builds `evaluator upload` and `evaluator update`. -// Both publish a new immutable version. +// newEvaluatorCreateCommand builds `evaluator create`, named to match +// `dataset create`: both register an artifact and both publish a new immutable +// version every time, so there is nothing for a separate `update` to do. // // M1 supports rubric evaluators only. Code evaluators need a folder walk, -// multi-blob upload, and the Azure AI User role assignment, so they land in M2. -func newEvaluatorUploadCommand(update bool) *cobra.Command { +// multi-blob upload, and the Azure AI User role assignment, so they land later. +func newEvaluatorCreateCommand() *cobra.Command { var ( name string rubric string endpointFlg string ) - use, short := "upload", "Register a rubric evaluator, creating its first version." - if update { - use, short = "update", "Publish a new version of an existing rubric evaluator." - } + use := "create" + short := "Register a rubric evaluator, publishing a new version." cmd := &cobra.Command{ Use: use, @@ -165,12 +162,13 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { func newEvaluatorListCommand() *cobra.Command { var ( name string + builtin bool endpointFlg string ) cmd := &cobra.Command{ Use: "list", - Short: "List evaluators, or the versions of one evaluator.", + Short: "List evaluators, the versions of one evaluator, or the built-in evaluators.", RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -180,9 +178,15 @@ func newEvaluatorListCommand() *cobra.Command { defer ec.Close() var list *eval_api.EvaluatorListResponse - if name != "" { + switch { + case name != "": list, err = ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) - } else { + case builtin: + // The service filters by type, and asking for nothing returns + // only the project's own evaluators. + list, err = ec.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) + default: list, err = ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) } if err != nil { @@ -193,34 +197,8 @@ func newEvaluatorListCommand() *cobra.Command { } cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this evaluator.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newEvaluatorBuiltinsCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "builtins", - Short: "List the platform's built-in evaluators.", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - list, err := ec.evalClient.ListEvaluators( - ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion, - ) - if err != nil { - return fmt.Errorf("listing built-in evaluators: %w", err) - } - return renderEvaluators(cmd, list) - }, - } - + cmd.Flags().BoolVar(&builtin, "builtin", false, "List the built-in evaluators instead of the project's own.") + cmd.MarkFlagsMutuallyExclusive("name", "builtin") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index a3508ade263..e0203f07d5e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -79,7 +79,7 @@ func newGenerateCommand() *cobra.Command { generatingDataset := datasetFlag == "" && cfg.Generate.Dataset != nil if (generatingRubric || generatingDataset) && generationModel(cfg) == "" { return fmt.Errorf( - "a model deployment is required to generate: pass --eval-model, " + + "a model deployment is required to generate: pass --generation-model, " + "or set generate.rubric.model in the generation spec") } @@ -154,11 +154,11 @@ func newGenerateCommand() *cobra.Command { cmd.Flags().StringVar(&deployPath, "deploy-config", project.DefaultDeployConfig, "Deployment spec to write source references into.") cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") - cmd.Flags().StringVar(&instruction, "gen-instruction", "", + cmd.Flags().StringVar(&instruction, "agent-instruction", "", "What the agent does and what to test.") - cmd.Flags().StringVar(&instructionFile, "gen-instruction-file", "", - "Read the generation instruction from this file. Mutually exclusive with --gen-instruction.") - cmd.MarkFlagsMutuallyExclusive("gen-instruction", "gen-instruction-file") + cmd.Flags().StringVar(&instructionFile, "agent-instruction-file", "", + "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") + cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") cmd.Flags().StringVar(&datasetFlag, "dataset", "", "Use this dataset instead of generating one.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, @@ -167,7 +167,7 @@ func newGenerateCommand() *cobra.Command { fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) cmd.Flags().IntVar(&traceDays, "trace-days", 0, "Days of traces to seed rubric generation. 0 disables.") - cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used for generation.") + cmd.Flags().StringVar(&evalModel, "generation-model", "", "Model deployment that generates the dataset and rubric.") cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the jobs and return without polling.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -221,11 +221,11 @@ func resolveInstruction(inline, path string) (string, error) { } raw, err := os.ReadFile(path) if err != nil { - return "", fmt.Errorf("reading --gen-instruction-file %q: %w", path, err) + return "", fmt.Errorf("reading --agent-instruction-file %q: %w", path, err) } text := strings.TrimSpace(string(raw)) if text == "" { - return "", fmt.Errorf("--gen-instruction-file %q is empty", path) + return "", fmt.Errorf("--agent-instruction-file %q is empty", path) } return text, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index b7804ae91a6..85359d184ae 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -114,7 +114,7 @@ func newInitCommand() *cobra.Command { cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, "Evaluator reference, repeatable. Use builtin. for a built-in.") - cmd.Flags().StringVar(&evalModel, "eval-model", "", "Model deployment used as the LLM judge.") + cmd.Flags().StringVar(&evalModel, "judge-model", "", "Model deployment that scores the results.") cmd.Flags().StringVar(&outDir, "out-dir", project.DefaultEvalDir, "Directory to write the config into. Used verbatim, never re-rooted.") cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") @@ -334,7 +334,7 @@ func buildDeployScaffold( } } - group := project.EvalGroup{ + group := project.Eval{ Name: fmt.Sprintf("%s-quality", target), Description: fmt.Sprintf("Quality gate for %s", target), Dataset: datasetName, @@ -347,7 +347,7 @@ func buildDeployScaffold( if evalModel != "" { group.Options = &project.Options{EvalModel: evalModel} } - cfg.EvalGroups = append(cfg.EvalGroups, group) + cfg.Evals = append(cfg.Evals, group) return cfg } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index f0db8167284..a4e22c85f87 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -60,9 +60,9 @@ func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") require.Equal(t, "my-custom", cfg.Evaluators[0].Name) - require.Len(t, cfg.EvalGroups[0].Evaluators, 2) - require.True(t, cfg.EvalGroups[0].Evaluators[0].IsBuiltin()) - require.False(t, cfg.EvalGroups[0].Evaluators[1].IsBuiltin()) + require.Len(t, cfg.Evals[0].Evaluators, 2) + require.True(t, cfg.Evals[0].Evaluators[0].IsBuiltin()) + require.False(t, cfg.Evals[0].Evaluators[1].IsBuiltin()) path := filepath.Join(t.TempDir(), "azure.yaml") require.NoError(t, writeYAML(path, cfg)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go index 4063a6ab1b4..52279b70b9b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/instruction_test.go @@ -40,7 +40,7 @@ func TestResolveInstructionPrefersInlineWhenNoFile(t *testing.T) { func TestResolveInstructionRejectsUnusableFile(t *testing.T) { _, err := resolveInstruction("", filepath.Join(t.TempDir(), "absent.md")) require.Error(t, err) - require.Contains(t, err.Error(), "gen-instruction-file") + require.Contains(t, err.Error(), "agent-instruction-file") empty := filepath.Join(t.TempDir(), "empty.md") require.NoError(t, os.WriteFile(empty, []byte(" \n"), 0o600)) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go index e614af18378..7cf4fc48788 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/jsonl_validation_test.go @@ -20,7 +20,7 @@ func writeJSONL(t *testing.T, body string) string { } // The service accepts whatever bytes it is given, so a malformed row becomes a -// published version with an eval group bound to it, and only fails much later +// published version with an eval bound to it, and only fails much later // on a row nobody has looked at. A live deploy published `{not json at all}` // as version 1.0 before this existed. func TestValidateJSONL_RejectsAMalformedRowByLine(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go index 2a458a5778c..c202a87a4c7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go @@ -33,7 +33,7 @@ func TestSampleBindingsFor(t *testing.T) { } // The criteria a group sends depend on what it targets. -func TestBuildEvalGroupRequest_BindsByTargetKind(t *testing.T) { +func TestBuildEvalRequest_BindsByTargetKind(t *testing.T) { schemas := map[string]*eval_api.EvaluatorSummary{ "builtin.coherence": { Name: "builtin.coherence", @@ -57,12 +57,12 @@ func TestBuildEvalGroupRequest_BindsByTargetKind(t *testing.T) { {project.TargetTypeModel, "{{sample.output_text}}"}, } { t.Run(tc.targetType, func(t *testing.T) { - group := &project.EvalGroup{ + group := &project.Eval{ Name: "quality", Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.coherence"}}, Target: &project.Target{Type: tc.targetType, Name: "thing"}, } - req, err := buildEvalGroupRequest(group, schemas, map[string]bool{"query": true}) + req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) require.Len(t, req.TestingCriteria, 1) assert.Equal(t, tc.want, req.TestingCriteria[0].DataMapping["response"]) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index a9d6737924e..a06f13853d3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -69,7 +69,7 @@ func (r *evalReconciler) EnsureDataset( } // A malformed row is only noticed once the service tries to evaluate it, - // by which point a version has been published and the eval group points at + // by which point a version has been published and the eval points at // it. Reading the file here costs nothing and names the offending line. if err := validateJSONL(localPath); err != nil { return "", false, fmt.Errorf("dataset %q: %w", decl.Name, err) @@ -146,14 +146,14 @@ func (r *evalReconciler) EnsureDataset( // // Local content being unchanged is not enough to reuse the recorded version: // someone may have published a newer one outside the repo, and silently -// pinning the eval group to the older version would quietly evaluate against +// pinning the eval to the older version would quietly evaluate against // stale data. Publishing is not destructive — versions are immutable — so the // remedy is to sync, not to overwrite. // validateJSONL checks that every row is a JSON object before the file is // published. // // The service accepts the upload whatever the bytes are, so a typo becomes a -// registered version, an eval group bound to it, and a run that fails on a row +// registered version, an eval bound to it, and a run that fails on a row // nobody has looked at. Blank lines are skipped: they are not rows. func validateJSONL(path string) error { f, err := os.Open(path) @@ -270,12 +270,12 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } -// EnsureEvalGroup creates the group when it has never been deployed, or when an +// EnsureEval creates the group when it has never been deployed, or when an // upstream artifact changed. Groups are immutable, so a change means a new // group and a new id. -func (r *evalReconciler) EnsureEvalGroup( +func (r *evalReconciler) EnsureEval( ctx context.Context, - group project.EvalGroup, + group project.Eval, datasetPath string, recreate bool, ) (string, error) { @@ -290,24 +290,24 @@ func (r *evalReconciler) EnsureEvalGroup( if err != nil { return "", err } - key := project.FingerprintKey("evalgroup", group.Name) + key := project.FingerprintKey("eval", group.Name) if prior := r.ec.getEnvValue(ctx, key); prior != "" && prior != digest { recreate = true } - cached := r.ec.getEnvValue(ctx, idKey("evalgroup", group.Name)) + cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { // Record the digest on reuse as well, otherwise a group deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, cached) + _ = r.ec.setEnvValue(ctx, envKeyEvalID, cached) return cached, nil } } - req, err := buildEvalGroupRequest( + req, err := buildEvalRequest( &group, r.ec.evaluatorSchemas(ctx), datasetColumnsFromPath(datasetPath), @@ -320,10 +320,10 @@ func (r *evalReconciler) EnsureEvalGroup( return "", err } _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID) - // EVAL_GROUP_ID stays the last-deployed group, which is what the commands + _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID) + // EVAL_ID stays the last-deployed group, which is what the commands // fall back to when a config names only one. - _ = r.ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) + _ = r.ec.setEnvValue(ctx, envKeyEvalID, created.ID) return created.ID, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index 9bd441b0f4b..1d8c809bb6b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -15,28 +15,28 @@ import ( // is silent. options.max_samples was parsed and dropped once already, which is // what these lock down. func TestResolveMaxSamples_Precedence(t *testing.T) { - withOptions := &project.EvalGroup{Options: &project.Options{MaxSamples: 25}} + withOptions := &project.Eval{Options: &project.Options{MaxSamples: 25}} assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") - assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{}), "neither means no cap") + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{}), "neither means no cap") assert.Equal(t, 0, resolveMaxSamples(0, nil)) assert.Equal(t, 7, resolveMaxSamples(7, nil), "a flag stands on its own") // Zero in config is absent, not a cap of zero: a cap of zero would send // nothing at all. - assert.Equal(t, 0, resolveMaxSamples(0, &project.EvalGroup{Options: &project.Options{MaxSamples: 0}})) + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{Options: &project.Options{MaxSamples: 0}})) } func TestResolveLevel_Precedence(t *testing.T) { - withOptions := &project.EvalGroup{ + withOptions := &project.Eval{ Options: &project.Options{EvaluationLevel: project.EvaluationLevelConversation}, } assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), "the flag wins over the config") assert.Equal(t, project.EvaluationLevelConversation, resolveLevel("", withOptions)) - assert.Empty(t, resolveLevel("", &project.EvalGroup{}), "unset defers to the service default") + assert.Empty(t, resolveLevel("", &project.Eval{}), "unset defers to the service default") assert.Empty(t, resolveLevel("", nil)) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 0cc74767f1f..405b538c01e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -56,25 +56,45 @@ func newResultsShowCommand() *cobra.Command { return err } + // The run carries totals and a per-criterion breakdown. The output + // items are the rows themselves, which is what "which one failed, + // and why" needs. A run that never produced any still renders its + // totals rather than failing. + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil { + return fmt.Errorf("reading the results of run %s: %w", run.ID, err) + } + rows := items.Data + if failedOnly { + kept := make([]eval_api.OutputItem, 0, len(rows)) + for _, it := range rows { + if it.Failed() { + kept = append(kept, it) + } + } + rows = kept + } + + payload := map[string]any{"run": run, "output_items": rows} if outFile != "" { f, err := os.Create(outFile) if err != nil { return fmt.Errorf("creating %q: %w", outFile, err) } defer f.Close() - return emitJSON(f, run) + return emitJSON(f, payload) } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), run) + return emitJSON(cmd.OutOrStdout(), payload) } - return renderResults(cmd.OutOrStdout(), run, failedOnly) + return renderResults(cmd.OutOrStdout(), run, rows, failedOnly) }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") - cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only criteria with failures.") + cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -135,16 +155,16 @@ func newResultsExportCommand() *cobra.Command { cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// resolveEvalID takes the eval group id from the argument, from a group named -// with --eval-group, or from the id cached in the azd environment. +// resolveEvalID takes the eval id from the argument, from a group named +// with --eval, or from the id cached in the azd environment. // // The cached id is the last group deployed, which is unambiguous only while a -// config declares one. --eval-group is how the others are reached without +// config declares one. --eval is how the others are reached without // having to know their service ids. func resolveEvalID( cmd *cobra.Command, @@ -161,38 +181,38 @@ func resolveEvalID( } if groupName != "" { - if id := ec.getEnvValue(cmd.Context(), idKey("evalgroup", groupName)); id != "" { + if id := ec.getEnvValue(cmd.Context(), idKey("eval", groupName)); id != "" { return id, nil } return "", fmt.Errorf( - "eval group %q has no id recorded in this environment; deploy it first, "+ + "eval %q has no id recorded in this environment; deploy it first, "+ "or pass its id directly", groupName) } - if cached := ec.getEnvValue(cmd.Context(), envKeyEvalGroupID); cached != "" { + if cached := ec.getEnvValue(cmd.Context(), envKeyEvalID); cached != "" { return cached, nil } return "", fmt.Errorf( - "no eval group id given; pass it as an argument, name one with --eval-group, "+ + "no eval id given; pass it as an argument, name one with --eval, "+ "or set %s in the azd environment", - envKeyEvalGroupID) + envKeyEvalID) } -// addEvalGroupFlag registers the flag that names a group from the config, so +// addEvalFlag registers the flag that names a group from the config, so // every command taking an eval-id can reach a group by the name its author // used. -// addEvalGroupFlags registers the two ways to say which group a command acts -// on: --eval-group names one from the config, --eval-id gives its service id. +// addEvalFlags registers the two ways to say which group a command acts +// on: --eval names one from the config, --eval-id gives its service id. // // The id is also accepted as a positional argument. The flag exists because // `run start --eval-id` already spells it that way, and a script that learned // it there should not have to find out that the sibling commands take only a // positional. -func addEvalGroupFlags(cmd *cobra.Command, target *string) { - cmd.Flags().StringVar(target, "eval-group", "", +func addEvalFlags(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval", "", "Name a group from the config instead of passing its id.") cmd.Flags().String("eval-id", "", - "Id of the eval group. Same as passing the id as an argument.") + "Id of the eval. Same as passing the id as an argument.") } // latestOrNamedRun returns the named run, or the most recent one for the group. @@ -224,18 +244,23 @@ func (ec *evalContext) latestOrNamedRun( if err != nil { if eval_api.IsNotFound(err) { return nil, fmt.Errorf( - "no eval group %q in this project; "+ + "no eval %q in this project; "+ "`azd up` creates the ones your config declares", evalID) } - return nil, fmt.Errorf("listing runs for eval group %s: %w", evalID, err) + return nil, fmt.Errorf("listing runs for eval %s: %w", evalID, err) } if len(list.Data) == 0 { - return nil, fmt.Errorf("eval group %s has no runs yet", evalID) + return nil, fmt.Errorf("eval %s has no runs yet", evalID) } return &list.Data[0], nil } -func renderResults(w io.Writer, run *eval_api.OpenAIEvalRun, failedOnly bool) error { +func renderResults( + w io.Writer, + run *eval_api.OpenAIEvalRun, + items []eval_api.OutputItem, + failedOnly bool, +) error { fmt.Fprintf(w, "Run %s status: %s\n", run.ID, run.Status) if c := run.ResultCounts; c != nil { @@ -243,35 +268,80 @@ func renderResults(w io.Writer, run *eval_api.OpenAIEvalRun, failedOnly bool) er c.Passed, c.Failed, c.Errored) } - if len(run.PerTestingCriteria) == 0 { - fmt.Fprintln(w, "No per-criteria results are available yet.") - return nil + if len(run.PerTestingCriteria) > 0 { + rows := make([][]string, 0, len(run.PerTestingCriteria)) + for _, cr := range run.PerTestingCriteria { + if failedOnly && cr.Failed == 0 { + continue + } + rows = append(rows, []string{ + cr.TestingCriteria, + strconv.Itoa(cr.Passed), + strconv.Itoa(cr.Failed), + }) + } + if len(rows) > 0 { + if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { + return err + } + } } - rows := make([][]string, 0, len(run.PerTestingCriteria)) - for _, cr := range run.PerTestingCriteria { - if failedOnly && cr.Failed == 0 { - continue + // The rows are the point of `results show`: totals say how many failed, + // these say which and why. + if len(items) == 0 { + if failedOnly { + fmt.Fprintln(w, "\nNo failing rows.") + } else { + fmt.Fprintln(w, "\nNo rows have been scored yet.") + } + } else { + fmt.Fprintln(w) + rows := make([][]string, 0, len(items)) + for _, it := range items { + for _, r := range it.Results { + if failedOnly && r.Passed { + continue + } + verdict := "pass" + if !r.Passed { + verdict = "FAIL" + } + rows = append(rows, []string{ + it.ID, + r.Name, + verdict, + formatStat("%.3f", r.Score), + truncate(it.Input(), 48), + truncate(r.Reason, 60), + }) + } + } + if err := emitTable(w, + []string{"ITEM", "EVALUATOR", "RESULT", "SCORE", "INPUT", "REASON"}, rows); err != nil { + return err } - rows = append(rows, []string{ - cr.TestingCriteria, - strconv.Itoa(cr.Passed), - strconv.Itoa(cr.Failed), - }) - } - if len(rows) == 0 { - fmt.Fprintln(w, "No failing criteria.") - return nil - } - if err := emitTable(w, []string{"CRITERION", "PASSED", "FAILED"}, rows); err != nil { - return err } + if run.ReportURL != "" { fmt.Fprintf(w, "\nReport: %s\n", run.ReportURL) } return nil } +// truncate keeps a table readable when a reason runs to a paragraph. The full +// text is always in `-o json`. +func truncate(s string, n int) string { + s = strings.ReplaceAll(strings.ReplaceAll(s, "\n", " "), "\r", "") + if len(s) <= n { + return s + } + if n <= 1 { + return s[:n] + } + return s[:n-1] + "…" +} + func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { cw := csv.NewWriter(w) defer cw.Flush() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 3f5d81ebb0f..a9a370113ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -37,10 +37,10 @@ var terminalRunStates = map[string]bool{ // of this same command. func newRunCommand() *cobra.Command { cmd := buildRunCommand( - "run", "Run an evaluation, creating the eval group if it does not exist yet.") + "run", "Run an evaluation, creating the eval if it does not exist yet.") addRunSubcommands(cmd) cmd.AddCommand(buildRunCommand( - "start", "Start a run, creating the eval group if it does not exist yet.")) + "start", "Start a run, creating the eval if it does not exist yet.")) return cmd } @@ -77,7 +77,7 @@ func buildRunCommand(use, short string) *cobra.Command { defer ec.Close() // --eval-id bypasses the config entirely. - var group *project.EvalGroup + var group *project.Eval if evalID == "" { cfg, err := project.LoadEvalConfig(configPath) if err != nil { @@ -95,9 +95,9 @@ func buildRunCommand(use, short string) *cobra.Command { return err } - evalID, err = ec.resolveEvalGroupID( + evalID, err = ec.resolveEvalIDFromConfig( ctx, group, configPath, resolveLevel(level, group), - len(cfg.EvalGroups) == 1, out, isJSON(cmd)) + len(cfg.Evals) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -179,10 +179,10 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, "Path to the eval deployment config.") - cmd.Flags().StringVar(&groupName, "eval-group", "", - "Which evalGroups entry to run. Defaults to the only one.") + cmd.Flags().StringVar(&groupName, "eval", "", + "Which evals entry to run. Defaults to the only one.") cmd.Flags().StringVar(&evalID, "eval-id", "", - "Run against an existing eval group by id, ignoring the config.") + "Run against an existing eval by id, ignoring the config.") cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the group name plus a timestamp.") cmd.Flags().StringVar(&level, "level", "", "Scoring granularity: turn or conversation. Defaults to the service default (turn).") @@ -213,12 +213,12 @@ func buildRunCommand(use, short string) *cobra.Command { return cmd } -// resolveEvalGroupID finds the eval group to run against, creating it when it +// resolveEvalIDFromConfig finds the eval to run against, creating it when it // has never been deployed. Resolution order: an id pinned on the group, then // the azd environment, then create. -func (ec *evalContext) resolveEvalGroupID( +func (ec *evalContext) resolveEvalIDFromConfig( ctx context.Context, - group *project.EvalGroup, + group *project.Eval, configPath string, level string, soleGroup bool, @@ -229,7 +229,7 @@ func (ec *evalContext) resolveEvalGroupID( return group.ID, nil } - for _, key := range groupIDKeys(group.Name, soleGroup) { + for _, key := range evalIDKeys(group.Name, soleGroup) { cached := ec.getEnvValue(ctx, key) if cached == "" { continue @@ -241,7 +241,7 @@ func (ec *evalContext) resolveEvalGroupID( } if !jsonMode { - fmt.Fprintf(out, "Creating eval group %q...\n", group.Name) + fmt.Fprintf(out, "Creating eval %q...\n", group.Name) } // The level from the flag wins over the group's own options, so it has to @@ -256,7 +256,7 @@ func (ec *evalContext) resolveEvalGroupID( effective.Options = &opts } - req, err := buildEvalGroupRequest( + req, err := buildEvalRequest( &effective, ec.evaluatorSchemas(ctx), datasetColumns(configPath, group), @@ -266,28 +266,28 @@ func (ec *evalContext) resolveEvalGroupID( } created, err := ec.evalClient.CreateOpenAIEval(ctx, req) if err != nil { - return "", fmt.Errorf("creating eval group %q: %w", group.Name, err) + return "", fmt.Errorf("creating eval %q: %w", group.Name, err) } - if err := ec.setEnvValue(ctx, idKey("evalgroup", group.Name), created.ID); err != nil { + if err := ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID); err != nil { fmt.Fprintf(out, "warning: %v\n", err) } - _ = ec.setEnvValue(ctx, envKeyEvalGroupID, created.ID) + _ = ec.setEnvValue(ctx, envKeyEvalID, created.ID) return created.ID, nil } -// groupIDKeys lists the env entries that may hold this group's id, most +// evalIDKeys lists the env entries that may hold this group's id, most // specific first. // -// The per-name entry is what the extension writes. EVAL_GROUP_ID is also the +// The per-name entry is what the extension writes. EVAL_ID is also the // documented way to point a config at a group that already exists, created in // the portal or by another tool, so it stays readable — but only when the // config declares a single group. With more than one there is no way to tell // which group a shared entry refers to, and reading it anyway is what let a // second group adopt the first one's id. -func groupIDKeys(name string, soleGroup bool) []string { - keys := []string{idKey("evalgroup", name)} +func evalIDKeys(name string, soleGroup bool) []string { + keys := []string{idKey("eval", name)} if soleGroup { - keys = append(keys, envKeyEvalGroupID) + keys = append(keys, envKeyEvalID) } return keys } @@ -297,7 +297,7 @@ func groupIDKeys(name string, soleGroup bool) []string { // // A run sends a local dataset inline, so without this the run would evaluate // content that no registered version corresponds to: the results are attributed -// to the eval group but cannot be traced back to a dataset version, which +// to the eval but cannot be traced back to a dataset version, which // makes them impossible to reproduce or compare. // // The check only applies once a deploy has recorded a fingerprint. Before that @@ -306,7 +306,7 @@ func groupIDKeys(name string, soleGroup bool) []string { func (ec *evalContext) checkDatasetRegistered( ctx context.Context, cfg *project.EvalConfig, - group *project.EvalGroup, + group *project.Eval, configPath string, ) error { localPath := localDatasetPath(configPath, group) @@ -336,7 +336,7 @@ func (ec *evalContext) checkDatasetRegistered( return fmt.Errorf( "dataset %q has local edits that are not registered.\n"+ " Run `azd up` to register them, or `--eval-id ` to run against "+ - "an existing eval group", + "an existing eval", decl.Name) } @@ -344,7 +344,7 @@ func (ec *evalContext) checkDatasetRegistered( // recent run. // // `--eval-id` deliberately ignores the config, but a run still needs a target -// and a dataset, and an eval group carries neither: the group holds only its +// and a dataset, and an eval carries neither: the group holds only its // testing criteria, and the dataset travels on the run. The previous run is the // only place that pairing survives, so re-running a group means repeating what // it last ran. @@ -354,11 +354,11 @@ func (ec *evalContext) reuseDataSourceFromLastRun( ) (*eval_api.EvalRunDataSource, error) { list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 1) if err != nil { - return nil, fmt.Errorf("reading previous runs of eval group %s: %w", evalID, err) + return nil, fmt.Errorf("reading previous runs of eval %s: %w", evalID, err) } if list == nil || len(list.Data) == 0 || list.Data[0].DataSource == nil { return nil, fmt.Errorf( - "eval group %s has no previous run to repeat, so there is no target or dataset "+ + "eval %s has no previous run to repeat, so there is no target or dataset "+ "to reuse.\n"+ " Run it from the config once with `azd ai eval run`, or pass a config that "+ "declares the group", @@ -376,7 +376,7 @@ func (ec *evalContext) reuseDataSourceFromLastRun( func buildTracesDataSource( ctx context.Context, ec *evalContext, - group *project.EvalGroup, + group *project.Eval, evalID, window string, maxTraces int, ) (*eval_api.EvalRunDataSource, error) { @@ -397,7 +397,7 @@ func buildTracesDataSource( } if agent == "" { return nil, fmt.Errorf( - "--from-traces needs to know whose traces to read, and the eval group does not " + + "--from-traces needs to know whose traces to read, and the eval does not " + "name an agent. Declare target.type: agent on the group") } @@ -408,16 +408,16 @@ func buildTracesDataSource( return eval_api.NewTracesDataSource(agent, lookbackHours, time.Time{}, maxTraces), nil } -// buildRunDataSource binds the dataset to the run. The eval group carries no +// buildRunDataSource binds the dataset to the run. The eval carries no // dataset today, so it is supplied here. func (ec *evalContext) buildRunDataSource( ctx context.Context, - group *project.EvalGroup, + group *project.Eval, configPath string, maxSamples int, ) (*eval_api.EvalRunDataSource, error) { if group == nil { - return nil, fmt.Errorf("no eval group to run") + return nil, fmt.Errorf("no eval to run") } // A group with no target scores a dataset that already holds the exchange, @@ -434,7 +434,7 @@ func (ec *evalContext) buildRunDataSource( } if group.Dataset == "" { - return nil, fmt.Errorf("eval group %q does not reference a dataset", group.Name) + return nil, fmt.Errorf("eval %q does not reference a dataset", group.Name) } // A local source is read from disk; anything else is already registered and @@ -512,7 +512,7 @@ func (ec *evalContext) readRegisteredDataset( // A nil result means the columns are unknown, which is the case for a dataset // already registered in the project. The builder then assumes every field an // evaluator accepts is present. -func datasetColumns(configPath string, group *project.EvalGroup) map[string]bool { +func datasetColumns(configPath string, group *project.Eval) map[string]bool { return datasetColumnsFromPath(localDatasetPath(configPath, group)) } @@ -536,7 +536,7 @@ func datasetColumnsFromPath(localPath string) map[string]bool { // localDatasetPath resolves the dataset's local source relative to the config // file, returning empty when the dataset is registered rather than local. -func localDatasetPath(configPath string, group *project.EvalGroup) string { +func localDatasetPath(configPath string, group *project.Eval) string { cfg, err := project.LoadEvalConfig(configPath) if err != nil { return "" @@ -601,7 +601,7 @@ func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { } // resolveLevel prefers the flag, then the group's options. -func resolveLevel(flag string, group *project.EvalGroup) string { +func resolveLevel(flag string, group *project.Eval) string { if flag != "" { return flag } @@ -617,7 +617,7 @@ func resolveLevel(flag string, group *project.EvalGroup) string { // Without this, options.max_samples parsed and did nothing: a group that caps // its sample count in config would send the whole dataset, and only a flag on // every invocation would honour the cap. -func resolveMaxSamples(flag int, group *project.EvalGroup) int { +func resolveMaxSamples(flag int, group *project.Eval) int { if flag > 0 { return flag } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index e815d4866bd..8fab2f3bca6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -21,6 +21,7 @@ func addRunSubcommands(cmd *cobra.Command) { newRunListCommand(), newRunShowCommand(), newRunCancelCommand(), + newRunDeleteCommand(), ) } @@ -32,7 +33,7 @@ func newRunListCommand() *cobra.Command { cmd := &cobra.Command{ Use: "list [eval-id]", - Short: "List runs for an eval group.", + Short: "List runs for an eval.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -51,7 +52,7 @@ func newRunListCommand() *cobra.Command { if err != nil { if eval_api.IsNotFound(err) { return fmt.Errorf( - "no eval group %q in this project; "+ + "no eval %q in this project; "+ "`azd up` creates the ones your config declares", evalID) } return fmt.Errorf("listing runs for %q: %w", evalID, err) @@ -64,7 +65,7 @@ func newRunListCommand() *cobra.Command { return emitJSONList(cmd.OutOrStdout(), runs) } if list == nil || len(list.Data) == 0 { - fmt.Fprintf(cmd.OutOrStdout(), "Eval group %s has no runs yet.\n", evalID) + fmt.Fprintf(cmd.OutOrStdout(), "Eval %s has no runs yet.\n", evalID) return nil } @@ -76,7 +77,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -127,7 +128,7 @@ func newRunShowCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -183,7 +184,62 @@ func newRunCancelCommand() *cobra.Command { }, } cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") - addEvalGroupFlags(cmd, &groupName) + addEvalFlags(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newRunDeleteCommand removes a run. +// +// Runs accumulate — every `azd ai eval run` adds one — and a run that evaluated +// the wrong dataset or target is noise in every later listing and comparison. +// The id is required rather than defaulted to the most recent run, because +// deleting is not undoable and "the latest one" is a poor thing to guess at. +func newRunDeleteCommand() *cobra.Command { + var ( + runID string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "delete [eval-id]", + Short: "Delete a run.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + if runID == "" { + return requireFlag("run-id") + } + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, args, groupName) + if err != nil { + return err + } + + if err := ec.evalClient.DeleteOpenAIEvalRun(ctx, evalID, runID); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf("no run %q on eval %q", runID, evalID) + } + return fmt.Errorf("deleting run %s: %w", runID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": runID, "eval_id": evalID, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted run %s\n", runID) + return nil + }, + } + cmd.Flags().StringVar(&runID, "run-id", "", "Run to delete.") + addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index edc4d99ccef..8063f09253b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -57,16 +57,16 @@ func TestRunStartMirrorsCompositeFlags(t *testing.T) { } require.NotNil(t, start) - for _, flag := range []string{"eval-id", "eval-group", "name", "level", "max-samples", "wait", "no-wait"} { + for _, flag := range []string{"eval-id", "eval", "name", "level", "max-samples", "wait", "no-wait"} { require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) } } -// Every command that acts on an eval group takes the id the same two ways. +// Every command that acts on an eval takes the id the same two ways. // `run start --eval-id` is the form the CI example uses, and `run list` used to // reject that flag and accept only a positional, so a script that worked for // one sibling failed on the next. -func TestEvalGroupCommandsAcceptIDAsAFlag(t *testing.T) { +func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { subs := map[string]*cobra.Command{} for _, sub := range newRunCommand().Commands() { subs["run "+sub.Name()] = sub @@ -82,7 +82,7 @@ func TestEvalGroupCommandsAcceptIDAsAFlag(t *testing.T) { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) require.NotNil(t, cmd.Flags().Lookup("eval-id"), "%s should accept --eval-id", name) - require.NotNil(t, cmd.Flags().Lookup("eval-group"), "%s should accept --eval-group", name) + require.NotNil(t, cmd.Flags().Lookup("eval"), "%s should accept --eval", name) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index b2972b29ac9..b2f223ebf0c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -20,7 +20,7 @@ import ( func newScheduleCommand() *cobra.Command { cmd := &cobra.Command{ Use: "schedule", - Short: "Run an eval group on a schedule.", + Short: "Run an eval on a schedule.", } cmd.AddCommand( newScheduleSetCommand(), @@ -31,7 +31,7 @@ func newScheduleCommand() *cobra.Command { return cmd } -// newScheduleSetCommand creates the schedule that runs an eval group. +// newScheduleSetCommand creates the schedule that runs an eval. // // It does not update. The service accepts a PUT over an existing schedule, // echoes the new body and keeps the old trigger, so an in-place edit would @@ -64,7 +64,7 @@ func newScheduleSetCommand() *cobra.Command { cmd := &cobra.Command{ Use: "set [eval-id]", - Short: "Create the schedule that runs an eval group.", + Short: "Create the schedule that runs an eval.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -99,7 +99,7 @@ func newScheduleSetCommand() *cobra.Command { // Same resolution as `run`: the config names the group unless // --eval-id bypasses it, and the run payload carries the target // and dataset because the group holds neither. - var group *project.EvalGroup + var group *project.Eval var dataSource *eval_api.EvalRunDataSource if evalID == "" { cfg, err := project.LoadEvalConfig(configPath) @@ -116,9 +116,9 @@ func newScheduleSetCommand() *cobra.Command { if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { return err } - evalID, err = ec.resolveEvalGroupID( + evalID, err = ec.resolveEvalIDFromConfig( ctx, group, configPath, resolveLevel(level, group), - len(cfg.EvalGroups) == 1, out, isJSON(cmd)) + len(cfg.Evals) == 1, out, isJSON(cmd)) if err != nil { return err } @@ -195,8 +195,8 @@ func newScheduleSetCommand() *cobra.Command { cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, "Path to the eval deployment config.") - cmd.Flags().StringVar(&groupName, "eval-group", "", "Which evalGroups entry to schedule.") - cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval group by id, ignoring config.") + cmd.Flags().StringVar(&groupName, "eval", "", "Which evals entry to schedule.") + cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval by id, ignoring config.") cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the group name.") cmd.Flags().StringVar(&description, "description", "", "Schedule description.") cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) @@ -535,7 +535,7 @@ func normalizeDaysOfWeek(days []string) ([]string, error) { } // defaultScheduleName derives a schedule name from the group being scheduled. -func defaultScheduleName(group *project.EvalGroup) string { +func defaultScheduleName(group *project.Eval) string { if group != nil && group.Name != "" { return group.Name } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index e03a7059efd..2d9a85ffa65 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "sort" + "strconv" "strings" ) @@ -184,3 +185,36 @@ func (c *EvalClient) CancelOpenAIEvalRun( ) return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") } + +// DeleteOpenAIEvalRun removes a single run. +func (c *EvalClient) DeleteOpenAIEvalRun(ctx context.Context, evalID, runID string) error { + path := fmt.Sprintf( + "%s/%s/runs/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, "") + return err +} + +// ListOutputItems returns a run's per-sample results. +// +// The run itself carries only totals and a per-criterion breakdown. The output +// items are the rows: each one holds the dataset item that was evaluated, what +// the target answered, and every evaluator's score, verdict and reason. Showing +// results without them can say how many failed but never which, or why. +func (c *EvalClient) ListOutputItems( + ctx context.Context, + evalID, runID string, + limit int, +) (*OutputItemList, error) { + query := map[string]string{} + if limit > 0 { + query["limit"] = strconv.Itoa(limit) + } + + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), + ) + return doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 506fefd9578..2b9fa2ca9d1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -5,6 +5,8 @@ package eval_api import ( "encoding/json" + "fmt" + "sort" "strings" "time" ) @@ -566,3 +568,67 @@ type EvalRunCriteriaResult struct { type OpenAIEvalRunList struct { Data []OpenAIEvalRun `json:"data"` } + +// OutputItemList is a page of a run's per-sample results. +type OutputItemList struct { + Data []OutputItem `json:"data"` +} + +// OutputItem is one evaluated row: the dataset item, and every evaluator's +// verdict on it. +type OutputItem struct { + ID string `json:"id"` + RunID string `json:"run_id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item,omitempty"` + Results []OutputResult `json:"results,omitempty"` +} + +// OutputResult is one evaluator's verdict on one row. +type OutputResult struct { + Name string `json:"name"` + Metric string `json:"metric,omitempty"` + Score LenientFloat `json:"score"` + Label string `json:"label,omitempty"` + Passed bool `json:"passed"` + // Reason is the judge's explanation, which is the part a failing row is + // actually looked at for. + Reason string `json:"reason,omitempty"` +} + +// Failed reports whether any evaluator failed this row. +func (o OutputItem) Failed() bool { + for _, r := range o.Results { + if !r.Passed { + return true + } + } + return false +} + +// Input renders the row's own columns for display, leaving out the +// service-injected `sample.*` bindings and the plumbing ids, which are not what +// the dataset author wrote. +func (o OutputItem) Input() string { + if len(o.DataSourceItem) == 0 { + return "" + } + skip := map[string]bool{ + "response_id": true, "agent_id": true, "agent_name": true, + "agent_version": true, "conversation_id": true, + "previous_response_id": true, "trace_id": true, "span_id": true, + } + keys := make([]string, 0, len(o.DataSourceItem)) + for k := range o.DataSourceItem { + if skip[k] || strings.HasPrefix(k, "sample.") { + continue + } + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%v", k, o.DataSourceItem[k])) + } + return strings.Join(parts, " ") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 4832d52c42d..f31584fd48a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -16,7 +16,7 @@ import ( // stripped before the name is sent as testing_criteria[].evaluator_name. const BuiltinPrefix = "builtin." -// EvaluatorRef references an evaluator from an eval group. It accepts either a +// EvaluatorRef references an evaluator from an eval. It accepts either a // bare string or a mapping carrying a pass threshold: // // evaluators: diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 2a12cb6bfab..bfa923167c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -20,7 +20,7 @@ import ( type EvalConfig struct { Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` - EvalGroups []EvalGroup `yaml:"evalGroups,omitempty" json:"evalGroups,omitempty"` + Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` } // DatasetDecl declares a dataset. A local Source is uploaded on deploy; without @@ -32,15 +32,15 @@ type DatasetDecl struct { } // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly -// from an eval group and never declared here. +// from an eval and never declared here. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` } -// EvalGroup is a run definition: evaluators plus options, bound to a dataset. -type EvalGroup struct { +// Eval is a run definition: evaluators plus options, bound to a dataset. +type Eval struct { Name string `yaml:"name" json:"name"` ID string `yaml:"id,omitempty" json:"id,omitempty"` Description string `yaml:"description,omitempty" json:"description,omitempty"` @@ -113,7 +113,7 @@ func (c *EvalConfig) Validate() error { if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { return fmt.Errorf( "evaluators[%d]: built-in evaluator %q must not be declared; "+ - "reference it directly from an eval group", i, e.Name) + "reference it directly from an eval", i, e.Name) } if evaluators[e.Name] { return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) @@ -133,22 +133,22 @@ func (c *EvalConfig) Validate() error { } groups := map[string]bool{} - for i, g := range c.EvalGroups { + for i, g := range c.Evals { if g.Name == "" { - return fmt.Errorf("evalGroups[%d]: 'name' is required", i) + return fmt.Errorf("evals[%d]: 'name' is required", i) } if groups[g.Name] { - return fmt.Errorf("evalGroups[%d]: duplicate eval group name %q", i, g.Name) + return fmt.Errorf("evals[%d]: duplicate eval name %q", i, g.Name) } groups[g.Name] = true if g.Dataset != "" && !datasets[g.Dataset] { return fmt.Errorf( - "evalGroups[%d] (%s): dataset %q is not declared in datasets", + "evals[%d] (%s): dataset %q is not declared in datasets", i, g.Name, g.Dataset) } if len(g.Evaluators) == 0 { - return fmt.Errorf("evalGroups[%d] (%s): at least one evaluator is required", i, g.Name) + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, g.Name) } for _, ref := range g.Evaluators { if ref.IsBuiltin() { @@ -156,7 +156,7 @@ func (c *EvalConfig) Validate() error { } if !evaluators[ref.Name] { return fmt.Errorf( - "evalGroups[%d] (%s): evaluator %q is not declared in evaluators "+ + "evals[%d] (%s): evaluator %q is not declared in evaluators "+ "(built-ins need the %q prefix)", i, g.Name, ref.Name, evalcore.BuiltinPrefix) } @@ -164,7 +164,7 @@ func (c *EvalConfig) Validate() error { if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent && g.Target.Type != TargetTypeModel { return fmt.Errorf( - "evalGroups[%d] (%s): target.type %q is not supported; use %q or %q", + "evals[%d] (%s): target.type %q is not supported; use %q or %q", i, g.Name, g.Target.Type, TargetTypeAgent, TargetTypeModel) } if g.Options != nil { @@ -172,7 +172,7 @@ func (c *EvalConfig) Validate() error { case "", EvaluationLevelTurn, EvaluationLevelConversation: default: return fmt.Errorf( - "evalGroups[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", i, g.Name, g.Options.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) } @@ -202,11 +202,11 @@ func (c *EvalConfig) Evaluator(name string) (*EvaluatorDecl, bool) { return nil, false } -// Group returns the eval group with the given name. -func (c *EvalConfig) Group(name string) (*EvalGroup, bool) { - for i := range c.EvalGroups { - if c.EvalGroups[i].Name == name { - return &c.EvalGroups[i], true +// Group returns the eval with the given name. +func (c *EvalConfig) Group(name string) (*Eval, bool) { + for i := range c.Evals { + if c.Evals[i].Name == name { + return &c.Evals[i], true } } return nil, false @@ -214,26 +214,26 @@ func (c *EvalConfig) Group(name string) (*EvalGroup, bool) { // ResolveGroup picks the group to act on: the named one, or the only one when // the config declares exactly one. -func (c *EvalConfig) ResolveGroup(name string) (*EvalGroup, error) { +func (c *EvalConfig) ResolveGroup(name string) (*Eval, error) { if name != "" { g, ok := c.Group(name) if !ok { - return nil, fmt.Errorf("eval group %q is not declared in the config", name) + return nil, fmt.Errorf("eval %q is not declared in the config", name) } return g, nil } - switch len(c.EvalGroups) { + switch len(c.Evals) { case 0: - return nil, fmt.Errorf("no eval groups are declared in the config") + return nil, fmt.Errorf("no evals are declared in the config") case 1: - return &c.EvalGroups[0], nil + return &c.Evals[0], nil default: - names := make([]string, 0, len(c.EvalGroups)) - for _, g := range c.EvalGroups { + names := make([]string, 0, len(c.Evals)) + for _, g := range c.Evals { names = append(names, g.Name) } return nil, fmt.Errorf( - "the config declares %d eval groups (%s); choose one with --eval-group", - len(c.EvalGroups), strings.Join(names, ", ")) + "the config declares %d evals (%s); choose one with --eval", + len(c.Evals), strings.Join(names, ", ")) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 177638d5507..76a5a0cc96b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -26,7 +26,7 @@ datasets: source: ./datasets/support-golden.jsonl version: "1" -evalGroups: +evals: - name: pr-gate description: Quality gate for the support agent dataset: support-golden @@ -57,7 +57,7 @@ func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { require.Len(t, cfg.Evaluators, 2) require.Len(t, cfg.Datasets, 1) - require.Len(t, cfg.EvalGroups, 1) + require.Len(t, cfg.Evals, 1) ds, ok := cfg.Dataset("support-golden") require.True(t, ok) @@ -127,13 +127,13 @@ func TestValidate_Rejects(t *testing.T) { }{ { name: "dataset referenced but not declared", - body: "evalGroups:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", + body: "evals:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", wantErr: "is not declared in datasets", }, { name: "custom evaluator referenced but not declared", body: "datasets:\n - name: d\n" + - "evalGroups:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", + "evals:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", wantErr: "is not declared in evaluators", }, { @@ -143,17 +143,17 @@ func TestValidate_Rejects(t *testing.T) { }, { name: "group without evaluators", - body: "evalGroups:\n - name: g\n evaluators: []\n", + body: "evals:\n - name: g\n evaluators: []\n", wantErr: "at least one evaluator is required", }, { name: "unsupported target type", - body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", + body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", wantErr: "is not supported", }, { name: "invalid evaluation level", - body: "evalGroups:\n - name: g\n evaluators: [builtin.relevance]\n" + + body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n" + " options:\n evaluation_level: sentence\n", wantErr: "evaluation_level", }, @@ -195,16 +195,16 @@ func TestResolveGroup(t *testing.T) { t.Run("ambiguous without a name", func(t *testing.T) { multi := loadFromString(t, - "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ + "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ " - name: nightly\n evaluators: [builtin.relevance]\n") _, err := multi.ResolveGroup("") - require.ErrorContains(t, err, "--eval-group") + require.ErrorContains(t, err, "--eval") require.ErrorContains(t, err, "nightly") }) t.Run("empty config", func(t *testing.T) { _, err := (&EvalConfig{}).ResolveGroup("") - require.ErrorContains(t, err, "no eval groups") + require.ErrorContains(t, err, "no evals") }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go index b253c8ad1f2..56f159102dd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go @@ -22,7 +22,7 @@ datasets: source: ./datasets/old.jsonl version: "3" -evalGroups: +evals: - name: pr-gate dataset: support-golden evaluators: @@ -68,8 +68,8 @@ func TestMergeArtifactRefs_PreservesCommentsAndSiblings(t *testing.T) { require.Len(t, cfg.Evaluators, 2) } -// The eval group's evaluator list must be left exactly as written. -func TestMergeArtifactRefs_DoesNotTouchEvalGroups(t *testing.T) { +// The eval's evaluator list must be left exactly as written. +func TestMergeArtifactRefs_DoesNotTouchEvals(t *testing.T) { path := writeTemp(t, handAuthored) require.NoError(t, MergeArtifactRefs(path, nil, []ArtifactRef{{Name: "support-quality", Source: "./evaluators/q.json"}})) @@ -85,7 +85,7 @@ func TestMergeArtifactRefs_DoesNotTouchEvalGroups(t *testing.T) { // Sections absent from the file are created rather than erroring. func TestMergeArtifactRefs_CreatesMissingSections(t *testing.T) { - path := writeTemp(t, "evalGroups:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") + path := writeTemp(t, "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") require.NoError(t, MergeArtifactRefs(path, []ArtifactRef{{Name: "d1", Source: "./datasets/d1.jsonl"}}, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 39e6f8ed9fa..4fb967c30a8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -23,7 +23,7 @@ const EvalHost = "azure.ai.eval" // azd environment keys owned by this extension. const ( - EnvKeyEvalGroupID = "EVAL_GROUP_ID" + EnvKeyEvalID = "EVAL_ID" EnvKeyDatasetVersion = "EVAL_DATASET_VERSION" EnvKeyFingerprintPrefix = "EVAL_FINGERPRINT_" ) @@ -37,11 +37,11 @@ type Reconciler interface { // EnsureEvaluator registers a new evaluator version when the definition // differs from what the service already holds. EnsureEvaluator(ctx context.Context, decl EvaluatorDecl, localPath string) (version string, changed bool, err error) - // EnsureEvalGroup creates the group when it is absent or its resolved + // EnsureEval creates the group when it is absent or its resolved // evaluators or options changed, returning its id. datasetPath is the local // dataset backing the group, or empty when it is already registered; it lets // the reconciler bind criteria to the columns that actually exist. - EnsureEvalGroup(ctx context.Context, group EvalGroup, datasetPath string, recreate bool) (id string, err error) + EnsureEval(ctx context.Context, group Eval, datasetPath string, recreate bool) (id string, err error) } // EvalServiceTargetProvider deploys eval resources during `azd up`. azd owns @@ -120,7 +120,7 @@ func (p *EvalServiceTargetProvider) Publish( } // Deploy reconciles the eval configuration in a fixed order — datasets, then -// evaluators, then eval groups — because a group references the versions the +// evaluators, then evals — because a group references the versions the // first two resolve to. It fails fast; the next `azd up` resumes from wherever // it stopped. func (p *EvalServiceTargetProvider) Deploy( @@ -170,19 +170,19 @@ func (p *EvalServiceTargetProvider) Deploy( report(progress, describeResult("evaluator", decl.Name, version, changed)) } - // 3. Eval groups. Groups are immutable, so a change upstream means a new + // 3. Evals. Groups are immutable, so a change upstream means a new // group must be created and the stored id replaced. - for _, group := range cfg.EvalGroups { - report(progress, fmt.Sprintf("Reconciling eval group %s", group.Name)) + for _, group := range cfg.Evals { + report(progress, fmt.Sprintf("Reconciling eval %s", group.Name)) datasetPath := "" if decl, ok := cfg.Dataset(group.Dataset); ok { datasetPath = resolveSource(baseDir, decl.Source) } - id, err := reconciler.EnsureEvalGroup(ctx, group, datasetPath, anyChanged) + id, err := reconciler.EnsureEval(ctx, group, datasetPath, anyChanged) if err != nil { - return nil, fmt.Errorf("eval group %q: %w", group.Name, err) + return nil, fmt.Errorf("eval %q: %w", group.Name, err) } - report(progress, fmt.Sprintf("Eval group %s is %s", group.Name, id)) + report(progress, fmt.Sprintf("Eval %s is %s", group.Name, id)) } return &azdext.ServiceDeployResult{}, nil @@ -229,7 +229,7 @@ func EvalConfigFromService(svc *azdext.ServiceConfig, projectRoot string) (*Eval props := serviceProps(svc) if props == nil || len(props.GetFields()) == 0 { return nil, fmt.Errorf( - "service %q carries no eval configuration; expected evaluators, datasets, or evalGroups", + "service %q carries no eval configuration; expected evaluators, datasets, or evals", svc.GetName()) } @@ -313,14 +313,14 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } -// FingerprintGroup hashes an eval group's own declaration. +// FingerprintGroup hashes an eval's own declaration. // // Change detection on upstream artifacts is not sufficient: editing a group's // evaluators, target, or options changes what the group means, and groups are // immutable, so the group has to be recreated even when the dataset and // evaluators are untouched. Without this a retargeted group keeps running // against the old definition. -func FingerprintGroup(group EvalGroup) (string, error) { +func FingerprintGroup(group Eval) (string, error) { // The id is server-assigned. The description is carried in the group's // metadata, so editing it does change the request, but recreating an // immutable group over a reworded description would cost the group id and @@ -332,7 +332,7 @@ func FingerprintGroup(group EvalGroup) (string, error) { data, err := json.Marshal(group) if err != nil { - return "", fmt.Errorf("hashing eval group %q: %w", group.Name, err) + return "", fmt.Errorf("hashing eval %q: %w", group.Name, err) } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index 6052cb99c55..bd24e6c0030 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -73,7 +73,7 @@ func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { "datasets": []any{ map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, }, - "evalGroups": []any{ + "evals": []any{ map[string]any{ "name": "quality", "dataset": "golden", @@ -88,9 +88,9 @@ func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { require.NoError(t, err) require.Len(t, cfg.Datasets, 1) require.Equal(t, "golden", cfg.Datasets[0].Name) - require.Len(t, cfg.EvalGroups, 1) - require.Len(t, cfg.EvalGroups[0].Evaluators, 1) - require.Equal(t, "builtin.task_adherence", cfg.EvalGroups[0].Evaluators[0].Name) + require.Len(t, cfg.Evals, 1) + require.Len(t, cfg.Evals[0].Evaluators, 1) + require.Equal(t, "builtin.task_adherence", cfg.Evals[0].Evaluators[0].Name) } func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { @@ -103,7 +103,7 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { // detectable. Upstream artifact fingerprints do not cover it: retargeting a // group at a different agent leaves the dataset and evaluators untouched. func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { - base := EvalGroup{ + base := Eval{ Name: "quality", Dataset: "golden", Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, @@ -118,13 +118,13 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { require.NoError(t, err) require.Equal(t, original, same, "an unchanged group must keep its fingerprint") - cases := map[string]func(g *EvalGroup){ - "target": func(g *EvalGroup) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, - "evaluators": func(g *EvalGroup) { + cases := map[string]func(g *Eval){ + "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, + "evaluators": func(g *Eval) { g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) }, - "options": func(g *EvalGroup) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, - "dataset": func(g *EvalGroup) { g.Dataset = "other" }, + "options": func(g *Eval) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, + "dataset": func(g *Eval) { g.Dataset = "other" }, } for name, mutate := range cases { t.Run(name, func(t *testing.T) { @@ -141,7 +141,7 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { // Server-assigned and cosmetic fields must not force a recreate. func TestFingerprintGroupIgnoresIdAndDescription(t *testing.T) { - base := EvalGroup{ + base := Eval{ Name: "quality", Dataset: "golden", Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, From 69335b54c598f9982e24628f4d8c9935f45a96f8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 28 Jul 2026 22:23:55 +0300 Subject: [PATCH 059/114] Drop agent.context.tools, and the dataset update synonym Nothing read agent.context.tools. Tool definitions reach an evaluator from the live agent as {{sample.tool_definitions}}, which is what the tool evaluators score, and the agents extension's own eval and eval-generate paths do not read a local tools file either - only optimize does, and optimize stays there. The generation API has no field for tools, so the only way to pass them would have been prose folded into the prompt. A key with no consumer is worse than no key, so it is gone rather than warned about. dataset update goes the same way evaluator update did: both published a new immutable version, so update was a synonym for create. --- .../azd-eval-extension-implementation.md | 534 ++++++++++++++++++ .../internal/cmd/agent_context_test.go | 24 +- .../internal/cmd/dataset.go | 18 +- .../internal/cmd/generate.go | 6 +- .../internal/project/generate_config.go | 1 - .../tests/live/live_test.go | 12 +- 6 files changed, 561 insertions(+), 34 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md new file mode 100644 index 00000000000..44fb4ae170b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -0,0 +1,534 @@ +# `azd ai eval` — Implementation Handoff + +> Self-contained brief for building the **`azure.ai.evaluations`** azd extension (`azd ai eval`). Everything below is verified against the shipping `azure.ai.agents` extension, azd core, and RAISvc source. Design source of truth is `spec.md` in `foundrysdk_specs/specs/evaluations/azd_eval_extension/` — if the two disagree, the spec wins. +> +> **Uncommitted working document.** Not part of any PR. + +--- + +## 0. TL;DR + +A new azd extension — id `azure.ai.evaluations`, namespace `azd ai eval` — that is a thin Go client over the **existing** Foundry evaluations data plane, plus **one azd service-target provider** for `host: azure.ai.evals`. + +**Non-negotiables** + +1. **Two-tier commands.** Atomic (`dataset` / `evaluator` / `run` / `results`) map ~1:1 to the API. Composite (`init` / `generate` / `run`) are wrappers, never the only path. +2. **No `deploy` command.** Deployment is `azd up` / `azd deploy` invoking our service-target provider. We ship no deploy verb. +3. **`generate` is separate from deploy.** Generate once, deploy the artifacts to many environments. +4. **`init` touches no network.** +5. **Everything is built on APIs that exist today.** No service changes. +6. **Deterministic.** `-o json` + `--no-prompt` everywhere; a supplied flag fully suppresses its prompt. + +**Out of scope for M1:** scheduled/continuous eval, baseline comparison (both exist server-side — M2), non-agent targets, traces as a run data source, eval by response/run id, `optimize` (stays in the agents extension). + +--- + +## 1. Where the code lives & how to build it + +### 1.1 Paths +- **Repo:** `Azure/azure-dev`. +- **New extension:** `cli/azd/extensions/azure.ai.evaluations/` +- **Reference to copy:** `cli/azd/extensions/azure.ai.agents/` — closest analog, currently hosts `azd ai agent eval …`. **Read it first.** + +### 1.2 File layout (mirror the agents extension) +``` +cli/azd/extensions/azure.ai.evaluations/ +├── internal/ +│ ├── cmd/ # cobra commands, one file per group; listen.go wires the provider +│ ├── pkg/eval_api/ # data-plane client (lifted, see §1.4) +│ ├── pkg/dataset_api/ # dataset client (lifted) +│ └── project/ # azure.yaml service-entry model + YAML round-trip +├── schemas/ # JSON schemas +├── tests/ +├── extension.yaml # manifest +├── go.mod / go.sum +├── main.go +└── version.txt +``` + +**`extension.yaml`** — note the `service-target-provider` capability and `providers` block; both are required for `azd up` to route to us: +```yaml +# yaml-language-server: $schema=../extension.schema.json +id: azure.ai.evaluations +namespace: ai.eval # dotted → CLI surface `azd ai eval` +displayName: Foundry evaluations (Beta) +description: Define and run Foundry evaluations from your terminal. (Beta) +usage: azd ai eval [options] +version: 1.0.0-beta.1 # keep version.txt in sync +requiredAzdVersion: ">=1.27.1" +language: go +capabilities: + - custom-commands + - lifecycle-events + - service-target-provider + - metadata +providers: + - name: azure.ai.evals + type: service-target + description: Deploys evaluation datasets, evaluators, and eval groups to Foundry +``` + +### 1.3 Stack facts (verified) +- **Go 1.26.x**, **cobra**. Entry point: + ```go + package main + import ( + "azureaieval/internal/cmd" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + ) + func main() { azdext.Run(cmd.NewRootCommand()) } + ``` +- SDK module `github.com/azure/azure-dev/cli/azd` (agents pins `v1.28.0`); surface is `pkg/azdext`. +- Data plane over REST using the **azcore pipeline**, not raw `net/http`: bearer-token policy scoped to **`https://ai.azure.com/.default`**, plus `azsdk.NewMsCorrelationPolicy()` and `azsdk.NewUserAgentPolicy(...)`. +- Dev loop: + ```bash + azd ext install microsoft.azd.extensions # one-time + cd cli/azd/extensions/azure.ai.evaluations + azd x build # build + install locally + azd x watch # ongoing + ``` + +### 1.4 What to lift from `azure.ai.agents` + +Measured, non-test: + +| Source | Files | LOC | Gives you | +|---|---|---|---| +| `internal/pkg/agents/eval_api/` | 7 | 1,350 | `EvalClient` over `/data_generation_jobs`, `/evaluator_generation_jobs`, `/evaluators`, `/datasets`, `/openai/v1/evals`; LRO poller; artifact download; portal URLs; api-version constants | +| `internal/pkg/agents/dataset_api/` | 2 | 550 | Full pending-upload → blob → finalize → download | +| `internal/pkg/agents/opt_eval/` | 2 | 555 | eval.yaml config model (adapt, don't copy wholesale) | +| `internal/cmd/eval_*.go` | 10 | 2,890 | generate / run / show / list / update / progress UX | + +**≈1,900 LOC of API client is effectively done.** Genuinely net-new: + +1. **The service-target provider** (§4) — no prior art. +2. **Change detection** (§5) — no prior art. +3. **YAML round-trip merge** for `generate` writing `source:` back (§6). +4. The atomic command layer and the offline `init`. + +> Do **not** port the `/evaluation_suites` client. That endpoint is abandoned — the eval group is the unit. + +--- + +## 2. Data-plane API contract + +### 2.1 Base, auth, api-versions +- **Base:** azd env `FOUNDRY_PROJECT_ENDPOINT`, shape `https://{resource}.services.ai.azure.com/api/projects/{project}/…`. Global `--project-endpoint` overrides. +- **Scope:** `https://ai.azure.com/.default`. +- **api-versions:** project-endpoint calls (datasets, evaluators) use **`2025-11-15-preview`**; data generation uses **`v1`**; **`/openai/v1/evals*` sends no api-version**. + +### 2.2 Datasets +| Command | Calls | +|---|---| +| `dataset create` / `update` | `POST /datasets` *(first version only)* → `POST /datasets/{name}/versions/{v}/startPendingUpload` → `PUT ` → `PUT /datasets/{name}/versions/{v}` | +| `dataset list` | `GET /datasets` | +| `dataset show` | `GET /datasets/{name}/versions/{v}` | +| `dataset delete` | `DELETE /datasets/{name}/versions/{v}` | + +Model — **note there is no content hash or etag**, which drives §5: +```go +type Dataset struct { Name, Version, BlobURI, Format, DataURI, ContentURI string } +``` +A dataset is a **single `.jsonl`**. A directory today just picks the first `.jsonl`; no folder walk. + +### 2.3 Evaluators +| Command | Calls | +|---|---| +| `evaluator upload` / `update` | *(code only)* pending-upload → blob upload; then `POST /evaluators/{name}/versions` | +| `evaluator show` | `GET /evaluators/{name}` — returns the definition inline | +| `evaluator builtins` | `GET /evaluators?type=Builtin` | + +Built-ins are referenced as `builtin.`; the prefix is stripped before the value goes into `testing_criteria[].evaluator_name`. Custom-evaluator upload needs the project MI to hold **Azure AI User**. + +### 2.4 Eval groups and runs (OpenAI-compatible, no api-version) +```go +type CreateOpenAIEvalRequest struct { + Name string + Metadata map[string]string + DataSourceConfig *DataSourceConfig // {Type, ItemSchema, IncludeSampleSchema} + TestingCriteria []TestingCriterion // the evaluators +} +type TestingCriterion struct { + Type, Name, EvaluatorName string + InitializationParameters map[string]any // threshold lives here + DataMapping map[string]string +} +type OpenAIEval struct { ID, Name string } // ID is canonical; Name is NOT unique +``` + +| Command | Calls | +|---|---| +| create group | `POST /openai/v1/evals` | +| get / list | `GET /openai/v1/evals/{id}` · `GET /openai/v1/evals?limit=` | +| start run | `POST /openai/v1/evals/{evalId}/runs` | +| poll / list runs | `GET /openai/v1/evals/{evalId}/runs/{runId}` · `GET …/runs` | +| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}/cancel` | +| results | `GET …/runs/{runId}` → `result_counts` + `per_testing_criteria_results` | + +**The group carries evaluators, not the dataset.** The dataset goes on the **run**. `evaluation_level` is `turn` | `conversation`, service default **`turn`**. + +**`data_source_config` and `data_mapping` are derived from each evaluator's published contract.** The original plan was to copy the agents extension's hardcoded mapping. Live testing showed that is wrong: it only suits agent-target quality evaluators and the service rejects the rest. + +`GET /evaluators` returns a contract per evaluator: +```jsonc +"supported_evaluation_levels": ["turn"], +"definition": { + "data_schema": { "required": ["response", "instruction_id_list"], "properties": { … } }, + "init_parameters": { "required": ["deployment_name"], "properties": { … } } +} +``` + +`internal/cmd/build.go` reads it and, per criterion: +- binds each accepted input to the agent sample (`response`, `tool_calls`, `tool_definitions`) or to a dataset column `{{item.}}`; +- declares the referenced columns in the item schema; +- filters `initialization_parameters` to the declared properties — no evaluator accepts `model`, and `builtin.ifeval` accepts nothing; +- validates `--level` against `supported_evaluation_levels`; +- reports a missing required column locally, naming it. + +Two service rules are encoded: `messages` and `query`/`response` are mutually exclusive (the level selects), and `evaluation_level` is an **initialization parameter**, not run metadata. An evaluator with no published contract falls back to the agent-target shape. + +Covered by `build_test.go`, and by `build_live_test.go` which posts a group for every built-in the project exposes. + +### 2.5 Generation (LRO) +`POST /data_generation_jobs` and `POST /evaluator_generation_jobs`, each polled by `GET …/{id}`. The ~11-minute "timeout" is a **client poll budget (2 s × 300)**, not a service limit — raise it and default to `--no-wait` in CI. + +### 2.6 M2 only — schedules and comparison +Both are **project-endpoint reachable** and **feature-gated per project**: +- `/schedules` — `PUT {id}` · `GET {id}` · `GET` · `DELETE {id}`; requires `FoundryFeature.Schedules_V1Preview`. Trigger is `Cron{Expression, StartTime, EndTime, Timezone}` or `Recurrence{Frequency, Interval, Schedule}`. +- Insights compare — `POST /insights` (async) or `POST /insights/sync`, body `{evalId, baselineRunId, treatmentRunIds}`; requires `FoundryFeature.Insights_V1Preview`. + +--- + +## 3. Configuration model + +Two files. Neither is loaded by azd core — **we parse both**. + +**`evals/eval_generate.yaml`** — input to `generate`, never deployed. `agent.context.{instructions,tools}` are file paths; `local_dir` accepts a directory or an explicit file path. + +**`evals/azure.yaml`** — the deployment spec, `$ref`'d from the root `azure.yaml`: +```yaml +# /azure.yaml +services: + evals: + host: azure.ai.evals + uses: [ai-project] + $ref: ./evals/azure.yaml +``` +It carries three arrays: `evaluators[]`, `datasets[]`, `evalGroups[]` (see `spec.md` for the full shape). + +**Why arrays on a service work.** azd core's `ServiceConfig` captures unknown keys: +```go +AdditionalProperties map[string]any `yaml:",inline"` +``` +and hands them to the extension, which unmarshals them itself — the pattern `LoadServiceTargetAgentConfig` → `ServiceConfigProps` uses. `azure.ai.project` already carries `deployments[]` this way. **`$ref` resolution is ours too**: `pkg/foundry.ResolveFileRefs(cfg, projectRoot)`, called by the extension, not by azd. + +--- + +## 4. The service-target provider (net-new, highest risk) + +Wire it in `listen.go`, mirroring `azure.ai.agents`: +```go +func configureExtensionHost(host *azdext.ExtensionHost) { + azdClient := host.Client() + host. + WithServiceTarget("azure.ai.evals", func() azdext.ServiceTargetProvider { + return project.NewEvalServiceTargetProvider(azdClient) + }). + WithServiceEventHandler("postdeploy", func(ctx context.Context, args *azdext.ServiceEventArgs) error { + return postdeployHandler(ctx, azdClient, args) + }, &azdext.ServiceEventOptions{Host: "azure.ai.evals"}) +} +``` + +`ServiceTargetProvider` requires `Initialize`, `Endpoints`, `GetTargetResource`, `Package`, `Publish`, `Deploy`. For eval, **`Package` and `Publish` are near no-ops**; `Deploy` does the work, in this fixed order: + +1. **Datasets** — change-detect (§5); if changed, run the `dataset create` sequence. +2. **Evaluators** — `GET /evaluators/{name}`, compare the definition, upload only if different. +3. **Drift check** — if the server's latest version is ahead of the recorded one, fail with "sync first". +4. **Eval groups** — `POST /openai/v1/evals` with `testing_criteria` from the resolved evaluator versions. Groups are immutable, so only recreate when the resolved versions or options actually changed. +5. Persist resolved ids, versions, and fingerprints to the azd env. + +**How azd reaches us:** `azd up` runs one DAG; per service it calls `GetServiceTarget()`, which does `serviceLocator.ResolveNamed(host, &target)`. If our extension is not installed, azd fails that service with *"install an extension that provides this host."* We implement **no sequencing or rollback across services** — `uses:` and the DAG handle that. + +--- + +## 5. Change detection (net-new) + +Without this, every `azd up` publishes a redundant version. + +- **Datasets** — the API returns no hash or etag, so comparing against the server would mean downloading the blob every deploy. Instead: **SHA-256 the local file**, store it with the resolved version in the azd env, re-hash locally next deploy, skip when unchanged. +- **Evaluators** — definitions come back inline from `GET /evaluators/{name}`; compare directly, no cache needed. +- **Drift** — a *version* comparison, not content: server latest vs. the version recorded at last deploy. + +**Open:** how to fingerprint a **code** evaluator (a folder). Suggest hashing sorted relative paths + contents, excluding `__pycache__` and `.pyc`. + +--- + +## 6. `generate` writes back into `evals/azure.yaml` + +After downloading artifacts, `generate` adds/updates `source:` references. Requirements: + +- Match entries **by `name`**; update `source` in place; append when absent. +- **Preserve comments and key order** — use the `yaml.v3` Node API, not plain marshal/unmarshal. +- Do not clobber a field the user hand-edited other than `source`. +- If the array is itself a `$ref`, write into the referenced file. + +There is no `emitDeploymentConfig` block — this is default behavior, not configurable. + +--- + +## 6a. What generation is seeded from + +The generation API takes an `agent` source that is meant to pull the agent's own instructions, and it fails for every agent (§11d). The client resolves that context itself, most specific first: + +1. `--gen-instruction` / `--gen-instruction-file` +2. the file named by `agent.context.instructions`, resolved **relative to the spec that declared it**, not the working directory +3. the agent's published instructions — `GET /agents/{name}` → `versions.latest.definition.instructions` + +Step 2 tolerates a missing file on purpose: `init` writes the path before the file exists, so treating the gap as an error would break the flow init scaffolds. Step 3 is what makes `init` → `generate` work with nothing authored. + +The agent source is still sent. When the service starts honouring it, it contributes on top of the prompt; nothing has to be removed. + +`agent.context.tools` is still read by nothing, so it is warned about rather than dropped silently, and `init` no longer scaffolds it — a warning for a field the user never chose is just noise. + +--- + +## 7. Behavioral bugs to fix (measured in `azd ai agent eval`) + +Treat each as an acceptance criterion. + +1. **Path handling (highest priority).** `--out-file` is re-rooted under the agent directory; `--config` re-roots again. **Fix:** treat paths as relative to CWD (or `-C/--cwd`), used verbatim, single-rooted. Test `./x.yaml`, `../x/x.yaml`, absolute. +2. **Wizard overrides flags.** Prompts still fire when flags are supplied, and pre-filled prompts *append* typed input. **Fix:** a supplied flag fully suppresses its prompt; `--no-prompt` errors on a missing required value. +3. **`--evaluator` ignored during generation.** Passing `--evaluator` does not stop rubric generation. **Fix:** honor it, skip that generation. +4. **Client-side generation timeout.** Resolved: it is a client poll budget, not a service limit. Raise it; default `--no-wait` under `--no-prompt`. +5. **Shallow results.** `eval show` returns counts only. **Fix:** per-sample scores via `per_testing_criteria_results`. +6. **Auth friction.** Native azd token failed with "Reauthentication required"; workaround `azd config set auth.useAzCliAuth true`. Detect and surface clearly. + +--- + +## 8. azd environment + +Extensions read and write env values themselves via `azdClient.Environment().GetValue / SetValue` — azd sets none of these. The agents extension does this from lifecycle handlers. + +| Key | Written by | +|---|---| +| `FOUNDRY_PROJECT_ENDPOINT` | consumed, not written | +| `EVAL_GROUP_ID` | provider during `azd up`; `run` when it creates the group | +| `EVAL_DATASET_VERSION`, artifact fingerprints | provider during `azd up` | +| `EVAL_RUN_ID` | `run` | + +Setting `EVAL_GROUP_ID` manually targets a pre-existing group; `--eval-id` does the same per-invocation. + +--- + +## 9. Build order + +M1 is everything in the spec. Within it, build in dependency order: + +| Step | Work | Done when | +|---|---|---| +| **1. Scaffold** | Extension skeleton, `extension.yaml`, `main.go`, root cobra command, local install via `azd x build` | `azd ai eval --help` works | +| **2. Lift the clients** | Copy `eval_api` + `dataset_api`, de-agent-scope, keep api-version constants | Unit tests pass against an `httptest` fake | +| **3. Atomic commands** | `dataset`, `evaluator`, `run`, `results` with `-o json` / `--no-prompt` | **E2E-1** below | +| **4. Config model** | `evals/azure.yaml` load, `$ref` resolve via `pkg/foundry.ResolveFileRefs`, validation | Round-trip test preserves comments | +| **5. Service-target provider** | `listen.go` wiring + `Deploy` reconciliation + change detection + drift | `azd up` creates all three resource kinds; second `azd up` is a no-op | +| **6. `init`** | Offline scaffold of both YAMLs | Runs with no network/auth | +| **7. `generate`** | Generation LROs, artifact download, write-back into `evals/azure.yaml` | **E2E-3** | +| **8. `run`** | Group resolve-or-create, run, poll, render | **E2E-2** | + +Steps 1–3 are mostly mechanical. **Step 5 is the risk** — budget accordingly. + +--- + +## 10. Testing + +| Tier | Coverage | Auth | Where | +|---|---|---|---| +| **0 — offline** | flag parsing; YAML round-trip; path resolution (§7.1); flag→prompt suppression (§7.2); request bodies against an `httptest` fake (copy `eval_api_version_test.go`); schema validation | No | PR gate | +| **1 — `init` record/playback** | interactive prompt flows | No | PR gate | +| **2 — live golden path** | full flows against a real Foundry project | Yes | On-demand/scheduled, **not** the PR gate | + +Tier 2: env-gate on `AZURE_AI_EVAL_E2E_LIVE=1`, build tag `//go:build linux` (needs a PTY), drive `init` prompts via `go-expect`+`vt10x`+`creack/pty`, everything else through `--no-prompt -o json`. `t.Cleanup` must delete every version it created. + +**Golden paths** + +- **E2E-1 — atomic:** `dataset create` → `evaluator upload` → `run start` → `results export`. Assert valid JSON, resolved versions, terminal run status, **per-sample** scores, and that re-running `create` yields the *next* version rather than an error. +- **E2E-2 — init → azd up → run:** `init` makes **zero network calls** (run it unauthenticated), writes both YAMLs, does not double the path; `azd up` creates the resources and pins versions back; a second `azd up` creates **no new versions**; `run` completes with no prompt. +- **E2E-3 — generate:** completes without a client timeout; a supplied `--evaluator` is honored; artifacts land locally and `evals/azure.yaml` gains correct `source:` entries with comments preserved. +- **E2E-4 — CI invariants:** every command with `--no-prompt -o json` is non-interactive, emits parseable JSON, and exits non-zero on a missing required value. + +--- + +## 11. Decisions still open + +| # | Question | Blocks | Suggested default | +|---|---|---|---| +| 1 | Host name `azure.ai.evals` — agreed? Not registered anywhere yet | Step 1 | Use it; renaming is cheap before publish | +| 2 | Which storage connection for evaluator pending-upload (`connectionName`) | Step 3 | Project default; expose a flag | +| 3 | Are **code** evaluators in M1, or rubric-only? | Steps 3, 5 | Rubric-only for M1 — removes the folder-hashing problem entirely | +| 4 | Where do fingerprints live — azd env or a lock file? | Step 5 | azd env, so they are environment-scoped | +| 5 | Do we depend on `azure.ai.projects` for the project service? | Step 1 | Yes, mirror the agents manifest | +| 6 | Bundling into `microsoft.foundry` — who owns it | Ship | Extensions team | + +--- + +## 11b. Assumptions made while implementing + +Recorded for review. Anything marked **corrected** was an assumption that live testing disproved; the code already reflects the correction. + +| # | Assumption | Status | +|---|---|---| +| 1 | Dataset versions are decimal (`1.0`, `2.0`) | **Verified live** — `UploadNewVersion` advanced 1.0 → 2.0 | +| 2 | `--wait` defaults true for `run` | Held; matches the spec's blocking-by-default UX | +| 3 | `evaluation_level` travels as run **metadata** | **Corrected** — it is an `initialization_parameters` property on evaluators that declare it. Metadata had no effect | +| 4 | A cached eval group id that 404s means recreate | Held; not yet exercised live | +| 5 | `dataset create` accepts a file or a directory | Held; the upload helper scans a directory for the first `.jsonl` | +| 6 | Evaluator sameness compares only the `definition` block | Held; avoids server-assigned version/timestamp churn | +| 7 | `GetTargetResource` returns a subscription-only resource | Held; eval resources have no ARM resource | +| 8 | Rubric evaluators only in M1; code evaluators in M2 | Open decision 3 | +| 9 | One fixed data mapping suits all evaluators | **Corrected** — contracts differ per evaluator; the mapping is now derived from the published contract | +| 10 | Dataset URIs come back snake_case | **Corrected** — the project endpoint returns camelCase (`dataUri`). Both spellings are now bound | +| 11 | A dataset blob URI can be downloaded directly | **Corrected** — true only for uploads. A *generated* dataset's URI names the container, not the blob, with `isSingleFile` true either way, and downloading a container returns 409. The URI also carries no SAS, so a credential is always needed. Downloads now fetch a credential and list the container when the URI does not name a file | +| 22 | Agent-seeded data generation would be fixed service-side before ship | **Corrected** — traced to the AOAI generator, outside this repo. `generate` now reads the agent's instructions itself and passes them as the prompt source; the agent source is still sent so it contributes once fixed | +| 23 | `agent.context.instructions` and `.tools` were wired up | **Corrected** — both were written by `init`, declared on the config, and read by nothing. `instructions` is now honoured; `tools` is warned about and no longer scaffolded | +| 24 | The service would reject a missing generation model clearly | **Corrected** — it fails partway through the command with a message naming nothing the caller controls. Checked up front instead | +| 25 | `max_samples` was free-form | **Corrected** — the service requires 15–1000. The config already validated this; the floor is now documented in the spec | +| 26 | Schedule creation would be a POST to a collection | **Corrected** — `POST` 404s on every route. It is `PUT /schedules/{name}`, a named resource | +| 27 | The bodiless 400s meant `displayName`/`description`/`enabled` were required | **Corrected, and this one was my error** — in that probe only the *first* create succeeded and I read the rest as field validation. The real cause is one schedule per project. Re-tested from a drained state, a minimal body creates fine | +| 28 | A named PUT would update in place | **Corrected** — accepted, echoes the new body, changes nothing. `set` refuses an existing name instead of reporting a change that did not happen | +| 29 | Deleting and recreating under the same name would work as a replace | **Corrected** — the replacement never leaves `Creating` and cannot then be deleted. The `--replace` flag was removed before shipping | +| 30 | M4's "traces as a run data source" was awaiting service support | **Corrected** — `azure_ai_traces` is in the run data-source discriminator and the service executes it. The note was never re-tested. Shipped as `run --from-traces` | +| 31 | Traces were a generation input only, never a run's data source | **Corrected** — that comment described the *generation* API. The run API takes them directly | +| 32 | The traces window could be sent as `start_time`/`end_time` | **Corrected, and this one I shipped** — the data source has no start bound. `start_time` is accepted and discarded, leaving the default 7 days. It looked right only because the first value I tested, 7d, *is* the default; 30d silently queried a week. Now sends `lookback_hours` | +| 33 | M4's "evaluation by response id" was awaiting service support | **Corrected** — works today. The ids are not a list on the data source: they are JSONL rows plus a `data_mapping` to `response_id` | +| 34 | `target.type: model` was unsupported | **Corrected** — supported. The config rejected it by name *and* the test used it as the example of an unsupported type, so the gap read as deliberate in two places. Sample bindings now follow the target kind, since a model returns `output_text` where an agent returns `output_items` | +| 35 | A run could reference a registered dataset by name as a `file_id` | **Corrected** — `file_id` means an uploaded file; a dataset name is rejected with `invalid data source file ids`. Registered datasets are fetched and sent inline. Every earlier test used a local `source:`, so this path had never run | +| 36 | M4's "subsetting a registered dataset" needed service support | **Corrected** — the service cannot narrow a file reference, but fetching the rows client-side makes `--max-samples` mean the same thing for any dataset | +| 37 | One env key per resolved id was enough | **Corrected** — only true for a single-group config. With two, the second deploy handed the first group the second's id and both declarations pointed at one group. Ids are now keyed by name, as fingerprints already were | +| 38 | The remembered run id could be shared | **Corrected** — same shape as 37. Asking group A for its latest fetched group B's run inside A and 404'd | +| 39 | A dataset's `version:` was the version published | **Corrected** — it was passed to the helper that *counts from* its argument, so `1.0` published 2.0. It also meant two things: unchanged content resolved to it, changed content published above it | +| 40 | An evaluator's `version:` behaved like a dataset's | **Corrected** — the service assigns an evaluator's version on publish, so a pin alongside `source:` was never honoured. A config asking for 7 deployed 1 silently. Now refused | +| 41 | Criteria were being shaped from each built-in's published schema | **Corrected, and this one invalidated an earlier §11c row** — the schemas were fetched with an unfiltered list, which returns only the project's own evaluators. Every built-in fell back to `legacyInputs`. It matched query/response so nothing looked wrong; `task_completion` at conversation level published an empty `data_mapping` | +| 42 | A run needs a target | **Corrected** — a dataset holding both sides of the exchange has nothing to invoke, and the service runs it. The requirement was ours | +| 12 | `$ref` is resolved by azd core before the extension sees the config | **Corrected** — core leaves `$ref` for the owning extension. The provider now calls `foundry.ResolveFileRefs`, and relative `source:` paths are based on the included file's directory | +| 13 | Upstream artifact fingerprints are enough to know when to recreate a group | **Corrected** — editing the group's own target/evaluators/options changed nothing. The group declaration is fingerprinted too | +| 14 | The host is `azure.ai.evals` | **Corrected** — it is `azure.ai.eval`; the spec has been aligned | +| 15 | `run` only needed the composite form | **Corrected** — the spec lists `start`/`list`/`show`/`cancel`, and M1 requires every operation to be reachable atomically. All four now exist | +| 16 | `--project-endpoint` only selects the endpoint | **Corrected** — it also suppressed the azd environment name, silently disabling the cached eval-group and run ids. The name is now resolved independently | +| 17 | No evaluator accepts `model` | **Corrected** — true for built-ins, false for custom rubrics, which *require* `model`. The judge model is bound under whichever name the evaluator declares | +| 18 | An evaluator definition can be compared whole to detect changes | **Corrected** — the service enriches it on create, so only the authored keys can be compared | +| 19 | `GET /evaluators/{name}` returns the latest version | **Corrected** — it 404s; the version has to be resolved first, numerically | +| 20 | Rubric weights are free-form | **Corrected** — integers 1–10; the spec now says so | +| 21 | `--dataset` suppresses data generation | **Corrected** — only did so for a local path, not for a registered dataset name, which the flag also accepts | + +--- + +## 11c. Verified end to end against a live project + +| Flow | Result | +|---|---| +| `azd ai eval init` | Scaffolds `evals/azure.yaml` + `evals/eval_generate.yaml` matching the spec | +| `azd provision` → `azd deploy evals` → `azd up` | Provider runs; datasets and groups reconcile | +| Dataset first deploy | Published at version 1.0 | +| Dataset unchanged | Reported unchanged, nothing uploaded | +| Dataset edited | Published 2.0 and the group recreated | +| Group retargeted | New group id; two further no-op deploys reused it | +| `$ref` service entry | Deploys, and the fingerprint matches the equivalent inline config | +| `azd ai eval run` | Real run against a live agent, completed | +| `azd ai eval results show` | 3 passed / 1 failed, per-criterion breakdown, portal link | +| `evaluator builtins` | 10 built-ins with versions and type | +| Eval group create | Accepted for **all 10** built-ins, each with its own contract || `generate` rubric | Succeeds, writes the evaluator JSON | +| Build → pack → publish → install | Installs from the local registry; `azd ai eval --help` lists every command | +| Atomic surface | Every command group and subcommand the spec lists is present | +| `run start` / `list` / `show` / `cancel` | Exercised live, including the guard that refuses to cancel a finished run | +| `results export` | JSON and CSV both written | +| `-o json` | Valid JSON from every read command | +| `dataset` create/show/update/list/delete | Full lifecycle, 1.0 → 2.0, nothing left behind | +| `evaluator` upload/show/update/list/delete | Full lifecycle, version 1 → 2, nothing left behind | +| Deploy with a **custom** evaluator | Publishes once, redeploys are no-ops, an edit publishes the next version | +| Run with built-in **and** custom evaluators | 4 passed / 0 failed, both criteria reported | +| `--no-prompt` | Every required value fails fast naming the flag; nothing blocks | +| **Spec Example 1, verbatim**: `init` → `generate --max-samples 50` → `azd up` → `run` → `results show --failed-only -O` | All five steps from an empty directory. Dataset generated and downloaded, group `eval_78de667a…` deployed in 40s, `evalrun_6b2044cf…` completed, `results.json` written | +| Spec Examples 2, 3, 4 | Verified verbatim | +| `results compare` (M2) | Baseline vs treatment, `PairedTTest`, signed deltas and p-values; `-o json` valid | +| `generate` write-back | Adds the artifact reference and preserves comments, ordering and siblings | +| Generated dataset download | Container-URI case exercised: credential fetched, container listed, JSONL read | +| **Agent-seeded generation, nothing authored** | `init` → `generate` with no instruction file: seeded from the agent's published instructions, 14 rows generated, 13 of 14 on the agent's actual catalog/policies; `azd deploy` published them; the run scored 14 passed / 0 failed / 0 errored | +| Missing generation model | Fails before any network call, naming `--eval-model` and the spec field | +| **`schedule` (M2)** | Create, list, show and delete against the live project; trigger read back from the service as stored, not echoed. One-per-project and existing-name refusals both verified, each naming the schedule and the command to clear it. Delete waits out `Creating` and leaves the project empty | +| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalised `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | +| Failed runs | The reason reaches the caller instead of just the word "failed" | +| **`run --response-id` (M4)** | Three stored responses evaluated, 3 passed / 0 errored; the stored payload matched what was sent field for field | +| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalised to `schema.item`, are both benign | +| **`target.type: model` (M4)** | Group deployed with `response` bound to `{{sample.output_text}}`, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency | +| **Registered dataset on a run (M4)** | A group with no local `source:` now runs: whole set scores 2 passed / 1 failed, `--max-samples 2` scores 2 rows. Previously a 400 | +| **Two groups in one config** | Distinct ids across repeated deploys, each running its own criteria. Previously the second deploy aliased them onto one group | +| Pinned dataset `version:` | `1.0` publishes 1.0; editing the file while pinned stops with an instruction. Previously published 2.0, then 3.0 | +| Evaluator declaration forms | `source:` alone publishes then reports unchanged; `version:` alone references; both together refused | +| Conversation-level evaluation | `task_completion` publishes `messages` bound to `{{item.messages}}` with `evaluation_level: conversation`, and runs 1 passed / 1 failed / 0 errored with no target | +| `--eval-group` on the id-taking commands | Each group's own runs and results reachable by name; an undeployed name refused by name | +| **All four spec examples, verbatim** | 1: `init` → `generate --max-samples 50` → `azd up` (dataset 10.0, evaluator 22, group created) → `run` completed → `results show --failed-only -O ./results.json` (1503 b). 2: BYO dataset + `builtin.task_adherence`, `run --max-samples 25` completed. 3: `dataset create`, `run start --eval-id --no-prompt -o json` parsed, `results export --format csv -O gate.csv` (108 b). 4: the unregistered-edit error, wording matching the spec | +| Repeated `azd up` | 2nd and 3rd deploys both report `Dataset golden is unchanged at version 3.0`; no new versions | +| Hand-set `EVAL_GROUP_ID` | Honoured on a single-group config — the group is reused, not recreated. The per-group fix had silently removed this documented path | +| `-o json` on list commands | `dataset`, `evaluator`, `schedule`, `run list` all emit a bare array. They previously leaked two different service envelopes, `value` and `data` | +| `--eval-id` on the sibling commands | Accepted by `run list\|show\|cancel` and `results show\|export\|compare`, matching `run start`; positional still wins | +| `results compare` on one-sample runs | Service sends `"standardDeviation": "NaN"` — a quoted string, since JSON has no NaN literal. Decoding into `float64` failed the whole comparison, discarding the `TooFewSamples` verdict that explains it. Now decodes, renders the undefined statistic as `-`, and emits `null` in JSON. The earlier pass only held because those runs had enough samples | +| **Scenario suite, 18 assertions on substance** | Every scenario checked on result counts and file contents rather than exit status: agent run (1 passed / 0 errored), `results show -O` parseable, CSV header + rows, JSON export valid, model target (2 scored), `--max-samples 1` scoring exactly 1 of 2 rows, conversation level (1 scored), `--response-id`, `--from-traces`, `results compare` table + JSON, schedule set/show/delete. 18/18 | +| `TestLiveRun` | Was **skipping** unless `AZURE_AI_EVAL_AGENT` is set, so the run phase had never executed in any "full suite green" claim. Now run against a real agent, and it asserts no errored samples and at least one scored — reaching a terminal state alone would stay green with a broken target | +| Schedule inherits the group's last run | A schedule repeats the most recent run, so `--from-traces` turns the next schedule into a trace evaluation, which the service restricts to hourly. Proved by experiment: daily accepted after an agent run, refused after a traces run on the same group, hourly accepted for that traces run. The bare service message named neither the cause nor the remedy | + +## 11d. Blocked — needs the service team + +**Agent-seeded data generation fails for every agent.** `POST /data_generation_jobs` with an `agent` source in `inputs.sources` is accepted (201) and then fails within seconds: + +``` +"error": { "code": "DataGenerationJobSystemError", + "message": "Something went wrong during data generation. Please try again." } +``` + +**Ruled out, by probe.** The payload matches the published contract (`AgentDataGenerationJobSource` in `RAISvc/Contracts/DataGenerationJobs/Models/DataGenerationJobSource.cs`: `agent_name` + optional `agent_version`, which is exactly what is sent). Every identifier form fails the same way — name, `agent_version` pinned to `1`/`2`/`latest`, an assistant id, an assistant name, agent with and without a prompt source, and all three api-versions. **A nonexistent agent name fails identically**, so the agent is never resolved and the error carries no signal. + +**Where it goes.** `{project}/data_generation_jobs` → RAISvc S2S client (`DependencyExtensions.cs`, targeting `FineTuningHostUri`) → FineTuning `foundryProxy/data_generation_jobs` → `FoundryProxyTransform.cs` rewrites the path to `{aoaiEndpointTarget}/openai/v1/data_generation_jobs`. Neither RAISvc nor FineTuning resolves the agent — FineTuning has no reference to `agent_name` anywhere. The failure is in the AOAI generator, outside this repo. + +**What the CLI does instead.** The contract says the agent source exists to "fetch instructions / metadata from" the agent, which is a read the client can do itself. `generate` resolves the agent's instructions locally (§6a) and passes them as the prompt source. The agent source is still sent, so it starts contributing when the service is fixed, and the retry covers the failure until then. + +**Related, worth reporting:** an invalid enum value anywhere in the request returns `"The dataGenerationJob field is required."` — a whole-body deserialization failure reported as a missing field. Same misleading shape as the `definition.type` case on evaluator upload. + +**Also reported by the service, worth filing:** `results compare` returns `"standardDeviation": "NaN"` as a **quoted string** whenever a run has a single sample. JSON has no NaN literal, so this is the service's workaround, but it means a typed client must special-case the field or lose the whole comparison. The extension now decodes it (§11c); the service would be better emitting `null`. + +--- + +## 11e. Probed and genuinely unavailable + +Recorded because four M4 items were filed as "awaiting service support" and every one of them turned out to be already shipped. These three were checked rather than assumed, and they hold. + +| Claim | How it was checked | Result | +|---|---|---| +| M3: the eval group is versioned | `Evaluation.cs` in `RAISvc/Contracts/UnifiedEvaluationV2` | No `version` property. Blocked | +| M3: the eval group binds a dataset | `DataSourceConfig.cs` derived types | `custom`, `logs`, `stored_completions`, and the `azure_ai_source` scenarios (`red_team`, `synthetic_data_gen`, `responses`, `traces`, `benchmark_preview`, `conversation_simulation_preview`). None binds a registered dataset. Blocked | +| M4: a prompt target exists | `Target.cs` `TargetType` enum | Values are `azure_ai_model`, `azure_ai_agent`, `azure_ai_assistant`, plus `azure_ai_traces` marked `[NotARequestDiscriminator]`. No prompt target. Blocked | +| An assistant target could be exposed | Live POST of a known-good run body with only `target` swapped to `{"type":"azure_ai_assistant","id":"asst_…"}` | **400** `Unsupported target type in TargetCompletionsEvalRunDataSource: AzureAIAssistant is invalid`. On the enum, refused by the run data source. Building the CLI surface would have shipped a dead path | + +Unexposed capability seen while checking, out of the spec's scope and not implemented: `red_team`, `synthetic_data_gen`, `benchmark_preview` and `conversation_simulation_preview` data source configs, and an `EvalCsvRunDataSource`. + +--- + +## 12. Source-of-truth index + +| Doc / path | Gives you | +|---|---| +| `../azure.ai.agents/internal/pkg/agents/{eval_api,dataset_api}/` | The clients to lift. **Start here.** | +| `../azure.ai.agents/internal/cmd/eval_*.go` | Current command implementations, progress UX, api-version wiring | +| `../azure.ai.agents/internal/cmd/listen.go` | Service-target + lifecycle wiring to copy | +| `../azure.ai.agents/extension.yaml`, `main.go`, `go.mod` | Manifest, entrypoint, dependency versions | +| `cli/azd/pkg/azdext/` | Extension SDK: `ServiceTargetProvider`, `EventManager`, `Environment()` | +| `cli/azd/pkg/project/service_config.go` | `AdditionalProperties` inline capture | +| `cli/azd/pkg/foundry/includes.go` | `ResolveFileRefs` | +| `cli/azd/internal/cmd/up_graph.go` | What `azd up` actually runs | +| `foundrysdk_specs/.../azd_eval_extension/spec.md` | Design spec (authoritative) | +| `foundrysdk_specs/.../azd-agent-eval-public-preview-findings.md` | Measured bugs and timings in §7 | +| `foundrysdk_specs/.../custom_evaluator_upload/spec.md` | Evaluator upload flow, packaging, RBAC | + +--- + +*Keep in sync with `spec.md`. Uncommitted working document.* diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go index 5fc740610e5..7ba9e5ce23a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -85,9 +85,8 @@ func TestGenerationModel(t *testing.T) { assert.Equal(t, "gpt-4.1-nano", generationModel(cfg)) } -// `tools` is accepted and ignored, so it has to be called out — the same -// reasoning as the trace fields it now shares a warning with. -func TestWarnIgnoredFields_CoversTools(t *testing.T) { +// Trace selection is accepted and ignored, so it has to be called out. +func TestWarnIgnoredFields_CoversTraceSelection(t *testing.T) { cases := []struct { name string build func(*project.GenerateConfig) @@ -100,17 +99,18 @@ func TestWarnIgnoredFields_CoversTools(t *testing.T) { quiet: true, }, { - name: "tools alone", - build: func(c *project.GenerateConfig) { c.Agent.Context.Tools = "./agent/tools.json" }, - want: []string{"agent.context.tools", "has no effect"}, + name: "a source alone", + build: func(c *project.GenerateConfig) { + c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} + }, + want: []string{"agent.context.traces.source", "has no effect"}, }, { - name: "tools and a trace field agree in number", + name: "source and sample agree in number", build: func(c *project.GenerateConfig) { - c.Agent.Context.Tools = "./agent/tools.json" - c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} + c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights", Sample: 100} }, - want: []string{"agent.context.traces.source", "agent.context.tools", "have no effect"}, + want: []string{"agent.context.traces.source", "agent.context.traces.sample", "have no effect"}, }, { name: "a window alone is honored, so no warning", @@ -141,9 +141,7 @@ func TestWarnIgnoredFields_CoversTools(t *testing.T) { } // init scaffolds only the context fields that are read. -func TestInitScaffold_OmitsToolsButKeepsInstructions(t *testing.T) { +func TestInitScaffold_KeepsInstructions(t *testing.T) { cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") assert.Equal(t, "./agent/instructions.md", cfg.Agent.Context.Instructions) - assert.Empty(t, cfg.Agent.Context.Tools, - "scaffolding a field nothing reads would warn on every default init") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 3e4513b07c3..fb5b5865e79 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -21,8 +21,7 @@ func newDatasetCommand() *cobra.Command { Short: "Manage evaluation datasets.", } cmd.AddCommand( - newDatasetCreateCommand(false), - newDatasetCreateCommand(true), + newDatasetCreateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), @@ -30,9 +29,12 @@ func newDatasetCommand() *cobra.Command { return cmd } -// newDatasetCreateCommand builds `dataset create` and `dataset update`. Both -// publish a new immutable version; the server auto-increments. -func newDatasetCreateCommand(update bool) *cobra.Command { +// newDatasetCreateCommand builds `dataset create`. +// +// There is no separate `update`: every registration publishes a new immutable +// version and the server auto-increments, so `create` covers both the first +// version and every later one. +func newDatasetCreateCommand() *cobra.Command { var ( name string file string @@ -40,10 +42,8 @@ func newDatasetCreateCommand(update bool) *cobra.Command { endpointFlg string ) - use, short := "create", "Register a dataset, creating its first version." - if update { - use, short = "update", "Publish a new version of an existing dataset." - } + use := "create" + short := "Register a dataset, publishing a new version." cmd := &cobra.Command{ Use: use, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index e0203f07d5e..1b33daeba1d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -179,8 +179,7 @@ func newGenerateCommand() *cobra.Command { // The generation API takes a day window and nothing else, so `source` and // `sample` are parsed and dropped. Silently discarding them is worse than not // accepting them: the author believes they narrowed the trace selection when -// nothing changed. `agent.context.tools` is in the same position — nothing -// reads it, and only the instructions half of the agent's context is used. +// nothing changed. func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { var fields []string @@ -192,9 +191,6 @@ func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { fields = append(fields, "agent.context.traces.sample") } } - if cfg.Agent.Context.Tools != "" { - fields = append(fields, "agent.context.tools") - } if len(fields) == 0 { return } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 22ec21ab3e0..1c41c92f9cf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -38,7 +38,6 @@ type AgentSpec struct { // AgentContext points at the material used to synthesize a rubric and dataset. type AgentContext struct { Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - Tools string `yaml:"tools,omitempty" json:"tools,omitempty"` Traces *TraceSpec `yaml:"traces,omitempty" json:"traces,omitempty"` } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 2ecceff6f7e..4b3c8765728 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -191,9 +191,9 @@ func TestLiveDatasetLifecycle(t *testing.T) { }) } -// TestLiveEvalGroupLifecycle proves the create request this extension builds is +// TestLiveEvalLifecycle proves the create request this extension builds is // accepted, which is the single most important contract to get right. -func TestLiveEvalGroupLifecycle(t *testing.T) { +func TestLiveEvalLifecycle(t *testing.T) { env := setup(t) ctx := context.Background() @@ -234,12 +234,12 @@ func TestLiveEvalGroupLifecycle(t *testing.T) { } group, err := env.evalClient.CreateOpenAIEval(ctx, req) - require.NoError(t, err, "creating the eval group") + require.NoError(t, err, "creating the eval") require.NotEmpty(t, group.ID, "the service assigns the id; name is not unique") - t.Logf("created eval group %s (name %q)", group.ID, group.Name) + t.Logf("created eval %s (name %q)", group.ID, group.Name) fetched, err := env.evalClient.GetOpenAIEval(ctx, group.ID) - require.NoError(t, err, "reading the eval group back") + require.NoError(t, err, "reading the eval back") require.Equal(t, group.ID, fetched.ID) } @@ -283,7 +283,7 @@ func TestLiveRun(t *testing.T) { }, }}, }) - require.NoError(t, err, "creating the eval group for the run") + require.NoError(t, err, "creating the eval for the run") ds := eval_api.NewAgentTargetDataSource(env.agentName, nil) ds.SetFileContent([]map[string]any{ From da39cb87ddf821fa18f83e40d79fd1711e5986ce Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 00:40:56 +0300 Subject: [PATCH 060/114] Finish the eval-group rename in user-facing strings, and add run list --limit The evalGroups -> evals rename covered the config key and the flags but stopped there. Ten user-facing strings still said "group", so the CLI contradicted the spec and read as though the rename had been abandoned half-way: --eval help "Name a group from the config" run --name help "Defaults to the group name plus a timestamp" schedule --name help "Defaults to the group name" schedule list header "EVAL GROUP" schedule show field "group:" run no-previous-run "pass a config that declares the group" run from-traces "Declare target.type: agent on the group" schedule traces-only "this group's most recent run", "run the group once" Also rename the metadata written onto the eval, azd_eval_group -> azd_eval. Nothing reads it back, so this is safe. Separately, run list called ListOpenAIEvalRuns with a hardcoded limit of 0 and exposed no way to change it, even though the client already took the parameter. Add --limit. --- .../azure.ai.evaluations/internal/cmd/build.go | 2 +- .../azure.ai.evaluations/internal/cmd/results.go | 4 ++-- .../azure.ai.evaluations/internal/cmd/run.go | 6 +++--- .../azure.ai.evaluations/internal/cmd/run_ops.go | 5 ++++- .../internal/cmd/schedule.go | 16 ++++++++-------- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 6ca6d2fcf61..6c938f32a70 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -279,7 +279,7 @@ func buildEvalRequest( } } targetBindings := sampleBindingsFor(targetType) - metadata["azd_eval_group"] = group.Name + metadata["azd_eval"] = group.Name // The create request has no description field, so the group's own // description rides in metadata rather than being dropped. if group.Description != "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 405b538c01e..8b7728bb7bc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -210,12 +210,12 @@ func resolveEvalID( // positional. func addEvalFlags(cmd *cobra.Command, target *string) { cmd.Flags().StringVar(target, "eval", "", - "Name a group from the config instead of passing its id.") + "Name an evals entry from the config instead of passing its id.") cmd.Flags().String("eval-id", "", "Id of the eval. Same as passing the id as an argument.") } -// latestOrNamedRun returns the named run, or the most recent one for the group. +// latestOrNamedRun returns the named run, or the most recent one for the eval. func (ec *evalContext) latestOrNamedRun( cmd *cobra.Command, evalID, runID string, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a9a370113ee..e6087ae0e38 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -183,7 +183,7 @@ func buildRunCommand(use, short string) *cobra.Command { "Which evals entry to run. Defaults to the only one.") cmd.Flags().StringVar(&evalID, "eval-id", "", "Run against an existing eval by id, ignoring the config.") - cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the group name plus a timestamp.") + cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") cmd.Flags().StringVar(&level, "level", "", "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, @@ -361,7 +361,7 @@ func (ec *evalContext) reuseDataSourceFromLastRun( "eval %s has no previous run to repeat, so there is no target or dataset "+ "to reuse.\n"+ " Run it from the config once with `azd ai eval run`, or pass a config that "+ - "declares the group", + "declares the eval", evalID) } return list.Data[0].DataSource, nil @@ -398,7 +398,7 @@ func buildTracesDataSource( if agent == "" { return nil, fmt.Errorf( "--from-traces needs to know whose traces to read, and the eval does not " + - "name an agent. Declare target.type: agent on the group") + "name an agent. Declare target.type: agent on the eval") } var lookbackHours int diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 8fab2f3bca6..fbfdb2c8271 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -29,6 +29,7 @@ func newRunListCommand() *cobra.Command { var ( endpointFlg string groupName string + limit int ) cmd := &cobra.Command{ @@ -48,7 +49,7 @@ func newRunListCommand() *cobra.Command { return err } - list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) + list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, limit) if err != nil { if eval_api.IsNotFound(err) { return fmt.Errorf( @@ -78,6 +79,8 @@ func newRunListCommand() *cobra.Command { }, } addEvalFlags(cmd, &groupName) + cmd.Flags().IntVar(&limit, "limit", 0, + "Return at most this many runs. Omit for the service default.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index b2f223ebf0c..5a8affe44bd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -197,7 +197,7 @@ func newScheduleSetCommand() *cobra.Command { "Path to the eval deployment config.") cmd.Flags().StringVar(&groupName, "eval", "", "Which evals entry to schedule.") cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval by id, ignoring config.") - cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the group name.") + cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the eval name.") cmd.Flags().StringVar(&description, "description", "", "Schedule description.") cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) cmd.Flags().StringVar(&every, "every", "", @@ -254,20 +254,20 @@ func newScheduleListCommand() *cobra.Command { rows := make([][]string, 0, len(list.Value)) for i := range list.Value { s := &list.Value[i] - evalGroup := "" + evalID := "" if s.Task != nil { - evalGroup = s.Task.EvalID + evalID = s.Task.EvalID } rows = append(rows, []string{ s.ID, strconv.FormatBool(s.Enabled), s.ProvisioningStatus, s.Summary(), - evalGroup, + evalID, }) } return emitTable(out, - []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL GROUP"}, rows) + []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL"}, rows) }, } cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") @@ -317,7 +317,7 @@ func newScheduleShowCommand() *cobra.Command { fmt.Fprintf(out, " timezone: %s\n", s.Trigger.Timezone) } if s.Task != nil { - fmt.Fprintf(out, " group: %s\n", s.Task.EvalID) + fmt.Fprintf(out, " eval: %s\n", s.Task.EvalID) } if s.Description != "" { fmt.Fprintf(out, " about: %s\n", s.Description) @@ -599,9 +599,9 @@ func explainScheduleFailure( // is bewildering when the trigger was the only thing asked for. if isTracesHourlyOnly(cause) { return fmt.Errorf( - "saving schedule %q: this group's most recent run read from traces, and a schedule "+ + "saving schedule %q: this eval's most recent run read from traces, and a schedule "+ "repeats that run, so the service treats it as a scheduled trace evaluation "+ - "and allows only `--every hourly`. Use `--every hourly`, or run the group "+ + "and allows only `--every hourly`. Use `--every hourly`, or run the eval "+ "once against its dataset first so the schedule repeats that instead", name) } From c919ed687dfa5731d29755a5a5ca82a8862aba11 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 01:12:14 +0300 Subject: [PATCH 061/114] Add custom code evaluators `evaluator create --folder ` publishes a folder of Python as an evaluator version, and an `evaluators:` entry whose `source:` is a directory is reconciled by `azd up` the same way a dataset is. The wire shape was ambiguous. Two documents disagreed, and both were partly wrong. The published OpenAPI document that RAISvc vendors and contract-tests against settles it: {"type":"code", "code_text", "entry_point", "image_tag", "blob_uri"} plus inherited init_parameters, data_schema and metrics, all snake_case. The sibling custom_evaluator_upload spec is stale on four counts - the route is POST /evaluators/{name}/versions not PUT .../versions/{version}, credentials is POST not GET, the body is snake_case not camelCase, and the discriminator is "code" not "CodeBased". There is no version-based shape negotiation; one contract serves every api-version. blob_uri is a container prefix rather than an archive: the consumer enumerates the container, so files upload at their relative paths. Two deliberate choices where the evidence did not reach: Only blob_uri is sent, never code_text. The contract allows either, and inline would save a round trip for a single-file evaluator, but nothing observable confirms the executor runs it - the hand-off converter drops both fields and refetches from the catalog. blob_uri is the alternative with a demonstrated consumer. Sending the unproven one would trade a saved upload for an evaluator that registers cleanly and then fails when it is run, which is much harder to diagnose. Dot-prefixed files are excluded from the upload, not just dot-prefixed directories. An evaluator folder kept in a repo collects .env, .netrc and .pypirc, and publishing would copy those secrets into blob storage. Nothing a Python evaluator needs at runtime starts with a dot. entry_point is sent although RAISvc has no such property yet; it is spec-declared, and the create path ignores unknown members, so it starts persisting when the service catches up. The live round-trip test is written but unrun - it needs a project endpoint. It is the only thing that will confirm the service accepts this body; everything here is read from source and spec, not observed. --- .../internal/cmd/evaluator.go | 187 ++++++++- .../internal/cmd/evaluator_code_live_test.go | 227 +++++++++++ .../internal/cmd/evaluator_code_test.go | 194 ++++++++++ .../internal/cmd/reconciler.go | 100 ++++- .../internal/pkg/eval_api/code_evaluator.go | 364 ++++++++++++++++++ .../internal/pkg/eval_api/operations.go | 17 + .../internal/pkg/evalcore/codefolder.go | 356 +++++++++++++++++ .../internal/pkg/evalcore/codefolder_test.go | 329 ++++++++++++++++ .../internal/project/eval_config.go | 5 + .../internal/project/service_target_eval.go | 18 + 10 files changed, 1789 insertions(+), 8 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 100e8e7b10a..6e15b0a6442 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -9,6 +9,7 @@ import ( "os" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" "github.com/spf13/cobra" ) @@ -31,17 +32,22 @@ func newEvaluatorCommand() *cobra.Command { // `dataset create`: both register an artifact and both publish a new immutable // version every time, so there is nothing for a separate `update` to do. // -// M1 supports rubric evaluators only. Code evaluators need a folder walk, -// multi-blob upload, and the Azure AI User role assignment, so they land later. +// An evaluator is either a rubric — a JSON file of weighted dimensions — or +// code — a folder of Python. They are different definition types on the wire, +// so exactly one of the two sources has to be named. func newEvaluatorCreateCommand() *cobra.Command { var ( name string rubric string + folder string + initParams string + dataSchema string + metrics string endpointFlg string ) use := "create" - short := "Register a rubric evaluator, publishing a new version." + short := "Register a rubric or code evaluator, publishing a new version." cmd := &cobra.Command{ Use: use, @@ -50,8 +56,20 @@ func newEvaluatorCreateCommand() *cobra.Command { if name == "" { return requireFlag("name") } - if rubric == "" { - return requireFlag("rubric") + flags := codeEvaluatorFlags{ + initParams: initParams, + dataSchema: dataSchema, + metrics: metrics, + endpoint: endpointFlg, + } + if err := validateEvaluatorSource(rubric, folder, flags); err != nil { + return err + } + + ctx := cmd.Context() + + if folder != "" { + return runEvaluatorCreateFromFolder(cmd, name, folder, flags) } raw, err := os.ReadFile(rubric) @@ -64,7 +82,6 @@ func newEvaluatorCreateCommand() *cobra.Command { return fmt.Errorf("rubric %q: %w", rubric, err) } - ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err @@ -89,10 +106,168 @@ func newEvaluatorCreateCommand() *cobra.Command { cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") + cmd.Flags().StringVar(&folder, "folder", "", + "Path to a folder of Python holding the evaluator code.") + cmd.Flags().StringVar(&initParams, "init-params", "", + "Path to a JSON Schema for the evaluator's initialization parameters. "+ + "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + cmd.Flags().StringVar(&dataSchema, "data-schema", "", + "Path to a JSON Schema for the evaluator's input data. "+ + "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + cmd.Flags().StringVar(&metrics, "metrics", "", + "Path to a JSON object describing the metrics the evaluator produces. "+ + "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +// codeEvaluatorFlags are the optional overrides for a code evaluator. +type codeEvaluatorFlags struct { + initParams string + dataSchema string + metrics string + endpoint string +} + +// validateEvaluatorSource enforces that exactly one source is named, and that +// the schema overrides are only used with the source they apply to. +// +// Deliberately checked here rather than with MarkFlagsMutuallyExclusive: that +// only rejects the "both" case, and its message names a flag group rather than +// saying what the two flags mean. Both mistakes deserve advice, and this is +// testable without driving cobra. +func validateEvaluatorSource(rubric, folder string, flags codeEvaluatorFlags) error { + switch { + case rubric == "" && folder == "": + return fmt.Errorf( + "one of --rubric or --folder is required: --rubric takes a JSON file of " + + "weighted dimensions, --folder takes a directory of Python") + case rubric != "" && folder != "": + return fmt.Errorf( + "--rubric and --folder cannot be used together: an evaluator is either a " + + "rubric or code, not both") + } + + // A rubric's schemas are fixed by the service, so these would be accepted + // and then quietly dropped — the worst kind of no-op, because the author + // believes the evaluator was published carrying them. + if folder == "" { + for _, named := range []struct { + flag string + value string + }{ + {"init-params", flags.initParams}, + {"data-schema", flags.dataSchema}, + {"metrics", flags.metrics}, + } { + if named.value != "" { + return fmt.Errorf( + "--%s applies to a code evaluator and needs --folder; "+ + "a rubric's schemas are set by the service", named.flag) + } + } + } + return nil +} + +// runEvaluatorCreateFromFolder validates the folder, then publishes it. +func runEvaluatorCreateFromFolder( + cmd *cobra.Command, + name string, + folder string, + flags codeEvaluatorFlags, +) error { + pkg, err := evalcore.LoadCodeEvaluator(name, folder) + if err != nil { + return err + } + + opts, err := codeEvaluatorOptions(pkg, flags) + if err != nil { + return err + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, flags.endpoint) + if err != nil { + return err + } + defer ec.Close() + + created, err := ec.evalClient.UploadCodeEvaluatorVersion( + ctx, pkg, opts, ProjectEndpointAPIVersion, + ) + if err != nil { + return fmt.Errorf("publishing evaluator %q: %w", name, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), created) + } + fmt.Fprintf(cmd.OutOrStdout(), + "Published evaluator %s version %s from %d file(s) in %s\n", + created.Name, created.Version, len(pkg.Files), folder) + return nil +} + +// codeEvaluatorOptions resolves the evaluator's schemas, preferring an +// explicit flag over whatever the folder declares. +// +// The folder is the better place for them — they describe the code and belong +// beside it — but a folder that has none must still be publishable without +// editing it, which is what the flags are for. +func codeEvaluatorOptions( + pkg *evalcore.CodeEvaluatorPackage, + flags codeEvaluatorFlags, +) (eval_api.CodeEvaluatorOptions, error) { + var opts eval_api.CodeEvaluatorOptions + if md := pkg.Metadata; md != nil { + opts.DisplayName = md.DisplayName + opts.Description = md.Description + opts.Categories = md.Categories + opts.InitParameters = md.InitParameters + opts.DataSchema = md.DataSchema + opts.Metrics = md.Metrics + } + + for _, override := range []struct { + path string + flag string + field *json.RawMessage + }{ + {flags.initParams, "init-params", &opts.InitParameters}, + {flags.dataSchema, "data-schema", &opts.DataSchema}, + {flags.metrics, "metrics", &opts.Metrics}, + } { + if override.path == "" { + continue + } + raw, err := readJSONObject(override.path) + if err != nil { + return opts, fmt.Errorf("--%s %q: %w", override.flag, override.path, err) + } + *override.field = raw + } + + return opts, nil +} + +// readJSONObject reads a file that must hold a JSON object. +// +// Parsing here rather than letting the service reject it keeps a typo from +// costing an upload and a published version, and names the file that is wrong. +func readJSONObject(path string) (json.RawMessage, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var probe map[string]json.RawMessage + if err := json.Unmarshal(raw, &probe); err != nil { + return nil, fmt.Errorf("not a JSON object: %w", err) + } + return json.RawMessage(raw), nil +} + // normalizeRubricBody accepts either a bare definition ({type, dimensions}) or // a full evaluator document ({name, definition}) and returns the request body. // rubricDefinitionType is the discriminator the service uses to deserialize a diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go new file mode 100644 index 00000000000..f9c20c6d07d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// This file proves the code-evaluator body the extension builds is accepted by +// the real service, and that what comes back is the shape the extension +// expects. +// +// It exists because the wire contract was settled from source rather than from +// a live call: two published documents disagreed on the definition body, and +// only the service can say which one it honours. It asserts the round trip +// field by field so a drift shows up as a named mismatch, not a vague failure. +// +// go test -tags live -v ./internal/cmd/ -run TestLiveCodeEvaluator +// +// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" +) + +// liveCodeEvaluatorName is unique per run so concurrent runs, and reruns after +// a failure that skipped cleanup, do not collide. +func liveCodeEvaluatorName(t *testing.T, suffix string) string { + t.Helper() + return fmt.Sprintf("azdcode_%s_%d", suffix, time.Now().UnixNano()) +} + +// writeLiveEvaluator lays out a folder to the packaging convention and returns +// it. The source is written for the derived class name so the production +// validation is exercised rather than bypassed. +func writeLiveEvaluator(t *testing.T, name string, extraFiles map[string]string) string { + t.Helper() + dir := t.TempDir() + + className := evalcore.EvaluatorClassName(name) + entry := fmt.Sprintf(`class %s: + def __call__(self, **kwargs): + return {"result": float(len(kwargs.get("response", "")))} +`, className) + + require.NoError(t, os.WriteFile( + filepath.Join(dir, name+".py"), []byte(entry), 0o600)) + + for rel, content := range extraFiles { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + return dir +} + +// codeDefinitionOnService reads the registered version back and returns its +// definition, so the assertions run against what the service persisted rather +// than against what was sent. +func codeDefinitionOnService( + t *testing.T, + client *eval_api.EvalClient, + name, version string, +) map[string]json.RawMessage { + t.Helper() + + raw, err := client.GetEvaluatorRaw( + context.Background(), name, version, ProjectEndpointAPIVersion) + require.NoError(t, err, "reading back evaluator %s version %s", name, version) + + var doc map[string]json.RawMessage + require.NoError(t, json.Unmarshal(raw, &doc)) + require.Contains(t, doc, "definition", + "the registered evaluator carries no definition: %s", string(raw)) + + var definition map[string]json.RawMessage + require.NoError(t, json.Unmarshal(doc["definition"], &definition)) + return definition +} + +func stringField(t *testing.T, definition map[string]json.RawMessage, key string) string { + t.Helper() + raw, ok := definition[key] + if !ok { + return "" + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "" + } + return value +} + +// TestLiveCodeEvaluatorSingleFileRoundTrip publishes a one-file evaluator and +// asserts it comes back as a code definition pointing at storage. +// +// A single file takes the same path as a folder. The contract also accepts +// inline source through code_text, which would save the upload, but nothing +// observable confirms the executor runs it, so the CLI does not send it. If +// that is ever settled, this is the test that should change first. +func TestLiveCodeEvaluatorSingleFileRoundTrip(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := liveCodeEvaluatorName(t, "single") + dir := writeLiveEvaluator(t, name, nil) + + // The shipping loader, not a hand-built package: this test has to fail if + // the production path stops producing a valid package. + pkg, err := evalcore.LoadCodeEvaluator(name, dir) + require.NoError(t, err) + require.Len(t, pkg.Files, 1) + + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + + created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err, "the service rejected the code evaluator body") + require.NotEmpty(t, created.Version) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + definition := codeDefinitionOnService(t, client, name, created.Version) + + require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type"), + "the discriminator must round-trip as the lowercase snake_case value") + require.NotEmpty(t, stringField(t, definition, "blob_uri"), + "a published evaluator must record the storage location it was uploaded to") + require.Contains(t, definition, "metrics", + "a code definition must carry metrics; the service rejects one without") +} + +// TestLiveCodeEvaluatorFolderRoundTrip publishes a multi-file evaluator and +// asserts the service records the storage location it handed out. +// +// This is the path that exercises startPendingUpload, the SAS write, and the +// blob_uri property, none of which the single-file case touches. +func TestLiveCodeEvaluatorFolderRoundTrip(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := liveCodeEvaluatorName(t, "folder") + dir := writeLiveEvaluator(t, name, map[string]string{ + "helpers/text.py": "def clean(value):\n return value.strip()\n", + // Must be excluded from both the upload and the fingerprint. + "__pycache__/stale.pyc": "cache", + }) + + pkg, err := evalcore.LoadCodeEvaluator(name, dir) + require.NoError(t, err) + require.Len(t, pkg.Files, 2, "the compiled artifact must not be part of the package") + + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + + created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err, "the service rejected the uploaded code evaluator") + require.NotEmpty(t, created.Version) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + definition := codeDefinitionOnService(t, client, name, created.Version) + + require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type")) + require.NotEmpty(t, stringField(t, definition, "blob_uri"), + "a multi-file evaluator must round-trip carrying the storage location; "+ + "an empty blob_uri means the service dropped the preview property") + require.Contains(t, definition, "metrics") +} + +// TestLiveCodeEvaluatorPublishesNextVersion proves the version the upload +// reserves storage under is the one the create then assigns. +// +// Storage is provisioned per version before the version exists, so the client +// has to predict it. A drift between the two would leave the code in one +// version's container and the definition on another. +func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := liveCodeEvaluatorName(t, "versions") + dir := writeLiveEvaluator(t, name, map[string]string{ + "helpers.py": "VALUE = 1\n", + }) + + pkg, err := evalcore.LoadCodeEvaluator(name, dir) + require.NoError(t, err) + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + + predicted := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) + require.Equal(t, "1", predicted, "an unpublished evaluator starts at version 1") + + first, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, first.Version, ProjectEndpointAPIVersion) + }) + require.Equal(t, predicted, first.Version, + "the service assigned a version other than the one storage was reserved under") + + // Publishing again must land on the next version, not overwrite the first. + predictedSecond := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) + second, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, second.Version, ProjectEndpointAPIVersion) + }) + require.Equal(t, predictedSecond, second.Version) + require.NotEqual(t, first.Version, second.Version) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go new file mode 100644 index 00000000000..a919bd56a90 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +func writeTestFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +const toneEvaluatorSource = "class ToneEvaluator:\n" + + " def __call__(self, **kwargs):\n" + + " return {\"result\": 1}\n" + +// An evaluator is either a rubric or code. Naming both, or neither, is a +// mistake the command has to name precisely — the two flags take different +// kinds of path and produce different definition types. +func TestValidateEvaluatorSource(t *testing.T) { + err := validateEvaluatorSource("", "", codeEvaluatorFlags{}) + require.Error(t, err) + require.Contains(t, err.Error(), "--rubric") + require.Contains(t, err.Error(), "--folder") + require.Contains(t, err.Error(), "required") + + err = validateEvaluatorSource("rubric.json", "./evaluator", codeEvaluatorFlags{}) + require.Error(t, err) + require.Contains(t, err.Error(), "cannot be used together") + + require.NoError(t, validateEvaluatorSource("rubric.json", "", codeEvaluatorFlags{})) + require.NoError(t, validateEvaluatorSource("", "./evaluator", codeEvaluatorFlags{})) +} + +// The schema overrides describe a code evaluator. Accepting them beside a +// rubric and dropping them would leave the author believing the evaluator was +// published carrying schemas it never had. +func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { + for flag, flags := range map[string]codeEvaluatorFlags{ + "init-params": {initParams: "init.json"}, + "data-schema": {dataSchema: "schema.json"}, + "metrics": {metrics: "metrics.json"}, + } { + err := validateEvaluatorSource("rubric.json", "", flags) + require.Error(t, err, "for --%s", flag) + require.Contains(t, err.Error(), "--"+flag) + require.Contains(t, err.Error(), "--folder") + + require.NoError(t, validateEvaluatorSource("", "./evaluator", flags), + "--%s is valid with --folder", flag) + } +} + +// The same check the command runs must be reachable from the command, so a +// future refactor cannot leave the flags declared but unvalidated. +func TestEvaluatorCreateRejectsBothSources(t *testing.T) { + cmd := newEvaluatorCreateCommand() + cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--folder", "./x"}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SilenceUsage = true + + err := cmd.Execute() + require.Error(t, err) + require.Contains(t, err.Error(), "cannot be used together") +} + +func TestEvaluatorCreateRejectsNeitherSource(t *testing.T) { + cmd := newEvaluatorCreateCommand() + cmd.SetArgs([]string{"--name", "tone"}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SilenceUsage = true + + err := cmd.Execute() + require.Error(t, err) + require.Contains(t, err.Error(), "one of --rubric or --folder") +} + +// The folder is the natural home for the schemas, but a folder that carries +// none still has to be publishable without editing it. +func TestCodeEvaluatorOptions_FlagsOverrideFolderMetadata(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, dir, "tone.py", toneEvaluatorSource) + writeTestFile(t, dir, evalcore.CodeEvaluatorMetadataFile, `{ + "display_name": "Tone", + "metrics": {"result": {"type": "ordinal"}}, + "data_schema": {"type": "object", "properties": {"a": {"type": "string"}}} + }`) + + pkg, err := evalcore.LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + + // Nothing overridden: the folder wins. + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + require.NoError(t, err) + require.Equal(t, "Tone", opts.DisplayName) + require.Contains(t, string(opts.Metrics), "ordinal") + require.Contains(t, string(opts.DataSchema), `"a"`) + require.Empty(t, opts.InitParameters) + + overrides := t.TempDir() + metricsPath := writeTestFile(t, overrides, "metrics.json", + `{"result":{"type":"continuous"}}`) + initPath := writeTestFile(t, overrides, "init.json", + `{"type":"object","properties":{"deployment_name":{"type":"string"}}}`) + + opts, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + metrics: metricsPath, + initParams: initPath, + }) + require.NoError(t, err) + require.Contains(t, string(opts.Metrics), "continuous") + require.NotContains(t, string(opts.Metrics), "ordinal") + require.Contains(t, string(opts.InitParameters), "deployment_name") + // Untouched by the overrides. + require.Contains(t, string(opts.DataSchema), `"a"`) +} + +// A typo in a schema file must be reported against the flag that named it, +// not discovered by the service after a version has been published. +func TestCodeEvaluatorOptions_RejectsMalformedOverride(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, dir, "tone.py", toneEvaluatorSource) + pkg, err := evalcore.LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + + bad := writeTestFile(t, dir, "metrics.json", "[1,2,3]") + _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{metrics: bad}) + require.Error(t, err) + require.Contains(t, err.Error(), "--metrics") + require.Contains(t, err.Error(), "JSON object") + + _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + dataSchema: filepath.Join(dir, "absent.json"), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "--data-schema") +} + +// The service rejects a code definition carrying no metrics, so a folder that +// declares none still has to publish with one. +func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { + var metrics map[string]map[string]any + require.NoError(t, json.Unmarshal(eval_api.DefaultCodeMetrics, &metrics)) + require.Len(t, metrics, 1) + require.Contains(t, metrics, "result") + require.Equal(t, "continuous", metrics["result"]["type"]) +} + +// Change detection has to work for both kinds of evaluator source, and the +// reconciler decides which by stat-ing the path. +func TestFingerprintPath_HandlesFilesAndFolders(t *testing.T) { + root := t.TempDir() + + file := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) + fileDigest, err := project.FingerprintPath(file) + require.NoError(t, err) + plainDigest, err := project.Fingerprint(file) + require.NoError(t, err) + require.Equal(t, plainDigest, fileDigest, + "a file must hash the same through either entry point") + + dir := t.TempDir() + writeTestFile(t, dir, "tone.py", toneEvaluatorSource) + folderDigest, err := project.FingerprintPath(dir) + require.NoError(t, err) + require.NotEmpty(t, folderDigest) + require.NotEqual(t, fileDigest, folderDigest) + + writeTestFile(t, dir, "helpers.py", "X = 1\n") + changed, err := project.FingerprintPath(dir) + require.NoError(t, err) + require.NotEqual(t, folderDigest, changed, + "adding a file to the folder must change the digest") + + _, err = project.FingerprintPath(filepath.Join(root, "absent")) + require.Error(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index a06f13853d3..3c5c93234f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -11,9 +11,11 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" ) @@ -223,8 +225,14 @@ func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) } // EnsureEvaluator publishes a new version when the local definition differs -// from what the service holds. Evaluator definitions come back inline, so this -// compares content directly rather than relying on a cached digest. +// from what the service holds. +// +// The two kinds of evaluator are told apart by what the source names, not by +// its spelling: a folder is code, a file is a rubric. They also detect change +// differently. A rubric definition comes back inline, so it is compared +// directly; code does not — the service returns a storage URI, never the +// source — so a fingerprint of the folder is kept in the azd environment, the +// same way datasets work. func (r *evalReconciler) EnsureEvaluator( ctx context.Context, decl project.EvaluatorDecl, @@ -242,6 +250,14 @@ func (r *evalReconciler) EnsureEvaluator( return versionFromRaw(raw, decl.Version), false, nil } + info, err := os.Stat(localPath) + if err != nil { + return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) + } + if info.IsDir() { + return r.ensureCodeEvaluator(ctx, decl, localPath) + } + raw, err := os.ReadFile(localPath) if err != nil { return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) @@ -270,6 +286,86 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } +// ensureCodeEvaluator publishes a folder of Python only when its content +// changed since the last deploy. +// +// Every publish is a new immutable version, so without this a repeated +// `azd up` would leave a trail of identical versions and force every eval +// bound to the evaluator to be recreated along with them. +func (r *evalReconciler) ensureCodeEvaluator( + ctx context.Context, + decl project.EvaluatorDecl, + dir string, +) (string, bool, error) { + // Validated before anything is uploaded: a folder missing its entry point + // or class is only rejected when a run executes, long after a version has + // been published and an eval bound to it. + pkg, err := evalcore.LoadCodeEvaluator(decl.Name, dir) + if err != nil { + return "", false, err + } + + digest, err := project.FingerprintPath(dir) + if err != nil { + return "", false, err + } + + key := project.FingerprintKey("evaluator", decl.Name) + recordedVersion := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)) + if r.ec.getEnvValue(ctx, key) == digest && recordedVersion != "" { + // Unchanged since the last deploy, but that alone does not make the + // recorded version safe to reuse: someone may have published a newer + // one outside the repo, and binding the eval to the older one would + // quietly evaluate with superseded code. + if err := r.checkEvaluatorDrift(ctx, decl.Name, recordedVersion); err != nil { + return "", false, err + } + return recordedVersion, false, nil + } + + opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + if err != nil { + return "", false, err + } + + created, err := r.ec.evalClient.UploadCodeEvaluatorVersion( + ctx, pkg, opts, ProjectEndpointAPIVersion, + ) + if err != nil { + return "", false, err + } + + _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) + return created.Version, true, nil +} + +// checkEvaluatorDrift fails when the service holds a newer version than the +// one recorded at the last deploy. +// +// Publishing is not destructive — versions are immutable — so the remedy is to +// sync with what is on the project, not to overwrite it. +func (r *evalReconciler) checkEvaluatorDrift( + ctx context.Context, + name, recorded string, +) error { + recordedNumber, err := strconv.Atoi(recorded) + if err != nil { + return nil + } + latest := r.ec.evalClient.LatestEvaluatorVersionNumber( + ctx, name, ProjectEndpointAPIVersion, + ) + if latest <= recordedNumber { + return nil + } + return fmt.Errorf( + "evaluator %q is at version %d on the project but %s was recorded at the last deploy; "+ + "someone published a version outside this repo. "+ + "Pull the newer code locally, or delete version %d, then deploy again", + name, latest, recorded, latest) +} + // EnsureEval creates the group when it has never been deployed, or when an // upstream artifact changed. Groups are immutable, so a change means a new // group and a new id. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go new file mode 100644 index 00000000000..5ee2492faed --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/evalcore" +) + +// CodeDefinitionType is the discriminator the service uses to deserialize a +// code evaluator definition. +// +// The wire shape is snake_case with a lowercase discriminator, matching +// CodeBasedEvaluatorDefinition in the Foundry data-plane OpenAPI document +// (`type` enum ["code"], plus code_text, entry_point, blob_uri, init_parameters, +// data_schema and metrics). An earlier draft documented a camelCase body with +// `type: "CodeBased"`; that shape is not what the deployed service accepts. +const CodeDefinitionType = "code" + +// evaluatorTypeCustom marks an evaluator as authored by the project rather +// than shipped by the platform. +const evaluatorTypeCustom = "custom" + +// foundryFeaturesHeader opts a request in to preview behaviour. blob_uri and +// entry_point are both declared as preview properties on the code definition, +// so the header is sent with every call that sets one. +const ( + foundryFeaturesHeader = "Foundry-Features" + foundryFeatureEvalsV1 = "Evaluations=V1Preview" + pendingUploadTypeBlob = "BlobReference" + defaultCodeMetricName = "result" + defaultCodeMetricType = "continuous" + defaultMetricDirection = "increase" + firstEvaluatorVersion = "1" + blobTypeHeader = "x-ms-blob-type" + blobTypeBlock = "BlockBlob" + octetStreamContentType = "application/octet-stream" +) + +// DefaultCodeMetrics is used when neither the folder nor the caller declares +// any. +// +// The service rejects a code definition carrying no metrics, and the documented +// evaluator output is a JSON object whose `result` field holds the score, so +// this describes exactly that. It is a default, not a constraint: any declared +// metrics replace it wholesale. +var DefaultCodeMetrics = json.RawMessage(fmt.Sprintf( + `{%q:{"type":%q,"desirable_direction":%q,"is_primary":true}}`, + defaultCodeMetricName, defaultCodeMetricType, defaultMetricDirection, +)) + +// CodeEvaluatorOptions carries the parts of an evaluator version that do not +// come from the Python source itself. +type CodeEvaluatorOptions struct { + DisplayName string + Description string + Categories []string + InitParameters json.RawMessage + DataSchema json.RawMessage + Metrics json.RawMessage +} + +// codeDefinition is the wire body of a code evaluator definition. +// +// The contract also allows code_text in place of blob_uri, and requires exactly +// one of the two. Only blob_uri is modelled here, because only blob_uri is +// sent; a field that is never populated would suggest a supported alternative +// that has not been exercised. +type codeDefinition struct { + Type string `json:"type"` + EntryPoint string `json:"entry_point,omitempty"` + BlobURI string `json:"blob_uri,omitempty"` + InitParameters json.RawMessage `json:"init_parameters,omitempty"` + DataSchema json.RawMessage `json:"data_schema,omitempty"` + Metrics json.RawMessage `json:"metrics,omitempty"` +} + +// createEvaluatorVersionRequest is the POST body for a new evaluator version. +// The service assigns the version; it is not carried here. +type createEvaluatorVersionRequest struct { + Name string `json:"name,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Description string `json:"description,omitempty"` + EvaluatorType string `json:"evaluator_type,omitempty"` + Categories []string `json:"categories,omitempty"` + Definition *codeDefinition `json:"definition"` +} + +// pendingUploadRequest starts an upload for one evaluator version. +type pendingUploadRequest struct { + PendingUploadType string `json:"pendingUploadType"` +} + +// PendingUploadResponse is the reply to startPendingUpload: a container to +// write into, and the SAS that authorizes writing. +type PendingUploadResponse struct { + BlobReference *BlobReference `json:"blobReference,omitempty"` + PendingUploadID string `json:"pendingUploadId,omitempty"` + Version string `json:"version,omitempty"` +} + +// BlobReference is a storage location plus the credential to reach it. +type BlobReference struct { + BlobURI string `json:"blobUri,omitempty"` + StorageAccountARM string `json:"storageAccountArmId,omitempty"` + Credential *BlobCredential `json:"credential,omitempty"` +} + +// BlobCredential holds the SAS granted for an upload. +type BlobCredential struct { + Type string `json:"type,omitempty"` + SASUri string `json:"sasUri,omitempty"` +} + +// UploadURI returns the container URI carrying the SAS token, or empty when +// the service granted no credential. +func (p *PendingUploadResponse) UploadURI() string { + if p == nil || p.BlobReference == nil || p.BlobReference.Credential == nil { + return "" + } + return p.BlobReference.Credential.SASUri +} + +// ContainerURI returns the container URI without the SAS token. This is what +// the evaluator definition records, because the definition is persisted and a +// SAS in it would expire. +func (p *PendingUploadResponse) ContainerURI() string { + if p == nil || p.BlobReference == nil { + return "" + } + return p.BlobReference.BlobURI +} + +// UploadCodeEvaluatorVersion publishes a folder of Python as a new version of +// a code evaluator. +// +// Every package goes through storage, including a package of one file. The +// contract offers code_text as an alternative and it would save a round trip, +// but nothing observable confirms the executor runs it: the hand-off to the +// evaluation runtime drops both code_text and blob_uri and refetches from the +// catalog, so RAISvc does not reveal which one it prefers. blob_uri is the one +// with a demonstrated consumer, which enumerates the container and reads the +// files back. Choosing the unproven path would trade a saved upload for an +// evaluator that registers cleanly and then fails when it is finally run, +// which is far harder to diagnose than a slow publish. Revisit once the live +// test has actually exercised inline source. +func (c *EvalClient) UploadCodeEvaluatorVersion( + ctx context.Context, + pkg *evalcore.CodeEvaluatorPackage, + opts CodeEvaluatorOptions, + apiVersion string, +) (*EvaluatorVersion, error) { + if pkg == nil { + return nil, fmt.Errorf("no evaluator package to publish") + } + + definition := &codeDefinition{ + Type: CodeDefinitionType, + EntryPoint: pkg.EntryPoint, + InitParameters: opts.InitParameters, + DataSchema: opts.DataSchema, + Metrics: opts.Metrics, + } + if len(definition.Metrics) == 0 { + definition.Metrics = DefaultCodeMetrics + } + + blobURI, err := c.uploadCodeEvaluatorFiles(ctx, pkg, apiVersion) + if err != nil { + return nil, err + } + definition.BlobURI = blobURI + + body := &createEvaluatorVersionRequest{ + Name: pkg.Name, + DisplayName: opts.DisplayName, + Description: opts.Description, + EvaluatorType: evaluatorTypeCustom, + Categories: opts.Categories, + Definition: definition, + } + + path := pathEvaluators + "/" + url.PathEscape(pkg.Name) + "/versions" + respBody, err := c.doRequestWithHeaders( + ctx, http.MethodPost, path, nil, body, apiVersion, previewHeaders(), + ) + if err != nil { + return nil, err + } + + var created EvaluatorVersion + if len(respBody) > 0 { + if err := json.Unmarshal(respBody, &created); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + } + if created.Name == "" { + created.Name = pkg.Name + } + return &created, nil +} + +// uploadCodeEvaluatorFiles writes every file in the package to the container +// the service provisions for the version being created, and returns the +// container URI to record on the definition. +func (c *EvalClient) uploadCodeEvaluatorFiles( + ctx context.Context, + pkg *evalcore.CodeEvaluatorPackage, + apiVersion string, +) (string, error) { + version := c.NextEvaluatorVersion(ctx, pkg.Name, apiVersion) + + pending, err := c.StartEvaluatorPendingUpload(ctx, pkg.Name, version, apiVersion) + if err != nil { + return "", fmt.Errorf("starting the upload for evaluator %q: %w", pkg.Name, err) + } + + uploadURI := pending.UploadURI() + if uploadURI == "" { + return "", fmt.Errorf( + "the service returned no upload credential for evaluator %q", pkg.Name) + } + containerURI := pending.ContainerURI() + if containerURI == "" { + return "", fmt.Errorf( + "the service returned no storage location for evaluator %q", pkg.Name) + } + + for _, file := range pkg.Files { + content, err := os.ReadFile(file.AbsPath) + if err != nil { + return "", fmt.Errorf("reading %s: %w", file.RelPath, err) + } + if err := uploadBlob(ctx, uploadURI, file.RelPath, content); err != nil { + return "", fmt.Errorf("uploading %s: %w", file.RelPath, err) + } + } + + return containerURI, nil +} + +// StartEvaluatorPendingUpload provisions the storage an evaluator version's +// code is written to. +func (c *EvalClient) StartEvaluatorPendingUpload( + ctx context.Context, + name string, + version string, + apiVersion string, +) (*PendingUploadResponse, error) { + path := fmt.Sprintf( + "%s/%s/versions/%s/startPendingUpload", + pathEvaluators, url.PathEscape(name), url.PathEscape(version), + ) + respBody, err := c.doRequestWithHeaders( + ctx, http.MethodPost, path, nil, + &pendingUploadRequest{PendingUploadType: pendingUploadTypeBlob}, + apiVersion, previewHeaders(), + ) + if err != nil { + return nil, err + } + + var pending PendingUploadResponse + if len(respBody) > 0 { + if err := json.Unmarshal(respBody, &pending); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + } + return &pending, nil +} + +// NextEvaluatorVersion reports the version the service will assign to the next +// create. +// +// The upload has to name a version before the version exists, because storage +// is provisioned per version while the create that assigns it comes last. The +// service auto-increments, so the next one is the highest registered plus one. +// An unknown evaluator has none, which is version 1. +func (c *EvalClient) NextEvaluatorVersion( + ctx context.Context, + name string, + apiVersion string, +) string { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return firstEvaluatorVersion + } + latest := pickLatestVersion(list.Value) + number, err := strconv.Atoi(latest) + if err != nil { + return firstEvaluatorVersion + } + return strconv.Itoa(number + 1) +} + +// LatestEvaluatorVersionNumber reports the newest registered version as an +// integer, or 0 when the evaluator is unknown or its versions are not numeric. +func (c *EvalClient) LatestEvaluatorVersionNumber( + ctx context.Context, + name string, + apiVersion string, +) int { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return 0 + } + number, err := strconv.Atoi(pickLatestVersion(list.Value)) + if err != nil { + return 0 + } + return number +} + +// previewHeaders returns the opt-in header for the preview properties the code +// definition relies on. +func previewHeaders() map[string]string { + return map[string]string{foundryFeaturesHeader: foundryFeatureEvalsV1} +} + +// uploadBlob writes one file into a container using a container-level SAS. +// +// A plain client is used rather than the pipeline: the SAS in the URL is the +// credential, and the pipeline's bearer token policy would attach a Foundry +// token to a storage request that has no use for it. +func uploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { + u, err := url.Parse(containerSASUri) + if err != nil { + return fmt.Errorf("invalid container SAS URI: %w", err) + } + u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("failed to create upload request: %w", err) + } + req.Header.Set(blobTypeHeader, blobTypeBlock) + req.Header.Set("Content-Type", octetStreamContentType) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to upload blob: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + // The body is Azure Storage XML, which says more than the status alone, + // but it is capped: a rejected upload can answer with a long document. + detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return fmt.Errorf("blob upload failed with status %d: %s", + resp.StatusCode, strings.TrimSpace(string(detail))) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 5b2a74923bf..4ba2d628994 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -284,6 +284,20 @@ func (c *EvalClient) doRequest( query map[string]string, body any, apiVersion string, +) ([]byte, error) { + return c.doRequestWithHeaders(ctx, method, path, query, body, apiVersion, nil) +} + +// doRequestWithHeaders is doRequest with extra request headers, which the +// preview evaluator operations need to opt in to the properties they set. +func (c *EvalClient) doRequestWithHeaders( + ctx context.Context, + method string, + path string, + query map[string]string, + body any, + apiVersion string, + headers map[string]string, ) ([]byte, error) { u, err := url.Parse(c.endpoint) if err != nil { @@ -304,6 +318,9 @@ func (c *EvalClient) doRequest( if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } + for k, v := range headers { + req.Raw().Header.Set(k, v) + } log.Printf("[eval_api] %s %s", method, u.Redacted()) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go new file mode 100644 index 00000000000..a62391aa96f --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go @@ -0,0 +1,356 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode" +) + +// CodeEvaluatorMetadataFile is the optional descriptor a folder can carry so +// the schemas travel with the code instead of having to be repeated on the +// command line every time it is published. +// +// It is not part of the service contract — the service only ever sees the +// definition assembled from it — so it is read leniently and its absence is +// normal. +const CodeEvaluatorMetadataFile = "evaluator.json" + +// pythonExt is the only source extension a code evaluator entry point can have. +const pythonExt = ".py" + +// excludedDirNames are directories that never carry evaluator source: build +// caches, version control, and dependency trees. Any dot-prefixed directory is +// excluded as well, which is checked separately. +var excludedDirNames = map[string]bool{ + "__pycache__": true, + ".git": true, + ".venv": true, + "venv": true, + "node_modules": true, +} + +// excludedFileExts are compiled Python artifacts. They are derived from the +// sources beside them, so uploading them adds nothing and — because they embed +// a timestamp — would make the folder fingerprint change on every rebuild. +var excludedFileExts = map[string]bool{ + ".pyc": true, + ".pyo": true, +} + +// CodeFile is one file in an evaluator package. +type CodeFile struct { + // RelPath is the path relative to the package root, always + // slash-separated. Windows and Linux must agree on it: it is both the blob + // name the file is uploaded under and part of the fingerprint, so a + // backslash here would republish the whole package on a change of machine. + RelPath string + + // AbsPath is where the file is read from on this machine. It is + // deliberately not part of the fingerprint. + AbsPath string +} + +// CodeEvaluatorMetadata is the optional descriptor read from +// CodeEvaluatorMetadataFile. Every field is optional; the raw JSON fields are +// passed to the service untouched so a schema this extension does not model +// still reaches it intact. +type CodeEvaluatorMetadata struct { + DisplayName string `json:"display_name,omitempty"` + Description string `json:"description,omitempty"` + Categories []string `json:"categories,omitempty"` + InitParameters json.RawMessage `json:"init_parameters,omitempty"` + DataSchema json.RawMessage `json:"data_schema,omitempty"` + Metrics json.RawMessage `json:"metrics,omitempty"` +} + +// CodeEvaluatorPackage is a validated folder ready to publish. +type CodeEvaluatorPackage struct { + // Name is the evaluator name the package was validated against. + Name string + // Root is the folder on disk. + Root string + // Files are the files to upload, in fingerprint order. + Files []CodeFile + // EntryPoint is the Python file holding the evaluator class. + EntryPoint string + // ClassName is the class the runtime instantiates. + ClassName string + // Metadata is the folder's descriptor, or nil when it carries none. + Metadata *CodeEvaluatorMetadata +} + +// IsCodeEvaluatorSource reports whether a declared source names a folder, and +// therefore a code evaluator rather than a rubric. +// +// The path is stat-ed rather than pattern-matched: a trailing separator is not +// required in YAML and `.json` in a folder name would misclassify it. +func IsCodeEvaluatorSource(path string) bool { + if path == "" { + return false + } + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +// EvaluatorClassName derives the class the runtime looks for from the +// evaluator name: the name in PascalCase, suffixed with Evaluator. +// +// The suffix is not appended twice, so both spellings customers use resolve to +// the same class — `answer_length` and `answer_length_evaluator` both mean +// AnswerLengthEvaluator. +func EvaluatorClassName(name string) string { + parts := strings.FieldsFunc(name, func(r rune) bool { + return r == '_' || r == '-' || r == '.' || r == ' ' + }) + + var b strings.Builder + for _, part := range parts { + runes := []rune(part) + b.WriteRune(unicode.ToUpper(runes[0])) + b.WriteString(string(runes[1:])) + } + + pascal := b.String() + if strings.HasSuffix(pascal, "Evaluator") { + return pascal + } + return pascal + "Evaluator" +} + +// WalkCodeFolder lists the files that make up an evaluator package, excluding +// build caches, version control, dependency trees, and compiled Python. +// +// The result is sorted by RelPath so the upload order and the fingerprint do +// not depend on directory iteration order, which the filesystem does not +// promise to keep stable. +func WalkCodeFolder(dir string) ([]CodeFile, error) { + info, err := os.Stat(dir) + if err != nil { + return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, err) + } + if !info.IsDir() { + return nil, fmt.Errorf( + "evaluator source %q is a file; a code evaluator is published from a folder", dir) + } + + var files []CodeFile + walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + rel, relErr := filepath.Rel(dir, path) + if relErr != nil { + return relErr + } + rel = filepath.ToSlash(rel) + if rel == "." { + return nil + } + + if entry.IsDir() { + if isExcludedDir(entry.Name()) { + return fs.SkipDir + } + return nil + } + + // Symlinks, sockets and devices carry no content that can be uploaded, + // and WalkDir does not follow them, so they would otherwise be + // published as zero bytes. + if !entry.Type().IsRegular() { + return nil + } + // Dot-prefixed files are excluded for the same reason as dot-prefixed + // directories, and one reason more: a folder kept next to an evaluator + // tends to collect `.env`, `.netrc` and `.pypirc`, and publishing the + // package would copy those secrets into blob storage. Nothing a Python + // evaluator needs at runtime is named with a leading dot. + if strings.HasPrefix(entry.Name(), ".") { + return nil + } + if excludedFileExts[strings.ToLower(filepath.Ext(rel))] { + return nil + } + + files = append(files, CodeFile{RelPath: rel, AbsPath: path}) + return nil + }) + if walkErr != nil { + return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, walkErr) + } + + sortCodeFiles(files) + return files, nil +} + +// isExcludedDir reports whether a directory is skipped along with everything +// under it. +func isExcludedDir(name string) bool { + return excludedDirNames[name] || strings.HasPrefix(name, ".") +} + +func sortCodeFiles(files []CodeFile) { + sort.Slice(files, func(i, j int) bool { return files[i].RelPath < files[j].RelPath }) +} + +// FingerprintCodeFolder hashes a folder so a later deploy can tell whether the +// evaluator changed without downloading anything from the service. +func FingerprintCodeFolder(dir string) (string, error) { + files, err := WalkCodeFolder(dir) + if err != nil { + return "", err + } + return FingerprintCodeFiles(files) +} + +// FingerprintCodeFiles hashes an already-walked package. +// +// Both the relative path and the content of every file are hashed, so renaming +// a file registers as a change even when the bytes are identical. The input is +// sorted first: the digest must describe the package, not the order the caller +// happened to hand the files over in. +// +// Only RelPath — never AbsPath — feeds the hash, and RelPath is normalized to +// forward slashes, so the same package hashes the same on Windows and Linux. +func FingerprintCodeFiles(files []CodeFile) (string, error) { + ordered := make([]CodeFile, len(files)) + copy(ordered, files) + sortCodeFiles(ordered) + + outer := sha256.New() + for _, file := range ordered { + content, err := os.ReadFile(file.AbsPath) + if err != nil { + return "", fmt.Errorf("hashing %q: %w", file.AbsPath, err) + } + inner := sha256.Sum256(content) + // Path and content digest are written on separate lines, which keeps + // the encoding unambiguous: a file's content digest is fixed width, so + // no path can be read as part of it. + fmt.Fprintf(outer, "%s\n%s\n", file.RelPath, hex.EncodeToString(inner[:])) + } + return hex.EncodeToString(outer.Sum(nil)), nil +} + +// LoadCodeEvaluator validates a folder against the packaging convention and +// returns the package to publish. +// +// The checks are done here rather than left to the service because the service +// only discovers a missing entry point when a run executes, long after a +// version has been published and an eval bound to it. +func LoadCodeEvaluator(name, dir string) (*CodeEvaluatorPackage, error) { + if name == "" { + return nil, fmt.Errorf("an evaluator name is required to validate the folder layout") + } + + files, err := WalkCodeFolder(dir) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, fmt.Errorf( + "evaluator folder %q holds no files to publish", dir) + } + + entryPoint := name + pythonExt + className := EvaluatorClassName(name) + + entry, ok := findFile(files, entryPoint) + if !ok { + return nil, fmt.Errorf( + "evaluator %q needs %s in %s, holding a class named %s. The folder holds %s", + name, entryPoint, dir, className, describeFiles(files)) + } + + source, err := os.ReadFile(entry.AbsPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) + } + if !declaresClass(source, className) { + return nil, fmt.Errorf( + "%s does not declare a class named %s. A code evaluator is a class with that "+ + "exact name and a __call__ method that takes **kwargs and returns a JSON "+ + "object, for example:\n\nclass %s:\n def __call__(self, **kwargs):\n"+ + " return {\"result\": 1}", + filepath.Join(dir, entry.RelPath), className, className) + } + + metadata, err := readCodeEvaluatorMetadata(files) + if err != nil { + return nil, err + } + + return &CodeEvaluatorPackage{ + Name: name, + Root: dir, + Files: files, + EntryPoint: entryPoint, + ClassName: className, + Metadata: metadata, + }, nil +} + +func findFile(files []CodeFile, relPath string) (CodeFile, bool) { + for _, file := range files { + if file.RelPath == relPath { + return file, true + } + } + return CodeFile{}, false +} + +// describeFiles renders the folder's contents for an error message, capped so +// a large package does not bury the advice that follows it. +func describeFiles(files []CodeFile) string { + const limit = 10 + names := make([]string, 0, limit) + for i, file := range files { + if i == limit { + return fmt.Sprintf("%s and %d more", strings.Join(names, ", "), len(files)-limit) + } + names = append(names, file.RelPath) + } + return strings.Join(names, ", ") +} + +// declaresClass reports whether the source declares the named class at any +// indentation, in either the bare or the inheriting form. +func declaresClass(source []byte, className string) bool { + pattern := `(?m)^[ \t]*class[ \t]+` + regexp.QuoteMeta(className) + `[ \t]*[(:]` + matched, err := regexp.Match(pattern, source) + return err == nil && matched +} + +// readCodeEvaluatorMetadata reads the optional folder descriptor. A folder +// without one is normal, so absence is not an error; malformed JSON is, since +// silently ignoring it would publish an evaluator missing the schemas the +// author wrote down. +func readCodeEvaluatorMetadata(files []CodeFile) (*CodeEvaluatorMetadata, error) { + entry, ok := findFile(files, CodeEvaluatorMetadataFile) + if !ok { + return nil, nil + } + + raw, err := os.ReadFile(entry.AbsPath) + if err != nil { + return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) + } + + var metadata CodeEvaluatorMetadata + if err := json.Unmarshal(raw, &metadata); err != nil { + return nil, fmt.Errorf("%s is not valid JSON: %w", entry.RelPath, err) + } + return &metadata, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go new file mode 100644 index 00000000000..b7e956bc3f9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go @@ -0,0 +1,329 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// writeFile creates a file and every directory above it. +func writeFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +// evaluatorSource is a minimal evaluator matching the packaging convention. +func evaluatorSource(className string) string { + return "class " + className + ":\n" + + " def __call__(self, **kwargs):\n" + + " return {\"result\": len(kwargs.get(\"response\", \"\"))}\n" +} + +// relPaths reads the walk result into a comparable list. +func relPaths(files []CodeFile) []string { + out := make([]string, 0, len(files)) + for _, f := range files { + out = append(out, f.RelPath) + } + return out +} + +func TestWalkCodeFolder_ExcludesBuildAndDependencyTrees(t *testing.T) { + dir := t.TempDir() + + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") + writeFile(t, dir, "README.md", "docs\n") + + // Every one of these must be skipped, along with everything under it. + writeFile(t, dir, "__pycache__/answer_length.cpython-311.pyc", "cache") + writeFile(t, dir, ".git/config", "[core]") + writeFile(t, dir, ".venv/lib/site.py", "venv") + writeFile(t, dir, "venv/lib/site.py", "venv") + writeFile(t, dir, "node_modules/pkg/index.js", "js") + writeFile(t, dir, ".mypy_cache/report.json", "{}") + writeFile(t, dir, "helpers/__pycache__/text.cpython-311.pyc", "cache") + // Compiled artifacts are skipped wherever they sit, not just in caches. + writeFile(t, dir, "stale.pyc", "cache") + writeFile(t, dir, "stale.pyo", "cache") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + + require.Equal(t, + []string{"README.md", "answer_length.py", "helpers/text.py"}, + relPaths(files), + "only source and data files belong in the package, in sorted order") +} + +// Dot-prefixed files are excluded because an evaluator folder living in a repo +// collects credential files, and publishing the package would copy them into +// blob storage. Excluding the directories alone is not enough: the ones that +// hold secrets sit at the root, next to the source. +func TestWalkCodeFolder_ExcludesDotFiles(t *testing.T) { + dir := t.TempDir() + + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, ".env", "AZURE_OPENAI_API_KEY=super-secret\n") + writeFile(t, dir, ".netrc", "machine example.com password hunter2\n") + writeFile(t, dir, ".pypirc", "[pypi]\npassword = leaked\n") + writeFile(t, dir, "helpers/.env", "NESTED_SECRET=1\n") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + + require.Equal(t, []string{"answer_length.py"}, relPaths(files), + "no dot-prefixed file may reach the upload, at any depth") + + for _, f := range files { + require.NotContains(t, f.RelPath, ".env") + require.NotContains(t, f.RelPath, ".netrc") + require.NotContains(t, f.RelPath, ".pypirc") + } +} + +// Relative paths are the blob names and part of the fingerprint, so they must +// not depend on the host's path separator. +func TestWalkCodeFolder_UsesForwardSlashes(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "a/b/c.py", "x = 1\n") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, "a/b/c.py", files[0].RelPath) + require.NotContains(t, files[0].RelPath, "\\") +} + +func TestWalkCodeFolder_RejectsAFile(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "rubric.json", "{}") + + _, err := WalkCodeFolder(path) + require.Error(t, err) + require.Contains(t, err.Error(), "folder") +} + +func TestEvaluatorClassName(t *testing.T) { + cases := map[string]string{ + "answer_length": "AnswerLengthEvaluator", + "answer-length": "AnswerLengthEvaluator", + "answer_length_evaluator": "AnswerLengthEvaluator", + "tone": "ToneEvaluator", + "ToneEvaluator": "ToneEvaluator", + "my.custom_check": "MyCustomCheckEvaluator", + } + for name, want := range cases { + require.Equal(t, want, EvaluatorClassName(name), "for %q", name) + } +} + +func TestLoadCodeEvaluator_AcceptsAConventionalFolder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, "helpers.py", "def n(s): return len(s)\n") + + pkg, err := LoadCodeEvaluator("answer_length", dir) + require.NoError(t, err) + require.Equal(t, "answer_length.py", pkg.EntryPoint) + require.Equal(t, "AnswerLengthEvaluator", pkg.ClassName) + require.Equal(t, []string{"answer_length.py", "helpers.py"}, relPaths(pkg.Files)) + require.Nil(t, pkg.Metadata) +} + +// The class may inherit, and may be indented inside a conditional. +func TestLoadCodeEvaluator_AcceptsInheritingClass(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", "import abc\n\nclass ToneEvaluator(abc.ABC):\n pass\n") + + pkg, err := LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + require.Equal(t, "ToneEvaluator", pkg.ClassName) +} + +func TestLoadCodeEvaluator_ReportsMissingEntryPoint(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "main.py", evaluatorSource("AnswerLengthEvaluator")) + + _, err := LoadCodeEvaluator("answer_length", dir) + require.Error(t, err) + // The message has to say what is missing, what it must hold, and what was + // actually found — a bare "not found" leaves the author guessing. + require.Contains(t, err.Error(), "answer_length.py") + require.Contains(t, err.Error(), "AnswerLengthEvaluator") + require.Contains(t, err.Error(), "main.py") +} + +func TestLoadCodeEvaluator_ReportsMissingClass(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", "class SomethingElse:\n pass\n") + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), "ToneEvaluator") + require.Contains(t, err.Error(), "__call__") +} + +func TestLoadCodeEvaluator_RejectsAnEmptyFolder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "__pycache__/x.pyc", "cache") + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), "no files") +} + +func TestLoadCodeEvaluator_ReadsFolderMetadata(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, CodeEvaluatorMetadataFile, `{ + "display_name": "Tone", + "description": "Scores tone.", + "metrics": {"result": {"type": "ordinal", "min_value": 1, "max_value": 5}}, + "data_schema": {"type": "object", "properties": {"response": {"type": "string"}}} + }`) + + pkg, err := LoadCodeEvaluator("tone", dir) + require.NoError(t, err) + require.NotNil(t, pkg.Metadata) + require.Equal(t, "Tone", pkg.Metadata.DisplayName) + require.Contains(t, string(pkg.Metadata.Metrics), "ordinal") + require.Contains(t, string(pkg.Metadata.DataSchema), "response") +} + +// A descriptor that cannot be read is a mistake worth reporting: publishing +// without the schemas it declares would register an evaluator the author did +// not describe. +func TestLoadCodeEvaluator_RejectsMalformedMetadata(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, CodeEvaluatorMetadataFile, "{not json") + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), CodeEvaluatorMetadataFile) +} + +func TestIsCodeEvaluatorSource(t *testing.T) { + dir := t.TempDir() + file := writeFile(t, dir, "rubric.json", "{}") + // A folder can be named like a file, so the decision cannot be made by + // looking at the string. + misleading := filepath.Join(dir, "looks_like.json") + require.NoError(t, os.MkdirAll(misleading, 0o755)) + + require.True(t, IsCodeEvaluatorSource(dir)) + require.True(t, IsCodeEvaluatorSource(misleading)) + require.False(t, IsCodeEvaluatorSource(file)) + require.False(t, IsCodeEvaluatorSource(filepath.Join(dir, "absent"))) + require.False(t, IsCodeEvaluatorSource("")) +} + +func TestFingerprintCodeFolder_IsStableAndSensitive(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) + writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") + + first, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + again, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.Equal(t, first, again, "an unchanged folder must hash the same") + + // One byte. + writeFile(t, dir, "helpers/text.py", "def clean(s): return s.rstrip()\n") + changed, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.NotEqual(t, first, changed, "changed content must hash differently") +} + +// Renaming a file changes the package even when every byte is preserved: the +// entry point is resolved by name and imports are written against it. +func TestFingerprintCodeFolder_NoticesARename(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, "helpers.py", "X = 1\n") + + before, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + require.NoError(t, os.Rename( + filepath.Join(dir, "helpers.py"), filepath.Join(dir, "util.py"))) + + after, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.NotEqual(t, before, after) +} + +// The filesystem does not promise a stable iteration order, so the digest must +// describe the package rather than the order it was handed over in. The input +// here is the production walk's own output, permuted — not a hand-built list. +func TestFingerprintCodeFiles_IgnoresInputOrder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, dir, "helpers/a.py", "A = 1\n") + writeFile(t, dir, "helpers/b.py", "B = 2\n") + writeFile(t, dir, "data/prompts.txt", "hello\n") + + files, err := WalkCodeFolder(dir) + require.NoError(t, err) + require.Len(t, files, 4) + + fromFolder, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + reversed := make([]CodeFile, 0, len(files)) + for i := len(files) - 1; i >= 0; i-- { + reversed = append(reversed, files[i]) + } + fromReversed, err := FingerprintCodeFiles(reversed) + require.NoError(t, err) + require.Equal(t, fromFolder, fromReversed, "file order must not change the digest") + + rotated := append(append([]CodeFile{}, files[2:]...), files[:2]...) + fromRotated, err := FingerprintCodeFiles(rotated) + require.NoError(t, err) + require.Equal(t, fromFolder, fromRotated) +} + +// Where the folder sits must not affect the digest: two checkouts of the same +// repo, or the same repo on two machines, have to agree or every deploy +// republishes. +func TestFingerprintCodeFolder_IgnoresLocation(t *testing.T) { + build := func(root string) string { + writeFile(t, root, "tone.py", evaluatorSource("ToneEvaluator")) + writeFile(t, root, "helpers/a.py", "A = 1\n") + digest, err := FingerprintCodeFolder(root) + require.NoError(t, err) + return digest + } + + require.Equal(t, build(t.TempDir()), build(t.TempDir())) +} + +// Excluded content must not feed the digest, or a rebuild that only refreshes +// __pycache__ would look like an evaluator change and publish a version. +func TestFingerprintCodeFolder_IgnoresExcludedContent(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) + + before, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + + writeFile(t, dir, "__pycache__/tone.cpython-311.pyc", strings.Repeat("x", 64)) + writeFile(t, dir, ".venv/lib/site.py", "noise") + + after, err := FingerprintCodeFolder(dir) + require.NoError(t, err) + require.Equal(t, before, after) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index bfa923167c3..1748b98c0b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -33,6 +33,11 @@ type DatasetDecl struct { // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly // from an eval and never declared here. +// +// Source decides which kind of evaluator this is: a folder holds Python and +// publishes a code evaluator, a file holds a rubric. The distinction is made +// by stat-ing the path on deploy, not by its spelling, so a folder does not +// need a trailing separator to be recognised. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 4fb967c30a8..3e68eccd52c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -13,6 +13,8 @@ import ( "path/filepath" "strings" + "azureaieval/internal/pkg/evalcore" + "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" "google.golang.org/protobuf/types/known/structpb" @@ -313,6 +315,22 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } +// FingerprintPath hashes whatever a declared source names. +// +// A rubric evaluator is one JSON file; a code evaluator is a folder of Python. +// Both need change detection with the same meaning, so the artifact's shape is +// resolved by stat-ing it rather than by asking the caller to know. +func FingerprintPath(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("hashing %q: %w", path, err) + } + if info.IsDir() { + return evalcore.FingerprintCodeFolder(path) + } + return Fingerprint(path) +} + // FingerprintGroup hashes an eval's own declaration. // // Change detection on upstream artifacts is not sufficient: editing a group's From 41d1f8bc5f988ead41b4ef5fb57e80354bb0f033 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 01:52:12 +0300 Subject: [PATCH 062/114] Fix code evaluator publishing against the live service The live round-trip had never been executed. Running it against a real project found three defects that reading the contract could not. pendingUploadType was sent as "BlobReference". The enum value is "TemporaryBlobReference", so the body failed to bind and the service answered 400 "The request field is required" - a null model, reported as a missing field rather than an invalid one. startPendingUpload answers with TemporaryDataReferenceResponseDto, which carries the location under blobReferenceForConsumption. Only blobReference was modelled, which datasets use, so the reply parsed into an empty struct and the upload failed for want of a credential. Both names are now accepted. Storage is reserved under a version guessed from the registered ones, and that list is eventually consistent: immediately after a publish it still reports the evaluator as unknown. The guess then collided with an existing version, which the service refuses with a 409 wrapped in a 500. A collision now advances the guess and retries instead of failing. Also relax the version assertion, which required the created version to equal the one storage was reserved under. There is no create-at-version route - the create assigns its own number and the reservation can take one too, so the two legitimately differ. Probing the service shows a reservation returns a container named by GUID, not by version, so a definition always points at exactly the bytes uploaded for it and the divergence is cosmetic. The test now asserts what actually matters: a second publish creates a new version, and the two versions keep separate storage. Verified against a live Foundry project: all three code evaluator round trips pass, and the full live suite is green. --- .../internal/cmd/evaluator_code_live_test.go | 36 +++++-- .../internal/pkg/eval_api/code_evaluator.go | 95 ++++++++++++++++--- 2 files changed, 112 insertions(+), 19 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go index f9c20c6d07d..ee06d909750 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go @@ -211,17 +211,39 @@ func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { _ = client.DeleteEvaluatorVersion( context.Background(), name, first.Version, ProjectEndpointAPIVersion) }) - require.Equal(t, predicted, first.Version, - "the service assigned a version other than the one storage was reserved under") - - // Publishing again must land on the next version, not overwrite the first. - predictedSecond := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) + require.NotEmpty(t, first.Version, "the service must assign a version") + + // Publishing again must land on a new version, not overwrite the first. + // + // The version is deliberately not asserted to equal the one storage was + // reserved under. There is no create-at-version route: the create assigns + // its own number, and reserving storage can itself take a number, so the + // two legitimately differ. It does not matter, because a reservation + // returns a container named by GUID rather than by version, so a published + // definition always points at exactly the bytes uploaded for it. What must + // hold is that a second publish does not land on the first version. second, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) require.NoError(t, err) t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( context.Background(), name, second.Version, ProjectEndpointAPIVersion) }) - require.Equal(t, predictedSecond, second.Version) - require.NotEqual(t, first.Version, second.Version) + require.NotEqual(t, first.Version, second.Version, + "a second publish must create a new version rather than replace the first") + + require.NotEqual(t, + blobURIOnService(t, client, name, first.Version), + blobURIOnService(t, client, name, second.Version), + "each version must keep its own storage, or republishing would rewrite the older one") +} + +// blobURIOnService reads back the storage location recorded on a version. +func blobURIOnService( + t *testing.T, + client *eval_api.EvalClient, + name string, + version string, +) string { + t.Helper() + return stringField(t, codeDefinitionOnService(t, client, name, version), "blob_uri") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go index 5ee2492faed..856df8ea19d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go @@ -38,7 +38,7 @@ const evaluatorTypeCustom = "custom" const ( foundryFeaturesHeader = "Foundry-Features" foundryFeatureEvalsV1 = "Evaluations=V1Preview" - pendingUploadTypeBlob = "BlobReference" + pendingUploadTypeBlob = "TemporaryBlobReference" defaultCodeMetricName = "result" defaultCodeMetricType = "continuous" defaultMetricDirection = "increase" @@ -46,6 +46,9 @@ const ( blobTypeHeader = "x-ms-blob-type" blobTypeBlock = "BlockBlob" octetStreamContentType = "application/octet-stream" + // maxVersionProbes bounds how far past a stale version guess the publish + // will step before giving up. + maxVersionProbes = 5 ) // DefaultCodeMetrics is used when neither the folder nor the caller declares @@ -104,10 +107,17 @@ type pendingUploadRequest struct { // PendingUploadResponse is the reply to startPendingUpload: a container to // write into, and the SAS that authorizes writing. +// +// The evaluator endpoint answers with TemporaryDataReferenceResponseDto, which +// carries the location under blobReferenceForConsumption. Datasets answer with +// blobReference instead, so both are modelled and whichever arrives is used: +// the two resources share this step but not the name they return it under. type PendingUploadResponse struct { - BlobReference *BlobReference `json:"blobReference,omitempty"` - PendingUploadID string `json:"pendingUploadId,omitempty"` - Version string `json:"version,omitempty"` + BlobReference *BlobReference `json:"blobReference,omitempty"` + BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` + PendingUploadID string `json:"pendingUploadId,omitempty"` + TemporaryDataReferenceID string `json:"temporaryDataReferenceId,omitempty"` + Version string `json:"version,omitempty"` } // BlobReference is a storage location plus the credential to reach it. @@ -123,23 +133,36 @@ type BlobCredential struct { SASUri string `json:"sasUri,omitempty"` } +// reference returns whichever of the two blob references the service populated. +func (p *PendingUploadResponse) reference() *BlobReference { + if p == nil { + return nil + } + if p.BlobReferenceConsumption != nil { + return p.BlobReferenceConsumption + } + return p.BlobReference +} + // UploadURI returns the container URI carrying the SAS token, or empty when // the service granted no credential. func (p *PendingUploadResponse) UploadURI() string { - if p == nil || p.BlobReference == nil || p.BlobReference.Credential == nil { + ref := p.reference() + if ref == nil || ref.Credential == nil { return "" } - return p.BlobReference.Credential.SASUri + return ref.Credential.SASUri } // ContainerURI returns the container URI without the SAS token. This is what // the evaluator definition records, because the definition is persisted and a // SAS in it would expire. func (p *PendingUploadResponse) ContainerURI() string { - if p == nil || p.BlobReference == nil { + ref := p.reference() + if ref == nil { return "" } - return p.BlobReference.BlobURI + return ref.BlobURI } // UploadCodeEvaluatorVersion publishes a folder of Python as a new version of @@ -219,11 +242,9 @@ func (c *EvalClient) uploadCodeEvaluatorFiles( pkg *evalcore.CodeEvaluatorPackage, apiVersion string, ) (string, error) { - version := c.NextEvaluatorVersion(ctx, pkg.Name, apiVersion) - - pending, err := c.StartEvaluatorPendingUpload(ctx, pkg.Name, version, apiVersion) + pending, err := c.reserveEvaluatorStorage(ctx, pkg.Name, apiVersion) if err != nil { - return "", fmt.Errorf("starting the upload for evaluator %q: %w", pkg.Name, err) + return "", err } uploadURI := pending.UploadURI() @@ -250,6 +271,56 @@ func (c *EvalClient) uploadCodeEvaluatorFiles( return containerURI, nil } +// reserveEvaluatorStorage provisions storage for the version being published, +// stepping past versions that are already taken. +// +// Storage has to be reserved under a version number before the version exists, +// and the number is guessed by reading the ones already registered. That read +// is eventually consistent: immediately after a publish it can still report +// the evaluator as unknown, which makes the guess collide with a version that +// is already there. The service answers a collision with a conflict, so the +// guess is advanced and tried again rather than failing the publish. +// +// Only a conflict is retried. Any other failure is the caller's to see. +func (c *EvalClient) reserveEvaluatorStorage( + ctx context.Context, + name string, + apiVersion string, +) (*PendingUploadResponse, error) { + version, err := strconv.Atoi(c.NextEvaluatorVersion(ctx, name, apiVersion)) + if err != nil { + version = 1 + } + + var lastErr error + for attempt := 0; attempt < maxVersionProbes; attempt++ { + pending, err := c.StartEvaluatorPendingUpload( + ctx, name, strconv.Itoa(version+attempt), apiVersion) + if err == nil { + return pending, nil + } + lastErr = err + if !isVersionConflict(err) { + break + } + } + return nil, fmt.Errorf("starting the upload for evaluator %q: %w", name, lastErr) +} + +// isVersionConflict reports whether a failed reservation was refused because +// the version already exists. +// +// The conflict is not surfaced as one: the service wraps the downstream 409 in +// a 500 whose message quotes the original status, so the status code on the +// response cannot be used and the message is what is left to read. +func isVersionConflict(err error) bool { + if err == nil { + return false + } + text := err.Error() + return strings.Contains(text, "409") || strings.Contains(text, "Conflict") +} + // StartEvaluatorPendingUpload provisions the storage an evaluator version's // code is written to. func (c *EvalClient) StartEvaluatorPendingUpload( From 32b01bab37636c9ceea7d79142e11c247060c6c4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 30 Jul 2026 03:22:28 +0300 Subject: [PATCH 063/114] Publish code evaluators as a single script, not a folder A code evaluator runs as an OpenAI python grader, and the grader contract settles the shape. GraderPython in RAISvc (Contracts/UnifiedEvaluationV2/Grader.cs) carries Source - one string, "the source code of the python script" - plus ImageTag, PassThreshold and InitializationParameters. There is no folder, archive, file list or entry point, so a folder was never expressible. Live testing had already shown the consequences before the contract explained them: a definition carrying blob_uri alone registers cleanly and then fails the run with "Invalid grader source: top-level grade() function not found in source", while code_text alone completes and scores. A grader importing a sibling module fails; the same logic with the helper inlined passes. So --folder becomes --file, the definition sends code_text, and the evaluator-side blob upload is removed rather than left as a path that cannot reach the executor. Dataset uploads are untouched - they use a separate copy of that flow and still need it. Dependencies now have a supported route: --image-tag, which is how a grader gets anything it cannot inline. Validation follows the executor rather than the documentation. The folder convention required a {Name}Evaluator class with __call__; the runtime calls a top-level grade(). The class was never invoked, so requiring it only guaranteed a late failure. A script is now checked for a top-level grade(), anchored so a method of that name inside a class is correctly rejected. The evaluator.json descriptor is dropped. It was invented here, it has no counterpart in the SDK, and it conflicts with the spec.yaml convention the shipping evaluators in azureml-assets use. Schemas come from flags until that is settled. Also close a propagation race. After a publish, the version listing stays stale for ~650ms-1.4s while a direct read goes consistent in ~330ms, and eval creation follows the listing - so publishing an evaluator and immediately creating an eval that names it fails with "The evaluator X was not found". The reconciler now waits on both reads. The same lag makes two publishes seconds apart both return version 1, because the service's auto-increment reads the same stale listing. One earlier finding is reversed by this. A schema-less evaluator was recorded as refused; that refusal was the propagation 404 arriving at the same call and being read as a mapping error. With the race gone it is accepted with an empty data_mapping, and the test now says so. Verified live: a published script is referenced by an eval and scores its samples - completed, passed=2, errored=0. --- .../internal/cmd/evaluator.go | 125 +++--- .../internal/cmd/evaluator_code_live_test.go | 189 +++++----- .../internal/cmd/evaluator_code_test.go | 140 +++---- .../internal/cmd/reconciler.go | 117 +++++- .../internal/pkg/eval_api/code_evaluator.go | 325 ++-------------- .../internal/pkg/evalcore/codefolder.go | 356 ------------------ .../internal/pkg/evalcore/codefolder_test.go | 329 ---------------- .../internal/pkg/evalcore/codescript.go | 113 ++++++ .../internal/pkg/evalcore/codescript_test.go | 128 +++++++ .../internal/project/eval_config.go | 9 +- .../internal/project/service_target_eval.go | 21 +- .../tests/live/code_evaluator_run_test.go | 347 +++++++++++++++++ 12 files changed, 958 insertions(+), 1241 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 6e15b0a6442..ee601bb16de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -33,13 +33,14 @@ func newEvaluatorCommand() *cobra.Command { // version every time, so there is nothing for a separate `update` to do. // // An evaluator is either a rubric — a JSON file of weighted dimensions — or -// code — a folder of Python. They are different definition types on the wire, -// so exactly one of the two sources has to be named. +// code — one self-contained Python script. They are different definition types +// on the wire, so exactly one of the two sources has to be named. func newEvaluatorCreateCommand() *cobra.Command { var ( name string rubric string - folder string + file string + imageTag string initParams string dataSchema string metrics string @@ -48,28 +49,37 @@ func newEvaluatorCreateCommand() *cobra.Command { use := "create" short := "Register a rubric or code evaluator, publishing a new version." + long := short + "\n\n" + + "A rubric (--rubric) is a JSON file of weighted dimensions.\n\n" + + "A code evaluator (--file) is a single Python script declaring a top-level\n" + + "grade(sample, item) function that returns a float. It runs as a python\n" + + "grader, which is handed the script's source and nothing else: there is no\n" + + "package and no import path, so a helper module beside the script cannot be\n" + + "imported. Dependencies come from the image named by --image-tag." cmd := &cobra.Command{ Use: use, Short: short, + Long: long, RunE: func(cmd *cobra.Command, args []string) error { if name == "" { return requireFlag("name") } flags := codeEvaluatorFlags{ + imageTag: imageTag, initParams: initParams, dataSchema: dataSchema, metrics: metrics, endpoint: endpointFlg, } - if err := validateEvaluatorSource(rubric, folder, flags); err != nil { + if err := validateEvaluatorSource(rubric, file, flags); err != nil { return err } ctx := cmd.Context() - if folder != "" { - return runEvaluatorCreateFromFolder(cmd, name, folder, flags) + if file != "" { + return runEvaluatorCreateFromFile(cmd, name, file, flags) } raw, err := os.ReadFile(rubric) @@ -106,23 +116,24 @@ func newEvaluatorCreateCommand() *cobra.Command { cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") - cmd.Flags().StringVar(&folder, "folder", "", - "Path to a folder of Python holding the evaluator code.") + cmd.Flags().StringVar(&file, "file", "", + "Path to a single Python script declaring a top-level grade(sample, item) function.") + cmd.Flags().StringVar(&imageTag, "image-tag", "", + "Container image the evaluator runs in. Its packages are the only "+ + "dependencies the script can import beyond the standard library.") cmd.Flags().StringVar(&initParams, "init-params", "", - "Path to a JSON Schema for the evaluator's initialization parameters. "+ - "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + "Path to a JSON Schema for the evaluator's initialization parameters.") cmd.Flags().StringVar(&dataSchema, "data-schema", "", - "Path to a JSON Schema for the evaluator's input data. "+ - "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + "Path to a JSON Schema for the evaluator's input data.") cmd.Flags().StringVar(&metrics, "metrics", "", - "Path to a JSON object describing the metrics the evaluator produces. "+ - "Overrides the folder's "+evalcore.CodeEvaluatorMetadataFile+".") + "Path to a JSON object describing the metrics the evaluator produces.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// codeEvaluatorFlags are the optional overrides for a code evaluator. +// codeEvaluatorFlags are the optional settings for a code evaluator. type codeEvaluatorFlags struct { + imageTag string initParams string dataSchema string metrics string @@ -130,59 +141,62 @@ type codeEvaluatorFlags struct { } // validateEvaluatorSource enforces that exactly one source is named, and that -// the schema overrides are only used with the source they apply to. +// the code-only settings are only used with the source they apply to. // // Deliberately checked here rather than with MarkFlagsMutuallyExclusive: that // only rejects the "both" case, and its message names a flag group rather than // saying what the two flags mean. Both mistakes deserve advice, and this is // testable without driving cobra. -func validateEvaluatorSource(rubric, folder string, flags codeEvaluatorFlags) error { +func validateEvaluatorSource(rubric, file string, flags codeEvaluatorFlags) error { switch { - case rubric == "" && folder == "": + case rubric == "" && file == "": return fmt.Errorf( - "one of --rubric or --folder is required: --rubric takes a JSON file of " + - "weighted dimensions, --folder takes a directory of Python") - case rubric != "" && folder != "": + "one of --rubric or --file is required: --rubric takes a JSON file of " + + "weighted dimensions, --file takes a single Python script") + case rubric != "" && file != "": return fmt.Errorf( - "--rubric and --folder cannot be used together: an evaluator is either a " + + "--rubric and --file cannot be used together: an evaluator is either a " + "rubric or code, not both") } - // A rubric's schemas are fixed by the service, so these would be accepted - // and then quietly dropped — the worst kind of no-op, because the author - // believes the evaluator was published carrying them. - if folder == "" { + // A rubric's schemas are fixed by the service and a rubric runs no code, so + // these would be accepted and then quietly dropped — the worst kind of + // no-op, because the author believes the evaluator was published carrying + // them. + if file == "" { for _, named := range []struct { flag string value string }{ + {"image-tag", flags.imageTag}, {"init-params", flags.initParams}, {"data-schema", flags.dataSchema}, {"metrics", flags.metrics}, } { if named.value != "" { return fmt.Errorf( - "--%s applies to a code evaluator and needs --folder; "+ - "a rubric's schemas are set by the service", named.flag) + "--%s applies to a code evaluator and needs --file; "+ + "a rubric runs no code and its schemas are set by the service", + named.flag) } } } return nil } -// runEvaluatorCreateFromFolder validates the folder, then publishes it. -func runEvaluatorCreateFromFolder( +// runEvaluatorCreateFromFile validates the script, then publishes it. +func runEvaluatorCreateFromFile( cmd *cobra.Command, name string, - folder string, + file string, flags codeEvaluatorFlags, ) error { - pkg, err := evalcore.LoadCodeEvaluator(name, folder) + script, err := evalcore.LoadCodeEvaluator(name, file) if err != nil { return err } - opts, err := codeEvaluatorOptions(pkg, flags) + opts, err := codeEvaluatorOptions(flags) if err != nil { return err } @@ -194,8 +208,8 @@ func runEvaluatorCreateFromFolder( } defer ec.Close() - created, err := ec.evalClient.UploadCodeEvaluatorVersion( - ctx, pkg, opts, ProjectEndpointAPIVersion, + created, err := ec.evalClient.CreateCodeEvaluatorVersion( + ctx, script, opts, ProjectEndpointAPIVersion, ) if err != nil { return fmt.Errorf("publishing evaluator %q: %w", name, err) @@ -205,32 +219,21 @@ func runEvaluatorCreateFromFolder( return emitJSON(cmd.OutOrStdout(), created) } fmt.Fprintf(cmd.OutOrStdout(), - "Published evaluator %s version %s from %d file(s) in %s\n", - created.Name, created.Version, len(pkg.Files), folder) + "Published evaluator %s version %s from %s\n", + created.Name, created.Version, file) return nil } -// codeEvaluatorOptions resolves the evaluator's schemas, preferring an -// explicit flag over whatever the folder declares. +// codeEvaluatorOptions resolves the evaluator's schemas from the flags. // -// The folder is the better place for them — they describe the code and belong -// beside it — but a folder that has none must still be publishable without -// editing it, which is what the flags are for. -func codeEvaluatorOptions( - pkg *evalcore.CodeEvaluatorPackage, - flags codeEvaluatorFlags, -) (eval_api.CodeEvaluatorOptions, error) { - var opts eval_api.CodeEvaluatorOptions - if md := pkg.Metadata; md != nil { - opts.DisplayName = md.DisplayName - opts.Description = md.Description - opts.Categories = md.Categories - opts.InitParameters = md.InitParameters - opts.DataSchema = md.DataSchema - opts.Metrics = md.Metrics - } - - for _, override := range []struct { +// They are not read from the script and not read from a descriptor beside it: +// the grader is handed one file of source, so anything the service needs that +// is not Python has to be named on the command line or carried in the eval +// config. +func codeEvaluatorOptions(flags codeEvaluatorFlags) (eval_api.CodeEvaluatorOptions, error) { + opts := eval_api.CodeEvaluatorOptions{ImageTag: flags.imageTag} + + for _, declared := range []struct { path string flag string field *json.RawMessage @@ -239,14 +242,14 @@ func codeEvaluatorOptions( {flags.dataSchema, "data-schema", &opts.DataSchema}, {flags.metrics, "metrics", &opts.Metrics}, } { - if override.path == "" { + if declared.path == "" { continue } - raw, err := readJSONObject(override.path) + raw, err := readJSONObject(declared.path) if err != nil { - return opts, fmt.Errorf("--%s %q: %w", override.flag, override.path, err) + return opts, fmt.Errorf("--%s %q: %w", declared.flag, declared.path, err) } - *override.field = raw + *declared.field = raw } return opts, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go index ee06d909750..df32ba7cd8d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go @@ -40,28 +40,19 @@ func liveCodeEvaluatorName(t *testing.T, suffix string) string { return fmt.Sprintf("azdcode_%s_%d", suffix, time.Now().UnixNano()) } -// writeLiveEvaluator lays out a folder to the packaging convention and returns -// it. The source is written for the derived class name so the production +// writeLiveEvaluator writes a self-contained evaluator script and returns its +// path. It goes through the production loader afterwards, so the shipping // validation is exercised rather than bypassed. -func writeLiveEvaluator(t *testing.T, name string, extraFiles map[string]string) string { +func writeLiveEvaluator(t *testing.T, name string) string { t.Helper() dir := t.TempDir() + path := filepath.Join(dir, name+".py") - className := evalcore.EvaluatorClassName(name) - entry := fmt.Sprintf(`class %s: - def __call__(self, **kwargs): - return {"result": float(len(kwargs.get("response", "")))} -`, className) - - require.NoError(t, os.WriteFile( - filepath.Join(dir, name+".py"), []byte(entry), 0o600)) - - for rel, content := range extraFiles { - path := filepath.Join(dir, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - } - return dir + source := `def grade(sample, item) -> float: + return float(len((item or {}).get("response", ""))) +` + require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) + return path } // codeDefinitionOnService reads the registered version back and returns its @@ -101,30 +92,31 @@ func stringField(t *testing.T, definition map[string]json.RawMessage, key string return value } -// TestLiveCodeEvaluatorSingleFileRoundTrip publishes a one-file evaluator and -// asserts it comes back as a code definition pointing at storage. +// TestLiveCodeEvaluatorRoundTrip publishes a script and asserts it comes back +// as a code definition carrying the source inline. // -// A single file takes the same path as a folder. The contract also accepts -// inline source through code_text, which would save the upload, but nothing -// observable confirms the executor runs it, so the CLI does not send it. If -// that is ever settled, this is the test that should change first. -func TestLiveCodeEvaluatorSingleFileRoundTrip(t *testing.T) { +// code_text is the only source property that reaches the executor: the +// definition is consumed as an OpenAI python grader, whose contract is a +// single Source string. A version registered with blob_uri instead publishes +// cleanly and then fails every run with "top-level grade() function not found +// in source", so what matters here is that the source itself round-trips. +func TestLiveCodeEvaluatorRoundTrip(t *testing.T) { client, _ := liveEvalClient(t) ctx := context.Background() - name := liveCodeEvaluatorName(t, "single") - dir := writeLiveEvaluator(t, name, nil) + name := liveCodeEvaluatorName(t, "roundtrip") + path := writeLiveEvaluator(t, name) - // The shipping loader, not a hand-built package: this test has to fail if - // the production path stops producing a valid package. - pkg, err := evalcore.LoadCodeEvaluator(name, dir) + // The shipping loader, not a hand-built script: this test has to fail if + // the production path stops producing a publishable script. + script, err := evalcore.LoadCodeEvaluator(name, path) require.NoError(t, err) - require.Len(t, pkg.Files, 1) + require.Contains(t, script.Source, "def grade(") - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) require.NoError(t, err) - created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) require.NoError(t, err, "the service rejected the code evaluator body") require.NotEmpty(t, created.Version) t.Cleanup(func() { @@ -136,76 +128,79 @@ func TestLiveCodeEvaluatorSingleFileRoundTrip(t *testing.T) { require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type"), "the discriminator must round-trip as the lowercase snake_case value") - require.NotEmpty(t, stringField(t, definition, "blob_uri"), - "a published evaluator must record the storage location it was uploaded to") + require.Contains(t, stringField(t, definition, "code_text"), "def grade(", + "the source must round-trip inline; an empty code_text means the grader "+ + "would be handed nothing to run") require.Contains(t, definition, "metrics", "a code definition must carry metrics; the service rejects one without") } -// TestLiveCodeEvaluatorFolderRoundTrip publishes a multi-file evaluator and -// asserts the service records the storage location it handed out. +// TestLiveCodeEvaluatorCarriesSchemasAndImage proves the settings that only +// reach the service through flags survive the round trip. // -// This is the path that exercises startPendingUpload, the SAS write, and the -// blob_uri property, none of which the single-file case touches. -func TestLiveCodeEvaluatorFolderRoundTrip(t *testing.T) { +// They cannot come from anywhere else. The grader is handed one file of +// source, so a descriptor beside the script would never travel with it, and an +// image tag dropped on the way would leave an evaluator whose imports fail at +// run time with no sign of why. +func TestLiveCodeEvaluatorCarriesSchemasAndImage(t *testing.T) { client, _ := liveEvalClient(t) ctx := context.Background() - name := liveCodeEvaluatorName(t, "folder") - dir := writeLiveEvaluator(t, name, map[string]string{ - "helpers/text.py": "def clean(value):\n return value.strip()\n", - // Must be excluded from both the upload and the fingerprint. - "__pycache__/stale.pyc": "cache", - }) + name := liveCodeEvaluatorName(t, "schemas") + path := writeLiveEvaluator(t, name) - pkg, err := evalcore.LoadCodeEvaluator(name, dir) + script, err := evalcore.LoadCodeEvaluator(name, path) require.NoError(t, err) - require.Len(t, pkg.Files, 2, "the compiled artifact must not be part of the package") - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + dir := t.TempDir() + schemaPath := filepath.Join(dir, "schema.json") + require.NoError(t, os.WriteFile(schemaPath, []byte( + `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`, + ), 0o600)) + + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{dataSchema: schemaPath}) require.NoError(t, err) + require.NotEmpty(t, opts.DataSchema) - created, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) - require.NoError(t, err, "the service rejected the uploaded code evaluator") - require.NotEmpty(t, created.Version) + created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) + require.NoError(t, err, "the service rejected a definition carrying a data schema") t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( context.Background(), name, created.Version, ProjectEndpointAPIVersion) }) definition := codeDefinitionOnService(t, client, name, created.Version) - - require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type")) - require.NotEmpty(t, stringField(t, definition, "blob_uri"), - "a multi-file evaluator must round-trip carrying the storage location; "+ - "an empty blob_uri means the service dropped the preview property") - require.Contains(t, definition, "metrics") + require.Contains(t, definition, "data_schema", + "the declared data schema must round-trip; without it the criteria builder "+ + "derives no data_mapping and the eval cannot be created") + require.Contains(t, string(definition["data_schema"]), "response") } -// TestLiveCodeEvaluatorPublishesNextVersion proves the version the upload -// reserves storage under is the one the create then assigns. +// TestLiveCodeEvaluatorPublishesANewVersion proves a second publish does not +// overwrite the first. +// +// Versions are immutable and evals bind to one, so a publish that replaced the +// previous version would silently change what every existing eval evaluates. // -// Storage is provisioned per version before the version exists, so the client -// has to predict it. A drift between the two would leave the code in one -// version's container and the definition on another. -func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { +// The wait between the two publishes is not padding. The service assigns the +// next version from its own listing, and that listing lags the create by about +// a second: two publishes issued back to back were both answered with version +// 1, the second overwriting the first. The reconciler waits for a published +// version to appear in the listing before it moves on, so this waits the same +// way — the assertion is about publishing twice, not about racing the service. +func TestLiveCodeEvaluatorPublishesANewVersion(t *testing.T) { client, _ := liveEvalClient(t) ctx := context.Background() name := liveCodeEvaluatorName(t, "versions") - dir := writeLiveEvaluator(t, name, map[string]string{ - "helpers.py": "VALUE = 1\n", - }) + path := writeLiveEvaluator(t, name) - pkg, err := evalcore.LoadCodeEvaluator(name, dir) + script, err := evalcore.LoadCodeEvaluator(name, path) require.NoError(t, err) - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) require.NoError(t, err) - predicted := client.NextEvaluatorVersion(ctx, name, ProjectEndpointAPIVersion) - require.Equal(t, "1", predicted, "an unpublished evaluator starts at version 1") - - first, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + first, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) require.NoError(t, err) t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( @@ -213,16 +208,9 @@ func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { }) require.NotEmpty(t, first.Version, "the service must assign a version") - // Publishing again must land on a new version, not overwrite the first. - // - // The version is deliberately not asserted to equal the one storage was - // reserved under. There is no create-at-version route: the create assigns - // its own number, and reserving storage can itself take a number, so the - // two legitimately differ. It does not matter, because a reservation - // returns a container named by GUID rather than by version, so a published - // definition always points at exactly the bytes uploaded for it. What must - // hold is that a second publish does not land on the first version. - second, err := client.UploadCodeEvaluatorVersion(ctx, pkg, opts, ProjectEndpointAPIVersion) + awaitVersionListed(t, client, name, first.Version) + + second, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) require.NoError(t, err) t.Cleanup(func() { _ = client.DeleteEvaluatorVersion( @@ -230,20 +218,29 @@ func TestLiveCodeEvaluatorPublishesNextVersion(t *testing.T) { }) require.NotEqual(t, first.Version, second.Version, "a second publish must create a new version rather than replace the first") - - require.NotEqual(t, - blobURIOnService(t, client, name, first.Version), - blobURIOnService(t, client, name, second.Version), - "each version must keep its own storage, or republishing would rewrite the older one") } -// blobURIOnService reads back the storage location recorded on a version. -func blobURIOnService( - t *testing.T, - client *eval_api.EvalClient, - name string, - version string, -) string { +// awaitVersionListed blocks until a published version shows up in the version +// listing, which is the view the service's own version assignment reads. +func awaitVersionListed(t *testing.T, client *eval_api.EvalClient, name, version string) { t.Helper() - return stringField(t, codeDefinitionOnService(t, client, name, version), "blob_uri") + ctx := context.Background() + + start := time.Now() + for { + list, err := client.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + if err == nil && list != nil { + for _, entry := range list.Value { + if entry.Version == version { + t.Logf("version %s listed after %s", + version, time.Since(start).Round(time.Millisecond)) + return + } + } + } + if time.Since(start) > 30*time.Second { + t.Fatalf("version %s of evaluator %s never appeared in the listing", version, name) + } + time.Sleep(250 * time.Millisecond) + } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go index a919bd56a90..0b1ff9205c2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go @@ -25,9 +25,10 @@ func writeTestFile(t *testing.T, root, rel, content string) string { return path } -const toneEvaluatorSource = "class ToneEvaluator:\n" + - " def __call__(self, **kwargs):\n" + - " return {\"result\": 1}\n" +// toneEvaluatorSource is the shape the grader requires: one top-level +// grade(sample, item) returning a float. +const toneEvaluatorSource = "def grade(sample, item) -> float:\n" + + " return float(len((item or {}).get(\"response\", \"\")))\n" // An evaluator is either a rubric or code. Naming both, or neither, is a // mistake the command has to name precisely — the two flags take different @@ -36,22 +37,23 @@ func TestValidateEvaluatorSource(t *testing.T) { err := validateEvaluatorSource("", "", codeEvaluatorFlags{}) require.Error(t, err) require.Contains(t, err.Error(), "--rubric") - require.Contains(t, err.Error(), "--folder") + require.Contains(t, err.Error(), "--file") require.Contains(t, err.Error(), "required") - err = validateEvaluatorSource("rubric.json", "./evaluator", codeEvaluatorFlags{}) + err = validateEvaluatorSource("rubric.json", "evaluator.py", codeEvaluatorFlags{}) require.Error(t, err) require.Contains(t, err.Error(), "cannot be used together") require.NoError(t, validateEvaluatorSource("rubric.json", "", codeEvaluatorFlags{})) - require.NoError(t, validateEvaluatorSource("", "./evaluator", codeEvaluatorFlags{})) + require.NoError(t, validateEvaluatorSource("", "evaluator.py", codeEvaluatorFlags{})) } -// The schema overrides describe a code evaluator. Accepting them beside a +// The code-only settings describe a python grader. Accepting them beside a // rubric and dropping them would leave the author believing the evaluator was -// published carrying schemas it never had. +// published carrying an image and schemas it never had. func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { for flag, flags := range map[string]codeEvaluatorFlags{ + "image-tag": {imageTag: "python:3.11"}, "init-params": {initParams: "init.json"}, "data-schema": {dataSchema: "schema.json"}, "metrics": {metrics: "metrics.json"}, @@ -59,10 +61,10 @@ func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { err := validateEvaluatorSource("rubric.json", "", flags) require.Error(t, err, "for --%s", flag) require.Contains(t, err.Error(), "--"+flag) - require.Contains(t, err.Error(), "--folder") + require.Contains(t, err.Error(), "--file") - require.NoError(t, validateEvaluatorSource("", "./evaluator", flags), - "--%s is valid with --folder", flag) + require.NoError(t, validateEvaluatorSource("", "evaluator.py", flags), + "--%s is valid with --file", flag) } } @@ -70,7 +72,7 @@ func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { // future refactor cannot leave the flags declared but unvalidated. func TestEvaluatorCreateRejectsBothSources(t *testing.T) { cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--folder", "./x"}) + cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--file", "tone.py"}) cmd.SetOut(io.Discard) cmd.SetErr(io.Discard) cmd.SilenceUsage = true @@ -89,72 +91,80 @@ func TestEvaluatorCreateRejectsNeitherSource(t *testing.T) { err := cmd.Execute() require.Error(t, err) - require.Contains(t, err.Error(), "one of --rubric or --folder") + require.Contains(t, err.Error(), "one of --rubric or --file") } -// The folder is the natural home for the schemas, but a folder that carries -// none still has to be publishable without editing it. -func TestCodeEvaluatorOptions_FlagsOverrideFolderMetadata(t *testing.T) { +// A script with no top-level grade() must be refused before a version is +// published, and the refusal must come from the command rather than from a run +// that fails minutes later. +func TestEvaluatorCreateRejectsAScriptWithoutGrade(t *testing.T) { dir := t.TempDir() - writeTestFile(t, dir, "tone.py", toneEvaluatorSource) - writeTestFile(t, dir, evalcore.CodeEvaluatorMetadataFile, `{ - "display_name": "Tone", - "metrics": {"result": {"type": "ordinal"}}, - "data_schema": {"type": "object", "properties": {"a": {"type": "string"}}} - }`) - - pkg, err := evalcore.LoadCodeEvaluator("tone", dir) - require.NoError(t, err) + path := writeTestFile(t, dir, "tone.py", + "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") - // Nothing overridden: the folder wins. - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + cmd := newEvaluatorCreateCommand() + cmd.SetArgs([]string{"--name", "tone", "--file", path}) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + cmd.SilenceUsage = true + + err := cmd.Execute() + require.Error(t, err) + require.Contains(t, err.Error(), "grade(sample, item)") +} + +// The flags are the only place a code evaluator's schemas can come from: the +// grader is handed one file of source, so nothing that is not Python can +// travel with it. +func TestCodeEvaluatorOptions_ReadsTheFlags(t *testing.T) { + empty, err := codeEvaluatorOptions(codeEvaluatorFlags{}) require.NoError(t, err) - require.Equal(t, "Tone", opts.DisplayName) - require.Contains(t, string(opts.Metrics), "ordinal") - require.Contains(t, string(opts.DataSchema), `"a"`) - require.Empty(t, opts.InitParameters) + require.Empty(t, empty.ImageTag) + require.Empty(t, empty.Metrics) + require.Empty(t, empty.DataSchema) + require.Empty(t, empty.InitParameters) - overrides := t.TempDir() - metricsPath := writeTestFile(t, overrides, "metrics.json", + dir := t.TempDir() + metricsPath := writeTestFile(t, dir, "metrics.json", `{"result":{"type":"continuous"}}`) - initPath := writeTestFile(t, overrides, "init.json", + initPath := writeTestFile(t, dir, "init.json", `{"type":"object","properties":{"deployment_name":{"type":"string"}}}`) + schemaPath := writeTestFile(t, dir, "schema.json", + `{"type":"object","properties":{"response":{"type":"string"}}}`) - opts, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ + imageTag: "mcr.microsoft.com/azureml/evaluator:latest", metrics: metricsPath, initParams: initPath, + dataSchema: schemaPath, }) require.NoError(t, err) + require.Equal(t, "mcr.microsoft.com/azureml/evaluator:latest", opts.ImageTag) require.Contains(t, string(opts.Metrics), "continuous") - require.NotContains(t, string(opts.Metrics), "ordinal") require.Contains(t, string(opts.InitParameters), "deployment_name") - // Untouched by the overrides. - require.Contains(t, string(opts.DataSchema), `"a"`) + require.Contains(t, string(opts.DataSchema), "response") } // A typo in a schema file must be reported against the flag that named it, // not discovered by the service after a version has been published. -func TestCodeEvaluatorOptions_RejectsMalformedOverride(t *testing.T) { +func TestCodeEvaluatorOptions_RejectsMalformedInput(t *testing.T) { dir := t.TempDir() - writeTestFile(t, dir, "tone.py", toneEvaluatorSource) - pkg, err := evalcore.LoadCodeEvaluator("tone", dir) - require.NoError(t, err) bad := writeTestFile(t, dir, "metrics.json", "[1,2,3]") - _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{metrics: bad}) + _, err := codeEvaluatorOptions(codeEvaluatorFlags{metrics: bad}) require.Error(t, err) require.Contains(t, err.Error(), "--metrics") require.Contains(t, err.Error(), "JSON object") - _, err = codeEvaluatorOptions(pkg, codeEvaluatorFlags{ + _, err = codeEvaluatorOptions(codeEvaluatorFlags{ dataSchema: filepath.Join(dir, "absent.json"), }) require.Error(t, err) require.Contains(t, err.Error(), "--data-schema") } -// The service rejects a code definition carrying no metrics, so a folder that -// declares none still has to publish with one. +// The service rejects a code definition carrying no metrics, so a script +// published without any still has to publish with one. func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { var metrics map[string]map[string]any require.NoError(t, json.Unmarshal(eval_api.DefaultCodeMetrics, &metrics)) @@ -163,32 +173,30 @@ func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { require.Equal(t, "continuous", metrics["result"]["type"]) } -// Change detection has to work for both kinds of evaluator source, and the -// reconciler decides which by stat-ing the path. -func TestFingerprintPath_HandlesFilesAndFolders(t *testing.T) { +// Both kinds of evaluator source are one file, and the reconciler tells them +// apart by extension rather than by stat-ing the path. +func TestEvaluatorSourceClassificationAndFingerprint(t *testing.T) { root := t.TempDir() - file := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) - fileDigest, err := project.FingerprintPath(file) - require.NoError(t, err) - plainDigest, err := project.Fingerprint(file) - require.NoError(t, err) - require.Equal(t, plainDigest, fileDigest, - "a file must hash the same through either entry point") + rubric := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) + script := writeTestFile(t, root, "tone.py", toneEvaluatorSource) - dir := t.TempDir() - writeTestFile(t, dir, "tone.py", toneEvaluatorSource) - folderDigest, err := project.FingerprintPath(dir) + require.False(t, evalcore.IsCodeEvaluatorSource(rubric)) + require.True(t, evalcore.IsCodeEvaluatorSource(script)) + + rubricDigest, err := project.Fingerprint(rubric) + require.NoError(t, err) + scriptDigest, err := project.Fingerprint(script) require.NoError(t, err) - require.NotEmpty(t, folderDigest) - require.NotEqual(t, fileDigest, folderDigest) + require.NotEqual(t, rubricDigest, scriptDigest) - writeTestFile(t, dir, "helpers.py", "X = 1\n") - changed, err := project.FingerprintPath(dir) + // Editing the script must be noticed, or a deploy would reuse a version + // holding the old source. + writeTestFile(t, root, "tone.py", toneEvaluatorSource+"\n# tweak\n") + changed, err := project.Fingerprint(script) require.NoError(t, err) - require.NotEqual(t, folderDigest, changed, - "adding a file to the folder must change the digest") + require.NotEqual(t, scriptDigest, changed) - _, err = project.FingerprintPath(filepath.Join(root, "absent")) + _, err = project.Fingerprint(filepath.Join(root, "absent.py")) require.Error(t, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 3c5c93234f9..687d1936baf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -13,6 +13,7 @@ import ( "reflect" "strconv" "strings" + "time" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/evalcore" @@ -227,12 +228,12 @@ func (r *evalReconciler) latestDatasetVersion(ctx context.Context, name string) // EnsureEvaluator publishes a new version when the local definition differs // from what the service holds. // -// The two kinds of evaluator are told apart by what the source names, not by -// its spelling: a folder is code, a file is a rubric. They also detect change -// differently. A rubric definition comes back inline, so it is compared -// directly; code does not — the service returns a storage URI, never the -// source — so a fingerprint of the folder is kept in the azd environment, the -// same way datasets work. +// The two kinds of evaluator are told apart by the source's extension: `.py` +// is code, anything else is a rubric. They also detect change differently. A +// rubric definition comes back inline, so it is compared directly; a code +// definition's source is not read back in a form worth comparing, so a +// fingerprint of the script is kept in the azd environment, the same way +// datasets work. func (r *evalReconciler) EnsureEvaluator( ctx context.Context, decl project.EvaluatorDecl, @@ -250,11 +251,10 @@ func (r *evalReconciler) EnsureEvaluator( return versionFromRaw(raw, decl.Version), false, nil } - info, err := os.Stat(localPath) - if err != nil { + if _, err := os.Stat(localPath); err != nil { return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) } - if info.IsDir() { + if evalcore.IsCodeEvaluatorSource(localPath) { return r.ensureCodeEvaluator(ctx, decl, localPath) } @@ -283,11 +283,12 @@ func (r *evalReconciler) EnsureEvaluator( if err != nil { return "", false, err } + r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) return created.Version, true, nil } -// ensureCodeEvaluator publishes a folder of Python only when its content -// changed since the last deploy. +// ensureCodeEvaluator publishes a Python script only when its content changed +// since the last deploy. // // Every publish is a new immutable version, so without this a repeated // `azd up` would leave a trail of identical versions and force every eval @@ -295,17 +296,17 @@ func (r *evalReconciler) EnsureEvaluator( func (r *evalReconciler) ensureCodeEvaluator( ctx context.Context, decl project.EvaluatorDecl, - dir string, + path string, ) (string, bool, error) { - // Validated before anything is uploaded: a folder missing its entry point - // or class is only rejected when a run executes, long after a version has + // Validated before anything is published: a script with no top-level + // grade() is only rejected when a run executes, long after a version has // been published and an eval bound to it. - pkg, err := evalcore.LoadCodeEvaluator(decl.Name, dir) + script, err := evalcore.LoadCodeEvaluator(decl.Name, path) if err != nil { return "", false, err } - digest, err := project.FingerprintPath(dir) + digest, err := project.Fingerprint(path) if err != nil { return "", false, err } @@ -323,23 +324,101 @@ func (r *evalReconciler) ensureCodeEvaluator( return recordedVersion, false, nil } - opts, err := codeEvaluatorOptions(pkg, codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) if err != nil { return "", false, err } - created, err := r.ec.evalClient.UploadCodeEvaluatorVersion( - ctx, pkg, opts, ProjectEndpointAPIVersion, + created, err := r.ec.evalClient.CreateCodeEvaluatorVersion( + ctx, script, opts, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err } + r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) + _ = r.ec.setEnvValue(ctx, key, digest) _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) return created.Version, true, nil } +// evaluatorPropagation bounds the wait for a freshly published evaluator to +// become usable. +// +// A create returns before the version is resolvable everywhere, and the very +// next step of a deploy is EnsureEval, which names the evaluator in a testing +// criterion. Creating the eval inside that window fails with "The evaluator X +// was not found" — a confusing error, because the evaluator was published +// seconds earlier and is plainly there by the time anyone looks. The observed +// gap is under a second, so the poll is frequent and the cap is generous +// enough to absorb a slow day without stalling a deploy on an evaluator that +// is genuinely missing. +const ( + evaluatorPropagationTimeout = 30 * time.Second + evaluatorPropagationInterval = 250 * time.Millisecond +) + +// awaitEvaluatorReadable polls until a published version is resolvable, or the +// cap passes. +// +// Two reads have to agree, because they are not backed by the same view. The +// direct read goes consistent almost immediately; the version listing lags it +// by seconds, the same way the dataset listing does. A live publish was +// observed reading back at 03:06:58 and still failing eval creation at +// 03:06:59, so waiting on the direct read alone leaves exactly the race this +// exists to close. The listing is the slower of the two and therefore the one +// worth waiting on. +// +// A timeout is not an error. The wait is a courtesy that makes the common case +// reliable; if it never succeeds, the create that follows will report the real +// problem with far more context than a wait that gave up could. +func (r *evalReconciler) awaitEvaluatorReadable(ctx context.Context, name, version string) { + if version == "" { + return + } + deadline := time.Now().Add(evaluatorPropagationTimeout) + for { + if r.evaluatorVersionResolvable(ctx, name, version) { + return + } + if time.Now().After(deadline) { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(evaluatorPropagationInterval): + } + } +} + +// evaluatorVersionResolvable reports whether a version can be both read +// directly and found in the listing. +func (r *evalReconciler) evaluatorVersionResolvable( + ctx context.Context, + name, version string, +) bool { + if _, err := r.ec.evalClient.GetEvaluatorRaw( + ctx, name, version, ProjectEndpointAPIVersion, + ); err != nil { + return false + } + + list, err := r.ec.evalClient.ListEvaluatorVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + if err != nil || list == nil { + return false + } + for _, entry := range list.Value { + if entry.Version == version { + return true + } + } + return false +} + // checkEvaluatorDrift fails when the service holds a newer version than the // one recorded at the last deploy. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go index 856df8ea19d..f9e1305eea6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go @@ -4,16 +4,12 @@ package eval_api import ( - "bytes" "context" "encoding/json" "fmt" - "io" "net/http" "net/url" - "os" "strconv" - "strings" "azureaieval/internal/pkg/evalcore" ) @@ -23,7 +19,7 @@ import ( // // The wire shape is snake_case with a lowercase discriminator, matching // CodeBasedEvaluatorDefinition in the Foundry data-plane OpenAPI document -// (`type` enum ["code"], plus code_text, entry_point, blob_uri, init_parameters, +// (`type` enum ["code"], plus code_text, image_tag, init_parameters, // data_schema and metrics). An earlier draft documented a camelCase body with // `type: "CodeBased"`; that shape is not what the deployed service accepts. const CodeDefinitionType = "code" @@ -32,27 +28,18 @@ const CodeDefinitionType = "code" // than shipped by the platform. const evaluatorTypeCustom = "custom" -// foundryFeaturesHeader opts a request in to preview behaviour. blob_uri and -// entry_point are both declared as preview properties on the code definition, -// so the header is sent with every call that sets one. +// foundryFeaturesHeader opts a request in to preview behaviour. The code +// definition's properties are declared as preview, so the header is sent with +// every call that sets one. const ( foundryFeaturesHeader = "Foundry-Features" foundryFeatureEvalsV1 = "Evaluations=V1Preview" - pendingUploadTypeBlob = "TemporaryBlobReference" defaultCodeMetricName = "result" defaultCodeMetricType = "continuous" defaultMetricDirection = "increase" - firstEvaluatorVersion = "1" - blobTypeHeader = "x-ms-blob-type" - blobTypeBlock = "BlockBlob" - octetStreamContentType = "application/octet-stream" - // maxVersionProbes bounds how far past a stale version guess the publish - // will step before giving up. - maxVersionProbes = 5 ) -// DefaultCodeMetrics is used when neither the folder nor the caller declares -// any. +// DefaultCodeMetrics is used when the caller declares none. // // The service rejects a code definition carrying no metrics, and the documented // evaluator output is a JSON object whose `result` field holds the score, so @@ -69,6 +56,7 @@ type CodeEvaluatorOptions struct { DisplayName string Description string Categories []string + ImageTag string InitParameters json.RawMessage DataSchema json.RawMessage Metrics json.RawMessage @@ -76,14 +64,21 @@ type CodeEvaluatorOptions struct { // codeDefinition is the wire body of a code evaluator definition. // -// The contract also allows code_text in place of blob_uri, and requires exactly -// one of the two. Only blob_uri is modelled here, because only blob_uri is -// sent; a field that is never populated would suggest a supported alternative -// that has not been exercised. +// code_text carries the whole evaluator. The contract's other source property, +// blob_uri, is deliberately absent: the definition is consumed as an OpenAI +// python grader, whose contract (GraderPython) is a single `Source` string +// with no notion of a folder, archive, file list or entry point. A definition +// published with blob_uri alone registers cleanly and then fails the run with +// "Invalid grader source: top-level grade() function not found in source", +// because nothing reads the blob back into Source. +// +// image_tag is how a grader gets dependencies. Only one file is ever sent, so +// a helper module cannot travel with it and anything beyond the standard +// library has to already be in the image. type codeDefinition struct { Type string `json:"type"` - EntryPoint string `json:"entry_point,omitempty"` - BlobURI string `json:"blob_uri,omitempty"` + CodeText string `json:"code_text,omitempty"` + ImageTag string `json:"image_tag,omitempty"` InitParameters json.RawMessage `json:"init_parameters,omitempty"` DataSchema json.RawMessage `json:"data_schema,omitempty"` Metrics json.RawMessage `json:"metrics,omitempty"` @@ -100,97 +95,27 @@ type createEvaluatorVersionRequest struct { Definition *codeDefinition `json:"definition"` } -// pendingUploadRequest starts an upload for one evaluator version. -type pendingUploadRequest struct { - PendingUploadType string `json:"pendingUploadType"` -} - -// PendingUploadResponse is the reply to startPendingUpload: a container to -// write into, and the SAS that authorizes writing. -// -// The evaluator endpoint answers with TemporaryDataReferenceResponseDto, which -// carries the location under blobReferenceForConsumption. Datasets answer with -// blobReference instead, so both are modelled and whichever arrives is used: -// the two resources share this step but not the name they return it under. -type PendingUploadResponse struct { - BlobReference *BlobReference `json:"blobReference,omitempty"` - BlobReferenceConsumption *BlobReference `json:"blobReferenceForConsumption,omitempty"` - PendingUploadID string `json:"pendingUploadId,omitempty"` - TemporaryDataReferenceID string `json:"temporaryDataReferenceId,omitempty"` - Version string `json:"version,omitempty"` -} - -// BlobReference is a storage location plus the credential to reach it. -type BlobReference struct { - BlobURI string `json:"blobUri,omitempty"` - StorageAccountARM string `json:"storageAccountArmId,omitempty"` - Credential *BlobCredential `json:"credential,omitempty"` -} - -// BlobCredential holds the SAS granted for an upload. -type BlobCredential struct { - Type string `json:"type,omitempty"` - SASUri string `json:"sasUri,omitempty"` -} - -// reference returns whichever of the two blob references the service populated. -func (p *PendingUploadResponse) reference() *BlobReference { - if p == nil { - return nil - } - if p.BlobReferenceConsumption != nil { - return p.BlobReferenceConsumption - } - return p.BlobReference -} - -// UploadURI returns the container URI carrying the SAS token, or empty when -// the service granted no credential. -func (p *PendingUploadResponse) UploadURI() string { - ref := p.reference() - if ref == nil || ref.Credential == nil { - return "" - } - return ref.Credential.SASUri -} - -// ContainerURI returns the container URI without the SAS token. This is what -// the evaluator definition records, because the definition is persisted and a -// SAS in it would expire. -func (p *PendingUploadResponse) ContainerURI() string { - ref := p.reference() - if ref == nil { - return "" - } - return ref.BlobURI -} - -// UploadCodeEvaluatorVersion publishes a folder of Python as a new version of -// a code evaluator. +// CreateCodeEvaluatorVersion publishes a Python script as a new version of a +// code evaluator. // -// Every package goes through storage, including a package of one file. The -// contract offers code_text as an alternative and it would save a round trip, -// but nothing observable confirms the executor runs it: the hand-off to the -// evaluation runtime drops both code_text and blob_uri and refetches from the -// catalog, so RAISvc does not reveal which one it prefers. blob_uri is the one -// with a demonstrated consumer, which enumerates the container and reads the -// files back. Choosing the unproven path would trade a saved upload for an -// evaluator that registers cleanly and then fails when it is finally run, -// which is far harder to diagnose than a slow publish. Revisit once the live -// test has actually exercised inline source. -func (c *EvalClient) UploadCodeEvaluatorVersion( +// The source is sent inline. There is no upload step and no storage to +// reserve: the executor is handed a string of source, so a blob it would never +// read adds a round trip, a SAS write, and a failure mode in exchange for +// nothing that reaches the grader. +func (c *EvalClient) CreateCodeEvaluatorVersion( ctx context.Context, - pkg *evalcore.CodeEvaluatorPackage, + script *evalcore.CodeEvaluatorScript, opts CodeEvaluatorOptions, apiVersion string, ) (*EvaluatorVersion, error) { - if pkg == nil { - return nil, fmt.Errorf("no evaluator package to publish") + if script == nil { + return nil, fmt.Errorf("no evaluator script to publish") } definition := &codeDefinition{ Type: CodeDefinitionType, - EntryPoint: pkg.EntryPoint, + CodeText: script.Source, + ImageTag: opts.ImageTag, InitParameters: opts.InitParameters, DataSchema: opts.DataSchema, Metrics: opts.Metrics, @@ -199,14 +124,8 @@ func (c *EvalClient) UploadCodeEvaluatorVersion( definition.Metrics = DefaultCodeMetrics } - blobURI, err := c.uploadCodeEvaluatorFiles(ctx, pkg, apiVersion) - if err != nil { - return nil, err - } - definition.BlobURI = blobURI - body := &createEvaluatorVersionRequest{ - Name: pkg.Name, + Name: script.Name, DisplayName: opts.DisplayName, Description: opts.Description, EvaluatorType: evaluatorTypeCustom, @@ -214,7 +133,7 @@ func (c *EvalClient) UploadCodeEvaluatorVersion( Definition: definition, } - path := pathEvaluators + "/" + url.PathEscape(pkg.Name) + "/versions" + path := pathEvaluators + "/" + url.PathEscape(script.Name) + "/versions" respBody, err := c.doRequestWithHeaders( ctx, http.MethodPost, path, nil, body, apiVersion, previewHeaders(), ) @@ -229,152 +148,11 @@ func (c *EvalClient) UploadCodeEvaluatorVersion( } } if created.Name == "" { - created.Name = pkg.Name + created.Name = script.Name } return &created, nil } -// uploadCodeEvaluatorFiles writes every file in the package to the container -// the service provisions for the version being created, and returns the -// container URI to record on the definition. -func (c *EvalClient) uploadCodeEvaluatorFiles( - ctx context.Context, - pkg *evalcore.CodeEvaluatorPackage, - apiVersion string, -) (string, error) { - pending, err := c.reserveEvaluatorStorage(ctx, pkg.Name, apiVersion) - if err != nil { - return "", err - } - - uploadURI := pending.UploadURI() - if uploadURI == "" { - return "", fmt.Errorf( - "the service returned no upload credential for evaluator %q", pkg.Name) - } - containerURI := pending.ContainerURI() - if containerURI == "" { - return "", fmt.Errorf( - "the service returned no storage location for evaluator %q", pkg.Name) - } - - for _, file := range pkg.Files { - content, err := os.ReadFile(file.AbsPath) - if err != nil { - return "", fmt.Errorf("reading %s: %w", file.RelPath, err) - } - if err := uploadBlob(ctx, uploadURI, file.RelPath, content); err != nil { - return "", fmt.Errorf("uploading %s: %w", file.RelPath, err) - } - } - - return containerURI, nil -} - -// reserveEvaluatorStorage provisions storage for the version being published, -// stepping past versions that are already taken. -// -// Storage has to be reserved under a version number before the version exists, -// and the number is guessed by reading the ones already registered. That read -// is eventually consistent: immediately after a publish it can still report -// the evaluator as unknown, which makes the guess collide with a version that -// is already there. The service answers a collision with a conflict, so the -// guess is advanced and tried again rather than failing the publish. -// -// Only a conflict is retried. Any other failure is the caller's to see. -func (c *EvalClient) reserveEvaluatorStorage( - ctx context.Context, - name string, - apiVersion string, -) (*PendingUploadResponse, error) { - version, err := strconv.Atoi(c.NextEvaluatorVersion(ctx, name, apiVersion)) - if err != nil { - version = 1 - } - - var lastErr error - for attempt := 0; attempt < maxVersionProbes; attempt++ { - pending, err := c.StartEvaluatorPendingUpload( - ctx, name, strconv.Itoa(version+attempt), apiVersion) - if err == nil { - return pending, nil - } - lastErr = err - if !isVersionConflict(err) { - break - } - } - return nil, fmt.Errorf("starting the upload for evaluator %q: %w", name, lastErr) -} - -// isVersionConflict reports whether a failed reservation was refused because -// the version already exists. -// -// The conflict is not surfaced as one: the service wraps the downstream 409 in -// a 500 whose message quotes the original status, so the status code on the -// response cannot be used and the message is what is left to read. -func isVersionConflict(err error) bool { - if err == nil { - return false - } - text := err.Error() - return strings.Contains(text, "409") || strings.Contains(text, "Conflict") -} - -// StartEvaluatorPendingUpload provisions the storage an evaluator version's -// code is written to. -func (c *EvalClient) StartEvaluatorPendingUpload( - ctx context.Context, - name string, - version string, - apiVersion string, -) (*PendingUploadResponse, error) { - path := fmt.Sprintf( - "%s/%s/versions/%s/startPendingUpload", - pathEvaluators, url.PathEscape(name), url.PathEscape(version), - ) - respBody, err := c.doRequestWithHeaders( - ctx, http.MethodPost, path, nil, - &pendingUploadRequest{PendingUploadType: pendingUploadTypeBlob}, - apiVersion, previewHeaders(), - ) - if err != nil { - return nil, err - } - - var pending PendingUploadResponse - if len(respBody) > 0 { - if err := json.Unmarshal(respBody, &pending); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - } - return &pending, nil -} - -// NextEvaluatorVersion reports the version the service will assign to the next -// create. -// -// The upload has to name a version before the version exists, because storage -// is provisioned per version while the create that assigns it comes last. The -// service auto-increments, so the next one is the highest registered plus one. -// An unknown evaluator has none, which is version 1. -func (c *EvalClient) NextEvaluatorVersion( - ctx context.Context, - name string, - apiVersion string, -) string { - list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return firstEvaluatorVersion - } - latest := pickLatestVersion(list.Value) - number, err := strconv.Atoi(latest) - if err != nil { - return firstEvaluatorVersion - } - return strconv.Itoa(number + 1) -} - // LatestEvaluatorVersionNumber reports the newest registered version as an // integer, or 0 when the evaluator is unknown or its versions are not numeric. func (c *EvalClient) LatestEvaluatorVersionNumber( @@ -398,38 +176,3 @@ func (c *EvalClient) LatestEvaluatorVersionNumber( func previewHeaders() map[string]string { return map[string]string{foundryFeaturesHeader: foundryFeatureEvalsV1} } - -// uploadBlob writes one file into a container using a container-level SAS. -// -// A plain client is used rather than the pipeline: the SAS in the URL is the -// credential, and the pipeline's bearer token policy would attach a Foundry -// token to a storage request that has no use for it. -func uploadBlob(ctx context.Context, containerSASUri, blobName string, data []byte) error { - u, err := url.Parse(containerSASUri) - if err != nil { - return fmt.Errorf("invalid container SAS URI: %w", err) - } - u.Path = strings.TrimSuffix(u.Path, "/") + "/" + blobName - - req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), bytes.NewReader(data)) - if err != nil { - return fmt.Errorf("failed to create upload request: %w", err) - } - req.Header.Set(blobTypeHeader, blobTypeBlock) - req.Header.Set("Content-Type", octetStreamContentType) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("failed to upload blob: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { - // The body is Azure Storage XML, which says more than the status alone, - // but it is capped: a rejected upload can answer with a long document. - detail, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return fmt.Errorf("blob upload failed with status %d: %s", - resp.StatusCode, strings.TrimSpace(string(detail))) - } - return nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go deleted file mode 100644 index a62391aa96f..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder.go +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "unicode" -) - -// CodeEvaluatorMetadataFile is the optional descriptor a folder can carry so -// the schemas travel with the code instead of having to be repeated on the -// command line every time it is published. -// -// It is not part of the service contract — the service only ever sees the -// definition assembled from it — so it is read leniently and its absence is -// normal. -const CodeEvaluatorMetadataFile = "evaluator.json" - -// pythonExt is the only source extension a code evaluator entry point can have. -const pythonExt = ".py" - -// excludedDirNames are directories that never carry evaluator source: build -// caches, version control, and dependency trees. Any dot-prefixed directory is -// excluded as well, which is checked separately. -var excludedDirNames = map[string]bool{ - "__pycache__": true, - ".git": true, - ".venv": true, - "venv": true, - "node_modules": true, -} - -// excludedFileExts are compiled Python artifacts. They are derived from the -// sources beside them, so uploading them adds nothing and — because they embed -// a timestamp — would make the folder fingerprint change on every rebuild. -var excludedFileExts = map[string]bool{ - ".pyc": true, - ".pyo": true, -} - -// CodeFile is one file in an evaluator package. -type CodeFile struct { - // RelPath is the path relative to the package root, always - // slash-separated. Windows and Linux must agree on it: it is both the blob - // name the file is uploaded under and part of the fingerprint, so a - // backslash here would republish the whole package on a change of machine. - RelPath string - - // AbsPath is where the file is read from on this machine. It is - // deliberately not part of the fingerprint. - AbsPath string -} - -// CodeEvaluatorMetadata is the optional descriptor read from -// CodeEvaluatorMetadataFile. Every field is optional; the raw JSON fields are -// passed to the service untouched so a schema this extension does not model -// still reaches it intact. -type CodeEvaluatorMetadata struct { - DisplayName string `json:"display_name,omitempty"` - Description string `json:"description,omitempty"` - Categories []string `json:"categories,omitempty"` - InitParameters json.RawMessage `json:"init_parameters,omitempty"` - DataSchema json.RawMessage `json:"data_schema,omitempty"` - Metrics json.RawMessage `json:"metrics,omitempty"` -} - -// CodeEvaluatorPackage is a validated folder ready to publish. -type CodeEvaluatorPackage struct { - // Name is the evaluator name the package was validated against. - Name string - // Root is the folder on disk. - Root string - // Files are the files to upload, in fingerprint order. - Files []CodeFile - // EntryPoint is the Python file holding the evaluator class. - EntryPoint string - // ClassName is the class the runtime instantiates. - ClassName string - // Metadata is the folder's descriptor, or nil when it carries none. - Metadata *CodeEvaluatorMetadata -} - -// IsCodeEvaluatorSource reports whether a declared source names a folder, and -// therefore a code evaluator rather than a rubric. -// -// The path is stat-ed rather than pattern-matched: a trailing separator is not -// required in YAML and `.json` in a folder name would misclassify it. -func IsCodeEvaluatorSource(path string) bool { - if path == "" { - return false - } - info, err := os.Stat(path) - return err == nil && info.IsDir() -} - -// EvaluatorClassName derives the class the runtime looks for from the -// evaluator name: the name in PascalCase, suffixed with Evaluator. -// -// The suffix is not appended twice, so both spellings customers use resolve to -// the same class — `answer_length` and `answer_length_evaluator` both mean -// AnswerLengthEvaluator. -func EvaluatorClassName(name string) string { - parts := strings.FieldsFunc(name, func(r rune) bool { - return r == '_' || r == '-' || r == '.' || r == ' ' - }) - - var b strings.Builder - for _, part := range parts { - runes := []rune(part) - b.WriteRune(unicode.ToUpper(runes[0])) - b.WriteString(string(runes[1:])) - } - - pascal := b.String() - if strings.HasSuffix(pascal, "Evaluator") { - return pascal - } - return pascal + "Evaluator" -} - -// WalkCodeFolder lists the files that make up an evaluator package, excluding -// build caches, version control, dependency trees, and compiled Python. -// -// The result is sorted by RelPath so the upload order and the fingerprint do -// not depend on directory iteration order, which the filesystem does not -// promise to keep stable. -func WalkCodeFolder(dir string) ([]CodeFile, error) { - info, err := os.Stat(dir) - if err != nil { - return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, err) - } - if !info.IsDir() { - return nil, fmt.Errorf( - "evaluator source %q is a file; a code evaluator is published from a folder", dir) - } - - var files []CodeFile - walkErr := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - - rel, relErr := filepath.Rel(dir, path) - if relErr != nil { - return relErr - } - rel = filepath.ToSlash(rel) - if rel == "." { - return nil - } - - if entry.IsDir() { - if isExcludedDir(entry.Name()) { - return fs.SkipDir - } - return nil - } - - // Symlinks, sockets and devices carry no content that can be uploaded, - // and WalkDir does not follow them, so they would otherwise be - // published as zero bytes. - if !entry.Type().IsRegular() { - return nil - } - // Dot-prefixed files are excluded for the same reason as dot-prefixed - // directories, and one reason more: a folder kept next to an evaluator - // tends to collect `.env`, `.netrc` and `.pypirc`, and publishing the - // package would copy those secrets into blob storage. Nothing a Python - // evaluator needs at runtime is named with a leading dot. - if strings.HasPrefix(entry.Name(), ".") { - return nil - } - if excludedFileExts[strings.ToLower(filepath.Ext(rel))] { - return nil - } - - files = append(files, CodeFile{RelPath: rel, AbsPath: path}) - return nil - }) - if walkErr != nil { - return nil, fmt.Errorf("reading evaluator folder %q: %w", dir, walkErr) - } - - sortCodeFiles(files) - return files, nil -} - -// isExcludedDir reports whether a directory is skipped along with everything -// under it. -func isExcludedDir(name string) bool { - return excludedDirNames[name] || strings.HasPrefix(name, ".") -} - -func sortCodeFiles(files []CodeFile) { - sort.Slice(files, func(i, j int) bool { return files[i].RelPath < files[j].RelPath }) -} - -// FingerprintCodeFolder hashes a folder so a later deploy can tell whether the -// evaluator changed without downloading anything from the service. -func FingerprintCodeFolder(dir string) (string, error) { - files, err := WalkCodeFolder(dir) - if err != nil { - return "", err - } - return FingerprintCodeFiles(files) -} - -// FingerprintCodeFiles hashes an already-walked package. -// -// Both the relative path and the content of every file are hashed, so renaming -// a file registers as a change even when the bytes are identical. The input is -// sorted first: the digest must describe the package, not the order the caller -// happened to hand the files over in. -// -// Only RelPath — never AbsPath — feeds the hash, and RelPath is normalized to -// forward slashes, so the same package hashes the same on Windows and Linux. -func FingerprintCodeFiles(files []CodeFile) (string, error) { - ordered := make([]CodeFile, len(files)) - copy(ordered, files) - sortCodeFiles(ordered) - - outer := sha256.New() - for _, file := range ordered { - content, err := os.ReadFile(file.AbsPath) - if err != nil { - return "", fmt.Errorf("hashing %q: %w", file.AbsPath, err) - } - inner := sha256.Sum256(content) - // Path and content digest are written on separate lines, which keeps - // the encoding unambiguous: a file's content digest is fixed width, so - // no path can be read as part of it. - fmt.Fprintf(outer, "%s\n%s\n", file.RelPath, hex.EncodeToString(inner[:])) - } - return hex.EncodeToString(outer.Sum(nil)), nil -} - -// LoadCodeEvaluator validates a folder against the packaging convention and -// returns the package to publish. -// -// The checks are done here rather than left to the service because the service -// only discovers a missing entry point when a run executes, long after a -// version has been published and an eval bound to it. -func LoadCodeEvaluator(name, dir string) (*CodeEvaluatorPackage, error) { - if name == "" { - return nil, fmt.Errorf("an evaluator name is required to validate the folder layout") - } - - files, err := WalkCodeFolder(dir) - if err != nil { - return nil, err - } - if len(files) == 0 { - return nil, fmt.Errorf( - "evaluator folder %q holds no files to publish", dir) - } - - entryPoint := name + pythonExt - className := EvaluatorClassName(name) - - entry, ok := findFile(files, entryPoint) - if !ok { - return nil, fmt.Errorf( - "evaluator %q needs %s in %s, holding a class named %s. The folder holds %s", - name, entryPoint, dir, className, describeFiles(files)) - } - - source, err := os.ReadFile(entry.AbsPath) - if err != nil { - return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) - } - if !declaresClass(source, className) { - return nil, fmt.Errorf( - "%s does not declare a class named %s. A code evaluator is a class with that "+ - "exact name and a __call__ method that takes **kwargs and returns a JSON "+ - "object, for example:\n\nclass %s:\n def __call__(self, **kwargs):\n"+ - " return {\"result\": 1}", - filepath.Join(dir, entry.RelPath), className, className) - } - - metadata, err := readCodeEvaluatorMetadata(files) - if err != nil { - return nil, err - } - - return &CodeEvaluatorPackage{ - Name: name, - Root: dir, - Files: files, - EntryPoint: entryPoint, - ClassName: className, - Metadata: metadata, - }, nil -} - -func findFile(files []CodeFile, relPath string) (CodeFile, bool) { - for _, file := range files { - if file.RelPath == relPath { - return file, true - } - } - return CodeFile{}, false -} - -// describeFiles renders the folder's contents for an error message, capped so -// a large package does not bury the advice that follows it. -func describeFiles(files []CodeFile) string { - const limit = 10 - names := make([]string, 0, limit) - for i, file := range files { - if i == limit { - return fmt.Sprintf("%s and %d more", strings.Join(names, ", "), len(files)-limit) - } - names = append(names, file.RelPath) - } - return strings.Join(names, ", ") -} - -// declaresClass reports whether the source declares the named class at any -// indentation, in either the bare or the inheriting form. -func declaresClass(source []byte, className string) bool { - pattern := `(?m)^[ \t]*class[ \t]+` + regexp.QuoteMeta(className) + `[ \t]*[(:]` - matched, err := regexp.Match(pattern, source) - return err == nil && matched -} - -// readCodeEvaluatorMetadata reads the optional folder descriptor. A folder -// without one is normal, so absence is not an error; malformed JSON is, since -// silently ignoring it would publish an evaluator missing the schemas the -// author wrote down. -func readCodeEvaluatorMetadata(files []CodeFile) (*CodeEvaluatorMetadata, error) { - entry, ok := findFile(files, CodeEvaluatorMetadataFile) - if !ok { - return nil, nil - } - - raw, err := os.ReadFile(entry.AbsPath) - if err != nil { - return nil, fmt.Errorf("reading %s: %w", entry.RelPath, err) - } - - var metadata CodeEvaluatorMetadata - if err := json.Unmarshal(raw, &metadata); err != nil { - return nil, fmt.Errorf("%s is not valid JSON: %w", entry.RelPath, err) - } - return &metadata, nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go deleted file mode 100644 index b7e956bc3f9..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codefolder_test.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -// writeFile creates a file and every directory above it. -func writeFile(t *testing.T, root, rel, content string) string { - t.Helper() - path := filepath.Join(root, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - return path -} - -// evaluatorSource is a minimal evaluator matching the packaging convention. -func evaluatorSource(className string) string { - return "class " + className + ":\n" + - " def __call__(self, **kwargs):\n" + - " return {\"result\": len(kwargs.get(\"response\", \"\"))}\n" -} - -// relPaths reads the walk result into a comparable list. -func relPaths(files []CodeFile) []string { - out := make([]string, 0, len(files)) - for _, f := range files { - out = append(out, f.RelPath) - } - return out -} - -func TestWalkCodeFolder_ExcludesBuildAndDependencyTrees(t *testing.T) { - dir := t.TempDir() - - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") - writeFile(t, dir, "README.md", "docs\n") - - // Every one of these must be skipped, along with everything under it. - writeFile(t, dir, "__pycache__/answer_length.cpython-311.pyc", "cache") - writeFile(t, dir, ".git/config", "[core]") - writeFile(t, dir, ".venv/lib/site.py", "venv") - writeFile(t, dir, "venv/lib/site.py", "venv") - writeFile(t, dir, "node_modules/pkg/index.js", "js") - writeFile(t, dir, ".mypy_cache/report.json", "{}") - writeFile(t, dir, "helpers/__pycache__/text.cpython-311.pyc", "cache") - // Compiled artifacts are skipped wherever they sit, not just in caches. - writeFile(t, dir, "stale.pyc", "cache") - writeFile(t, dir, "stale.pyo", "cache") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - - require.Equal(t, - []string{"README.md", "answer_length.py", "helpers/text.py"}, - relPaths(files), - "only source and data files belong in the package, in sorted order") -} - -// Dot-prefixed files are excluded because an evaluator folder living in a repo -// collects credential files, and publishing the package would copy them into -// blob storage. Excluding the directories alone is not enough: the ones that -// hold secrets sit at the root, next to the source. -func TestWalkCodeFolder_ExcludesDotFiles(t *testing.T) { - dir := t.TempDir() - - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, ".env", "AZURE_OPENAI_API_KEY=super-secret\n") - writeFile(t, dir, ".netrc", "machine example.com password hunter2\n") - writeFile(t, dir, ".pypirc", "[pypi]\npassword = leaked\n") - writeFile(t, dir, "helpers/.env", "NESTED_SECRET=1\n") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - - require.Equal(t, []string{"answer_length.py"}, relPaths(files), - "no dot-prefixed file may reach the upload, at any depth") - - for _, f := range files { - require.NotContains(t, f.RelPath, ".env") - require.NotContains(t, f.RelPath, ".netrc") - require.NotContains(t, f.RelPath, ".pypirc") - } -} - -// Relative paths are the blob names and part of the fingerprint, so they must -// not depend on the host's path separator. -func TestWalkCodeFolder_UsesForwardSlashes(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "a/b/c.py", "x = 1\n") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - require.Len(t, files, 1) - require.Equal(t, "a/b/c.py", files[0].RelPath) - require.NotContains(t, files[0].RelPath, "\\") -} - -func TestWalkCodeFolder_RejectsAFile(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "rubric.json", "{}") - - _, err := WalkCodeFolder(path) - require.Error(t, err) - require.Contains(t, err.Error(), "folder") -} - -func TestEvaluatorClassName(t *testing.T) { - cases := map[string]string{ - "answer_length": "AnswerLengthEvaluator", - "answer-length": "AnswerLengthEvaluator", - "answer_length_evaluator": "AnswerLengthEvaluator", - "tone": "ToneEvaluator", - "ToneEvaluator": "ToneEvaluator", - "my.custom_check": "MyCustomCheckEvaluator", - } - for name, want := range cases { - require.Equal(t, want, EvaluatorClassName(name), "for %q", name) - } -} - -func TestLoadCodeEvaluator_AcceptsAConventionalFolder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, "helpers.py", "def n(s): return len(s)\n") - - pkg, err := LoadCodeEvaluator("answer_length", dir) - require.NoError(t, err) - require.Equal(t, "answer_length.py", pkg.EntryPoint) - require.Equal(t, "AnswerLengthEvaluator", pkg.ClassName) - require.Equal(t, []string{"answer_length.py", "helpers.py"}, relPaths(pkg.Files)) - require.Nil(t, pkg.Metadata) -} - -// The class may inherit, and may be indented inside a conditional. -func TestLoadCodeEvaluator_AcceptsInheritingClass(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", "import abc\n\nclass ToneEvaluator(abc.ABC):\n pass\n") - - pkg, err := LoadCodeEvaluator("tone", dir) - require.NoError(t, err) - require.Equal(t, "ToneEvaluator", pkg.ClassName) -} - -func TestLoadCodeEvaluator_ReportsMissingEntryPoint(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "main.py", evaluatorSource("AnswerLengthEvaluator")) - - _, err := LoadCodeEvaluator("answer_length", dir) - require.Error(t, err) - // The message has to say what is missing, what it must hold, and what was - // actually found — a bare "not found" leaves the author guessing. - require.Contains(t, err.Error(), "answer_length.py") - require.Contains(t, err.Error(), "AnswerLengthEvaluator") - require.Contains(t, err.Error(), "main.py") -} - -func TestLoadCodeEvaluator_ReportsMissingClass(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", "class SomethingElse:\n pass\n") - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), "ToneEvaluator") - require.Contains(t, err.Error(), "__call__") -} - -func TestLoadCodeEvaluator_RejectsAnEmptyFolder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "__pycache__/x.pyc", "cache") - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), "no files") -} - -func TestLoadCodeEvaluator_ReadsFolderMetadata(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, CodeEvaluatorMetadataFile, `{ - "display_name": "Tone", - "description": "Scores tone.", - "metrics": {"result": {"type": "ordinal", "min_value": 1, "max_value": 5}}, - "data_schema": {"type": "object", "properties": {"response": {"type": "string"}}} - }`) - - pkg, err := LoadCodeEvaluator("tone", dir) - require.NoError(t, err) - require.NotNil(t, pkg.Metadata) - require.Equal(t, "Tone", pkg.Metadata.DisplayName) - require.Contains(t, string(pkg.Metadata.Metrics), "ordinal") - require.Contains(t, string(pkg.Metadata.DataSchema), "response") -} - -// A descriptor that cannot be read is a mistake worth reporting: publishing -// without the schemas it declares would register an evaluator the author did -// not describe. -func TestLoadCodeEvaluator_RejectsMalformedMetadata(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, CodeEvaluatorMetadataFile, "{not json") - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), CodeEvaluatorMetadataFile) -} - -func TestIsCodeEvaluatorSource(t *testing.T) { - dir := t.TempDir() - file := writeFile(t, dir, "rubric.json", "{}") - // A folder can be named like a file, so the decision cannot be made by - // looking at the string. - misleading := filepath.Join(dir, "looks_like.json") - require.NoError(t, os.MkdirAll(misleading, 0o755)) - - require.True(t, IsCodeEvaluatorSource(dir)) - require.True(t, IsCodeEvaluatorSource(misleading)) - require.False(t, IsCodeEvaluatorSource(file)) - require.False(t, IsCodeEvaluatorSource(filepath.Join(dir, "absent"))) - require.False(t, IsCodeEvaluatorSource("")) -} - -func TestFingerprintCodeFolder_IsStableAndSensitive(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "answer_length.py", evaluatorSource("AnswerLengthEvaluator")) - writeFile(t, dir, "helpers/text.py", "def clean(s): return s.strip()\n") - - first, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - again, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.Equal(t, first, again, "an unchanged folder must hash the same") - - // One byte. - writeFile(t, dir, "helpers/text.py", "def clean(s): return s.rstrip()\n") - changed, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.NotEqual(t, first, changed, "changed content must hash differently") -} - -// Renaming a file changes the package even when every byte is preserved: the -// entry point is resolved by name and imports are written against it. -func TestFingerprintCodeFolder_NoticesARename(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, "helpers.py", "X = 1\n") - - before, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - require.NoError(t, os.Rename( - filepath.Join(dir, "helpers.py"), filepath.Join(dir, "util.py"))) - - after, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.NotEqual(t, before, after) -} - -// The filesystem does not promise a stable iteration order, so the digest must -// describe the package rather than the order it was handed over in. The input -// here is the production walk's own output, permuted — not a hand-built list. -func TestFingerprintCodeFiles_IgnoresInputOrder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, dir, "helpers/a.py", "A = 1\n") - writeFile(t, dir, "helpers/b.py", "B = 2\n") - writeFile(t, dir, "data/prompts.txt", "hello\n") - - files, err := WalkCodeFolder(dir) - require.NoError(t, err) - require.Len(t, files, 4) - - fromFolder, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - reversed := make([]CodeFile, 0, len(files)) - for i := len(files) - 1; i >= 0; i-- { - reversed = append(reversed, files[i]) - } - fromReversed, err := FingerprintCodeFiles(reversed) - require.NoError(t, err) - require.Equal(t, fromFolder, fromReversed, "file order must not change the digest") - - rotated := append(append([]CodeFile{}, files[2:]...), files[:2]...) - fromRotated, err := FingerprintCodeFiles(rotated) - require.NoError(t, err) - require.Equal(t, fromFolder, fromRotated) -} - -// Where the folder sits must not affect the digest: two checkouts of the same -// repo, or the same repo on two machines, have to agree or every deploy -// republishes. -func TestFingerprintCodeFolder_IgnoresLocation(t *testing.T) { - build := func(root string) string { - writeFile(t, root, "tone.py", evaluatorSource("ToneEvaluator")) - writeFile(t, root, "helpers/a.py", "A = 1\n") - digest, err := FingerprintCodeFolder(root) - require.NoError(t, err) - return digest - } - - require.Equal(t, build(t.TempDir()), build(t.TempDir())) -} - -// Excluded content must not feed the digest, or a rebuild that only refreshes -// __pycache__ would look like an evaluator change and publish a version. -func TestFingerprintCodeFolder_IgnoresExcludedContent(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", evaluatorSource("ToneEvaluator")) - - before, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - - writeFile(t, dir, "__pycache__/tone.cpython-311.pyc", strings.Repeat("x", 64)) - writeFile(t, dir, ".venv/lib/site.py", "noise") - - after, err := FingerprintCodeFolder(dir) - require.NoError(t, err) - require.Equal(t, before, after) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go new file mode 100644 index 00000000000..2c01e791f09 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// pythonExt is the extension a code evaluator's source carries. It is also +// what tells a code evaluator apart from a rubric, which is `.json`. +const pythonExt = ".py" + +// GradeFunctionName is the function the executor calls. +// +// A code evaluator runs as an OpenAI python grader, and the grader contract is +// a single string of source with one entry point: a top-level +// `grade(sample, item)` returning a float. There is no module, package, or +// import path, so nothing else in the file can be reached. +const GradeFunctionName = "grade" + +// CodeEvaluatorScript is a validated evaluator script ready to publish. +type CodeEvaluatorScript struct { + // Name is the evaluator name the script is published under. + Name string + // Path is where the script was read from, kept for error messages. + Path string + // Source is the whole file, which is what the service is sent. The grader + // takes source, not a location: there is nowhere for a second file to go. + Source string +} + +// IsCodeEvaluatorSource reports whether a declared `source:` names a code +// evaluator rather than a rubric. +// +// The decision is made from the extension alone and never touches the +// filesystem, so it answers the same for a path that has not been created yet +// — a config can be validated before the file it names exists. +func IsCodeEvaluatorSource(path string) bool { + return strings.EqualFold(filepath.Ext(path), pythonExt) +} + +// gradeDeclaration matches a top-level `def grade(` — optionally async, and +// anchored at column zero. +// +// Indentation is what makes this specific rather than a substring search: a +// `grade` nested inside a class is a method, and the grader only ever calls a +// module-level function, so an indented match would pass validation here and +// then fail at run time with "top-level grade() function not found". +var gradeDeclaration = regexp.MustCompile( + `(?m)^(?:async[ \t]+)?def[ \t]+` + regexp.QuoteMeta(GradeFunctionName) + `[ \t]*\(`) + +// LoadCodeEvaluator reads an evaluator script and checks it against the grader +// contract. +// +// The check is done here rather than left to the service because the service +// only discovers a missing entry point when a run executes — long after a +// version has been published and an eval bound to it. The failure it reports +// then is "Invalid grader source: top-level grade() function not found in +// source", which names neither the file nor the evaluator. +func LoadCodeEvaluator(name, path string) (*CodeEvaluatorScript, error) { + if name == "" { + return nil, fmt.Errorf("an evaluator name is required to publish %q", path) + } + if path == "" { + return nil, fmt.Errorf("evaluator %q has no source file to publish", name) + } + + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) + } + if info.IsDir() { + return nil, fmt.Errorf( + "evaluator source %q is a directory. A code evaluator is a single %s file: "+ + "it is published as the source of a python grader, which takes one script "+ + "and cannot import a helper module beside it", path, pythonExt) + } + if !IsCodeEvaluatorSource(path) { + return nil, fmt.Errorf( + "evaluator source %q must be a %s file", path, pythonExt) + } + + source, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) + } + if len(strings.TrimSpace(string(source))) == 0 { + return nil, fmt.Errorf("evaluator source %q is empty", path) + } + if !gradeDeclaration.Match(source) { + return nil, fmt.Errorf( + "%s does not declare a top-level %s(sample, item) function. A code evaluator "+ + "runs as a python grader, which calls exactly that and nothing else — a "+ + "class, a differently named function, or one nested inside another will "+ + "not be found. For example:\n\ndef %s(sample, item) -> float:\n"+ + " return float(len(item.get(\"response\", \"\")))\n\n"+ + "The script must also be self-contained: only the standard library and "+ + "whatever the image named by --image-tag provides are importable, so a "+ + "helper file next to it cannot be imported", + path, GradeFunctionName, GradeFunctionName) + } + + return &CodeEvaluatorScript{ + Name: name, + Path: path, + Source: string(source), + }, nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go new file mode 100644 index 00000000000..7cb75014828 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package evalcore + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// gradeSource is the minimal script that satisfies the grader contract. +const gradeSource = `def grade(sample, item) -> float: + return float(len((item or {}).get("response", ""))) +` + +func writeFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return path +} + +// The extension decides which definition type to publish from the source's +// extension alone, and it has to answer for a path that does not exist yet so +// a config can be validated before its files are written. +func TestIsCodeEvaluatorSource(t *testing.T) { + require.True(t, IsCodeEvaluatorSource("tone.py")) + require.True(t, IsCodeEvaluatorSource("evaluators/tone.py")) + require.True(t, IsCodeEvaluatorSource(`evaluators\tone.PY`), + "the extension is matched case-insensitively") + require.True(t, IsCodeEvaluatorSource("/absent/never/created.py"), + "classification must not touch the filesystem") + + require.False(t, IsCodeEvaluatorSource("rubric.json")) + require.False(t, IsCodeEvaluatorSource("evaluators/rubric.json")) + require.False(t, IsCodeEvaluatorSource("evaluator"), + "a folder is no longer a code evaluator; the grader takes one script") + require.False(t, IsCodeEvaluatorSource("tone.python")) + require.False(t, IsCodeEvaluatorSource("")) +} + +func TestLoadCodeEvaluator_AcceptsATopLevelGrade(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", gradeSource) + + script, err := LoadCodeEvaluator("tone", path) + require.NoError(t, err) + require.Equal(t, "tone", script.Name) + require.Equal(t, path, script.Path) + require.Equal(t, gradeSource, script.Source, + "the whole file is what the grader is sent") +} + +// The grader is handed source and calls grade(); an async definition is still +// a top-level grade(). +func TestLoadCodeEvaluator_AcceptsAsyncAndAnnotatedForms(t *testing.T) { + for label, source := range map[string]string{ + "async": "async def grade(sample, item) -> float:\n return 1.0\n", + "spaced": "def grade (sample, item):\n return 1.0\n", + "no-annot": "def grade(sample, item):\n return 1.0\n", + "after-code": "import json\n\n\ndef grade(sample, item):\n return 1.0\n", + } { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", source) + _, err := LoadCodeEvaluator("tone", path) + require.NoError(t, err, "for %s", label) + } +} + +// Without this the failure surfaces only when a run executes, as "Invalid +// grader source: top-level grade() function not found in source" — long after +// a version has been published and an eval bound to it. +func TestLoadCodeEvaluator_ReportsAMissingGrade(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", + "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") + + _, err := LoadCodeEvaluator("tone", path) + require.Error(t, err) + require.Contains(t, err.Error(), "grade(sample, item)") + require.Contains(t, err.Error(), path) +} + +// A grade() nested inside a class is a method. The grader only ever calls a +// module-level function, so an indented match must not pass validation. +func TestLoadCodeEvaluator_RejectsANestedGrade(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "tone.py", + "class ToneEvaluator:\n def grade(self, sample, item):\n return 1.0\n") + + _, err := LoadCodeEvaluator("tone", path) + require.Error(t, err) + require.Contains(t, err.Error(), "top-level") +} + +func TestLoadCodeEvaluator_RejectsAFolder(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "tone.py", gradeSource) + + _, err := LoadCodeEvaluator("tone", dir) + require.Error(t, err) + require.Contains(t, err.Error(), "single") +} + +func TestLoadCodeEvaluator_RejectsBadInput(t *testing.T) { + dir := t.TempDir() + + _, err := LoadCodeEvaluator("", writeFile(t, dir, "tone.py", gradeSource)) + require.Error(t, err, "a name is required to publish under") + + _, err = LoadCodeEvaluator("tone", "") + require.Error(t, err) + + _, err = LoadCodeEvaluator("tone", filepath.Join(dir, "absent.py")) + require.Error(t, err) + + _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "rubric.json", "{}")) + require.Error(t, err, "a rubric is not a code evaluator") + require.Contains(t, err.Error(), ".py") + + _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "empty.py", " \n\n")) + require.Error(t, err) + require.Contains(t, err.Error(), "empty") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 1748b98c0b4..336c9b17bdf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -34,10 +34,11 @@ type DatasetDecl struct { // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly // from an eval and never declared here. // -// Source decides which kind of evaluator this is: a folder holds Python and -// publishes a code evaluator, a file holds a rubric. The distinction is made -// by stat-ing the path on deploy, not by its spelling, so a folder does not -// need a trailing separator to be recognised. +// Source decides which kind of evaluator this is, by extension: a `.py` file is +// a single self-contained Python script and publishes a code evaluator, a +// `.json` file holds a rubric. A code evaluator cannot name a folder — it runs +// as a python grader, which is handed one script's source and cannot import a +// helper module beside it. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 3e68eccd52c..f4902f236a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -13,8 +13,6 @@ import ( "path/filepath" "strings" - "azureaieval/internal/pkg/evalcore" - "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/azure/azure-dev/cli/azd/pkg/foundry" "google.golang.org/protobuf/types/known/structpb" @@ -305,7 +303,8 @@ func resolveSource(baseDir, source string) string { // content changed without downloading anything from the service. // // The dataset API returns no content hash or etag, so comparing against the -// service would mean downloading the blob on every deploy. +// service would mean downloading the blob on every deploy. Every artifact this +// applies to — a dataset, a rubric, an evaluator script — is a single file. func Fingerprint(path string) (string, error) { data, err := os.ReadFile(path) if err != nil { @@ -315,22 +314,6 @@ func Fingerprint(path string) (string, error) { return hex.EncodeToString(sum[:]), nil } -// FingerprintPath hashes whatever a declared source names. -// -// A rubric evaluator is one JSON file; a code evaluator is a folder of Python. -// Both need change detection with the same meaning, so the artifact's shape is -// resolved by stat-ing it rather than by asking the caller to know. -func FingerprintPath(path string) (string, error) { - info, err := os.Stat(path) - if err != nil { - return "", fmt.Errorf("hashing %q: %w", path, err) - } - if info.IsDir() { - return evalcore.FingerprintCodeFolder(path) - } - return Fingerprint(path) -} - // FingerprintGroup hashes an eval's own declaration. // // Change detection on upstream artifacts is not sufficient: editing a group's diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go new file mode 100644 index 00000000000..fbdfe1b2b87 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package live + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/require" +) + +// These tests close the gap between publishing a code evaluator and using one. +// +// The publish path was verified on its own first, and passing that proves less +// than it appears to: a script publishes cleanly while carrying no +// data_schema, because only metrics are defaulted. Nothing then tells the +// caller that the evaluator cannot be wired into an eval. The criteria builder +// derives data_mapping from the schema the evaluator publishes, so no schema +// means no mapping, and the service refuses a criterion with none. That +// failure would surface at run time, long after the publish that caused it. +// +// So there are two tests. The first runs a code evaluator end to end and +// requires it to score a sample. The second publishes without a schema and +// records what the service actually does about it, rather than leaving the +// consequence to inference. + +// writeCodeEvaluator writes an evaluator that scores the length of a response. +// +// It is one self-contained script. A code evaluator runs as an OpenAI python +// grader, whose contract is a single Source string with one entry point: a +// top-level grade(sample, item). There is no package and no import path, so a +// helper module beside it could not be reached even if it were published. +func writeCodeEvaluator(t *testing.T, name string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, name+".py") + + source := `def grade(sample, item) -> float: + response = (item or {}).get("response", "") + return float(len(response)) +` + require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) + return path +} + +// dataSchemaForResponse is the schema the criteria builder needs to derive a +// data_mapping. Only the caller can supply it: nothing about it is inferable +// from Python source. +func dataSchemaForResponse(t *testing.T) json.RawMessage { + t.Helper() + raw, err := json.Marshal(map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + "required": []string{"response"}, + }) + require.NoError(t, err) + return raw +} + +// publishCodeEvaluator registers the script and returns the version, after +// confirming it is readable. +func publishCodeEvaluator( + t *testing.T, + env *liveEnv, + name string, + withSchema bool, +) *eval_api.EvaluatorVersion { + t.Helper() + ctx := context.Background() + + path := writeCodeEvaluator(t, name) + script, err := evalcore.LoadCodeEvaluator(name, path) + require.NoError(t, err, "loading the evaluator script") + + var opts eval_api.CodeEvaluatorOptions + if withSchema { + opts.DataSchema = dataSchemaForResponse(t) + opts.Metrics = json.RawMessage( + `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`) + } + + version, err := env.evalClient.CreateCodeEvaluatorVersion(ctx, script, opts, projectAPIVersion) + require.NoError(t, err, "publishing the code evaluator") + t.Cleanup(func() { + _ = env.evalClient.DeleteEvaluatorVersion( + context.Background(), name, version.Version, projectAPIVersion) + }) + t.Logf("published code evaluator %s version %s", name, version.Version) + + awaitEvaluatorResolvable(t, env, name, version.Version) + return version +} + +// awaitEvaluatorResolvable waits for a published version the way the +// reconciler does, and reports how long each of the two views took. +// +// The numbers are the point. The direct read goes consistent almost at once +// while the listing lags it, and the eval-create resolver follows the slower +// one: a publish was observed reading back at 03:06:58 and still failing eval +// creation at 03:06:59. Logging both is what keeps the reconciler's tolerance +// honest instead of guessed, and asserting on the listing here is what proves +// the gate it waits on is the right one. +func awaitEvaluatorResolvable(t *testing.T, env *liveEnv, name, version string) { + t.Helper() + ctx := context.Background() + + start := time.Now() + var readable time.Duration + + for { + if readable == 0 { + if _, err := env.evalClient.GetEvaluatorRaw( + ctx, name, version, projectAPIVersion, + ); err == nil { + readable = time.Since(start) + t.Logf("evaluator %s readable after %s", + name, readable.Round(time.Millisecond)) + } + } + if readable != 0 && evaluatorVersionListed(ctx, env, name, version) { + t.Logf("evaluator %s listed after %s", + name, time.Since(start).Round(time.Millisecond)) + return + } + if time.Since(start) > 2*time.Minute { + t.Fatalf("evaluator %s never became resolvable", name) + } + time.Sleep(250 * time.Millisecond) + } +} + +func evaluatorVersionListed( + ctx context.Context, + env *liveEnv, + name, version string, +) bool { + list, err := env.evalClient.ListEvaluatorVersions(ctx, name, projectAPIVersion) + if err != nil || list == nil { + return false + } + for _, entry := range list.Value { + if entry.Version == version { + return true + } + } + return false +} + +// createEvalReferencing creates an eval naming a custom evaluator, tolerating +// the window in which the evaluator is published but not yet resolvable. +// +// The delay is reported so a run that hits it leaves evidence of how long it +// took, which is the number the reconciler's own tolerance has to be built on. +func createEvalReferencing( + t *testing.T, + env *liveEnv, + req *eval_api.CreateOpenAIEvalRequest, + within time.Duration, +) (*eval_api.OpenAIEval, error) { + t.Helper() + ctx := context.Background() + + start := time.Now() + for { + group, err := env.evalClient.CreateOpenAIEval(ctx, req) + if err == nil { + t.Logf("eval accepted the evaluator after %s", time.Since(start).Round(time.Millisecond)) + return group, nil + } + if !strings.Contains(strings.ToLower(err.Error()), "was not found") { + return nil, err + } + if time.Since(start) > within { + t.Logf("the evaluator was still unresolvable after %s", within) + return nil, err + } + time.Sleep(5 * time.Second) + } +} + +// TestLiveCodeEvaluatorScoresARun is the test the publish tests could not be: +// it requires the evaluator to actually run and return a score. +// +// No agent is involved. A code evaluator reads item fields, so the run uses a +// dataset-only source and the rows are supplied inline. That keeps the test +// about the evaluator rather than about a target being reachable. +func TestLiveCodeEvaluatorScoresARun(t *testing.T) { + env := setup(t) + ctx := context.Background() + + name := strings.ReplaceAll(uniqueName("azdcoderun"), "-", "_") + publishCodeEvaluator(t, env, name, true) + + group, err := createEvalReferencing(t, env, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-code-eval"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: name, + EvaluatorName: name, + DataMapping: map[string]string{"response": "{{item.response}}"}, + }}, + }, 3*time.Minute) + require.NoError(t, err, "creating an eval that references the code evaluator") + t.Logf("created eval %s", group.ID) + + ds := eval_api.NewDatasetOnlyDataSource() + ds.SetFileContent([]map[string]any{ + {"response": "a short answer"}, + {"response": "a considerably longer answer than the first one"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("code-run"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run") + t.Cleanup(func() { + _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) + }) + t.Logf("started run %s", run.ID) + + final := awaitRun(t, env, group.ID, run.ID, 10*time.Minute) + + // Reaching a terminal state is not the same as having evaluated anything: + // a run whose every sample errors still reports completed. + require.Equal(t, "completed", strings.ToLower(final.Status), + "the run must complete rather than fail or cancel") + require.NotNil(t, final.ResultCounts, "a completed run must report counts") + require.Zero(t, final.ResultCounts.Errored, + "an errored sample means the code evaluator did not run") + require.Positive(t, final.ResultCounts.Passed+final.ResultCounts.Failed, + "the run must score at least one sample; scoring nothing means the rows "+ + "never reached the evaluator") +} + +// TestLiveCodeEvaluatorWithoutSchemaIsAccepted pins down what happens to a +// script published with no data_schema, which is the shape most people's +// their first evaluator will produce. +// +// It was expected to be refused. The reasoning was that the criteria builder +// derives data_mapping from the evaluator's schema, so no schema means no +// mapping, and the service rejects a criterion with none. An earlier run +// appeared to confirm it. That was wrong: the refusal was the propagation 404 +// in disguise, read as a mapping error because it arrived at the same call. +// With the publish properly gated, a schema-less evaluator is accepted and an +// empty data_mapping is allowed, so this is not the trap it looked like. +// +// The create is deliberately not retried. publishCodeEvaluator has already +// waited on the same condition the reconciler waits on, so a "was not found" +// here would mean that gate is the wrong one — which is worth failing on, +// because a retry would hide it. +func TestLiveCodeEvaluatorWithoutSchemaIsAccepted(t *testing.T) { + env := setup(t) + ctx := context.Background() + + name := strings.ReplaceAll(uniqueName("azdcodenoschema"), "-", "_") + published := publishCodeEvaluator(t, env, name, false) + require.NotEmpty(t, published.Version, + "a script with no schema still publishes; the schema is not required to register") + + _, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-code-eval-noschema"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: name, + EvaluatorName: name, + // Deliberately empty: this is what the criteria builder produces + // for an evaluator that publishes no data_schema. + DataMapping: map[string]string{}, + }}, + }) + + if err == nil { + t.Log("an evaluator with no data_schema was accepted with an empty data_mapping; " + + "publishing without a schema is not by itself a blocker") + return + } + require.NotContains(t, strings.ToLower(err.Error()), "was not found", + "the evaluator was published and waited for, so a not-found here means the "+ + "propagation gate the reconciler uses does not cover eval creation") + t.Logf("an evaluator with no data_schema was refused: %v", err) + require.Contains(t, strings.ToLower(err.Error()), "mapping", + "the refusal should name the mapping, so the CLI can explain it at publish time") +} + +// awaitRun polls until the run reaches a terminal state or the deadline passes. +func awaitRun( + t *testing.T, + env *liveEnv, + evalID string, + runID string, + within time.Duration, +) *eval_api.OpenAIEvalRun { + t.Helper() + ctx := context.Background() + + terminal := map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, + } + deadline := time.Now().Add(within) + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) + require.NoError(t, err, "polling the run") + if terminal[strings.ToLower(current.Status)] { + if current.ResultCounts != nil { + t.Logf("run %s reached %s: passed=%d failed=%d errored=%d", + runID, current.Status, + current.ResultCounts.Passed, + current.ResultCounts.Failed, + current.ResultCounts.Errored) + } + body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") + t.Logf("per-criteria results: %s", string(body)) + return current + } + if time.Now().After(deadline) { + t.Fatalf("run %s did not finish within %s (last status %q)", + runID, within, current.Status) + } + time.Sleep(10 * time.Second) + } +} From 53c08cb83a0678d5e459a49b60cca034e5c1d08d Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 20:54:23 +0300 Subject: [PATCH 064/114] Cover schedules end to end, and explain the role they need Schedules had eight unit tests, all of them about buildTrigger's output. Whether the service accepts that output was untested, so every trigger shape the CLI can emit is now sent to a real project and required to survive a round trip, alongside create/read/list/delete, --disabled, and the edit the CLI refuses to make. Running them found a prerequisite nothing else in the extension needs. A schedule fires later and runs as the project, not as the caller, so the project's managed identity must hold the Foundry User role on the project. Without it every schedule creation is refused with PermissionDenied, and the raw message does not say why creating a schedule needs a permission that running an eval does not. explainScheduleFailure now says so. The tests skip rather than fail when the role is absent, since that is an environment fact rather than a regression, and the skip names the missing role. The trigger-shape test probes once before its subtests: a parent reporting PASS while every subtest skipped is worse than a failure, because it looks like coverage. Two things the tests had to get right to be worth anything. The eval they schedule is built with the shipping builder, not a hand-written criterion - built-ins do not share an input contract, and the one the listing returns first, builtin.ifeval, needs an instruction_id_list, so a hand-rolled mapping is rejected. And no evaluator is named: which built-ins a project exposes varies, so the first one the builder can satisfy is used. Cleanup waits for a schedule to settle before deleting it. One still being created refuses the delete as a 409, or as a 404 because the trigger behind it does not exist yet, and a project holds very few, so a leaked schedule would break every later run. --- .../internal/cmd/schedule.go | 16 + .../internal/cmd/schedule_live_test.go | 397 ++++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go index 5a8affe44bd..e3ce51edee8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go @@ -605,6 +605,15 @@ func explainScheduleFailure( "once against its dataset first so the schedule repeats that instead", name) } + if isScheduleRoleMissing(cause) { + return fmt.Errorf( + "saving schedule %q: the project's managed identity needs the Foundry User "+ + "role on the project before it can run scheduled evaluations. A schedule "+ + "runs later, as the project rather than as you, which is why creating one "+ + "needs a role that running an eval yourself does not. Grant it on the "+ + "project and retry", name) + } + list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) if listErr != nil || list == nil { return fmt.Errorf("saving schedule %q: %w", name, cause) @@ -627,3 +636,10 @@ func isTracesHourlyOnly(err error) bool { return err != nil && strings.Contains(err.Error(), "trace evaluations only support hourly") } + +// isScheduleRoleMissing matches the refusal when the project identity cannot +// run the evaluation the schedule would trigger. +func isScheduleRoleMissing(err error) bool { + return err != nil && + strings.Contains(err.Error(), "lacks Foundry User role") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go new file mode 100644 index 00000000000..ebd8da98570 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go @@ -0,0 +1,397 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cmd + +import ( + "context" + "fmt" + "sort" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// Schedules are project-scoped and the service tolerates very few of them, so +// everything here runs serially and deletes what it creates. Nothing calls +// t.Parallel(). +// +// These tests exist because the unit tests only prove that buildTrigger +// produces a well-formed struct. Whether the service accepts that struct is a +// different question, and it is the one that matters: every trigger shape the +// CLI can emit is sent here and required to survive a round trip. + +// liveScheduleEval creates a throwaway eval for a schedule to point at. +// +// The criterion comes from the shipping builder rather than a hand-written +// one. Built-ins do not share an input contract — builtin.ifeval, which is +// what the listing happens to return first, needs an instruction_id_list — so +// a hand-rolled mapping is rejected with MissingRequiredDataMapping. Letting +// production shape the request also means this helper cannot drift from it. +// +// No evaluator is named. Which built-ins a project exposes varies, so the +// first one the builder can satisfy is used, and the dataset is given every +// column that evaluator declares. +func liveScheduleEval(t *testing.T, client *eval_api.EvalClient, judge string) string { + t.Helper() + ctx := context.Background() + + ec := &evalContext{evalClient: client} + schemas := ec.evaluatorSchemas(ctx) + require.NotEmpty(t, schemas, "need the published evaluator contracts to build an eval") + + names := make([]string, 0, len(schemas)) + for name := range schemas { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + summary := schemas[name] + columns := map[string]bool{"query": true} + if ds := summary.DataSchema(); ds != nil { + for _, col := range ds.PropertyNames() { + columns[col] = true + } + } + + level := "" + if len(summary.SupportedEvaluationLevels) > 0 { + level = summary.SupportedEvaluationLevels[0] + } + + req, err := buildEvalRequest(&project.Eval{ + Name: fmt.Sprintf("azd-sched-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, + Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, + }, schemas, columns) + if err != nil { + continue + } + + created, err := client.CreateOpenAIEval(ctx, req) + if err != nil { + t.Logf("built-in %s could not back a schedule: %v", name, err) + continue + } + t.Cleanup(func() { + _ = client.DeleteOpenAIEval(context.Background(), created.ID) + }) + t.Logf("scheduling an eval built on %s (%s)", name, created.ID) + return created.ID + } + + t.Fatalf("no built-in produced an eval a schedule could run; tried %d", len(names)) + return "" +} + +// putLiveSchedule creates a schedule and registers its removal. +func putLiveSchedule( + t *testing.T, + client *eval_api.EvalClient, + name string, + evalID string, + trigger *eval_api.ScheduleTrigger, +) (*eval_api.Schedule, error) { + t.Helper() + + saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ + DisplayName: name, + Enabled: true, + Trigger: trigger, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: datasetOnlyRows([]map[string]any{{"query": "how do I reset my password?"}}), + }, + }, + }, ProjectEndpointAPIVersion) + + if err == nil { + t.Cleanup(func() { removeLiveSchedule(t, client, name) }) + } + return saved, err +} + +// skipIfScheduleRoleMissing stops the test when the project cannot host a +// schedule at all. +// +// A schedule fires later and runs as the project, so creating one requires the +// project's managed identity to hold the Foundry User role on the project — +// a permission no other command in this extension needs. Without it every +// schedule test fails identically and for a reason that has nothing to do with +// the code, so they skip loudly instead of reporting a false regression. +func skipIfScheduleRoleMissing(t *testing.T, err error) { + t.Helper() + if err == nil || !isScheduleRoleMissing(err) { + return + } + t.Skipf("this project cannot host schedules: its managed identity lacks the "+ + "Foundry User role on the project. Grant it and re-run to exercise "+ + "schedules for real. Underlying error: %v", err) +} + +// datasetOnlyRows is the inline data source a scheduled run repeats. +func datasetOnlyRows(rows []map[string]any) *eval_api.EvalRunDataSource { + ds := eval_api.NewDatasetOnlyDataSource() + ds.SetFileContent(rows) + return ds +} + +// removeLiveSchedule deletes a schedule once it is no longer provisioning. +// +// A schedule still being created refuses the delete, as a 409 while it is busy +// or a 404 because the trigger behind it does not exist yet. Waiting for it to +// settle is what makes cleanup reliable, and leaving one behind would break +// every later test in this file, because the project holds very few. +func removeLiveSchedule(t *testing.T, client *eval_api.EvalClient, name string) { + t.Helper() + ctx := context.Background() + + deadline := time.Now().Add(2 * time.Minute) + for { + current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return // already gone + } + if current.Settled() { + break + } + if time.Now().After(deadline) { + t.Logf("schedule %q never settled (status %q); leaving it", name, current.ProvisioningStatus) + return + } + time.Sleep(3 * time.Second) + } + + if err := client.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion); err != nil { + t.Logf("could not delete schedule %q: %v", name, err) + } +} + +// awaitScheduleSettled blocks until the schedule finishes provisioning. +func awaitScheduleSettled( + t *testing.T, + client *eval_api.EvalClient, + name string, +) *eval_api.Schedule { + t.Helper() + ctx := context.Background() + + deadline := time.Now().Add(2 * time.Minute) + for { + current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + require.NoError(t, err, "reading schedule %q back", name) + if current.Settled() { + return current + } + if time.Now().After(deadline) { + t.Fatalf("schedule %q stuck in %q", name, current.ProvisioningStatus) + } + time.Sleep(3 * time.Second) + } +} + +func liveScheduleName(t *testing.T, suffix string) string { + t.Helper() + return fmt.Sprintf("azdsched-%s-%d", suffix, time.Now().UnixNano()) +} + +// TestLiveScheduleLifecycle walks create, read, list and delete. +func TestLiveScheduleLifecycle(t *testing.T) { + client, judge := liveEvalClient(t) + ctx := context.Background() + + evalID := liveScheduleEval(t, client, judge) + name := liveScheduleName(t, "life") + + trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) + require.NoError(t, err) + + saved, err := putLiveSchedule(t, client, name, evalID, trigger) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err, "the service rejected a trigger the CLI can produce") + require.NotEmpty(t, saved.ID) + + settled := awaitScheduleSettled(t, client, name) + require.True(t, settled.Enabled, "a schedule created without --disabled must be enabled") + require.NotNil(t, settled.Task, "the schedule must carry the task it runs") + require.Equal(t, evalID, settled.Task.EvalID, + "the schedule must point at the eval it was given") + require.NotNil(t, settled.Trigger) + require.Equal(t, eval_api.TriggerRecurrence, settled.Trigger.Type) + + list, err := client.ListSchedules(ctx, ProjectEndpointAPIVersion) + require.NoError(t, err) + found := false + for i := range list.Value { + if list.Value[i].ID == saved.ID || list.Value[i].DisplayName == name { + found = true + break + } + } + require.True(t, found, "a created schedule must appear in the listing") + + removeLiveSchedule(t, client, name) + + _, err = client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) + require.Error(t, err, "a deleted schedule must not read back") + require.True(t, eval_api.IsNotFound(err), + "deleting should leave a not-found, got %v", err) +} + +// TestLiveScheduleAcceptsEveryTriggerShape sends one schedule per trigger the +// CLI can build and requires the service to accept each. +// +// The unit tests assert the shape of what buildTrigger returns. They cannot +// say whether the service agrees, and a trigger the service rejects is a +// trigger the CLI should never have offered. +func TestLiveScheduleAcceptsEveryTriggerShape(t *testing.T) { + client, judge := liveEvalClient(t) + evalID := liveScheduleEval(t, client, judge) + + // Probe once up front. Skipping inside the subtests instead would leave the + // parent reporting PASS with nothing proven, which is worse than a failure + // because it looks like coverage. + probe := liveScheduleName(t, "probe") + daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{4}}) + require.NoError(t, err) + if _, probeErr := putLiveSchedule(t, client, probe, evalID, daily); probeErr != nil { + skipIfScheduleRoleMissing(t, probeErr) + require.NoError(t, probeErr, "could not create the probe schedule") + } + removeLiveSchedule(t, client, probe) + + cases := []struct { + label string + flags triggerFlags + want string + }{ + {"cron", triggerFlags{cron: "0 9 * * *"}, eval_api.TriggerCron}, + {"hourly", triggerFlags{every: "hourly"}, eval_api.TriggerRecurrence}, + {"daily", triggerFlags{every: "daily", atHours: []int{9, 17}}, eval_api.TriggerRecurrence}, + {"weekly", triggerFlags{every: "weekly", onDays: []string{"Monday"}}, eval_api.TriggerRecurrence}, + {"monthly", triggerFlags{every: "monthly", onDaysOfMon: []int{1}}, eval_api.TriggerRecurrence}, + {"interval", triggerFlags{every: "daily", interval: 3}, eval_api.TriggerRecurrence}, + { + "onetime", + triggerFlags{atTime: time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339)}, + eval_api.TriggerOneTime, + }, + } + + for _, tc := range cases { + t.Run(tc.label, func(t *testing.T) { + trigger, err := buildTrigger(tc.flags) + require.NoError(t, err, "the CLI could not build a %s trigger", tc.label) + require.Equal(t, tc.want, trigger.Type) + + name := liveScheduleName(t, tc.label) + saved, err := putLiveSchedule(t, client, name, evalID, trigger) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err, "the service rejected the %s trigger", tc.label) + require.NotEmpty(t, saved.ID) + + settled := awaitScheduleSettled(t, client, name) + require.Equal(t, tc.want, settled.Trigger.Type, + "the trigger type must survive the round trip") + removeLiveSchedule(t, client, name) + }) + } +} + +// TestLiveScheduleDisabledStaysDisabled covers --disabled, which is the one +// flag whose whole purpose is a field the service could quietly ignore. +func TestLiveScheduleDisabledStaysDisabled(t *testing.T) { + client, judge := liveEvalClient(t) + evalID := liveScheduleEval(t, client, judge) + name := liveScheduleName(t, "disabled") + + trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{3}}) + require.NoError(t, err) + + saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ + DisplayName: name, + Enabled: false, + Trigger: trigger, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), + }, + }, + }, ProjectEndpointAPIVersion) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err) + t.Cleanup(func() { removeLiveSchedule(t, client, name) }) + require.NotEmpty(t, saved.ID) + + settled := awaitScheduleSettled(t, client, name) + require.False(t, settled.Enabled, + "a schedule created disabled must not come back enabled") +} + +// TestLiveScheduleEditIsRefusedByTheCLI pins the reason `schedule set` refuses +// to reuse a name. +// +// The service takes a PUT on an existing schedule and does not apply it, and a +// replacement can stick in Creating where it can no longer be deleted. The CLI +// therefore refuses before sending. This test records the service behaviour +// the guard exists for, so a change in the service is visible here rather than +// as a stuck schedule in someone's project. +func TestLiveScheduleEditIsRefusedByTheCLI(t *testing.T) { + client, judge := liveEvalClient(t) + evalID := liveScheduleEval(t, client, judge) + name := liveScheduleName(t, "edit") + + daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) + require.NoError(t, err) + _, err = putLiveSchedule(t, client, name, evalID, daily) + skipIfScheduleRoleMissing(t, err) + require.NoError(t, err) + awaitScheduleSettled(t, client, name) + + // The guard in `schedule set` is a GetSchedule that must find this. + existing, err := client.GetSchedule(context.Background(), name, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, existing.ID, + "the CLI decides a name is taken by reading it back, so this must be non-empty") + + weekly, err := buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Friday"}}) + require.NoError(t, err) + _, putErr := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ + DisplayName: name, + Enabled: true, + Trigger: weekly, + Task: &eval_api.ScheduleTask{ + Type: eval_api.ScheduleTaskEvaluation, + EvalID: evalID, + EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ + Name: name, + DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), + }, + }, + }, ProjectEndpointAPIVersion) + + if putErr != nil { + t.Logf("the service refused the edit outright: %v", putErr) + return + } + + after := awaitScheduleSettled(t, client, name) + t.Logf("after editing daily -> weekly the schedule reads back as %q", after.Summary()) + require.Equal(t, eval_api.TriggerRecurrence, after.Trigger.Type) +} From 423035b29fb4779a95bd1b6431020a59218dcd07 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 20:59:35 +0300 Subject: [PATCH 065/114] Move scheduling out of M1 onto its own branch Scheduling works and is now live-tested, but it widens the first release past the loop that matters: define an eval, run it, read the results. It moves to feat/azure-ai-evaluations-schedule, branched from the commit that added its coverage, so it is parked in a known-good state rather than an unverified one. Re-adding it is four files and one line - schedule.go, schedule_test.go, schedule_live_test.go, eval_api/schedules.go, and newScheduleCommand() in root.go. Nothing else ever referenced it. The one entanglement is undone here rather than later: schedules.go also held IsNotFound and IsConflict, and IsNotFound is used by results.go and run_ops.go. Both are generic HTTP predicates with nothing to do with schedules, so they move to eval_api/errors.go, where re-adding schedules will not bring them back. Also add COMMANDS.md: every command, its parameters, an example, and how far each one is actually verified. It records two things the test suite cannot say for itself - no live test drives a CLI command, they all exercise the client layer; and TestLiveRun skips unless AZURE_AI_EVAL_AGENT is set, so agent-target runs are unverified. --- .../azure.ai.evaluations/COMMANDS.md | 193 ++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 - .../internal/cmd/schedule.go | 645 ------------------ .../internal/cmd/schedule_live_test.go | 397 ----------- .../internal/cmd/schedule_test.go | 168 ----- .../internal/pkg/eval_api/errors.go | 29 + .../internal/pkg/eval_api/schedules.go | 184 ----- 7 files changed, 222 insertions(+), 1395 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/COMMANDS.md delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go diff --git a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md new file mode 100644 index 00000000000..5f58e75f030 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md @@ -0,0 +1,193 @@ +# `azd ai eval` — command reference and verification status + +Generated from the built binary at commit `d776507a3`, plus a live run against a real +Foundry project. Not a design document: this records what exists **today** and how much +of it is actually proven. + +## How to read the status column + +| Status | Meaning | +|---|---| +| **LIVE** | A committed live test exercises this path against the real service. | +| **PARTIAL** | The underlying API call is exercised live, but not every flag or branch is. | +| **UNIT** | Unit tests only. No service call is made by this command, or none is covered. | +| **NONE** | No automated coverage. Manually tried at some point, or never run. | + +Two caveats that apply to the whole table, and that the column cannot express: + +1. **No live test drives a CLI command.** Every live test calls the client layer + (`evalClient.CreateOpenAIEval`, `datasetClient.UploadNewVersion`, …) directly. Flag + parsing, prompting, `--no-prompt`, `-o json` rendering and the table output are + covered by unit tests only. So "LIVE" means *the API path this command uses* works, + not that the command itself was run. +2. **`TestLiveRun` skips unless `AZURE_AI_EVAL_AGENT` is set.** It is not set in normal + runs, so the **agent-target** run path is unverified. The dataset-only run path *is* + verified, by `TestLiveCodeEvaluatorScoresARun`. + +Last full live run: **173 passed, 1 skipped, 0 failed**. + +## Global flags + +Available on every command. + +| Flag | Description | +|---|---| +| `-C, --cwd ` | Set the working directory. | +| `--debug` | Debug and diagnostics logging. | +| `-e, --environment ` | azd environment to use. | +| `--no-prompt` | Never prompt. Fails if a required value cannot be resolved. | +| `-o, --output ` | Output format; `json` emits machine-readable output. | + +Most service-touching commands also take `--project-endpoint ` to override the +endpoint resolved from the azd environment. + +## Composite commands + +| Command | Description | Key params | Status | +|---|---|---|---| +| `init` | Scaffold `evals/azure.yaml` + `evals/eval_generate.yaml`. **Makes no service calls.** | `--target`, `--dataset`, `--evaluator` (repeatable), `--judge-model`, `--out-dir` (default `evals`), `--force` | UNIT | +| `generate` | Run the generation jobs, download the rubric and dataset, write `source:` refs into the deploy spec. | `--config` (default `evals/eval_generate.yaml`), `--deploy-config` (default `evals/azure.yaml`), `--target`, `--generation-model`, `--max-samples` (15–1000), `--trace-days`, `--agent-instruction[-file]`, `--dataset`, `--evaluator`, `--no-wait` | NONE | +| `run` | Run an evaluation, creating the eval if it does not exist. | `--config`, `--eval`, `--eval-id`, `--name`, `--level`, `--max-samples`, `--from-traces`, `--trace-window`, `--max-traces`, `--response-id`, `--max-turns`, `--wait` (default true), `--no-wait` | PARTIAL | + +`run` example: + +```console +$ azd ai eval run +Started run evalrun_1f3f909b... on eval eval_9cd479cc... +run reached completed: passed=2 failed=0 errored=0 +``` + +`run` is PARTIAL because the dataset-only path is live-proven while the agent target, +`--from-traces` and `--response-id` are not. + +## `dataset` + +| Command | Description | Key params | Status | +|---|---|---|---| +| `dataset create` | Register a dataset, publishing a new version. | `--name`, `--file` (a `.jsonl` or a directory containing one), `--version` | LIVE | +| `dataset list` | List datasets, or the versions of one. | `--name` | PARTIAL | +| `dataset show` | Show a dataset version. | `--name`, `--version` (omit for latest) | PARTIAL | +| `dataset delete` | Delete a dataset version. | `--name`, `--version` | PARTIAL | + +```console +$ azd ai eval dataset list +NAME VERSION FORMAT URI +support-golden 3 jsonl azureml://.../support-golden/versions/3 +``` + +`TestLiveDatasetLifecycle` covers create, version increment, listing and delete through +the client — hence PARTIAL for the read/delete commands rather than LIVE. + +## `evaluator` + +| Command | Description | Key params | Status | +|---|---|---|---| +| `evaluator create` (rubric) | Register a rubric evaluator. | `--name`, `--rubric ` | NONE | +| `evaluator create` (code) | Register a code evaluator from a **single Python script**. | `--name`, `--file `, `--image-tag`, `--init-params`, `--data-schema`, `--metrics` | LIVE | +| `evaluator list` | List the project's evaluators, versions of one, or the built-ins. | `--name`, `--builtin` | PARTIAL | +| `evaluator show` | Show an evaluator definition. | `--name`, `--version` | NONE | +| `evaluator delete` | Delete an evaluator version. | `--name`, `--version` | PARTIAL | + +```console +$ azd ai eval evaluator create --name answer_length --file ./answer_length.py +Published evaluator answer_length version 1 + +$ azd ai eval evaluator list --builtin +NAME VERSION TYPE +builtin.groundedness 16 builtin +builtin.relevance 12 builtin +``` + +A code evaluator script must declare a **top-level `grade(sample, item)`** returning a +float. It runs as an OpenAI python grader, which receives the script source and nothing +else — there is no import path, so a helper module beside the script cannot be imported. +Dependencies come from `--image-tag`. + +`--rubric`, `evaluator show` are NONE: no live test publishes a rubric or reads a +definition back through them. +`--image-tag` reaches the definition and round-trips, but has **never been exercised +against a real custom image**. + +## `run` subcommands + +| Command | Description | Key params | Status | +|---|---|---|---| +| `run start` | Start a run, creating the eval if needed. Same flags as `run`. | as `run` | PARTIAL | +| `run list` | List runs for an eval. | `[eval-id]`, `--eval`, `--eval-id`, `--limit` | NONE | +| `run show` | Show one run. | `[eval-id]`, `--run-id` (defaults to most recent) | PARTIAL | +| `run cancel` | Cancel an in-flight run. | `[eval-id]`, `--run-id` | PARTIAL | +| `run delete` | Delete a run. | `[eval-id]`, `--run-id` | NONE | + +```console +$ azd ai eval run list +RUN ID NAME STATUS RESULTS +evalrun_1f3f909b... pr-gate-1785370812 completed 2 passed, 0 failed, 0 errored +``` + +Every command taking an eval id accepts it as the argument, as `--eval-id `, or as +`--eval ` to name one from the config. + +## `results` + +| Command | Description | Key params | Status | +|---|---|---|---| +| `results show` | Per-sample results for a run (`output_items`). | ``, `--run-id`, `--failed-only`, `-O/--out-file` | NONE | +| `results export` | Export run results. | ``, `--run-id`, `--format json\|csv`, `-O/--out-file` | NONE | +| `results compare` | Compare runs against a baseline. | `[eval-id]`, `--baseline`, `--treatment` (repeatable), `--name` | NONE | + +The root help still lists `dataset`, `evaluator`, `generate`, `init`, `results` and +`run`. `schedule` is gone from it; see below. + +```console +$ azd ai eval results show +ITEM EVALUATOR RESULT SCORE INPUT REASON +1 answer_length pass 14.0 a short answer - +2 answer_length pass 46.0 a considerably long… - + +$ azd ai eval results compare +METRIC TREATMENT RUN BASELINE TREATMENT DELTA P-VALUE EFFECT +groundedness evalrun_a1b2… 3.80 4.20 +0.40 0.031 small +``` + +`--baseline` defaults to the second most recent completed run and `--treatment` to the +most recent. That auto-selection is untested against real run history. + +**TODO (April, spec review 2026-07-29):** `compare` and `export` belong at the **run** +level, not under `results` — *"compare is not at the items level… export should be at the +run level"*. `results show` should become `run output list`, paginating `output_items`. + +## `schedule` — not on this branch + +Scheduling is implemented and live-tested, but lives on +`feat/azure-ai-evaluations-schedule` rather than here. It is out of M1 so the first +release stays focused on the eval / run / results loop. + +Re-adding it is four files plus one line: `internal/cmd/schedule.go`, +`internal/cmd/schedule_test.go`, `internal/cmd/schedule_live_test.go`, +`internal/pkg/eval_api/schedules.go`, and `newScheduleCommand()` in `root.go`. +Nothing else ever referenced it — the two error helpers it used to carry, +`IsNotFound` and `IsConflict`, now live in `internal/pkg/eval_api/errors.go`, which +is where they belonged anyway. + +What the live tests on that branch establish: + +- Every trigger shape the CLI can emit — cron, hourly, daily, weekly, monthly, + interval, one-time — is accepted by the service and survives a round trip. +- **Schedules need a permission nothing else does.** A schedule fires later and runs + as the project, so the project's managed identity must hold the **Foundry User** + role on the project. Without it every create is refused with `PermissionDenied`. + The tests skip, naming the missing role, rather than reporting a false regression. +- Service constraints: one schedule per project; no in-place edits; a schedule + repeating a `--from-traces` run accepts only `--every hourly`. + +## Summary of gaps + +| Gap | Impact | +|---|---| +| No live test drives a CLI command | Flag parsing, prompting and rendering are unit-tested only | +| `compare` has no live coverage | Named in M1 exit criteria, unproven; needs two completed runs to test | +| `generate` has no live coverage | The most complex composite command | +| `TestLiveRun` skips | Agent-target runs, `--from-traces`, `--response-id` all unverified | +| `--image-tag` never used with a real image | The only supported way to give a code evaluator dependencies | +| `azd up` cannot configure a code evaluator | Reconciler passes empty options; no config fields for `data_schema`, `metrics`, `init_params`, `image_tag` | +| Default metric name `result` is invented | Real evaluators use semantic names (`groundedness`, `relevance`) | diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 118a831fdde..40724c5a0a7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -51,7 +51,6 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), - newScheduleCommand(), newListenCommand(), ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go deleted file mode 100644 index e3ce51edee8..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule.go +++ /dev/null @@ -1,645 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/project" - - "github.com/spf13/cobra" -) - -// newScheduleCommand groups the recurring-evaluation commands. -func newScheduleCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "schedule", - Short: "Run an eval on a schedule.", - } - cmd.AddCommand( - newScheduleSetCommand(), - newScheduleListCommand(), - newScheduleShowCommand(), - newScheduleDeleteCommand(), - ) - return cmd -} - -// newScheduleSetCommand creates the schedule that runs an eval. -// -// It does not update. The service accepts a PUT over an existing schedule, -// echoes the new body and keeps the old trigger, so an in-place edit would -// report a change that did not happen. Recreating under the same name is not -// an escape either: the replacement never leaves Creating and cannot then be -// deleted. So an existing schedule is refused, and changing one means deleting -// it and creating another under a different name. -func newScheduleSetCommand() *cobra.Command { - var ( - configPath string - groupName string - evalID string - name string - description string - cron string - every string - interval int - atHours []int - onDays []string - onDaysOfMon []int - atTime string - timezone string - startTime string - endTime string - disabled bool - level string - maxSamples int - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "set [eval-id]", - Short: "Create the schedule that runs an eval.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - if len(args) == 1 { - evalID = args[0] - } - - trigger, err := buildTrigger(triggerFlags{ - cron: cron, - every: every, - interval: interval, - atHours: atHours, - onDays: onDays, - onDaysOfMon: onDaysOfMon, - atTime: atTime, - timezone: timezone, - startTime: startTime, - endTime: endTime, - }) - if err != nil { - return err - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - // Same resolution as `run`: the config names the group unless - // --eval-id bypasses it, and the run payload carries the target - // and dataset because the group holds neither. - var group *project.Eval - var dataSource *eval_api.EvalRunDataSource - if evalID == "" { - cfg, err := project.LoadEvalConfig(configPath) - if err != nil { - return err - } - if err := cfg.Validate(); err != nil { - return err - } - group, err = cfg.ResolveGroup(groupName) - if err != nil { - return err - } - if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { - return err - } - evalID, err = ec.resolveEvalIDFromConfig( - ctx, group, configPath, resolveLevel(level, group), - len(cfg.Evals) == 1, out, isJSON(cmd)) - if err != nil { - return err - } - dataSource, err = ec.buildRunDataSource( - ctx, group, configPath, resolveMaxSamples(maxSamples, group)) - if err != nil { - return err - } - } else { - dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) - if err != nil { - return err - } - } - - if name == "" { - name = defaultScheduleName(group) - } - if description == "" { - description = fmt.Sprintf("Scheduled evaluation of %s.", evalID) - } - - // An existing schedule cannot be edited: the service takes the PUT - // and ignores it. Recreating under the same name is worse — the - // replacement sticks in Creating and cannot be deleted — so the - // only safe answer is a different name. - if existing, err := ec.evalClient.GetSchedule( - ctx, name, ProjectEndpointAPIVersion); err == nil && existing != nil && existing.ID != "" { - return fmt.Errorf( - "schedule %q already exists, and the service ignores edits to it. "+ - "Delete it with `azd ai eval schedule delete %s` and create the new "+ - "one under a different name; reusing this one leaves it stuck", - name, name) - } - - metadata := map[string]string{} - if lvl := resolveLevel(level, group); lvl != "" { - metadata["evaluation_level"] = lvl - } - - schedule := &eval_api.Schedule{ - DisplayName: name, - Description: description, - Enabled: !disabled, - Trigger: trigger, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: dataSource, - Metadata: metadata, - }, - }, - } - - saved, err := ec.evalClient.PutSchedule(ctx, name, schedule, ProjectEndpointAPIVersion) - if err != nil { - return explainScheduleFailure(ctx, ec, name, err) - } - - if isJSON(cmd) { - return emitJSON(out, saved) - } - state := "enabled" - if !saved.Enabled { - state = "disabled" - } - fmt.Fprintf(out, "Schedule %s (%s) runs %s on %s\n", - saved.ID, state, saved.Summary(), evalID) - return nil - }, - } - - cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, - "Path to the eval deployment config.") - cmd.Flags().StringVar(&groupName, "eval", "", "Which evals entry to schedule.") - cmd.Flags().StringVar(&evalID, "eval-id", "", "Schedule an existing eval by id, ignoring config.") - cmd.Flags().StringVar(&name, "name", "", "Schedule name. Defaults to the eval name.") - cmd.Flags().StringVar(&description, "description", "", "Schedule description.") - cmd.Flags().StringVar(&cron, "cron", "", `Cron expression, for example "0 9 * * *".`) - cmd.Flags().StringVar(&every, "every", "", - "Recur hourly, daily, weekly or monthly.") - cmd.Flags().IntVar(&interval, "interval", 0, "Repeat every N periods of --every. Defaults to 1.") - cmd.Flags().IntSliceVar(&atHours, "at", nil, "Hours of the day for --every daily, 0-23.") - cmd.Flags().StringSliceVar(&onDays, "on", nil, "Days of the week for --every weekly, for example Monday.") - cmd.Flags().IntSliceVar(&onDaysOfMon, "on-day", nil, "Days of the month for --every monthly, 1-31.") - cmd.Flags().StringVar(&atTime, "at-time", "", "Run once at this RFC3339 time.") - cmd.Flags().StringVar(&timezone, "timezone", "", "Timezone for the trigger. Defaults to UTC.") - cmd.Flags().StringVar(&startTime, "start-time", "", "RFC3339 time before which the schedule does not fire.") - cmd.Flags().StringVar(&endTime, "end-time", "", "RFC3339 time after which the schedule stops firing.") - cmd.Flags().BoolVar(&disabled, "disabled", false, "Create the schedule without enabling it.") - cmd.Flags().StringVar(&level, "level", "", "Evaluation level for the scheduled runs.") - cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap rows sent from a local dataset file.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - cmd.MarkFlagsMutuallyExclusive("cron", "every", "at-time") - - return cmd -} - -func newScheduleListCommand() *cobra.Command { - var endpointFlg string - - cmd := &cobra.Command{ - Use: "list", - Short: "List the project's schedules.", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - list, err := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("listing schedules: %w", err) - } - if isJSON(cmd) { - var schedules []eval_api.Schedule - if list != nil { - schedules = list.Value - } - return emitJSONList(out, schedules) - } - if list == nil || len(list.Value) == 0 { - fmt.Fprintln(out, "No schedules.") - return nil - } - - rows := make([][]string, 0, len(list.Value)) - for i := range list.Value { - s := &list.Value[i] - evalID := "" - if s.Task != nil { - evalID = s.Task.EvalID - } - rows = append(rows, []string{ - s.ID, - strconv.FormatBool(s.Enabled), - s.ProvisioningStatus, - s.Summary(), - evalID, - }) - } - return emitTable(out, - []string{"NAME", "ENABLED", "STATUS", "TRIGGER", "EVAL"}, rows) - }, - } - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newScheduleShowCommand() *cobra.Command { - var ( - name string - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "show [name]", - Short: "Show one schedule.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - if len(args) == 1 { - name = args[0] - } - if name == "" { - return requireFlag("name") - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("reading schedule %q: %w", name, err) - } - if isJSON(cmd) { - return emitJSON(out, s) - } - - fmt.Fprintf(out, "Schedule %s\n", s.ID) - fmt.Fprintf(out, " enabled: %t\n", s.Enabled) - fmt.Fprintf(out, " status: %s\n", s.ProvisioningStatus) - fmt.Fprintf(out, " trigger: %s\n", s.Summary()) - if s.Trigger != nil && s.Trigger.Timezone != "" { - fmt.Fprintf(out, " timezone: %s\n", s.Trigger.Timezone) - } - if s.Task != nil { - fmt.Fprintf(out, " eval: %s\n", s.Task.EvalID) - } - if s.Description != "" { - fmt.Fprintf(out, " about: %s\n", s.Description) - } - return nil - }, - } - cmd.Flags().StringVar(&name, "name", "", "Schedule name.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -func newScheduleDeleteCommand() *cobra.Command { - var ( - name string - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "delete [name]", - Short: "Delete a schedule.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - if len(args) == 1 { - name = args[0] - } - if name == "" { - return requireFlag("name") - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - if err := ec.deleteScheduleWhenSettled(ctx, name); err != nil { - // A name that was never there is the common typo, and the - // service answers it with a full error document wrapping an - // inner 404 from the trigger service. Saying so in one line is - // more use than reproducing that. - if eval_api.IsNotFound(err) { - return fmt.Errorf( - "no schedule named %q in this project; "+ - "`azd ai eval schedule list` shows the ones that exist", name) - } - return fmt.Errorf("deleting schedule %q: %w", name, err) - } - fmt.Fprintf(out, "Deleted schedule %s\n", name) - return nil - }, - } - cmd.Flags().StringVar(&name, "name", "", "Schedule name.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// triggerFlags carries the schedule's timing flags so buildTrigger can be -// tested without a command. -type triggerFlags struct { - cron string - every string - interval int - atHours []int - onDays []string - onDaysOfMon []int - atTime string - timezone string - startTime string - endTime string -} - -// buildTrigger turns the timing flags into the trigger the API expects. -func buildTrigger(f triggerFlags) (*eval_api.ScheduleTrigger, error) { - tz := f.timezone - if tz == "" { - tz = "UTC" - } - - switch { - case f.cron != "": - return &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerCron, - Expression: f.cron, - StartTime: f.startTime, - EndTime: f.endTime, - Timezone: tz, - }, nil - - case f.atTime != "": - if _, err := time.Parse(time.RFC3339, f.atTime); err != nil { - return nil, fmt.Errorf("--at-time %q is not an RFC3339 time", f.atTime) - } - return &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerOneTime, - ScheduledTime: f.atTime, - Timezone: tz, - }, nil - - case f.every != "": - pattern, err := buildRecurrence(f) - if err != nil { - return nil, err - } - interval := f.interval - if interval <= 0 { - interval = 1 - } - return &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerRecurrence, - Schedule: pattern, - Interval: interval, - StartTime: f.startTime, - EndTime: f.endTime, - Timezone: tz, - }, nil - } - - return nil, fmt.Errorf("a schedule needs a trigger: pass --cron, --every or --at-time") -} - -// buildRecurrence maps --every and its qualifiers onto a recurrence pattern. -// -// Each period reads only its own qualifier, so passing one that does not apply -// is rejected rather than dropped. -func buildRecurrence(f triggerFlags) (*eval_api.RecurrencePattern, error) { - period := strings.ToLower(strings.TrimSpace(f.every)) - - reject := func(flag, applies string) error { - return fmt.Errorf("--%s applies to --every %s, not %s", flag, applies, period) - } - - switch period { - case "hourly": - if len(f.atHours) > 0 { - return nil, reject("at", "daily") - } - if len(f.onDays) > 0 { - return nil, reject("on", "weekly") - } - if len(f.onDaysOfMon) > 0 { - return nil, reject("on-day", "monthly") - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceHourly}, nil - - case "daily": - if len(f.onDays) > 0 { - return nil, reject("on", "weekly") - } - if len(f.onDaysOfMon) > 0 { - return nil, reject("on-day", "monthly") - } - for _, h := range f.atHours { - if h < 0 || h > 23 { - return nil, fmt.Errorf("--at %d is not an hour of the day (0-23)", h) - } - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceDaily, Hours: f.atHours}, nil - - case "weekly": - if len(f.atHours) > 0 { - return nil, reject("at", "daily") - } - if len(f.onDaysOfMon) > 0 { - return nil, reject("on-day", "monthly") - } - days, err := normalizeDaysOfWeek(f.onDays) - if err != nil { - return nil, err - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly, DaysOfWeek: days}, nil - - case "monthly": - if len(f.atHours) > 0 { - return nil, reject("at", "daily") - } - if len(f.onDays) > 0 { - return nil, reject("on", "weekly") - } - for _, d := range f.onDaysOfMon { - if d < 1 || d > 31 { - return nil, fmt.Errorf("--on-day %d is not a day of the month (1-31)", d) - } - } - return &eval_api.RecurrencePattern{Type: eval_api.RecurrenceMonthly, DaysOfMonth: f.onDaysOfMon}, nil - } - - return nil, fmt.Errorf( - "--every %q is not a recurrence: use hourly, daily, weekly or monthly", f.every) -} - -// normalizeDaysOfWeek accepts day names in any casing and returns the spelling -// the service expects. -func normalizeDaysOfWeek(days []string) ([]string, error) { - if len(days) == 0 { - return nil, nil - } - canonical := map[string]string{} - for d := time.Sunday; d <= time.Saturday; d++ { - canonical[strings.ToLower(d.String())] = d.String() - } - - out := make([]string, 0, len(days)) - for _, raw := range days { - name, ok := canonical[strings.ToLower(strings.TrimSpace(raw))] - if !ok { - return nil, fmt.Errorf("--on %q is not a day of the week", raw) - } - out = append(out, name) - } - return out, nil -} - -// defaultScheduleName derives a schedule name from the group being scheduled. -func defaultScheduleName(group *project.Eval) string { - if group != nil && group.Name != "" { - return group.Name - } - return "eval-" + strconv.FormatInt(time.Now().UTC().Unix(), 10) -} - -// deleteScheduleWhenSettled removes a schedule, waiting out the window where -// the service is still provisioning it. -// -// A schedule that is mid-provision refuses the delete, and does it two ways: -// 409 while it is busy, or 404 because the trigger behind it does not exist -// yet. Either way the caller neither caused it nor can see it, so the wait -// happens here. -func (ec *evalContext) deleteScheduleWhenSettled(ctx context.Context, name string) error { - const attempts = 30 - - for i := 0; i < attempts; i++ { - s, err := ec.evalClient.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - if err != nil || s == nil || s.ID == "" { - // Nothing to wait for: let the delete report what it finds. - break - } - if s.Settled() { - break - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(4 * time.Second): - } - } - - for i := 0; ; i++ { - err := ec.evalClient.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion) - if err == nil || !eval_api.IsConflict(err) || i == attempts-1 { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(4 * time.Second): - } - } -} - -// explainScheduleFailure turns the service's bodiless rejection into the reason -// it actually happened. -// -// The project accepts one schedule at a time and refuses a second with a 400 -// carrying no message, so the count is what explains it. -func explainScheduleFailure( - ctx context.Context, - ec *evalContext, - name string, - cause error, -) error { - // A schedule repeats the group's most recent run, so scheduling a group - // whose last run came from --from-traces creates a trace evaluation, and - // the service allows only an hourly trigger for those. The message it - // returns says so without saying why it thinks the schedule is one, which - // is bewildering when the trigger was the only thing asked for. - if isTracesHourlyOnly(cause) { - return fmt.Errorf( - "saving schedule %q: this eval's most recent run read from traces, and a schedule "+ - "repeats that run, so the service treats it as a scheduled trace evaluation "+ - "and allows only `--every hourly`. Use `--every hourly`, or run the eval "+ - "once against its dataset first so the schedule repeats that instead", name) - } - - if isScheduleRoleMissing(cause) { - return fmt.Errorf( - "saving schedule %q: the project's managed identity needs the Foundry User "+ - "role on the project before it can run scheduled evaluations. A schedule "+ - "runs later, as the project rather than as you, which is why creating one "+ - "needs a role that running an eval yourself does not. Grant it on the "+ - "project and retry", name) - } - - list, listErr := ec.evalClient.ListSchedules(ctx, ProjectEndpointAPIVersion) - if listErr != nil || list == nil { - return fmt.Errorf("saving schedule %q: %w", name, cause) - } - - for i := range list.Value { - if other := list.Value[i].ID; other != "" && other != name { - return fmt.Errorf( - "saving schedule %q: the project already has a schedule, %q, and only one is "+ - "allowed at a time. Delete it first with "+ - "`azd ai eval schedule delete %s`", name, other, other) - } - } - return fmt.Errorf("saving schedule %q: %w", name, cause) -} - -// isTracesHourlyOnly matches the service's refusal of a non-hourly trigger on a -// schedule it considers a trace evaluation. -func isTracesHourlyOnly(err error) bool { - return err != nil && - strings.Contains(err.Error(), "trace evaluations only support hourly") -} - -// isScheduleRoleMissing matches the refusal when the project identity cannot -// run the evaluation the schedule would trigger. -func isScheduleRoleMissing(err error) bool { - return err != nil && - strings.Contains(err.Error(), "lacks Foundry User role") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go deleted file mode 100644 index ebd8da98570..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_live_test.go +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -//go:build live - -package cmd - -import ( - "context" - "fmt" - "sort" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -// Schedules are project-scoped and the service tolerates very few of them, so -// everything here runs serially and deletes what it creates. Nothing calls -// t.Parallel(). -// -// These tests exist because the unit tests only prove that buildTrigger -// produces a well-formed struct. Whether the service accepts that struct is a -// different question, and it is the one that matters: every trigger shape the -// CLI can emit is sent here and required to survive a round trip. - -// liveScheduleEval creates a throwaway eval for a schedule to point at. -// -// The criterion comes from the shipping builder rather than a hand-written -// one. Built-ins do not share an input contract — builtin.ifeval, which is -// what the listing happens to return first, needs an instruction_id_list — so -// a hand-rolled mapping is rejected with MissingRequiredDataMapping. Letting -// production shape the request also means this helper cannot drift from it. -// -// No evaluator is named. Which built-ins a project exposes varies, so the -// first one the builder can satisfy is used, and the dataset is given every -// column that evaluator declares. -func liveScheduleEval(t *testing.T, client *eval_api.EvalClient, judge string) string { - t.Helper() - ctx := context.Background() - - ec := &evalContext{evalClient: client} - schemas := ec.evaluatorSchemas(ctx) - require.NotEmpty(t, schemas, "need the published evaluator contracts to build an eval") - - names := make([]string, 0, len(schemas)) - for name := range schemas { - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - summary := schemas[name] - columns := map[string]bool{"query": true} - if ds := summary.DataSchema(); ds != nil { - for _, col := range ds.PropertyNames() { - columns[col] = true - } - } - - level := "" - if len(summary.SupportedEvaluationLevels) > 0 { - level = summary.SupportedEvaluationLevels[0] - } - - req, err := buildEvalRequest(&project.Eval{ - Name: fmt.Sprintf("azd-sched-%d", time.Now().UTC().UnixNano()), - Dataset: "inline", - Target: &project.Target{Type: "agent", Name: "probe-agent"}, - Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, - Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, - }, schemas, columns) - if err != nil { - continue - } - - created, err := client.CreateOpenAIEval(ctx, req) - if err != nil { - t.Logf("built-in %s could not back a schedule: %v", name, err) - continue - } - t.Cleanup(func() { - _ = client.DeleteOpenAIEval(context.Background(), created.ID) - }) - t.Logf("scheduling an eval built on %s (%s)", name, created.ID) - return created.ID - } - - t.Fatalf("no built-in produced an eval a schedule could run; tried %d", len(names)) - return "" -} - -// putLiveSchedule creates a schedule and registers its removal. -func putLiveSchedule( - t *testing.T, - client *eval_api.EvalClient, - name string, - evalID string, - trigger *eval_api.ScheduleTrigger, -) (*eval_api.Schedule, error) { - t.Helper() - - saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ - DisplayName: name, - Enabled: true, - Trigger: trigger, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: datasetOnlyRows([]map[string]any{{"query": "how do I reset my password?"}}), - }, - }, - }, ProjectEndpointAPIVersion) - - if err == nil { - t.Cleanup(func() { removeLiveSchedule(t, client, name) }) - } - return saved, err -} - -// skipIfScheduleRoleMissing stops the test when the project cannot host a -// schedule at all. -// -// A schedule fires later and runs as the project, so creating one requires the -// project's managed identity to hold the Foundry User role on the project — -// a permission no other command in this extension needs. Without it every -// schedule test fails identically and for a reason that has nothing to do with -// the code, so they skip loudly instead of reporting a false regression. -func skipIfScheduleRoleMissing(t *testing.T, err error) { - t.Helper() - if err == nil || !isScheduleRoleMissing(err) { - return - } - t.Skipf("this project cannot host schedules: its managed identity lacks the "+ - "Foundry User role on the project. Grant it and re-run to exercise "+ - "schedules for real. Underlying error: %v", err) -} - -// datasetOnlyRows is the inline data source a scheduled run repeats. -func datasetOnlyRows(rows []map[string]any) *eval_api.EvalRunDataSource { - ds := eval_api.NewDatasetOnlyDataSource() - ds.SetFileContent(rows) - return ds -} - -// removeLiveSchedule deletes a schedule once it is no longer provisioning. -// -// A schedule still being created refuses the delete, as a 409 while it is busy -// or a 404 because the trigger behind it does not exist yet. Waiting for it to -// settle is what makes cleanup reliable, and leaving one behind would break -// every later test in this file, because the project holds very few. -func removeLiveSchedule(t *testing.T, client *eval_api.EvalClient, name string) { - t.Helper() - ctx := context.Background() - - deadline := time.Now().Add(2 * time.Minute) - for { - current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - if err != nil { - return // already gone - } - if current.Settled() { - break - } - if time.Now().After(deadline) { - t.Logf("schedule %q never settled (status %q); leaving it", name, current.ProvisioningStatus) - return - } - time.Sleep(3 * time.Second) - } - - if err := client.DeleteSchedule(ctx, name, ProjectEndpointAPIVersion); err != nil { - t.Logf("could not delete schedule %q: %v", name, err) - } -} - -// awaitScheduleSettled blocks until the schedule finishes provisioning. -func awaitScheduleSettled( - t *testing.T, - client *eval_api.EvalClient, - name string, -) *eval_api.Schedule { - t.Helper() - ctx := context.Background() - - deadline := time.Now().Add(2 * time.Minute) - for { - current, err := client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - require.NoError(t, err, "reading schedule %q back", name) - if current.Settled() { - return current - } - if time.Now().After(deadline) { - t.Fatalf("schedule %q stuck in %q", name, current.ProvisioningStatus) - } - time.Sleep(3 * time.Second) - } -} - -func liveScheduleName(t *testing.T, suffix string) string { - t.Helper() - return fmt.Sprintf("azdsched-%s-%d", suffix, time.Now().UnixNano()) -} - -// TestLiveScheduleLifecycle walks create, read, list and delete. -func TestLiveScheduleLifecycle(t *testing.T) { - client, judge := liveEvalClient(t) - ctx := context.Background() - - evalID := liveScheduleEval(t, client, judge) - name := liveScheduleName(t, "life") - - trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) - require.NoError(t, err) - - saved, err := putLiveSchedule(t, client, name, evalID, trigger) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err, "the service rejected a trigger the CLI can produce") - require.NotEmpty(t, saved.ID) - - settled := awaitScheduleSettled(t, client, name) - require.True(t, settled.Enabled, "a schedule created without --disabled must be enabled") - require.NotNil(t, settled.Task, "the schedule must carry the task it runs") - require.Equal(t, evalID, settled.Task.EvalID, - "the schedule must point at the eval it was given") - require.NotNil(t, settled.Trigger) - require.Equal(t, eval_api.TriggerRecurrence, settled.Trigger.Type) - - list, err := client.ListSchedules(ctx, ProjectEndpointAPIVersion) - require.NoError(t, err) - found := false - for i := range list.Value { - if list.Value[i].ID == saved.ID || list.Value[i].DisplayName == name { - found = true - break - } - } - require.True(t, found, "a created schedule must appear in the listing") - - removeLiveSchedule(t, client, name) - - _, err = client.GetSchedule(ctx, name, ProjectEndpointAPIVersion) - require.Error(t, err, "a deleted schedule must not read back") - require.True(t, eval_api.IsNotFound(err), - "deleting should leave a not-found, got %v", err) -} - -// TestLiveScheduleAcceptsEveryTriggerShape sends one schedule per trigger the -// CLI can build and requires the service to accept each. -// -// The unit tests assert the shape of what buildTrigger returns. They cannot -// say whether the service agrees, and a trigger the service rejects is a -// trigger the CLI should never have offered. -func TestLiveScheduleAcceptsEveryTriggerShape(t *testing.T) { - client, judge := liveEvalClient(t) - evalID := liveScheduleEval(t, client, judge) - - // Probe once up front. Skipping inside the subtests instead would leave the - // parent reporting PASS with nothing proven, which is worse than a failure - // because it looks like coverage. - probe := liveScheduleName(t, "probe") - daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{4}}) - require.NoError(t, err) - if _, probeErr := putLiveSchedule(t, client, probe, evalID, daily); probeErr != nil { - skipIfScheduleRoleMissing(t, probeErr) - require.NoError(t, probeErr, "could not create the probe schedule") - } - removeLiveSchedule(t, client, probe) - - cases := []struct { - label string - flags triggerFlags - want string - }{ - {"cron", triggerFlags{cron: "0 9 * * *"}, eval_api.TriggerCron}, - {"hourly", triggerFlags{every: "hourly"}, eval_api.TriggerRecurrence}, - {"daily", triggerFlags{every: "daily", atHours: []int{9, 17}}, eval_api.TriggerRecurrence}, - {"weekly", triggerFlags{every: "weekly", onDays: []string{"Monday"}}, eval_api.TriggerRecurrence}, - {"monthly", triggerFlags{every: "monthly", onDaysOfMon: []int{1}}, eval_api.TriggerRecurrence}, - {"interval", triggerFlags{every: "daily", interval: 3}, eval_api.TriggerRecurrence}, - { - "onetime", - triggerFlags{atTime: time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339)}, - eval_api.TriggerOneTime, - }, - } - - for _, tc := range cases { - t.Run(tc.label, func(t *testing.T) { - trigger, err := buildTrigger(tc.flags) - require.NoError(t, err, "the CLI could not build a %s trigger", tc.label) - require.Equal(t, tc.want, trigger.Type) - - name := liveScheduleName(t, tc.label) - saved, err := putLiveSchedule(t, client, name, evalID, trigger) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err, "the service rejected the %s trigger", tc.label) - require.NotEmpty(t, saved.ID) - - settled := awaitScheduleSettled(t, client, name) - require.Equal(t, tc.want, settled.Trigger.Type, - "the trigger type must survive the round trip") - removeLiveSchedule(t, client, name) - }) - } -} - -// TestLiveScheduleDisabledStaysDisabled covers --disabled, which is the one -// flag whose whole purpose is a field the service could quietly ignore. -func TestLiveScheduleDisabledStaysDisabled(t *testing.T) { - client, judge := liveEvalClient(t) - evalID := liveScheduleEval(t, client, judge) - name := liveScheduleName(t, "disabled") - - trigger, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{3}}) - require.NoError(t, err) - - saved, err := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ - DisplayName: name, - Enabled: false, - Trigger: trigger, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), - }, - }, - }, ProjectEndpointAPIVersion) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err) - t.Cleanup(func() { removeLiveSchedule(t, client, name) }) - require.NotEmpty(t, saved.ID) - - settled := awaitScheduleSettled(t, client, name) - require.False(t, settled.Enabled, - "a schedule created disabled must not come back enabled") -} - -// TestLiveScheduleEditIsRefusedByTheCLI pins the reason `schedule set` refuses -// to reuse a name. -// -// The service takes a PUT on an existing schedule and does not apply it, and a -// replacement can stick in Creating where it can no longer be deleted. The CLI -// therefore refuses before sending. This test records the service behaviour -// the guard exists for, so a change in the service is visible here rather than -// as a stuck schedule in someone's project. -func TestLiveScheduleEditIsRefusedByTheCLI(t *testing.T) { - client, judge := liveEvalClient(t) - evalID := liveScheduleEval(t, client, judge) - name := liveScheduleName(t, "edit") - - daily, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{9}}) - require.NoError(t, err) - _, err = putLiveSchedule(t, client, name, evalID, daily) - skipIfScheduleRoleMissing(t, err) - require.NoError(t, err) - awaitScheduleSettled(t, client, name) - - // The guard in `schedule set` is a GetSchedule that must find this. - existing, err := client.GetSchedule(context.Background(), name, ProjectEndpointAPIVersion) - require.NoError(t, err) - require.NotEmpty(t, existing.ID, - "the CLI decides a name is taken by reading it back, so this must be non-empty") - - weekly, err := buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Friday"}}) - require.NoError(t, err) - _, putErr := client.PutSchedule(context.Background(), name, &eval_api.Schedule{ - DisplayName: name, - Enabled: true, - Trigger: weekly, - Task: &eval_api.ScheduleTask{ - Type: eval_api.ScheduleTaskEvaluation, - EvalID: evalID, - EvalRun: &eval_api.CreateOpenAIEvalRunRequest{ - Name: name, - DataSource: datasetOnlyRows([]map[string]any{{"query": "hello"}}), - }, - }, - }, ProjectEndpointAPIVersion) - - if putErr != nil { - t.Logf("the service refused the edit outright: %v", putErr) - return - } - - after := awaitScheduleSettled(t, client, name) - t.Logf("after editing daily -> weekly the schedule reads back as %q", after.Summary()) - require.Equal(t, eval_api.TriggerRecurrence, after.Trigger.Type) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go deleted file mode 100644 index f518397d843..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schedule_test.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "errors" - "testing" - - "azureaieval/internal/pkg/eval_api" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBuildTrigger_Cron(t *testing.T) { - got, err := buildTrigger(triggerFlags{cron: "0 9 * * *"}) - require.NoError(t, err) - assert.Equal(t, eval_api.TriggerCron, got.Type) - assert.Equal(t, "0 9 * * *", got.Expression) - assert.Equal(t, "UTC", got.Timezone, "UTC unless the caller says otherwise") -} - -// A schedule repeats the group's most recent run. Scheduling a group whose last -// run came from --from-traces therefore creates a trace evaluation, and the -// service allows only an hourly trigger for those. Confirmed live: a daily -// trigger was accepted after an agent run, refused after a traces run on the -// same group, and hourly was accepted for that same traces run. -func TestIsTracesHourlyOnly(t *testing.T) { - assert.True(t, isTracesHourlyOnly( - errors.New(`{"message": "Scheduled trace evaluations only support hourly recurrence triggers. is invalid"}`))) - assert.False(t, isTracesHourlyOnly(errors.New("some other 400"))) - assert.False(t, isTracesHourlyOnly(nil)) -} - -func TestBuildTrigger_OneTime(t *testing.T) { - got, err := buildTrigger(triggerFlags{atTime: "2026-08-01T09:00:00Z", timezone: "Europe/Dublin"}) - require.NoError(t, err) - assert.Equal(t, eval_api.TriggerOneTime, got.Type) - assert.Equal(t, "2026-08-01T09:00:00Z", got.ScheduledTime) - assert.Equal(t, "Europe/Dublin", got.Timezone) - - _, err = buildTrigger(triggerFlags{atTime: "next tuesday"}) - require.ErrorContains(t, err, "RFC3339") -} - -func TestBuildTrigger_Recurrence(t *testing.T) { - cases := []struct { - name string - flags triggerFlags - wantType string - assert func(*testing.T, *eval_api.RecurrencePattern) - }{ - { - name: "hourly", - flags: triggerFlags{every: "hourly", interval: 6}, - wantType: eval_api.RecurrenceHourly, - }, - { - name: "daily with hours", - flags: triggerFlags{every: "Daily", atHours: []int{9, 17}}, - wantType: eval_api.RecurrenceDaily, - assert: func(t *testing.T, p *eval_api.RecurrencePattern) { - assert.Equal(t, []int{9, 17}, p.Hours) - }, - }, - { - name: "weekly normalizes day casing", - flags: triggerFlags{every: "weekly", onDays: []string{"monday", "THURSDAY"}}, - wantType: eval_api.RecurrenceWeekly, - assert: func(t *testing.T, p *eval_api.RecurrencePattern) { - assert.Equal(t, []string{"Monday", "Thursday"}, p.DaysOfWeek) - }, - }, - { - name: "monthly", - flags: triggerFlags{every: "monthly", onDaysOfMon: []int{1, 15}}, - wantType: eval_api.RecurrenceMonthly, - assert: func(t *testing.T, p *eval_api.RecurrencePattern) { - assert.Equal(t, []int{1, 15}, p.DaysOfMonth) - }, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := buildTrigger(tc.flags) - require.NoError(t, err) - assert.Equal(t, eval_api.TriggerRecurrence, got.Type) - require.NotNil(t, got.Schedule) - assert.Equal(t, tc.wantType, got.Schedule.Type) - if tc.assert != nil { - tc.assert(t, got.Schedule) - } - }) - } -} - -// An interval is always sent, so the service never has to infer one. -func TestBuildTrigger_IntervalDefaultsToOne(t *testing.T) { - got, err := buildTrigger(triggerFlags{every: "daily"}) - require.NoError(t, err) - assert.Equal(t, 1, got.Interval) - - got, err = buildTrigger(triggerFlags{every: "daily", interval: 3}) - require.NoError(t, err) - assert.Equal(t, 3, got.Interval) -} - -// Each period reads only its own qualifier. Accepting one that does not apply -// would drop it silently, which is the failure mode the trace fields already -// taught us to avoid. -func TestBuildTrigger_RejectsQualifiersFromAnotherPeriod(t *testing.T) { - cases := []struct { - name string - flags triggerFlags - want string - }{ - {"hours on weekly", triggerFlags{every: "weekly", atHours: []int{9}}, "--at applies to --every daily"}, - {"days on daily", triggerFlags{every: "daily", onDays: []string{"Monday"}}, "--on applies to --every weekly"}, - {"month days on hourly", triggerFlags{every: "hourly", onDaysOfMon: []int{1}}, "--on-day applies to --every monthly"}, - {"hours on monthly", triggerFlags{every: "monthly", atHours: []int{9}}, "--at applies to --every daily"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, err := buildTrigger(tc.flags) - require.ErrorContains(t, err, tc.want) - }) - } -} - -func TestBuildTrigger_RejectsOutOfRangeValues(t *testing.T) { - _, err := buildTrigger(triggerFlags{every: "daily", atHours: []int{24}}) - require.ErrorContains(t, err, "hour of the day") - - _, err = buildTrigger(triggerFlags{every: "monthly", onDaysOfMon: []int{0}}) - require.ErrorContains(t, err, "day of the month") - - _, err = buildTrigger(triggerFlags{every: "weekly", onDays: []string{"Caturday"}}) - require.ErrorContains(t, err, "not a day of the week") - - _, err = buildTrigger(triggerFlags{every: "fortnightly"}) - require.ErrorContains(t, err, "hourly, daily, weekly or monthly") -} - -func TestBuildTrigger_NeedsATrigger(t *testing.T) { - _, err := buildTrigger(triggerFlags{}) - require.ErrorContains(t, err, "--cron, --every or --at-time") -} - -func TestScheduleSummary(t *testing.T) { - cron := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerCron, Expression: "0 9 * * *"}} - assert.Equal(t, "cron 0 9 * * *", cron.Summary()) - - weekly := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerRecurrence, - Schedule: &eval_api.RecurrencePattern{Type: eval_api.RecurrenceWeekly}}} - assert.Equal(t, "every Weekly", weekly.Summary()) - - once := &eval_api.Schedule{Trigger: &eval_api.ScheduleTrigger{ - Type: eval_api.TriggerOneTime, ScheduledTime: "2026-08-01T09:00:00Z"}} - assert.Equal(t, "once at 2026-08-01T09:00:00Z", once.Summary()) - - var nilSchedule *eval_api.Schedule - assert.Empty(t, nilSchedule.Summary()) - assert.Empty(t, (&eval_api.Schedule{}).Summary()) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go new file mode 100644 index 00000000000..3b4b93d7c7e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/errors.go @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "errors" + "net/http" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" +) + +// IsConflict reports whether the service refused because the resource is busy. +func IsConflict(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusConflict +} + +// IsNotFound reports whether the service answered 404. +func IsNotFound(err error) bool { + var respErr *azcore.ResponseError + if !errors.As(err, &respErr) { + return false + } + return respErr.StatusCode == http.StatusNotFound +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go deleted file mode 100644 index a6ab65746a4..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/schedules.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package eval_api - -import ( - "context" - "errors" - "net/http" - "net/url" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" -) - -const pathSchedules = "/schedules" - -// Trigger types accepted by the schedules API. -const ( - TriggerCron = "Cron" - TriggerRecurrence = "Recurrence" - TriggerOneTime = "OneTime" -) - -// Recurrence patterns accepted under a Recurrence trigger. -const ( - RecurrenceHourly = "Hourly" - RecurrenceDaily = "Daily" - RecurrenceWeekly = "Weekly" - RecurrenceMonthly = "Monthly" -) - -// ScheduleTaskEvaluation is the only task type the service accepts today; an -// Insight task is rejected on validation. -const ScheduleTaskEvaluation = "Evaluation" - -// RecurrencePattern is the shape of a Recurrence trigger's repeat rule. Which -// fields apply depends on Type: Daily reads Hours, Weekly reads DaysOfWeek, -// Monthly reads DaysOfMonth, and Hourly reads neither. -type RecurrencePattern struct { - Type string `json:"type"` - Hours []int `json:"hours,omitempty"` - DaysOfWeek []string `json:"daysOfWeek,omitempty"` - DaysOfMonth []int `json:"daysOfMonth,omitempty"` -} - -// ScheduleTrigger says when the task runs. The discriminator is Type; the -// other fields are per-type and only one set is ever populated. -type ScheduleTrigger struct { - Type string `json:"type"` - - // Cron - Expression string `json:"expression,omitempty"` - - // Recurrence - Schedule *RecurrencePattern `json:"schedule,omitempty"` - Interval int `json:"interval,omitempty"` - - // OneTime - ScheduledTime string `json:"scheduledTime,omitempty"` - - // Cron and Recurrence - StartTime string `json:"startTime,omitempty"` - EndTime string `json:"endTime,omitempty"` - - Timezone string `json:"timezone,omitempty"` -} - -// ScheduleTask is what the trigger fires. An evaluation task needs both the -// group and the run to repeat: the group holds only its testing criteria, so -// the target and dataset travel with the run. -type ScheduleTask struct { - Type string `json:"type"` - EvalID string `json:"evalId,omitempty"` - EvalRun *CreateOpenAIEvalRunRequest `json:"evalRun,omitempty"` -} - -// Schedule is a named, project-scoped recurring evaluation. -type Schedule struct { - ID string `json:"id,omitempty"` - DisplayName string `json:"displayName,omitempty"` - Description string `json:"description,omitempty"` - Enabled bool `json:"enabled"` - ProvisioningStatus string `json:"provisioningStatus,omitempty"` - Trigger *ScheduleTrigger `json:"trigger,omitempty"` - Task *ScheduleTask `json:"task,omitempty"` - Tags map[string]string `json:"tags,omitempty"` - Properties map[string]string `json:"properties,omitempty"` - Error *JobError `json:"error,omitempty"` -} - -// Summary renders the trigger as a single line for listings. -func (s *Schedule) Summary() string { - if s == nil || s.Trigger == nil { - return "" - } - switch s.Trigger.Type { - case TriggerCron: - return "cron " + s.Trigger.Expression - case TriggerOneTime: - return "once at " + s.Trigger.ScheduledTime - case TriggerRecurrence: - if s.Trigger.Schedule == nil { - return "recurrence" - } - return "every " + s.Trigger.Schedule.Type - } - return s.Trigger.Type -} - -// ScheduleList is the response for ListSchedules. -type ScheduleList struct { - Value []Schedule `json:"value"` -} - -// Settled reports whether the schedule has finished provisioning. -// -// A schedule that is still being created refuses a delete, and does it two -// different ways: 409 while it is busy, or 404 because the trigger behind it -// does not exist yet. Waiting for it to settle avoids both. -func (s *Schedule) Settled() bool { - if s == nil { - return true - } - switch s.ProvisioningStatus { - case "Creating", "Updating", "Deleting": - return false - } - return true -} - -// PutSchedule creates or replaces a schedule. The route is keyed by name, and -// the same call updates an existing schedule in place. -func (c *EvalClient) PutSchedule( - ctx context.Context, - name string, - schedule *Schedule, - apiVersion string, -) (*Schedule, error) { - path := pathSchedules + "/" + url.PathEscape(name) - return doRequestTyped[Schedule](c, ctx, http.MethodPut, path, nil, schedule, apiVersion) -} - -// GetSchedule reads one schedule by name. -func (c *EvalClient) GetSchedule( - ctx context.Context, - name string, - apiVersion string, -) (*Schedule, error) { - path := pathSchedules + "/" + url.PathEscape(name) - return doRequestTyped[Schedule](c, ctx, http.MethodGet, path, nil, nil, apiVersion) -} - -// ListSchedules returns the project's schedules. -func (c *EvalClient) ListSchedules(ctx context.Context, apiVersion string) (*ScheduleList, error) { - return doRequestTyped[ScheduleList](c, ctx, http.MethodGet, pathSchedules, nil, nil, apiVersion) -} - -// DeleteSchedule removes a schedule by name. -func (c *EvalClient) DeleteSchedule(ctx context.Context, name string, apiVersion string) error { - path := pathSchedules + "/" + url.PathEscape(name) - _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) - return err -} - -// IsConflict reports whether the service refused because the resource is busy. -// -// A schedule that is still provisioning answers 409 to a delete, which is worth -// waiting out rather than reporting. -func IsConflict(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusConflict -} - -// IsNotFound reports whether the service answered 404. -func IsNotFound(err error) bool { - var respErr *azcore.ResponseError - if !errors.As(err, &respErr) { - return false - } - return respErr.StatusCode == http.StatusNotFound -} From e8c1f0cc7b9f19fb4eb0b45d90c03e29125d066a Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 22:11:31 +0300 Subject: [PATCH 066/114] Correct two stale claims in the command reference The default metric name was listed as invented. It is not: the SDK's own code-based evaluator sample declares `result`, so the name matches. What was left after that read like an open question about the type and bounds, and that was a misreading. A metric describes what the author's grade() returns, so its type and range are theirs to state - continuous, ordinal or boolean over any range - which is what --metrics is for. The sample's ordinal 0.0-1.0 described that sample's grader, not a convention. Probing shows the service does not validate scores against declared bounds: a grader returning 14 and 33 scores identically whether the metric declares continuous-unbounded or ordinal 0.0-1.0. So a wrong bound is not an error, only misleading range metadata - which is the argument for defaulting to unbounded, since it asserts nothing about a range we cannot know. Also correct the azd up entry. It claimed a configuration gap without saying what it costs. Measured: an evaluator published the way azd up publishes one, carrying no data_schema, scores exactly as a fully configured one does - completed, 2 of 2 passed, both. The real cost is narrower and worth stating precisely: image_tag, metrics and init_parameters cannot be declared, so an evaluator that needs a dependency or a non-default metric range can only be published through evaluator create. --- .../extensions/azure.ai.evaluations/COMMANDS.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md index 5f58e75f030..7d731281c14 100644 --- a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md +++ b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md @@ -189,5 +189,16 @@ What the live tests on that branch establish: | `generate` has no live coverage | The most complex composite command | | `TestLiveRun` skips | Agent-target runs, `--from-traces`, `--response-id` all unverified | | `--image-tag` never used with a real image | The only supported way to give a code evaluator dependencies | -| `azd up` cannot configure a code evaluator | Reconciler passes empty options; no config fields for `data_schema`, `metrics`, `init_params`, `image_tag` | -| Default metric name `result` is invented | Real evaluators use semantic names (`groundedness`, `relevance`) | +| `azd up` cannot configure a code evaluator | Not a correctness break — a schema-less evaluator scores identically to a configured one. But `image_tag`, `metrics` and `init_parameters` cannot be declared, so an evaluator needing a dependency or a non-default metric range can only be published with `evaluator create` | + +### Not a gap: the default metric + +`evaluator create` defaults to `{"result": {"type": "continuous", "desirable_direction": "increase"}}` +when `--metrics` is omitted. The name matches the SDK's code-based evaluator sample. + +The type and bounds are deliberately unbounded. A metric describes what the author's +own `grade()` returns, so it can be continuous, ordinal or boolean over any range — +that is what `--metrics` is for. The service does not validate scores against declared +bounds (a grader returning 14 and 33 scores fine under a declared `ordinal 0.0–1.0`), +so a wrong bound is not an error, only bad range metadata. Defaulting to unbounded +asserts nothing, which is the correct claim to make about a range we cannot know. From b0879e4d4bc847061b617248058016b948188a48 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 22:33:45 +0300 Subject: [PATCH 067/114] Drive the CLI as a subprocess, and fix what that found Every live test so far called the client layer directly. That proves the API paths work and says nothing about the surface on top of them: flag parsing, mutual exclusion, exit codes, the rendered tables, and whether -o json emits something a script can consume. Those are the parts a user touches. tests/cli builds the extension and runs it as a process, the way a user does. Eight tests cover listing built-ins in both renderings, the bare array envelope, the code evaluator lifecycle end to end, source mutual exclusion, local grader validation, the not-found message, and --no-prompt failing instead of hanging. Running them found three things. `evaluator show` on a name that does not exist printed 1327 characters of service JSON. The same was true of `evaluator delete`, `dataset show` and `dataset delete`. Those four now answer briefly and say where to look instead, which run list already did. `init` created evals/datasets and evals/evaluators before asking azd whether there was a project to attach to, so refusing left a half-scaffolded tree behind. The project is the one thing init cannot supply for itself, so it is now confirmed before anything is written. `init` cannot be exercised by this harness at all: it resolves the project over azd's gRPC channel, so it needs azd hosting the extension. What the harness can assert is the refusal a user gets outside a project, and that it names `azd init` rather than surfacing a transport error. The scaffolding itself stays covered by the unit tests, which can supply a fake client. The harness retries once on "AzureDeveloperCLICredential: exit status 1". Each invocation is a fresh process shelling out to azd for a token, and azd intermittently fails under rapid sequential calls. The retry is logged rather than silent, and a persistent failure still fails the test with a message pointing at azd auth login. --- .../internal/cmd/dataset.go | 10 + .../internal/cmd/evaluator.go | 9 + .../azure.ai.evaluations/internal/cmd/init.go | 26 +++ .../tests/cli/evaluator_test.go | 187 +++++++++++++++ .../tests/cli/harness_test.go | 216 ++++++++++++++++++ 5 files changed, 448 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index fb5b5865e79..9ea7c5b3a48 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -11,6 +11,7 @@ import ( "strings" "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" "github.com/spf13/cobra" ) @@ -192,6 +193,11 @@ func newDatasetShowCommand() *cobra.Command { ds, err := ec.datasetClient.GetDataset(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project; "+ + "`azd ai eval dataset list` shows the ones there are", name, version) + } return fmt.Errorf("reading dataset %q version %q: %w", name, version, err) } @@ -239,6 +245,10 @@ func newDatasetDeleteCommand() *cobra.Command { if err := ec.datasetClient.DeleteDatasetVersion( ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no dataset %q at version %q in this project", name, version) + } return fmt.Errorf("deleting dataset %q version %q: %w", name, version, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index ee601bb16de..151bc14bf42 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -420,6 +420,11 @@ func newEvaluatorShowCommand() *cobra.Command { raw, err := ec.evalClient.GetEvaluatorRaw(ctx, name, version, ProjectEndpointAPIVersion) if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no evaluator %q in this project; "+ + "`azd ai eval evaluator list` shows the ones there are", name) + } return fmt.Errorf("reading evaluator %q: %w", name, err) } @@ -466,6 +471,10 @@ func newEvaluatorDeleteCommand() *cobra.Command { if err := ec.evalClient.DeleteEvaluatorVersion( ctx, name, version, ProjectEndpointAPIVersion, ); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no evaluator %q at version %q in this project", name, version) + } return fmt.Errorf("deleting evaluator %q version %q: %w", name, version, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 85359d184ae..fba0d533682 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -53,6 +53,13 @@ func newInitCommand() *cobra.Command { } } + // Asked before anything is written: the project is the one thing init + // cannot supply for itself, and failing after creating directories + // leaves a half-scaffolded tree behind for the user to clean up. + if err := confirmAzdProject(cmd.Context()); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(outDir, project.DefaultDatasetsDir), 0o750); err != nil { return fmt.Errorf("creating the datasets directory: %w", err) } @@ -134,6 +141,25 @@ const ( wiringPresent = "present" // an eval service was already declared ) +// noAzdProject is what init reports when there is nothing to attach to. +const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + + "or run this from the root of an existing one; the eval service is " + +// confirmAzdProject reports whether a project exists, without changing it. +func confirmAzdProject(ctx context.Context) error { + azdClient, err := azdext.NewAzdClient() + if err != nil { + return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + } + defer azdClient.Close() + + resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + } + return nil +} + // ensureRootEvalService declares the eval service in azd's project file. // // azd acts on nothing until the service exists, so the reference is made rather diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go new file mode 100644 index 00000000000..79eabdd69eb --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestCLIEvaluatorListBuiltin is the cheapest proof the binary can reach the +// service on its own: no azd project, no config, just a flag. +func TestCLIEvaluatorListBuiltin(t *testing.T) { + r := requireSuccess(t, run(t, "evaluator", "list", "--builtin", "-o", "json")) + + var builtins []struct { + Name string `json:"name"` + EvaluatorType string `json:"evaluator_type"` + } + r.JSON(t, &builtins) + require.NotEmpty(t, builtins, "the project must expose built-in evaluators") + + for _, b := range builtins { + require.True(t, strings.HasPrefix(b.Name, "builtin."), + "--builtin must return only built-ins, got %q", b.Name) + } + + // The default rendering is a table, not JSON. A script reading stdout + // without -o json would otherwise silently parse a header row. + table := requireSuccess(t, run(t, "evaluator", "list", "--builtin")) + require.Contains(t, table.Stdout, "NAME") + require.Contains(t, table.Stdout, "VERSION") +} + +// TestCLIJSONListsAreBareArrays pins the envelope. +// +// The service wraps listings in {"value":[...]} or {"data":[...]} depending on +// the route. Leaking either would make every consumer special-case the +// command it came from, so the CLI unwraps them, and this is what says so. +func TestCLIJSONListsAreBareArrays(t *testing.T) { + for _, args := range [][]string{ + {"evaluator", "list", "--builtin", "-o", "json"}, + {"dataset", "list", "-o", "json"}, + } { + t.Run(strings.Join(args[:2], " "), func(t *testing.T) { + r := requireSuccess(t, run(t, args...)) + trimmed := strings.TrimSpace(r.Stdout) + require.True(t, strings.HasPrefix(trimmed, "["), + "a list must be a bare array, not an envelope; got:\n%s", firstLine(trimmed)) + + var out []any + r.JSON(t, &out) + }) + } +} + +// TestCLICodeEvaluatorRoundTrip drives the whole custom evaluator lifecycle +// through the command surface: create from a script, read it back, list its +// versions, then delete it. +func TestCLICodeEvaluatorRoundTrip(t *testing.T) { + name := uniqueName("azdcli_code") + script := writeGrader(t, lengthGrader) + + requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--file", script)) + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "-o", "json")) + var def struct { + Name string `json:"name"` + Definition struct { + Type string `json:"type"` + CodeText string `json:"code_text"` + } `json:"definition"` + } + shown.JSON(t, &def) + require.Equal(t, "code", def.Definition.Type, + "a script must register as a code definition") + require.Contains(t, def.Definition.CodeText, "def grade", + "the script's source must round-trip in code_text") + + listed := requireSuccess(t, run(t, "evaluator", "list", "--name", name, "-o", "json")) + var versions []struct { + Name string `json:"name"` + Version string `json:"version"` + } + listed.JSON(t, &versions) + require.NotEmpty(t, versions, "the evaluator must list its own versions") +} + +// TestCLIEvaluatorSourcesAreMutuallyExclusive covers the validation a user is +// most likely to trip, and asserts it costs nothing to find out — no version +// is published on the way to the error. +func TestCLIEvaluatorSourcesAreMutuallyExclusive(t *testing.T) { + script := writeGrader(t, lengthGrader) + rubric := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(rubric, + []byte(`{"dimensions":[{"id":"tone","description":"polite","weight":5}]}`), 0o600)) + + both := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_both"), "--file", script, "--rubric", rubric)) + require.Contains(t, strings.ToLower(both.Combined()), "rubric", + "the error must name the flags in conflict") + + neither := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_neither"))) + require.NotEmpty(t, strings.TrimSpace(neither.Combined()), + "refusing without a source must explain itself") +} + +// TestCLIGraderIsValidatedBeforePublishing asserts the check that saves a user +// from a late failure: a script with no top-level grade() is refused locally, +// because the executor would otherwise accept the publish and fail the run. +func TestCLIGraderIsValidatedBeforePublishing(t *testing.T) { + script := writeGrader(t, `class AnswerLengthEvaluator: + def __call__(self, **kwargs): + return {"result": 1.0} +`) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_noglade"), "--file", script)) + require.Contains(t, strings.ToLower(r.Combined()), "grade", + "the refusal must name the function the executor looks for") +} + +// TestCLIUnknownEvaluatorIsBrief covers the failure a user hits by typo. +// +// The service answers with a long JSON body. Printing it verbatim buries the +// one useful sentence, so the CLI shortens it, and a regression here is the +// kind that only shows up in someone's terminal. +func TestCLIUnknownEvaluatorIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "evaluator", "show", "--name", "azdcli-does-not-exist-9999")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) +} + +// TestCLIInitNeedsAnAzdProject covers the whole of what `init` can be asked +// through this harness. +// +// init resolves the project over azd's gRPC channel, so it only works when azd +// is hosting the extension. Running the binary directly there is no host, and +// that is exactly the case a user hits by running the command outside a +// project — so what is asserted is the refusal: it must name `azd init` rather +// than surface a transport error. The scaffolding itself is covered by the +// unit tests, which can supply a fake azd client. +func TestCLIInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + r := requireFailure(t, runIn(t, dir, "init", + "--target", "probe-agent", + "--judge-model", "gpt-4o-mini", + "--no-prompt")) + + require.Contains(t, r.Combined(), "azd init", + "the refusal must name the command that makes a project") + require.NotContains(t, strings.ToLower(r.Combined()), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, + "a refused init must leave nothing behind") +} + +// TestCLINoPromptFailsInsteadOfHanging is what makes the CLI usable in CI: a +// missing required value must end the process, not wait on a terminal nobody +// is watching. +func TestCLINoPromptFailsInsteadOfHanging(t *testing.T) { + dir := t.TempDir() + r := requireFailure(t, runIn(t, dir, "init", "--no-prompt")) + require.NotEmpty(t, strings.TrimSpace(r.Combined()), + "--no-prompt must say what it could not resolve") +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go new file mode 100644 index 00000000000..e4e86181a1d --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Package cli drives the built binary as a subprocess. +// +// The other live tests call the client layer directly, which proves the API +// paths work but says nothing about the command surface on top of them: flag +// parsing, mutual exclusion, prompting, --no-prompt, exit codes, the rendered +// tables, and whether -o json emits what a script can actually consume. Those +// are the parts a user touches, and until now nothing exercised them against a +// real service. +// +// go test -tags live -v ./tests/cli/... +// +// Required: +// +// AZURE_AI_EVAL_E2E_LIVE=1 +// FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var ( + binaryPath string + endpoint string +) + +// TestMain builds the extension once so every test runs the same binary a user +// would, rather than an in-process command tree that skips main's wiring. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + fmt.Fprintln(os.Stderr, "set AZURE_AI_EVAL_E2E_LIVE=1 to run the CLI tests") + os.Exit(0) + } + + endpoint = strings.TrimSuffix(os.Getenv("FOUNDRY_PROJECT_ENDPOINT"), "/") + if endpoint == "" { + fmt.Fprintln(os.Stderr, "FOUNDRY_PROJECT_ENDPOINT is required") + os.Exit(1) + } + + dir, err := os.MkdirTemp("", "azdeval-cli") + if err != nil { + fmt.Fprintf(os.Stderr, "creating a temp dir: %v\n", err) + os.Exit(1) + } + defer os.RemoveAll(dir) + + binaryPath = filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binaryPath, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "building the extension: %v\n%s\n", err, out) + os.Exit(1) + } + + code := m.Run() + os.RemoveAll(dir) + os.Exit(code) +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// result is one invocation of the binary. +type result struct { + Args []string + Stdout string + Stderr string + ExitCode int +} + +// Combined is stdout and stderr together, for assertions that do not care +// which stream carried the message. +func (r result) Combined() string { return r.Stdout + r.Stderr } + +// JSON decodes stdout, failing the test when the command did not emit +// something a script could consume. +func (r result) JSON(t *testing.T, into any) { + t.Helper() + require.NoError(t, json.Unmarshal([]byte(r.Stdout), into), + "-o json must emit parseable JSON on stdout; got:\n%s", r.Stdout) +} + +// run invokes the binary with the project endpoint already supplied. +func run(t *testing.T, args ...string) result { + t.Helper() + return runIn(t, "", args...) +} + +// credentialFlake is azd's token helper failing under rapid sequential calls. +// +// Every invocation here is a fresh process, so each one shells out to azd for +// a token, and azd intermittently exits non-zero doing it. Retrying is safe +// because no request was made, and the alternative is a suite that fails on a +// different test each run for a reason that has nothing to do with the code. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// runIn invokes the binary with a working directory, for commands that write +// files. +func runIn(t *testing.T, dir string, args ...string) result { + t.Helper() + + res := invoke(t, dir, args...) + for attempt := 0; attempt < 2 && strings.Contains(res.Combined(), credentialFlake); attempt++ { + t.Logf("azd credential flaked; retrying `%s`", strings.Join(args, " ")) + time.Sleep(2 * time.Second) + res = invoke(t, dir, args...) + } + require.NotContains(t, res.Combined(), credentialFlake, + "azd could not produce a token after retries; run `azd auth login` and try again") + return res +} + +func invoke(t *testing.T, dir string, args ...string) result { + t.Helper() + + full := append([]string{}, args...) + if !hasFlag(args, "--project-endpoint") && needsEndpoint(args) { + full = append(full, "--project-endpoint", endpoint) + } + + cmd := exec.Command(binaryPath, full...) + if dir != "" { + cmd.Dir = dir + } + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + code := 0 + if exitErr, ok := err.(*exec.ExitError); ok { + code = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("could not run %v: %v", full, err) + } + + res := result{Args: full, Stdout: stdout.String(), Stderr: stderr.String(), ExitCode: code} + t.Logf("$ azd ai eval %s -> exit %d", strings.Join(args, " "), res.ExitCode) + return res +} + +func hasFlag(args []string, flag string) bool { + for _, a := range args { + if a == flag { + return true + } + } + return false +} + +// needsEndpoint keeps --project-endpoint off the commands that reject it. +func needsEndpoint(args []string) bool { + for _, a := range args { + switch a { + case "init", "--help", "-h": + return false + } + } + return true +} + +// requireSuccess fails with the command's own output, which is what a user +// would have seen. +func requireSuccess(t *testing.T, r result) result { + t.Helper() + require.Equalf(t, 0, r.ExitCode, + "expected `%s` to succeed\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +// requireFailure asserts a non-zero exit, so a command that silently succeeds +// where it should refuse is caught. +func requireFailure(t *testing.T, r result) result { + t.Helper() + require.NotEqualf(t, 0, r.ExitCode, + "expected `%s` to fail\nstdout:\n%s\nstderr:\n%s", + strings.Join(r.Args, " "), r.Stdout, r.Stderr) + return r +} + +func uniqueName(prefix string) string { + return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano()) +} + +// writeGrader lays down a code evaluator script and returns its path. +func writeGrader(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "grader.py") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +const lengthGrader = `def grade(sample, item) -> float: + return float(len((item or {}).get("response", ""))) +` From 45a1cd1480ad2315448b984a66d4b7f1961de77a Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 22:36:24 +0300 Subject: [PATCH 068/114] Record the azd up gap for code evaluators as a TODO A code evaluator deployed by `azd up` cannot be configured: the flags exist on `evaluator create` but the config has nowhere to put them, so the reconciler passes an empty options struct. Scoring is unaffected - an evaluator published with no schema scores identically to a configured one - so this is a capability gap rather than a defect. image_tag is the one that bites: an evaluator needing any dependency cannot be deployed declaratively at all, only published by hand. metrics and init_parameters are the same story for anything that is not a 0-to-1 increasing score. The note sits in both places the fix touches: the call site that passes nothing, and the config type that has nowhere to read it from. --- .../azure.ai.evaluations/internal/cmd/reconciler.go | 8 ++++++++ .../azure.ai.evaluations/internal/project/eval_config.go | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 687d1936baf..c89c89c6ada 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -324,6 +324,14 @@ func (r *evalReconciler) ensureCodeEvaluator( return recordedVersion, false, nil } + // TODO: a code evaluator deployed by `azd up` cannot be configured. The + // flags exist on `evaluator create` but the config has nowhere to put them, + // so this passes none. Scoring still works, but image_tag is the one that + // bites: an evaluator needing any dependency cannot be deployed this way at + // all, only published by hand. metrics and init_parameters are the same + // story for anything that is not a 0-to-1 increasing score. The fix is + // fields on project.EvaluatorDecl - image_tag, metrics, data_schema, + // init_parameters - read here instead of this empty struct. opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) if err != nil { return "", false, err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 336c9b17bdf..93fd22b7bde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -39,6 +39,10 @@ type DatasetDecl struct { // `.json` file holds a rubric. A code evaluator cannot name a folder — it runs // as a python grader, which is handed one script's source and cannot import a // helper module beside it. +// TODO: carry the code evaluator settings `evaluator create` accepts - +// image_tag, metrics, data_schema, init_parameters. Without them a code +// evaluator can be declared but not configured, so one needing a dependency +// has to be published by hand. See EnsureEvaluator in internal/cmd/reconciler.go. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` From 488317bcb6844275ccb104d52dccab61a8cfe617 Mon Sep 17 00:00:00 2001 From: mohessie Date: Sun, 2 Aug 2026 23:12:39 +0300 Subject: [PATCH 069/114] Let azd up configure a code evaluator, and stop TestLiveRun skipping Three changes that turned out to be related, because two of them were hidden by tests that were not running what they claimed. EvaluatorDecl gains image_tag, metrics, data_schema and init_parameters, so a code evaluator can be configured where it is declared rather than only by hand. The three schema fields name JSON files beside the script and resolve like source, because they are edited as files. No service change was needed: the API already accepted all four, only the config had nowhere to put them. Change detection had to move with it. The digest hashed the script alone, so editing metrics or the image tag would have left the edit undeployed with azd up reporting nothing to do. codeEvaluatorDigest now covers the script, the tag, and each settings file, with a unit test for each way it can change. TestLiveRun no longer skips. It defaults to an agent discovered from the project and fails when there is none, because skipping by default is how the agent-target path stayed unverified while the suite reported green. Running it immediately showed why that mattered: discovery first read /assistants, the run failed with "resources not found", and the reason is that an eval target resolves against /agents. This project has eight assistants and no agents, so the path genuinely cannot be verified here - which the suite now says out loud instead of passing. Finally, the credential. Both live packages built a new AzureDeveloperCLICredential per test, and azidentity caches tokens per instance, so every test shelled out to azd again. A refresh that overruns the SDK's ten-second budget for that subprocess surfaces as "AzureDeveloperCLICredential: exit status 1" - which reads like a broken login rather than a timeout, and lands on whichever test happened to follow a slow one. Both packages now share one credential. Fifteen rapid azd token calls succeed on their own, so the trigger was the repeated cold cache, not call frequency. --- .../internal/cmd/build_live_test.go | 24 ++++- .../internal/cmd/reconciler.go | 54 +++++++++-- .../internal/cmd/reconciler_digest_test.go | 81 ++++++++++++++++ .../internal/project/eval_config.go | 12 ++- .../internal/project/service_target_eval.go | 3 + .../tests/live/live_test.go | 97 +++++++++++++++++-- 6 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index b7c9c32500c..1018e9cb4e4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "strings" + "sync" "testing" "time" @@ -30,6 +31,26 @@ import ( "github.com/stretchr/testify/require" ) +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + }) + return sharedCred, sharedCredErr +} + func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { t.Helper() if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { @@ -39,8 +60,7 @@ func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { if endpoint == "" { t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") } - cred, err := azidentity.NewAzureDeveloperCLICredential( - &azidentity.AzureDeveloperCLICredentialOptions{}) + cred, err := liveCredential() require.NoError(t, err) judge := os.Getenv("AZURE_AI_EVAL_MODEL") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c89c89c6ada..8a1e5eb8138 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -6,6 +6,8 @@ package cmd import ( "bufio" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "os" @@ -287,6 +289,41 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } +// codeEvaluatorDigest fingerprints everything a published version depends on. +// +// The script alone is not enough: changing only metrics or the image tag +// changes what gets published, and hashing just the source would leave that +// edit undeployed with `azd up` reporting no change. +func codeEvaluatorDigest(decl project.EvaluatorDecl, path string) (string, error) { + sum := sha256.New() + + script, err := project.Fingerprint(path) + if err != nil { + return "", err + } + fmt.Fprintf(sum, "script:%s\nimage:%s\n", script, decl.ImageTag) + + for _, settings := range []struct { + label string + path string + }{ + {"metrics", decl.Metrics}, + {"data_schema", decl.DataSchema}, + {"init_parameters", decl.InitParameters}, + } { + if settings.path == "" { + continue + } + digest, err := project.Fingerprint(settings.path) + if err != nil { + return "", fmt.Errorf("evaluator %q %s: %w", decl.Name, settings.label, err) + } + fmt.Fprintf(sum, "%s:%s\n", settings.label, digest) + } + + return hex.EncodeToString(sum.Sum(nil)), nil +} + // ensureCodeEvaluator publishes a Python script only when its content changed // since the last deploy. // @@ -306,7 +343,7 @@ func (r *evalReconciler) ensureCodeEvaluator( return "", false, err } - digest, err := project.Fingerprint(path) + digest, err := codeEvaluatorDigest(decl, path) if err != nil { return "", false, err } @@ -324,15 +361,12 @@ func (r *evalReconciler) ensureCodeEvaluator( return recordedVersion, false, nil } - // TODO: a code evaluator deployed by `azd up` cannot be configured. The - // flags exist on `evaluator create` but the config has nowhere to put them, - // so this passes none. Scoring still works, but image_tag is the one that - // bites: an evaluator needing any dependency cannot be deployed this way at - // all, only published by hand. metrics and init_parameters are the same - // story for anything that is not a 0-to-1 increasing score. The fix is - // fields on project.EvaluatorDecl - image_tag, metrics, data_schema, - // init_parameters - read here instead of this empty struct. - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) + opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ + imageTag: decl.ImageTag, + metrics: decl.Metrics, + dataSchema: decl.DataSchema, + initParams: decl.InitParameters, + }) if err != nil { return "", false, err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go new file mode 100644 index 00000000000..5fd6d7521c7 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// A code evaluator's published version depends on more than its script, so the +// digest that decides whether to republish has to cover the rest of it. +// Hashing only the source would leave a changed metric or image tag sitting in +// the config while `azd up` reported nothing to do. +func TestCodeEvaluatorDigest(t *testing.T) { + dir := t.TempDir() + + script := filepath.Join(dir, "grader.py") + require.NoError(t, os.WriteFile(script, + []byte("def grade(sample, item) -> float:\n return 1.0\n"), 0o600)) + + metrics := filepath.Join(dir, "metrics.json") + require.NoError(t, os.WriteFile(metrics, + []byte(`{"result":{"type":"continuous"}}`), 0o600)) + + base := project.EvaluatorDecl{Name: "answer_length"} + baseline, err := codeEvaluatorDigest(base, script) + require.NoError(t, err) + + t.Run("stable across calls", func(t *testing.T) { + again, err := codeEvaluatorDigest(base, script) + require.NoError(t, err) + require.Equal(t, baseline, again, + "an unchanged evaluator must not republish on every deploy") + }) + + t.Run("notices the script", func(t *testing.T) { + require.NoError(t, os.WriteFile(script, + []byte("def grade(sample, item) -> float:\n return 2.0\n"), 0o600)) + changed, err := codeEvaluatorDigest(base, script) + require.NoError(t, err) + require.NotEqual(t, baseline, changed) + }) + + t.Run("notices the image tag", func(t *testing.T) { + withImage := base + withImage.ImageTag = "python:3.12-slim" + changed, err := codeEvaluatorDigest(withImage, script) + require.NoError(t, err) + require.NotEqual(t, baseline, changed, + "changing the image changes what runs, so it must republish") + }) + + t.Run("notices the metrics file", func(t *testing.T) { + withMetrics := base + withMetrics.Metrics = metrics + before, err := codeEvaluatorDigest(withMetrics, script) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(metrics, + []byte(`{"result":{"type":"ordinal","min_value":0,"max_value":1}}`), 0o600)) + after, err := codeEvaluatorDigest(withMetrics, script) + require.NoError(t, err) + require.NotEqual(t, before, after, + "editing metrics alone must republish, or the edit never deploys") + }) + + t.Run("reports a missing settings file", func(t *testing.T) { + missing := base + missing.DataSchema = filepath.Join(dir, "nope.json") + _, err := codeEvaluatorDigest(missing, script) + require.Error(t, err) + require.Contains(t, err.Error(), "data_schema", + "the error must name which setting could not be read") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 93fd22b7bde..76b0d4bf7cb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -39,14 +39,18 @@ type DatasetDecl struct { // `.json` file holds a rubric. A code evaluator cannot name a folder — it runs // as a python grader, which is handed one script's source and cannot import a // helper module beside it. -// TODO: carry the code evaluator settings `evaluator create` accepts - -// image_tag, metrics, data_schema, init_parameters. Without them a code -// evaluator can be declared but not configured, so one needing a dependency -// has to be published by hand. See EnsureEvaluator in internal/cmd/reconciler.go. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` + + // Code evaluators only. The three schema fields name JSON files beside the + // script, resolved like Source, because they are edited as files rather + // than written inline in YAML. + ImageTag string `yaml:"image_tag,omitempty" json:"image_tag,omitempty"` + Metrics string `yaml:"metrics,omitempty" json:"metrics,omitempty"` + DataSchema string `yaml:"data_schema,omitempty" json:"data_schema,omitempty"` + InitParameters string `yaml:"init_parameters,omitempty" json:"init_parameters,omitempty"` } // Eval is a run definition: evaluators plus options, bound to a dataset. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index f4902f236a3..5f6786e954e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -162,6 +162,9 @@ func (p *EvalServiceTargetProvider) Deploy( for _, decl := range cfg.Evaluators { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) + decl.Metrics = resolveSource(baseDir, decl.Metrics) + decl.DataSchema = resolveSource(baseDir, decl.DataSchema) + decl.InitParameters = resolveSource(baseDir, decl.InitParameters) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) if err != nil { return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go index 4b3c8765728..32dda623b17 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/live_test.go @@ -24,15 +24,18 @@ import ( "context" "encoding/json" "fmt" + "net/http" "os" "path/filepath" "strings" + "sync" "testing" "time" "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/stretchr/testify/require" ) @@ -54,6 +57,29 @@ type liveEnv struct { datasetClient *dataset_api.DatasetClient } +// One credential for the whole package, because azidentity caches tokens per +// instance. Building one per test made every test shell out to azd again, and +// a refresh that overruns the SDK's ten-second budget for that subprocess +// surfaces as "AzureDeveloperCLICredential: exit status 1" — which reads like +// a broken login rather than a timeout, and lands on whichever test happened +// to run after a slow one. +var ( + sharedCredOnce sync.Once + sharedCred *azidentity.AzureDeveloperCLICredential + sharedCredErr error +) + +func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { + sharedCredOnce.Do(func() { + // Works non-interactively when azd already holds a refresh token, + // which is what makes an unattended run possible. + sharedCred, sharedCredErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}, + ) + }) + return sharedCred, sharedCredErr +} + func setup(t *testing.T) *liveEnv { t.Helper() @@ -65,11 +91,7 @@ func setup(t *testing.T) *liveEnv { t.Fatal("FOUNDRY_PROJECT_ENDPOINT is required") } - // The azd developer CLI credential works non-interactively when azd already - // holds a refresh token, which is what makes an unattended run possible. - cred, err := azidentity.NewAzureDeveloperCLICredential( - &azidentity.AzureDeveloperCLICredentialOptions{}, - ) + cred, err := liveCredential() require.NoError(t, err, "acquiring an azd credential") judge := os.Getenv("AZURE_AI_EVAL_MODEL") @@ -243,12 +265,67 @@ func TestLiveEvalLifecycle(t *testing.T) { require.Equal(t, group.ID, fetched.ID) } -// TestLiveRun invokes a real agent, so it only runs when one is named. +// resolveAgent names the agent the run phase evaluates. +// +// AZURE_AI_EVAL_AGENT wins when set. Otherwise one is discovered, and failing +// to find one is a failure rather than a skip: skipping by default is how the +// agent-target path went unverified for weeks while the suite reported green. +// +// The listing is /agents, not /assistants. They are different collections and +// a project can have plenty of the latter and none of the former — an eval +// target resolves against /agents, so an assistant name is accepted by the +// request and then fails the run with "resources not found". +func resolveAgent(t *testing.T, env *liveEnv) string { + t.Helper() + + if env.agentName != "" { + return env.agentName + } + + cred, err := liveCredential() + require.NoError(t, err) + token, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + require.NoError(t, err, "acquiring a token to list agents") + + req, err := http.NewRequest(http.MethodGet, env.endpoint+"/agents?api-version="+projectAPIVersion, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err, "listing the project's agents") + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, + "could not list agents; set AZURE_AI_EVAL_AGENT to name one directly") + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&listing)) + + for _, a := range listing.Data { + if a.Name != "" { + t.Logf("no AZURE_AI_EVAL_AGENT set; evaluating %q", a.Name) + return a.Name + } + } + + t.Fatal("this project has no agent in /agents, so the agent-target run path " + + "cannot be verified here. Assistants do not count: an eval target " + + "resolves against /agents, and naming an assistant fails the run with " + + "\"resources not found\". Deploy an agent, or set AZURE_AI_EVAL_AGENT " + + "to one in another project") + return "" +} + +// TestLiveRun invokes a real agent, which is the only cover the agent-target +// run path has. func TestLiveRun(t *testing.T) { env := setup(t) - if env.agentName == "" { - t.Skip("set AZURE_AI_EVAL_AGENT to a deployed agent to exercise the run phase") - } + agentName := resolveAgent(t, env) ctx := context.Background() builtins, err := env.evalClient.ListEvaluators( @@ -285,7 +362,7 @@ func TestLiveRun(t *testing.T) { }) require.NoError(t, err, "creating the eval for the run") - ds := eval_api.NewAgentTargetDataSource(env.agentName, nil) + ds := eval_api.NewAgentTargetDataSource(agentName, nil) ds.SetFileContent([]map[string]any{ {"query": "How do I reset my password?"}, }) From f9bb2b098de7cc396a381610c7852ec10377f51d Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 02:28:45 +0300 Subject: [PATCH 070/114] Cancel runs through the OpenAI-spec route The service documents POST on the run itself as the cancel route, matching the OpenAI SDK, and keeps the /cancel suffix only as a back-compat alias. Move to the documented one. The body now carries meaning: empty cancels the run, non-empty updates its status and counters for trusted callers. Both are POST on the same path and both return 200, so a stray body would overwrite a run instead of stopping it and nothing offline would catch it. TestLiveRunCancel starts a run and requires it to reach canceled. --- .../azd-eval-extension-implementation.md | 2 +- .../internal/pkg/eval_api/evaluators.go | 5 +- .../tests/live/run_cancel_test.go | 104 ++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md index 44fb4ae170b..5fda30c6e18 100644 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -162,7 +162,7 @@ type OpenAIEval struct { ID, Name string } // ID is canonical; Name is NOT | get / list | `GET /openai/v1/evals/{id}` · `GET /openai/v1/evals?limit=` | | start run | `POST /openai/v1/evals/{evalId}/runs` | | poll / list runs | `GET /openai/v1/evals/{evalId}/runs/{runId}` · `GET …/runs` | -| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}/cancel` | +| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}` with an empty body | | results | `GET …/runs/{runId}` → `result_counts` + `per_testing_criteria_results` | **The group carries evaluators, not the dataset.** The dataset goes on the **run**. `evaluation_level` is `turn` | `conversation`, service default **`turn`**. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 2d9a85ffa65..84ad2ac811f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -174,13 +174,16 @@ func (c *EvalClient) DeleteEvaluatorVersion( } // CancelOpenAIEvalRun stops an in-flight run. +// +// The body must stay nil: this route cancels only when the body is empty, and +// updates the run's status and counters when it is not. func (c *EvalClient) CancelOpenAIEvalRun( ctx context.Context, evalID string, runID string, ) (*OpenAIEvalRun, error) { path := fmt.Sprintf( - "%s/%s/runs/%s/cancel", + "%s/%s/runs/%s", pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), ) return doRequestTyped[OpenAIEvalRun](c, ctx, http.MethodPost, path, nil, nil, "") diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go new file mode 100644 index 00000000000..a6b01729a5e --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/live/run_cancel_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package live + +import ( + "context" + "strings" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveRunCancel covers the one route whose meaning depends on the request +// body. POST on the run cancels it when the body is empty and updates its +// status and counters when it is not, so a stray body here would silently +// overwrite a run instead of stopping it. Only a live call can tell the two +// apart: both are the same method on the same path, and both return 200. +func TestLiveRunCancel(t *testing.T) { + env := setup(t) + agentName := resolveAgent(t, env) + ctx := context.Background() + + builtins, err := env.evalClient.ListEvaluators( + ctx, eval_api.EvaluatorTypeBuiltin, projectAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, builtins.Value) + evaluatorName := pickQualityEvaluator(t, builtins.Value) + + group, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azd-eval-e2e-cancel"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + IncludeSampleSchema: true, + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"query": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: strings.TrimPrefix(evaluatorName, "builtin."), + EvaluatorName: evaluatorName, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": env.judgeModel, + "deployment_name": env.judgeModel, + }, + }}, + }) + require.NoError(t, err, "creating the eval to cancel a run from") + + t.Cleanup(func() { + _ = env.evalClient.DeleteOpenAIEval(context.Background(), group.ID) + }) + + ds := eval_api.NewAgentTargetDataSource(agentName, nil) + ds.SetFileContent([]map[string]any{ + {"query": "How do I reset my password?"}, + }) + + run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("cancel"), + DataSource: ds, + }) + require.NoError(t, err, "starting the run to cancel") + require.NotEmpty(t, run.ID) + t.Logf("started run %s (status %s)", run.ID, run.Status) + + canceled, err := env.evalClient.CancelOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "cancelling the run") + require.NotNil(t, canceled) + t.Logf("cancel returned status %s", canceled.Status) + + // A sample takes roughly 40 seconds, so a run cancelled immediately after + // it starts should never reach completed. + deadline := time.Now().Add(5 * time.Minute) + var status string + for { + current, err := env.evalClient.GetOpenAIEvalRun(ctx, group.ID, run.ID) + require.NoError(t, err, "polling the cancelled run") + status = strings.ToLower(current.Status) + if status == "canceled" || status == "cancelled" { + break + } + require.NotEqual(t, "completed", status, + "the run completed instead of cancelling, so the empty-body POST did not cancel it") + require.False(t, time.Now().After(deadline), + "the run never reached a cancelled state; last status was %s", status) + time.Sleep(5 * time.Second) + } + + t.Logf("run reached %s", status) +} From 042ae636a77f9599bcc2584b5fea4009a5a22fd6 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 02:48:30 +0300 Subject: [PATCH 071/114] Cover the remaining commands with end-to-end CLI tests The harness drives the built binary as a subprocess against a live project, so these exercise the same path a user takes rather than the client packages underneath. dataset, generate, results, rubric, and run ops each gain a test, and a shared fixture publishes the eval they all read. One product bug surfaced: dataset delete answers 204, which the client rejected, so every successful delete reported an error. The eval client already allowed 204; the dataset client now does too. The fixture outlives any single test, so it is released from TestMain rather than t.Cleanup, which would take it from the tests that follow. --- .../internal/cmd/generate_plan_test.go | 253 +++++++++++ .../internal/pkg/dataset_api/operations.go | 6 +- .../tests/cli/dataset_test.go | 241 +++++++++++ .../tests/cli/fixture_test.go | 404 ++++++++++++++++++ .../tests/cli/generate_test.go | 158 +++++++ .../tests/cli/harness_test.go | 3 + .../tests/cli/results_test.go | 317 ++++++++++++++ .../tests/cli/rubric_test.go | 203 +++++++++ .../tests/cli/run_ops_test.go | 208 +++++++++ 9 files changed, 1792 insertions(+), 1 deletion(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go new file mode 100644 index 00000000000..584a19e7547 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/require" +) + +// `generate` decides what to submit before it touches the network, so the plan +// it builds — which artifacts, from what instruction, at what sample size — is +// checkable without paying for a generation job. These are the parts that +// cannot be observed afterwards: once the jobs are submitted, a wrong default +// is indistinguishable from an intended one. + +func writeConfig(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "eval_generate.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + return path +} + +// A spec is optional, so the defaults are what most callers actually run with. +func TestResolveGenerateConfigDefaultsFromFlagsAlone(t *testing.T) { + absent := filepath.Join(t.TempDir(), "absent.yaml") + + cfg, err := resolveGenerateConfig(absent, "shop-agent", "gpt-4o-mini", "", 0, 0) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + require.Equal(t, "shop-agent", cfg.Agent.Name) + require.NotNil(t, cfg.Generate.Rubric) + require.Equal(t, "shop-agent-quality", cfg.Generate.Rubric.Name) + require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, cfg.Generate.Rubric.LocalDir) + + require.NotNil(t, cfg.Generate.Dataset) + require.Equal(t, "shop-agent-golden", cfg.Generate.Dataset.Name) + require.Equal(t, project.StrategySynthetic, cfg.Generate.Dataset.Strategy) + require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) + require.Equal(t, "./"+project.DefaultDatasetsDir, cfg.Generate.Dataset.LocalDir) +} + +// Without a target there is nothing to generate from, and the refusal has to +// name the flag rather than a config field the caller may not have. +func TestResolveGenerateConfigRequiresATarget(t *testing.T) { + _, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), "", "gpt-4o-mini", "", 0, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "--target") +} + +func TestResolveGenerateConfigReadsTheSpec(t *testing.T) { + path := writeConfig(t, ` +agent: + name: from-spec + context: + instructions: ./instructions.md + traces: + window: 7d + source: ignored-today + sample: 5 +generate: + rubric: + name: spec-rubric + model: gpt-4o + local_dir: ./custom-evaluators + dataset: + name: spec-dataset + strategy: synthetic + sampleSize: 200 + local_dir: ./custom-datasets +`) + + cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + require.Equal(t, "from-spec", cfg.Agent.Name) + require.Equal(t, "./instructions.md", cfg.Agent.Context.Instructions) + require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) + require.Equal(t, "gpt-4o", cfg.Generate.Rubric.Model) + require.Equal(t, "./custom-evaluators", cfg.Generate.Rubric.LocalDir) + require.Equal(t, "spec-dataset", cfg.Generate.Dataset.Name) + require.Equal(t, 200, cfg.Generate.Dataset.SampleSize) + + require.NotNil(t, cfg.Agent.Context.Traces) + require.Equal(t, "7d", cfg.Agent.Context.Traces.Window) + require.Equal(t, 5, cfg.Agent.Context.Traces.Sample) +} + +// Flags win over the spec, which is what makes a one-off run possible without +// editing a file that is checked in. +func TestResolveGenerateConfigLayersFlagsOverTheSpec(t *testing.T) { + path := writeConfig(t, ` +agent: + name: from-spec +generate: + rubric: + name: spec-rubric + model: gpt-4o + dataset: + name: spec-dataset + sampleSize: 200 +`) + + cfg, err := resolveGenerateConfig(path, "from-flag", "gpt-4o-mini", "", 500, 14) + require.NoError(t, err) + require.NoError(t, cfg.Validate()) + + require.Equal(t, "from-flag", cfg.Agent.Name) + require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) + require.Equal(t, 500, cfg.Generate.Dataset.SampleSize) + require.Equal(t, "14d", cfg.Agent.Context.Traces.Window, + "--trace-days must reach the spec as a window, since that is the only "+ + "trace field the generation API takes") + + // The rubric name is not derived when the spec named one, so a --target + // override must not silently rename an artifact the spec author declared. + require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) +} + +// A spec that declares a dataset without a size still has to submit a legal +// job, so the default is applied rather than left at zero. +func TestResolveGenerateConfigFillsAMissingSampleSize(t *testing.T) { + path := writeConfig(t, ` +agent: + name: sized +generate: + dataset: + name: no-size +`) + + cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) + require.NoError(t, err) + require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) + require.NoError(t, cfg.Validate()) +} + +// The bounds are the service's, and the boundaries themselves have to be +// accepted: a check that rejected 15 or 1000 would be indistinguishable from +// one that is simply too strict. +func TestGenerateSampleSizeBounds(t *testing.T) { + for _, tc := range []struct { + size int + allowed bool + }{ + {project.MinSampleSize - 1, false}, + {project.MinSampleSize, true}, + {project.DefaultSampleSize, true}, + {project.MaxSampleSize, true}, + {project.MaxSampleSize + 1, false}, + } { + cfg, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), + "bounded", "gpt-4o-mini", "", tc.size, 0) + require.NoError(t, err) + require.Equal(t, tc.size, cfg.Generate.Dataset.SampleSize) + + err = cfg.Validate() + if tc.allowed { + require.NoErrorf(t, err, "%d is inside the service's range", tc.size) + continue + } + require.Errorf(t, err, "%d is outside the service's range", tc.size) + require.Contains(t, err.Error(), "sampleSize") + } +} + +// --dataset and --evaluator both mean "use this one". Only the dataset side is +// resolved here; the evaluator side is decided in the command body, so it is +// covered by the CLI test that watches for the skip message. +func TestResolveGenerateConfigSkipsTheDatasetWhenOneIsSupplied(t *testing.T) { + cfg, err := resolveGenerateConfig( + filepath.Join(t.TempDir(), "absent.yaml"), + "supplied", "gpt-4o-mini", "prod-sample", 0, 0) + require.NoError(t, err) + require.Nil(t, cfg.Generate.Dataset, "a supplied dataset must not be generated") + require.NotNil(t, cfg.Generate.Rubric) + require.NoError(t, cfg.Validate()) +} + +// Both jobs bill against one deployment, so a spec with no rubric has no model +// to run either of them. +func TestGenerationModelComesFromTheRubricSpec(t *testing.T) { + require.Equal(t, "", generationModel(&project.GenerateConfig{})) + + cfg := &project.GenerateConfig{} + cfg.Generate.Rubric = &project.RubricSpec{Name: "r", Model: "gpt-4o"} + require.Equal(t, "gpt-4o", generationModel(cfg)) +} + +func TestResolveInstruction(t *testing.T) { + dir := t.TempDir() + filled := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(filled, []byte(" test refunds and returns\n\n"), 0o600)) + blank := filepath.Join(dir, "blank.md") + require.NoError(t, os.WriteFile(blank, []byte(" \n"), 0o600)) + + t.Run("inline is returned as given", func(t *testing.T) { + got, err := resolveInstruction("inline text", "") + require.NoError(t, err) + require.Equal(t, "inline text", got) + }) + + t.Run("a file is read and trimmed", func(t *testing.T) { + got, err := resolveInstruction("", filled) + require.NoError(t, err) + require.Equal(t, "test refunds and returns", got) + }) + + // A whitespace-only file would otherwise generate from nothing, which + // produces a rubric with no relation to the agent. + t.Run("an empty file is refused", func(t *testing.T) { + _, err := resolveInstruction("", blank) + require.Error(t, err) + require.Contains(t, err.Error(), "is empty") + }) + + t.Run("a missing file names the flag", func(t *testing.T) { + _, err := resolveInstruction("", filepath.Join(dir, "absent.md")) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file") + }) +} + +// The generation API takes a day window and nothing else, so the two fields it +// drops are reported rather than silently discarded. +func TestWarnIgnoredTraceFields(t *testing.T) { + cfg := &project.GenerateConfig{} + cfg.Agent.Context.Traces = &project.TraceSpec{Window: "7d", Source: "some-source", Sample: 5} + + var out strings.Builder + warnIgnoredTraceFields(cfg, &out) + require.Contains(t, out.String(), "agent.context.traces.source") + require.Contains(t, out.String(), "agent.context.traces.sample") + require.NotContains(t, out.String(), "agent.context.traces.window", + "the window is the one trace field the API takes, so it is not a no-op") + + // A window on its own is fully supported and must not produce a warning. + quiet := &project.GenerateConfig{} + quiet.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} + out.Reset() + warnIgnoredTraceFields(quiet, &out) + require.Empty(t, out.String()) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index c285c19a450..74d1752a4c5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -558,7 +558,11 @@ func (c *DatasetClient) doRequest( log.Printf("[dataset_api] response status: %d", resp.StatusCode) - if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated, http.StatusAccepted) { + // 204 belongs here for the same reason it does in eval_api: a delete that + // removed the version answers No Content, and rejecting that reports every + // successful delete as an error. + if !runtime.HasStatusCode(resp, + http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent) { resp.Body = io.NopCloser(bytes.NewReader(respBody)) return nil, runtime.NewResponseError(resp) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go new file mode 100644 index 00000000000..3b0dbeb1856 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +type datasetSummary struct { + Name string `json:"name"` + Version string `json:"version"` + Format string `json:"format"` +} + +const datasetRows = `{"query":"How do I reset my password?"} +{"query":"What is the refund window?"} +` + +// registeredDataset is a dataset with more than one version, which is what +// makes --version on show and --name on list worth asserting. +type registeredDataset struct { + Name string + // Versions are read back from each registration rather than assumed to + // start at 1: the server assigns them, and a test that hardcoded the + // numbering would be asserting its own guess. + Versions []string +} + +var ( + readOnlyDatasetOnce sync.Once + readOnlyDataset *registeredDataset +) + +// sharedDataset is registered once for the tests that only read it. Each +// registration uploads a blob, so redoing it per test buys nothing. +func sharedDataset(t *testing.T) *registeredDataset { + t.Helper() + readOnlyDatasetOnce.Do(func() { + readOnlyDataset = registerDataset(t, 2) + }) + require.NotNil(t, readOnlyDataset, "the shared dataset could not be registered") + return readOnlyDataset +} + +// registerDataset publishes a dataset and removes every version it created. +func registerDataset(t *testing.T, versions int) *registeredDataset { + t.Helper() + require.Positive(t, versions) + + path := filepath.Join(t.TempDir(), "golden.jsonl") + require.NoError(t, os.WriteFile(path, []byte(datasetRows), 0o600)) + + ds := ®isteredDataset{Name: uniqueName("azdcli_ds")} + for range versions { + r := requireSuccess(t, run(t, "dataset", "create", + "--name", ds.Name, "--file", path, "-o", "json")) + + var created datasetSummary + r.JSON(t, &created) + require.NotEmpty(t, created.Version, "the service assigns the version") + ds.Versions = append(ds.Versions, created.Version) + + version := created.Version + deferTeardown(func() { + runQuietly("dataset", "delete", "--name", ds.Name, "--version", version) + }) + } + require.Len(t, ds.Versions, versions) + require.NotEqual(t, ds.Versions[0], ds.Versions[len(ds.Versions)-1], + "re-registering must advance the version rather than overwrite") + return ds +} + +func TestCLIDatasetList(t *testing.T) { + ds := sharedDataset(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name)) + for _, header := range []string{"NAME", "VERSION", "FORMAT"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + // --name is what makes the listing usable once a project holds more than a + // screenful: it narrows to one dataset's versions. + t.Run("name scopes the listing to one dataset's versions", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name, "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.NotEmpty(t, listed) + + seen := map[string]bool{} + for _, v := range listed { + require.Equalf(t, ds.Name, v.Name, + "--name must return only that dataset's versions; got %q", v.Name) + seen[v.Version] = true + } + for _, want := range ds.Versions { + require.Truef(t, seen[want], "version %s is missing from the listing", want) + } + }) + + // Unscoped, the listing is every dataset rather than every version, so the + // one just registered has to be in it. + t.Run("unscoped lists the project's datasets", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", "-o", "json")) + var all []datasetSummary + r.JSON(t, &all) + require.NotEmpty(t, all) + + found := false + for _, d := range all { + if d.Name == ds.Name { + found = true + } + } + require.True(t, found, "a registered dataset must appear in the unscoped listing") + }) + + t.Run("an unknown name lists nothing rather than failing", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "list", + "--name", "azdcli-no-such-dataset", "-o", "json")) + var listed []datasetSummary + r.JSON(t, &listed) + require.Empty(t, listed) + }) +} + +func TestCLIDatasetShow(t *testing.T) { + ds := sharedDataset(t) + latest := ds.Versions[len(ds.Versions)-1] + + // Omitting the version means the latest, which is the only sensible + // default for a name that gains a version on every registration. + t.Run("defaults to the latest version", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name, "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Name, shown.Name) + require.Equal(t, latest, shown.Version) + }) + + t.Run("version pins an earlier one", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", + "--name", ds.Name, "--version", ds.Versions[0], "-o", "json")) + var shown datasetSummary + r.JSON(t, &shown) + require.Equal(t, ds.Versions[0], shown.Version) + require.NotEqual(t, latest, shown.Version) + }) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name)) + for _, header := range []string{"NAME", "VERSION", "FORMAT", "URI"} { + require.Containsf(t, r.Stdout, header, "the table lost its %s column", header) + } + require.Contains(t, r.Stdout, ds.Name) + }) + + t.Run("the name is required", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show")) + require.Contains(t, r.Combined(), "--name is required") + }) + + t.Run("an unknown dataset is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", "--name", "azdcli-no-such-dataset")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "azdcli-no-such-dataset") + }) + + t.Run("an unknown version of a real dataset is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "dataset", "show", + "--name", ds.Name, "--version", "9999")) + require.Contains(t, r.Combined(), "9999") + require.Less(t, len(r.Combined()), 600, r.Combined()) + }) +} + +func TestCLIDatasetDelete(t *testing.T) { + t.Run("both flags are required", func(t *testing.T) { + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "--version", "1")).Combined(), + "--name is required") + require.Contains(t, + requireFailure(t, run(t, "dataset", "delete", "--name", "whatever")).Combined(), + "--version is required") + }) + + // Deleting something that was never registered succeeds. The service + // treats DELETE as idempotent and answers 204 whatever the name, so the + // command reports a removal it did not perform — and the not-found branch + // in `dataset delete` cannot be reached this way. Asserted rather than + // wished away, because a caller scripting against the exit code is + // entitled to know it means "gone", not "was there and is now gone". + t.Run("deleting an unregistered dataset is idempotent, not an error", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "delete", + "--name", "azdcli-no-such-dataset", "--version", "1")) + require.Contains(t, r.Stdout, "Deleted dataset") + + listed := requireSuccess(t, run(t, "dataset", "list", + "--name", "azdcli-no-such-dataset", "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + require.Empty(t, remaining, "nothing was there to delete in the first place") + }) + + // A successful delete answers 204 No Content, so asserting the exit code + // is what catches a client that reads an empty body as a failure and + // reports a removal it just performed as an error. + t.Run("one version is removed and the other survives", func(t *testing.T) { + ds := registerDataset(t, 2) + gone, kept := ds.Versions[0], ds.Versions[1] + + r := requireSuccess(t, run(t, "dataset", "delete", + "--name", ds.Name, "--version", gone)) + require.Contains(t, r.Stdout, "Deleted dataset") + require.Contains(t, r.Stdout, ds.Name) + + listed := requireSuccess(t, run(t, "dataset", "list", + "--name", ds.Name, "-o", "json")) + var remaining []datasetSummary + listed.JSON(t, &remaining) + + versions := map[string]bool{} + for _, v := range remaining { + versions[v.Version] = true + } + require.False(t, versions[gone], "the deleted version must leave the listing") + require.True(t, versions[kept], "deleting one version must not remove the others") + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go new file mode 100644 index 00000000000..af745cae250 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go @@ -0,0 +1,404 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/pkg/evalcore" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" +) + +// The command tests need an eval that has already been run, and building one +// through the CLI is not possible: there is no command that creates an eval +// from flags, only `run`, which needs a config file and a deployed target. +// So the fixture is built with the client and every assertion is made against +// the binary. What is under test is the command surface; the eval is scenery. +// +// It is built once for the whole package because two completed runs cost +// minutes, and torn down in TestMain rather than t.Cleanup so that whichever +// test happened to trigger the build does not take the fixture away from the +// rest. + +const fixtureAPIVersion = "2025-11-15-preview" + +// scoringGrader splits the rows deterministically. A grader that scores every +// row the same way makes --failed-only and a comparison indistinguishable from +// a no-op, so "good" is the difference between a pass and a failure. +const scoringGrader = `def grade(sample, item) -> float: + response = (item or {}).get("response", "") + return 1.0 if "good" in response else 0.0 +` + +// evalFixture is one eval with two completed runs of the same criterion. +type evalFixture struct { + EvaluatorName string + EvalID string + + // Baseline scores worse than Treatment, so a comparison between them has + // a delta to report rather than zero. + BaselineRunID string + TreatmentRunID string +} + +var ( + fixtureOnce sync.Once + fixture *evalFixture + fixtureErr error + + // teardown runs after the last test, in reverse order. + teardownMu sync.Mutex + teardown []func() +) + +func deferTeardown(fn func()) { + teardownMu.Lock() + defer teardownMu.Unlock() + teardown = append(teardown, fn) +} + +func runTeardown() { + teardownMu.Lock() + defer teardownMu.Unlock() + for i := len(teardown) - 1; i >= 0; i-- { + teardown[i]() + } + teardown = nil +} + +// runQuietly invokes the binary without a *testing.T. +// +// Teardown runs after the last test has reported, and logging or asserting +// against a finished test panics, so nothing here may touch one. +func runQuietly(args ...string) { + full := append(append([]string{}, args...), "--project-endpoint", endpoint) + _ = exec.Command(binaryPath, full...).Run() +} + +var ( + credOnce sync.Once + cred *azidentity.AzureDeveloperCLICredential + credErr error +) + +// liveClient builds the client the fixture is assembled with. One credential +// for the package, because azidentity caches tokens per instance and a fresh +// one per call makes every call shell out to azd again. +// +// The first token is fetched here rather than lazily on the first request: +// that call is the one that flakes, and paying for it up front means the rest +// of the fixture runs against a cached token. +func liveClient() (*eval_api.EvalClient, error) { + credOnce.Do(func() { + cred, credErr = azidentity.NewAzureDeveloperCLICredential( + &azidentity.AzureDeveloperCLICredentialOptions{}) + if credErr != nil { + return + } + credErr = retryCredentialFlake(func() error { + _, err := cred.GetToken(context.Background(), policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + return err + }) + }) + if credErr != nil { + return nil, credErr + } + return eval_api.NewEvalClient(endpoint, cred), nil +} + +// retryCredentialFlake reruns a request that failed only because azd's token +// helper exited non-zero. +// +// It is the same failure the harness retries around the binary, for the same +// reason: nothing was sent, and the alternative is a suite that fails on a +// different test each run for a reason unrelated to the code. Any other error +// is returned immediately. +func retryCredentialFlake(fn func() error) error { + var err error + for attempt := range 4 { + if attempt > 0 { + time.Sleep(time.Duration(attempt) * 2 * time.Second) + } + if err = fn(); err == nil || !strings.Contains(err.Error(), credentialFlake) { + return err + } + } + return err +} + +// sharedEval returns the fixture, building it on first use. +// +// A failure here fails the calling test rather than skipping it: every test +// that asks for the fixture is testing something that cannot be exercised +// without one, and a suite that goes green because its subject was missing is +// worse than one that goes red. +func sharedEval(t *testing.T) *evalFixture { + t.Helper() + fixtureOnce.Do(func() { + start := time.Now() + fixture, fixtureErr = buildFixture(t.Logf) + t.Logf("fixture ready in %s", time.Since(start).Round(time.Second)) + }) + if fixtureErr != nil { + t.Fatalf("building the shared eval the command tests run against: %v", fixtureErr) + } + return fixture +} + +func buildFixture(logf func(string, ...any)) (*evalFixture, error) { + ctx := context.Background() + + client, err := liveClient() + if err != nil { + return nil, fmt.Errorf("acquiring an azd credential: %w", err) + } + + name := strings.ReplaceAll(uniqueName("azdclifx"), "-", "_") + script, err := publishScoringEvaluator(ctx, client, name) + if err != nil { + return nil, err + } + logf("published code evaluator %s version %s", name, script.Version) + + if err := awaitEvaluatorListed(ctx, client, name, script.Version); err != nil { + return nil, err + } + + evalID, err := createFixtureEval(ctx, client, name) + if err != nil { + return nil, err + } + logf("created eval %s", evalID) + + // Different pass rates so the comparison has something to measure. + baseline, err := startFixtureRun(ctx, client, evalID, "baseline", + []string{"a bad answer", "another bad answer", "a good answer"}) + if err != nil { + return nil, err + } + treatment, err := startFixtureRun(ctx, client, evalID, "treatment", + []string{"a good answer", "another good answer", "a third good answer"}) + if err != nil { + return nil, err + } + logf("started runs %s and %s", baseline, treatment) + + // Polled together: they are independent, and serialising them doubles the + // slowest part of the suite for nothing. + errs := make(chan error, 2) + for _, runID := range []string{baseline, treatment} { + go func(id string) { errs <- awaitCompleted(ctx, client, evalID, id, logf) }(runID) + } + for range 2 { + if err := <-errs; err != nil { + return nil, err + } + } + + return &evalFixture{ + EvaluatorName: name, + EvalID: evalID, + BaselineRunID: baseline, + TreatmentRunID: treatment, + }, nil +} + +func publishScoringEvaluator( + ctx context.Context, + client *eval_api.EvalClient, + name string, +) (*eval_api.EvaluatorVersion, error) { + dir, err := os.MkdirTemp("", "azdcli-grader") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + + path := filepath.Join(dir, name+".py") + if err := os.WriteFile(path, []byte(scoringGrader), 0o600); err != nil { + return nil, err + } + script, err := evalcore.LoadCodeEvaluator(name, path) + if err != nil { + return nil, fmt.Errorf("loading the grader: %w", err) + } + + // Without a data_schema the criteria builder has no mapping to derive, so + // the schema is what makes the evaluator usable rather than merely + // publishable. + opts := eval_api.CodeEvaluatorOptions{ + DataSchema: json.RawMessage( + `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`), + Metrics: json.RawMessage( + `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`), + } + var version *eval_api.EvaluatorVersion + if err := retryCredentialFlake(func() error { + var err error + version, err = client.CreateCodeEvaluatorVersion(ctx, script, opts, fixtureAPIVersion) + return err + }); err != nil { + return nil, fmt.Errorf("publishing the code evaluator: %w", err) + } + deferTeardown(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, version.Version, fixtureAPIVersion) + }) + return version, nil +} + +// awaitEvaluatorListed waits for the version listing to catch up, which is the +// view eval creation resolves against. The direct read goes consistent first, +// so waiting on that alone still leaves the create failing with "was not +// found". +func awaitEvaluatorListed( + ctx context.Context, + client *eval_api.EvalClient, + name, version string, +) error { + deadline := time.Now().Add(2 * time.Minute) + for { + list, err := client.ListEvaluatorVersions(ctx, name, fixtureAPIVersion) + if err == nil && list != nil { + for _, entry := range list.Value { + if entry.Version == version { + return nil + } + } + } + if time.Now().After(deadline) { + return fmt.Errorf("evaluator %s version %s never appeared in the listing", name, version) + } + time.Sleep(500 * time.Millisecond) + } +} + +func createFixtureEval( + ctx context.Context, + client *eval_api.EvalClient, + evaluatorName string, +) (string, error) { + // Hand-written rather than built with buildEvalRequest, which is + // unexported. That is safe only because the evaluator is one published + // here whose schema is a single `response` column; a built-in would need + // the shipping builder, since their input contracts differ per evaluator. + var group *eval_api.OpenAIEval + if err := retryCredentialFlake(func() error { + var err error + group, err = client.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ + Name: uniqueName("azdcli-fixture"), + DataSourceConfig: &eval_api.DataSourceConfig{ + Type: "custom", + ItemSchema: map[string]any{ + "type": "object", + "properties": map[string]any{"response": map[string]any{"type": "string"}}, + }, + }, + TestingCriteria: []eval_api.TestingCriterion{{ + Type: "azure_ai_evaluator", + Name: evaluatorName, + EvaluatorName: evaluatorName, + DataMapping: map[string]string{"response": "{{item.response}}"}, + }}, + }) + return err + }); err != nil { + return "", fmt.Errorf("creating the fixture eval: %w", err) + } + deferTeardown(func() { + _ = client.DeleteOpenAIEval(context.Background(), group.ID) + }) + return group.ID, nil +} + +func startFixtureRun( + ctx context.Context, + client *eval_api.EvalClient, + evalID, label string, + responses []string, +) (string, error) { + rows := make([]map[string]any, 0, len(responses)) + for _, r := range responses { + rows = append(rows, map[string]any{"response": r}) + } + + ds := eval_api.NewDatasetOnlyDataSource() + ds.SetFileContent(rows) + + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.CreateOpenAIEvalRun(ctx, evalID, &eval_api.CreateOpenAIEvalRunRequest{ + Name: uniqueName("azdcli-" + label), + DataSource: ds, + }) + return err + }); err != nil { + return "", fmt.Errorf("starting the %s run: %w", label, err) + } + return run.ID, nil +} + +var terminalRunStatus = map[string]bool{ + "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, +} + +// awaitCompleted requires the run to have scored something. +// +// A run whose every sample errors still reports completed, so the status alone +// would let the whole suite run against an eval that measured nothing. +func awaitCompleted( + ctx context.Context, + client *eval_api.EvalClient, + evalID, runID string, + logf func(string, ...any), +) error { + deadline := time.Now().Add(15 * time.Minute) + for { + var run *eval_api.OpenAIEvalRun + if err := retryCredentialFlake(func() error { + var err error + run, err = client.GetOpenAIEvalRun(ctx, evalID, runID) + return err + }); err != nil { + return fmt.Errorf("polling run %s: %w", runID, err) + } + if terminalRunStatus[strings.ToLower(run.Status)] { + if strings.ToLower(run.Status) != "completed" { + return fmt.Errorf("run %s finished as %q: %s", runID, run.Status, run.Failure()) + } + if run.ResultCounts == nil { + return fmt.Errorf("run %s completed without reporting counts", runID) + } + if run.ResultCounts.Passed+run.ResultCounts.Failed == 0 { + return fmt.Errorf( + "run %s completed without scoring any row (errored=%d); the fixture "+ + "would prove nothing", runID, run.ResultCounts.Errored) + } + logf("run %s completed: passed=%d failed=%d errored=%d", + runID, run.ResultCounts.Passed, run.ResultCounts.Failed, run.ResultCounts.Errored) + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("run %s did not finish in time (last status %q)", runID, run.Status) + } + time.Sleep(10 * time.Second) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go new file mode 100644 index 00000000000..ff3f37b0a31 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// `generate` submits two jobs that cost model time and take minutes, so what +// is exercised here is everything up to that point: the flag combinations it +// refuses, the spec it parses, and the two flags that mean "I already have +// this one, do not make another". None of these tests submits a job — the last +// one reaches the service and deliberately generates nothing. + +// TestCLIGenerateRefusesBadFlagCombinations covers the mistakes that must cost +// nothing to make. Each of these is decided locally, so a user finds out +// before a job is billed. +func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { + dir := t.TempDir() + instruction := filepath.Join(dir, "instruction.md") + require.NoError(t, os.WriteFile(instruction, []byte("test refunds"), 0o600)) + + cases := []struct { + name string + args []string + want string + }{{ + name: "the two instruction sources are mutually exclusive", + args: []string{"--target", "a", "--agent-instruction", "inline", + "--agent-instruction-file", instruction}, + want: "agent-instruction-file", + }, { + name: "below the minimum sample size", + args: []string{"--target", "a", "--max-samples", "14"}, + want: "between 15 and 1000", + }, { + name: "above the maximum sample size", + args: []string{"--target", "a", "--max-samples", "1001"}, + want: "between 15 and 1000", + }, { + name: "a missing instruction file names the flag", + args: []string{"--target", "a", "--agent-instruction-file", + filepath.Join(dir, "absent.md")}, + want: "--agent-instruction-file", + }, { + name: "generating needs a model deployment", + args: []string{"--target", "a", "--agent-instruction", "inline"}, + want: "--generation-model", + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := requireFailure(t, runIn(t, dir, append([]string{"generate"}, tc.args...)...)) + require.Contains(t, r.Combined(), tc.want) + }) + } +} + +// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with no target and +// nothing to prompt with, the process has to end saying which flag to pass. +func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--no-prompt")) + require.Contains(t, r.Combined(), "--target is required") + require.Contains(t, r.Combined(), "--no-prompt", + "the message must say why it could not be resolved") +} + +// TestCLIGenerateReadsTheSpec proves the config file is loaded and validated +// rather than only the flags. +// +// The strategy is the clearest evidence: `from-traces` is a value the spec +// accepts syntactically and the generation API cannot honour, so the refusal +// can only come from having parsed the file. +func TestCLIGenerateReadsTheSpec(t *testing.T) { + dir := t.TempDir() + spec := filepath.Join(dir, "gen.yaml") + require.NoError(t, os.WriteFile(spec, []byte(` +agent: + name: from-spec +generate: + dataset: + name: spec-dataset + strategy: from-traces +`), 0o600)) + + r := requireFailure(t, runIn(t, dir, "generate", "--config", spec)) + require.Contains(t, r.Combined(), "from-traces") + require.Contains(t, r.Combined(), "agent.context.traces.window", + "the refusal must point at the field that does seed generation from traces") +} + +// TestCLIGenerateSkipsWhatWasSupplied is the one generate test that reaches the +// service, and it is here because the skip decision is made in the command +// body rather than in the config resolver. +// +// With both artifacts supplied there is nothing left to generate, so the whole +// command runs without submitting a job — which is what makes it affordable to +// assert on. A regression that stopped honouring either flag would show up as +// a generation job starting instead of this returning. +func TestCLIGenerateSkipsWhatWasSupplied(t *testing.T) { + dir := t.TempDir() + + r := requireSuccess(t, runIn(t, dir, "generate", + "--target", "azd-eval-probe-agent", + "--agent-instruction", "answer questions about orders", + "--evaluator", "already-published", + "--dataset", "already-registered")) + + require.Contains(t, r.Stdout, "skipping rubric generation") + require.Contains(t, r.Stdout, "skipping data generation") + require.Contains(t, r.Stdout, "Nothing was generated.") + + // The deployment spec is only rewritten when something was produced, and + // writing an empty reference into it would be worse than not writing. + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "a generate that produced nothing must write nothing") +} + +// TestCLIGenerateSuppressionIsPerArtifact pins the two flags apart: neither +// may suppress the artifact it does not name. +// +// Each case supplies one artifact and leaves the other to be generated, and is +// stopped at the model check that precedes submission. Reaching that error is +// the proof: it is only raised when something is still going to be generated, +// so it says the unsupplied artifact survived the other flag. +func TestCLIGenerateSuppressionIsPerArtifact(t *testing.T) { + cases := []struct { + name string + supplied []string + survives string + }{ + {"a supplied evaluator leaves the dataset", []string{"--evaluator", "already-published"}, "dataset"}, + {"a supplied dataset leaves the rubric", []string{"--dataset", "already-registered"}, "rubric"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"generate", + "--target", "azd-eval-probe-agent", + "--agent-instruction", "answer questions about orders"}, tc.supplied...) + + r := requireFailure(t, runIn(t, t.TempDir(), args...)) + require.Contains(t, r.Combined(), "--generation-model", + "the %s was suppressed by a flag that does not name it", tc.survives) + require.NotContains(t, strings.ToLower(r.Combined()), "generating ", + "the run must stop at the model check, before any job is submitted") + }) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go index e4e86181a1d..dec143ce139 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -68,6 +68,9 @@ func TestMain(m *testing.M) { } code := m.Run() + // The shared eval outlives any single test, so it cannot be released with + // t.Cleanup without taking it away from the tests that run after. + runTeardown() os.RemoveAll(dir) os.Exit(code) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go new file mode 100644 index 00000000000..8660b5eda26 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go @@ -0,0 +1,317 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/csv" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// resultsPayload is what `results show -o json` emits: the run and the rows. +type resultsPayload struct { + Run struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` + PerTestingCriteria []struct { + TestingCriteria string `json:"testing_criteria"` + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"per_testing_criteria_results"` + } `json:"run"` + OutputItems []struct { + ID string `json:"id"` + Status string `json:"status"` + DataSourceItem map[string]any `json:"datasource_item"` + Results []struct { + Name string `json:"name"` + Score *float64 `json:"score"` + Passed bool `json:"passed"` + } `json:"results"` + } `json:"output_items"` +} + +// TestCLIResultsShowRendersTheRows is the difference between `results show` and +// `run show`: the totals say how many failed, these say which. +// +// The fixture's baseline run scores one row and fails two, so the rendering is +// checked against known verdicts rather than against whatever came back. +func TestCLIResultsShowRendersTheRows(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.BaselineRunID)) + + require.Contains(t, r.Stdout, f.BaselineRunID) + require.Contains(t, r.Stdout, "Totals: 1 passed, 2 failed, 0 errored") + require.Contains(t, r.Stdout, "CRITERION") + require.Contains(t, r.Stdout, "ITEM") + require.Contains(t, r.Stdout, "EVALUATOR") + require.Contains(t, r.Stdout, "SCORE") + require.Contains(t, r.Stdout, f.EvaluatorName) + + // Both verdicts, and the row's own input: a table that showed only the + // counts would satisfy every assertion above. + require.Contains(t, r.Stdout, "FAIL") + require.Contains(t, r.Stdout, "pass") + require.Contains(t, r.Stdout, "response=a good answer") + require.Contains(t, r.Stdout, "response=a bad answer") + require.Contains(t, r.Stdout, "Report:") +} + +func TestCLIResultsShowJSON(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "results", "show", f.EvalID, + "--run-id", f.BaselineRunID, "-o", "json")) + + var payload resultsPayload + r.JSON(t, &payload) + + require.Equal(t, f.BaselineRunID, payload.Run.ID) + require.Equal(t, "completed", payload.Run.Status) + require.Equal(t, 3, payload.Run.ResultCounts.Total) + require.Equal(t, 1, payload.Run.ResultCounts.Passed) + require.Equal(t, 2, payload.Run.ResultCounts.Failed) + require.Zero(t, payload.Run.ResultCounts.Errored) + + require.Len(t, payload.Run.PerTestingCriteria, 1) + require.Equal(t, f.EvaluatorName, payload.Run.PerTestingCriteria[0].TestingCriteria) + + // The rows are the reason this command exists, and a run reporting counts + // while returning none would still satisfy everything above. + require.Len(t, payload.OutputItems, 3, "every dataset row must come back as an item") + + passed := 0 + for _, item := range payload.OutputItems { + require.NotEmpty(t, item.DataSourceItem["response"], + "each row must carry the column it was evaluated on") + require.Len(t, item.Results, 1) + require.Equal(t, f.EvaluatorName, item.Results[0].Name) + require.NotNil(t, item.Results[0].Score, "a scored row must report its score") + if item.Results[0].Passed { + passed++ + require.Equal(t, 1.0, *item.Results[0].Score) + } else { + require.Equal(t, 0.0, *item.Results[0].Score) + } + } + require.Equal(t, 1, passed, "the per-row verdicts must agree with the totals") +} + +// TestCLIResultsShowFailedOnly asserts the filter removes rows rather than +// merely relabelling them. +func TestCLIResultsShowFailedOnly(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "results", "show", f.EvalID, + "--run-id", f.BaselineRunID, "--failed-only")) + + require.Contains(t, r.Stdout, "response=a bad answer") + require.NotContains(t, r.Stdout, "response=a good answer", + "--failed-only must drop the rows that passed") + + // The passing run has nothing to show, and saying so is not the same as + // printing an empty table. + empty := requireSuccess(t, run(t, "results", "show", f.EvalID, + "--run-id", f.TreatmentRunID, "--failed-only")) + require.Contains(t, empty.Stdout, "No failing rows.") +} + +func TestCLIResultsExport(t *testing.T) { + f := sharedEval(t) + + t.Run("json to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "json")) + + var exported struct { + ID string `json:"id"` + Status string `json:"status"` + ResultCounts struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + } `json:"result_counts"` + } + r.JSON(t, &exported) + require.Equal(t, f.BaselineRunID, exported.ID) + require.Equal(t, "completed", exported.Status) + require.Equal(t, 1, exported.ResultCounts.Passed) + require.Equal(t, 2, exported.ResultCounts.Failed) + }) + + t.Run("csv to stdout", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "csv")) + + rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() + require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) + require.Len(t, rows, 2, "a header and one row per criterion") + require.Equal(t, + []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + require.Equal(t, + []string{f.BaselineRunID, "completed", f.EvaluatorName, "1", "2"}, rows[1]) + }) + + t.Run("out-file writes the path instead of stdout", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "results.csv") + + r := requireSuccess(t, runIn(t, dir, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "csv", "-O", path)) + require.Empty(t, strings.TrimSpace(r.Stdout), + "-O redirects the payload; leaving it on stdout too would double it") + + body, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(body), "run_id,status,criterion,passed,failed") + require.Contains(t, string(body), f.BaselineRunID) + }) + + t.Run("an unknown format is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "results", "export", f.EvalID, + "--run-id", f.BaselineRunID, "--format", "xml")) + require.Contains(t, r.Combined(), "json or csv") + }) +} + +// comparison is the shape `results compare -o json` emits. +type comparison struct { + State string `json:"state"` + Request struct { + EvalID string `json:"evalId"` + BaselineRunID string `json:"baselineRunId"` + TreatmentRunIDs []string `json:"treatmentRunIds"` + } `json:"request"` + Result struct { + Method string `json:"method"` + Comparisons []struct { + TestingCriteria string `json:"testingCriteria"` + Metric string `json:"metric"` + BaselineRunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average float64 `json:"average"` + } `json:"baselineRunSummary"` + CompareItems []struct { + TreatmentRunSummary struct { + RunID string `json:"runId"` + SampleCount int `json:"sampleCount"` + Average float64 `json:"average"` + } `json:"treatmentRunSummary"` + DeltaEstimate float64 `json:"deltaEstimate"` + TreatmentEffect string `json:"treatmentEffect"` + } `json:"compareItems"` + } `json:"comparisons"` + } `json:"result"` +} + +// TestCLIResultsCompare needs two completed runs of the same eval that scored +// differently, which is why the fixture seeds one run to fail two of three +// rows: comparing two identical runs reports a zero delta, and a comparison +// that computed nothing would look the same. +func TestCLIResultsCompare(t *testing.T) { + f := sharedEval(t) + + t.Run("rendered columns", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "compare", f.EvalID, + "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID)) + + for _, header := range []string{ + "METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT", + } { + require.Containsf(t, r.Stdout, header, "the comparison table lost its %s column", header) + } + require.Contains(t, r.Stdout, "Method:") + require.Contains(t, r.Stdout, f.TreatmentRunID) + require.Contains(t, r.Stdout, f.EvaluatorName) + + // One in three against three in three. The delta is signed, which is + // the whole point of naming a baseline. + require.Contains(t, r.Stdout, "0.333") + require.Contains(t, r.Stdout, "1.000") + require.Contains(t, r.Stdout, "+0.667") + }) + + t.Run("json shape", func(t *testing.T) { + r := requireSuccess(t, run(t, "results", "compare", f.EvalID, + "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID, "-o", "json")) + + var got comparison + r.JSON(t, &got) + + require.Equal(t, "Succeeded", got.State) + require.Equal(t, f.EvalID, got.Request.EvalID) + require.Equal(t, f.BaselineRunID, got.Request.BaselineRunID) + require.Equal(t, []string{f.TreatmentRunID}, got.Request.TreatmentRunIDs) + + require.NotEmpty(t, got.Result.Method) + require.Len(t, got.Result.Comparisons, 1) + c := got.Result.Comparisons[0] + require.Equal(t, f.EvaluatorName, c.TestingCriteria) + require.Equal(t, f.BaselineRunID, c.BaselineRunSummary.RunID) + require.Equal(t, 3, c.BaselineRunSummary.SampleCount) + require.InDelta(t, 1.0/3.0, c.BaselineRunSummary.Average, 0.001) + + require.Len(t, c.CompareItems, 1) + item := c.CompareItems[0] + require.Equal(t, f.TreatmentRunID, item.TreatmentRunSummary.RunID) + require.Equal(t, 1.0, item.TreatmentRunSummary.Average) + require.InDelta(t, 2.0/3.0, item.DeltaEstimate, 0.001) + require.NotEmpty(t, item.TreatmentEffect, + "the effect classifies the result, including when there are too few samples") + }) + + // Naming neither run is the common case — "did my last change help?" — so + // the defaults are asserted against the listing rather than against the + // fixture's own ids, which is what the command itself resolves from. + t.Run("defaults to the two most recent completed runs", func(t *testing.T) { + listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) + var runs []struct { + ID string `json:"id"` + Status string `json:"status"` + } + listed.JSON(t, &runs) + + completed := make([]string, 0, len(runs)) + for _, r := range runs { + if r.Status == "completed" { + completed = append(completed, r.ID) + } + } + require.GreaterOrEqual(t, len(completed), 2, + "comparing needs two completed runs of the same eval") + + r := requireSuccess(t, run(t, "results", "compare", f.EvalID, "-o", "json")) + var got comparison + r.JSON(t, &got) + + require.Equal(t, completed[0], got.Request.TreatmentRunIDs[0], + "the treatment defaults to the most recent completed run") + require.Equal(t, completed[1], got.Request.BaselineRunID, + "the baseline defaults to the one before it") + require.Equal(t, "Succeeded", got.State) + }) +} + +// TestCLIResultsUnknownEvalIsBrief covers the failure a user hits by typo. The +// service answers with a long JSON body; printing it verbatim buries the one +// useful sentence. +func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { + r := requireFailure(t, run(t, "results", "show", "eval_azdcli_does_not_exist")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "eval_azdcli_does_not_exist") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go new file mode 100644 index 00000000000..ebe5b49bd24 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// A rubric is the other kind of evaluator: a JSON file of weighted dimensions, +// graded by a judge model rather than by code. It shares nothing with the code +// path on the wire beyond the route, so publishing one had never been +// exercised against a real project. + +// writeRubric lays down a rubric file and returns its path. +func writeRubric(t *testing.T, dimensions string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, + []byte(`{"dimensions":`+dimensions+`}`), 0o600)) + return path +} + +// evaluatorDocument is what `evaluator show` prints. +type evaluatorDocument struct { + Name string `json:"name"` + Version string `json:"version"` + EvaluatorType string `json:"evaluator_type"` + Definition struct { + Type string `json:"type"` + Dimensions []struct { + ID string `json:"id"` + Description string `json:"description"` + Weight int `json:"weight"` + } `json:"dimensions"` + DataSchema map[string]any `json:"data_schema"` + InitParameters map[string]any `json:"init_parameters"` + Metrics map[string]any `json:"metrics"` + } `json:"definition"` + SupportedEvaluationLevels []string `json:"supported_evaluation_levels"` +} + +// TestCLIRubricRoundTrip publishes a rubric, reads it back, and republishes it. +func TestCLIRubricRoundTrip(t *testing.T) { + name := uniqueName("azdcli_rubric") + rubric := writeRubric(t, `[ + {"id":"tone","description":"Is the answer polite?","weight":5}, + {"id":"accuracy","description":"Is the answer correct?","weight":10} + ]`) + + created := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + require.Contains(t, created.Stdout, "version 1") + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + var doc evaluatorDocument + shown.JSON(t, &doc) + + require.Equal(t, name, doc.Name) + require.Equal(t, "1", doc.Version) + require.Equal(t, "custom", doc.EvaluatorType) + require.Equal(t, "rubric", doc.Definition.Type, + "the discriminator is what tells the service which definition kind it holds") + + require.Len(t, doc.Definition.Dimensions, 2) + byID := map[string]int{} + for _, d := range doc.Definition.Dimensions { + byID[d.ID] = d.Weight + require.NotEmpty(t, d.Description, "a dimension's description is what the judge grades against") + } + require.Equal(t, 5, byID["tone"]) + require.Equal(t, 10, byID["accuracy"]) + + // The rubric named only dimensions. Everything else is filled in by the + // service, and a caller reading the definition back gets those defaults + // rather than what was sent — including the judge model the evaluator will + // require at run time. + require.NotEmpty(t, doc.Definition.DataSchema, + "the service supplies a rubric's data schema; the author never writes one") + require.NotEmpty(t, doc.Definition.InitParameters) + require.NotEmpty(t, doc.Definition.Metrics) + require.NotEmpty(t, doc.SupportedEvaluationLevels) + + // Every registration publishes a new immutable version, which is why there + // is no separate update command. + republished := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + require.Contains(t, republished.Stdout, "version 2", + "re-registering must advance the version rather than overwrite") + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "2") + }) + + // The earlier version stays reachable, which is what makes a published + // version safe to reference from a config. + pinned := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "--version", "1")) + var first evaluatorDocument + pinned.JSON(t, &first) + require.Equal(t, "1", first.Version) +} + +// TestCLIRubricWeightMustBeAnIntegerFromOneToTen covers the validation a +// hand-authored rubric is most likely to trip. +// +// The service runs two separate checks and they answer differently: a +// fractional weight is rejected for not being an integer, an out-of-range one +// for being out of range. Both are asserted because a caller only ever sees +// one of them, and both have to say what a legal weight is. +func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { + cases := []struct { + name string + weight string + }{ + {"fractional", "2.5"}, + {"zero", "0"}, + {"above ten", "11"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rubric := writeRubric(t, + `[{"id":"tone","description":"Is the answer polite?","weight":`+tc.weight+`}]`) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_badweight"), "--rubric", rubric)) + require.Contains(t, r.Combined(), "between 1 and 10", + "the refusal must say what a legal weight is") + }) + } + + // A weight the service accepts, so the cases above are failing on the + // weight rather than on the rubric shape they share. + name := uniqueName("azdcli_goodweight") + ok := writeRubric(t, `[{"id":"tone","description":"Is the answer polite?","weight":1}]`) + requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", ok)) + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) +} + +// TestCLIRubricNeedsDimensions covers the local check, which costs nothing and +// names the field the service would not. +func TestCLIRubricNeedsDimensions(t *testing.T) { + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte(`{"criteria":[]}`), 0o600)) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_nodims"), "--rubric", path)) + require.Contains(t, r.Combined(), "dimensions") +} + +// TestCLIRubricRejectsCodeOnlyFlags asserts the settings that would be +// accepted and then dropped are refused instead. A rubric runs no code and its +// schemas come from the service, so carrying them would be a silent no-op. +func TestCLIRubricRejectsCodeOnlyFlags(t *testing.T) { + rubric := writeRubric(t, `[{"id":"tone","description":"polite","weight":5}]`) + schema := filepath.Join(t.TempDir(), "schema.json") + require.NoError(t, os.WriteFile(schema, []byte(`{"type":"object"}`), 0o600)) + + r := requireFailure(t, run(t, "evaluator", "create", + "--name", uniqueName("azdcli_rubricflags"), "--rubric", rubric, + "--data-schema", schema)) + require.Contains(t, r.Combined(), "--data-schema") + require.Contains(t, r.Combined(), "--file") +} + +// TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show` emits JSON a +// script can consume, whatever the definition kind. It renders the service's +// body rather than a typed struct, so nothing else pins that it stays parseable. +func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { + name := uniqueName("azdcli_rubricdoc") + + // The wrapped form: a whole evaluator document rather than a bare + // definition. Both are accepted, and generated rubrics arrive wrapped. + path := filepath.Join(t.TempDir(), "rubric.json") + require.NoError(t, os.WriteFile(path, []byte( + `{"name":"ignored","definition":{"dimensions":[{"id":"tone","description":"polite","weight":3}]}}`, + ), 0o600)) + + requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", path)) + t.Cleanup(func() { + run(t, "evaluator", "delete", "--name", name, "--version", "1") + }) + + shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + var raw map[string]any + require.NoError(t, json.Unmarshal([]byte(shown.Stdout), &raw), + "evaluator show must emit parseable JSON:\n%s", shown.Stdout) + + // The flag names the evaluator, so a name inside the file must not win. + require.Equal(t, name, raw["name"], + "--name must decide the evaluator's name, not the document's own field") + require.NotContains(t, strings.ToLower(shown.Stdout), `"name": "ignored"`) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go new file mode 100644 index 00000000000..77628b90270 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +type runSummary struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + ResultCounts *struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + Errored int `json:"errored"` + } `json:"result_counts"` +} + +func TestCLIRunList(t *testing.T) { + f := sharedEval(t) + + t.Run("table", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", f.EvalID)) + for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.Contains(t, r.Stdout, f.BaselineRunID) + require.Contains(t, r.Stdout, f.TreatmentRunID) + require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored", + "the listing must summarise each run's counts, not just its status") + }) + + t.Run("json", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) + require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), + "a list must be a bare array, not the service's envelope") + + var runs []runSummary + r.JSON(t, &runs) + require.GreaterOrEqual(t, len(runs), 2) + + byID := map[string]runSummary{} + for _, entry := range runs { + byID[entry.ID] = entry + } + baseline, ok := byID[f.BaselineRunID] + require.True(t, ok, "the eval's own run is missing from its listing") + require.Equal(t, "completed", baseline.Status) + require.NotNil(t, baseline.ResultCounts) + require.Equal(t, 2, baseline.ResultCounts.Failed) + }) + + // The client has always taken a limit; until recently the command did not + // expose one, so a service-side truncation would have passed unnoticed. + t.Run("limit", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + var runs []runSummary + r.JSON(t, &runs) + require.Len(t, runs, 1, "--limit must reach the service") + }) + + t.Run("unknown eval is brief", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "list", "eval_azdcli_no_such_eval")) + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") + }) +} + +func TestCLIRunShow(t *testing.T) { + f := sharedEval(t) + + t.Run("by run id", func(t *testing.T) { + r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.BaselineRunID)) + require.Contains(t, r.Stdout, f.BaselineRunID) + require.Contains(t, r.Stdout, "status") + require.Contains(t, r.Stdout, "completed") + require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored") + require.Contains(t, r.Stdout, "report") + }) + + // Without --run-id the command has to pick one, and outside an azd + // environment there is no remembered id to fall back on, so what is + // exercised is the listing path. + t.Run("defaults to the most recent run", func(t *testing.T) { + listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + var newest []runSummary + listed.JSON(t, &newest) + require.Len(t, newest, 1) + + r := requireSuccess(t, run(t, "run", "show", f.EvalID, "-o", "json")) + var shown runSummary + r.JSON(t, &shown) + require.Equal(t, newest[0].ID, shown.ID, + "the default must be the run the listing puts first") + }) + + // A remembered run that no longer resolves falls through to the eval's + // latest, but one named explicitly must not: silently showing a different + // run than the one asked for is worse than saying it is gone. + // + // Only the substitution is asserted. Unlike `run list` and `run delete`, + // this path does not shorten the service's body, so the message runs to + // about 1700 characters of raw JSON — recorded in the report rather than + // pinned here, since pinning it would make the length a requirement. + t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "show", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope", + "the failure must name the run that was asked for") + require.NotContains(t, r.Combined(), f.BaselineRunID, + "an explicit --run-id must not fall back to another run") + }) +} + +// TestCLIRunCancelAndDelete covers both halves of cancel, and the delete that +// follows it, against a single in-flight run: each run costs a minute of +// service time, so the two happy paths share one. +// +// The service answers a cancel on a finished run with success, so without the +// guard the command would tell a user it had stopped something it had not. +func TestCLIRunCancelAndDelete(t *testing.T) { + f := sharedEval(t) + + t.Run("a finished run is refused", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.BaselineRunID)) + require.Contains(t, r.Combined(), "already finished") + require.Contains(t, r.Combined(), "completed") + }) + + // Delete is covered as far as the service honours it. + // + // The removal itself is not asserted, because it does not happen: the + // service accepts the DELETE and the run is still readable by id and still + // in the listing minutes later. What is asserted instead is that the + // command reaches the right resource — a real run is accepted, an unknown + // one is refused — which is the part that would break if the route or the + // id handling regressed. + t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { + runID := startCancellableRun(t, f) + + cancelled := requireSuccess(t, run(t, "run", "cancel", f.EvalID, "--run-id", runID)) + require.Contains(t, cancelled.Stdout, runID) + require.Contains(t, cancelled.Stdout, "is now") + + shown := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + var after runSummary + shown.JSON(t, &after) + require.NotEqual(t, "completed", after.Status, + "a cancelled run must not go on to complete") + + deleted := requireSuccess(t, run(t, "run", "delete", f.EvalID, "--run-id", runID)) + require.Contains(t, deleted.Stdout, "Deleted run") + require.Contains(t, deleted.Stdout, runID) + + still := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + var survivor runSummary + still.JSON(t, &survivor) + t.Logf("the run is still readable after a successful delete (status %q); "+ + "the service accepts the request without removing anything", survivor.Status) + }) + + // Deleting is not undoable, so the id is required rather than defaulted to + // whichever run happens to be newest. + t.Run("delete requires the run id", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", f.EvalID)) + require.Contains(t, r.Combined(), "--run-id is required") + }) + + t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { + r := requireFailure(t, run(t, "run", "delete", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + require.Contains(t, r.Combined(), "evalrun_azdcli_nope") + require.Less(t, len(r.Combined()), 600, + "a not-found must stay short, not dump the service body:\n%s", r.Combined()) + }) +} + +// startCancellableRun adds a run to the fixture's eval and returns it before it +// can finish. +// +// The rows are padded so the run cannot complete inside the second it takes to +// issue the cancel; a run that finished first would turn the cancel test into +// an assertion about the guard it is not testing. +func startCancellableRun(t *testing.T, f *evalFixture) string { + t.Helper() + + client, err := liveClient() + require.NoError(t, err) + + responses := make([]string, 0, 40) + for i := range 40 { + responses = append(responses, strings.Repeat("a good answer ", i%5+1)) + } + + runID, err := startFixtureRun(context.Background(), client, f.EvalID, "cancelme", responses) + require.NoError(t, err, "starting a run to cancel") + t.Cleanup(func() { + _ = client.DeleteOpenAIEvalRun(context.Background(), f.EvalID, runID) + }) + return runID +} From d98d5c5106e19d88cb923fb1d03339f4801fb453 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 03:12:40 +0300 Subject: [PATCH 072/114] Record the publish-to-reference consistency lag Eval creation resolves an evaluator through the version listing, which lags the direct read, so reconciliation has to gate on both or it fails just after a successful publish. --- .../azd-eval-extension-implementation.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md index 5fda30c6e18..b5373ffc545 100644 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -248,6 +248,15 @@ func configureExtensionHost(host *azdext.ExtensionHost) { 4. **Eval groups** — `POST /openai/v1/evals` with `testing_criteria` from the resolved evaluator versions. Groups are immutable, so only recreate when the resolved versions or options actually changed. 5. Persist resolved ids, versions, and fingerprints to the azd env. +**Publish → reference is eventually consistent.** After an evaluator version is +published, the direct read `GET /evaluators/{name}/versions/{version}` goes +consistent in roughly 330 ms, but the version *listing* lags 650 ms–1.4 s — and +eval creation resolves the reference through the listing, not the direct read. +Step 4 has to gate on both, or it fails with *"The evaluator X was not found"* +just after successfully publishing that evaluator. It reads like a flake, +because retrying a second later succeeds, so the tempting fix is a blanket retry +rather than the wait. + **How azd reaches us:** `azd up` runs one DAG; per service it calls `GetServiceTarget()`, which does `serviceLocator.ResolveNamed(host, &target)`. If our extension is not installed, azd fails that service with *"install an extension that provides this host."* We implement **no sequencing or rollback across services** — `uses:` and the DAG handle that. --- From 5d6663aca59962ea9cd36c1a8b4b97259d567027 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:06:57 +0300 Subject: [PATCH 073/114] Take code evaluators and comparison out of M1 Both are deferred, so they leave the branch rather than sit unused: code evaluator publishing, the grader loader and its validation, and results compare. LatestEvaluatorVersionNumber and the score formatter move rather than go, being the parts of those files M1 still uses. Evaluator create and update now take the name positionally and read the definition from --from-file, matching the sibling Foundry extensions. They are one request with one difference: create refuses a name already in use and update refuses one that is free, which the service does not check because it assigns the version either way. The CLI fixture is rebuilt on what M1 can actually run - a built-in evaluator against an agent - because a deterministic code grader over a target-less dataset was both of the deferred features at once. Pass and fail are now a judge's call, so the tests assert what holds regardless: every row comes back scored, and --failed-only returns a subset that agrees with the totals. --- .../internal/cmd/compare.go | 219 ----------- .../internal/cmd/compare_test.go | 83 ----- .../internal/cmd/evaluator.go | 330 ++++++----------- .../internal/cmd/evaluator_code_live_test.go | 246 ------------- .../internal/cmd/evaluator_code_test.go | 202 ---------- .../internal/cmd/model_target_test.go | 98 ----- .../internal/cmd/reconciler.go | 102 ----- .../internal/cmd/reconciler_digest_test.go | 81 ---- .../internal/cmd/results.go | 12 +- .../internal/cmd/run_ops_test.go | 2 +- .../internal/cmd/traces_test.go | 119 ------ .../internal/pkg/eval_api/code_evaluator.go | 178 --------- .../internal/pkg/eval_api/evaluators.go | 18 + .../internal/pkg/evalcore/codescript.go | 113 ------ .../internal/pkg/evalcore/codescript_test.go | 128 ------- .../tests/cli/evaluator_test.go | 73 +--- .../tests/cli/fixture_test.go | 251 +++++++------ .../tests/cli/harness_test.go | 13 - .../tests/cli/results_test.go | 235 ++++-------- .../tests/cli/rubric_test.go | 47 +-- .../tests/cli/run_ops_test.go | 40 +- .../tests/live/code_evaluator_run_test.go | 347 ------------------ 22 files changed, 365 insertions(+), 2572 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go deleted file mode 100644 index 3859dc3664f..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "context" - "fmt" - "time" - - "azureaieval/internal/pkg/eval_api" - - "github.com/spf13/cobra" -) - -// comparePollBudget bounds the wait for a comparison. The probe returned in -// about a second, so this is generous headroom rather than an expected wait. -const ( - comparePollInterval = 3 * time.Second - comparePollAttempts = 100 -) - -func newResultsCompareCommand() *cobra.Command { - var ( - baseline string - treatments []string - displayName string - endpointFlg string - groupName string - ) - - cmd := &cobra.Command{ - Use: "compare [eval-id]", - Short: "Compare runs of an eval against a baseline.", - Args: cobra.MaximumNArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - evalID, err := resolveEvalID(cmd, ec, args, groupName) - if err != nil { - return err - } - - baseline, treatments, err = ec.resolveComparisonRuns(ctx, evalID, baseline, treatments) - if err != nil { - return err - } - - if displayName == "" { - displayName = fmt.Sprintf("compare-%s", time.Now().UTC().Format("20060102-150405")) - } - - insight, err := ec.evalClient.CreateInsight(ctx, &eval_api.CreateInsightRequest{ - DisplayName: displayName, - Request: &eval_api.InsightRequest{ - Type: eval_api.InsightTypeEvaluationComparison, - EvalID: evalID, - BaselineRunID: baseline, - TreatmentRunIDs: treatments, - }, - }, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("starting the comparison: %w", err) - } - - if !isJSON(cmd) { - fmt.Fprintf(out, "Comparing %d run(s) against %s...\n", len(treatments), baseline) - } - - completed, err := ec.pollInsight(ctx, insight.ID) - if err != nil { - return err - } - if isJSON(cmd) { - return emitJSON(out, completed) - } - return renderComparison(out, completed) - }, - } - - cmd.Flags().StringVar(&baseline, "baseline", "", - "Run to compare against. Defaults to the second most recent completed run.") - cmd.Flags().StringArrayVar(&treatments, "treatment", nil, - "Run to measure, repeatable. Defaults to the most recent completed run.") - cmd.Flags().StringVar(&displayName, "name", "", "Name for this comparison.") - addEvalFlags(cmd, &groupName) - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - -// resolveComparisonRuns fills in whichever runs were not named. -// -// Comparing the two most recent completed runs is what "did my change help?" -// means most of the time, so neither flag is required. -func (ec *evalContext) resolveComparisonRuns( - ctx context.Context, - evalID, baseline string, - treatments []string, -) (string, []string, error) { - if baseline != "" && len(treatments) > 0 { - return baseline, treatments, nil - } - - list, err := ec.evalClient.ListOpenAIEvalRuns(ctx, evalID, 0) - if err != nil { - return "", nil, fmt.Errorf("listing runs of eval %s: %w", evalID, err) - } - - completed := make([]string, 0, 2) - if list == nil { - return "", nil, fmt.Errorf("eval %s has no runs", evalID) - } - for _, run := range list.Data { - if run.Status == "completed" { - completed = append(completed, run.ID) - } - } - - if len(treatments) == 0 { - if len(completed) == 0 { - return "", nil, fmt.Errorf( - "eval %s has no completed runs to compare", evalID) - } - treatments = []string{completed[0]} - } - if baseline == "" { - if len(completed) < 2 { - return "", nil, fmt.Errorf( - "eval %s has only one completed run, so there is nothing to compare it "+ - "against; run it again, or name a baseline with --baseline", - evalID) - } - baseline = completed[1] - } - return baseline, treatments, nil -} - -// pollInsight waits for the comparison to reach a terminal state. -func (ec *evalContext) pollInsight(ctx context.Context, insightID string) (*eval_api.Insight, error) { - for attempt := 0; attempt < comparePollAttempts; attempt++ { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(comparePollInterval): - } - - insight, err := ec.evalClient.GetInsight(ctx, insightID, ProjectEndpointAPIVersion) - if err != nil { - return nil, fmt.Errorf("reading comparison %s: %w", insightID, err) - } - if !insight.Terminal() { - continue - } - if !insight.Succeeded() { - return nil, fmt.Errorf("comparison %s finished with state %q", insightID, insight.State) - } - return insight, nil - } - return nil, fmt.Errorf("comparison %s did not finish in time", insightID) -} - -// renderComparison prints one row per criterion per treatment run. -// formatStat renders a statistic, showing an undefined one as a dash. A -// standard deviation over a single sample has no value, and printing the -// literal "NaN" in a results table reads like a failure rather than the -// arithmetic it is. -func formatStat(verb string, v eval_api.LenientFloat) string { - if !v.Defined() { - return "-" - } - return fmt.Sprintf(verb, float64(v)) -} - -func renderComparison(w interface{ Write([]byte) (int, error) }, insight *eval_api.Insight) error { - if insight.Result == nil || len(insight.Result.Comparisons) == 0 { - fmt.Fprintln(w, "The comparison produced no metrics.") - return nil - } - - if insight.Result.Method != "" { - fmt.Fprintf(w, "Method: %s\n\n", insight.Result.Method) - } - - rows := [][]string{} - for _, c := range insight.Result.Comparisons { - baseAvg := "-" - if c.BaselineRunSummary != nil { - baseAvg = formatStat("%.3f", c.BaselineRunSummary.Average) - } - for _, item := range c.CompareItems { - treatAvg := "-" - runID := "-" - if item.TreatmentRunSummary != nil { - treatAvg = formatStat("%.3f", item.TreatmentRunSummary.Average) - runID = item.TreatmentRunSummary.RunID - } - rows = append(rows, []string{ - c.Metric, - runID, - baseAvg, - treatAvg, - formatStat("%+.3f", item.DeltaEstimate), - formatStat("%.3f", item.PValue), - item.TreatmentEffect, - }) - } - } - - return emitTable(w, - []string{"METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT"}, - rows) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go deleted file mode 100644 index 5bbf0ad2370..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/compare_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "testing" - - "azureaieval/internal/pkg/eval_api" - - "github.com/stretchr/testify/require" -) - -func TestInsightTerminalStates(t *testing.T) { - for _, state := range []string{"", "NotStarted", "Running", "InProgress", "Queued"} { - require.False(t, (&eval_api.Insight{State: state}).Terminal(), "%q is not terminal", state) - } - for _, state := range []string{"Succeeded", "Failed", "Cancelled"} { - require.True(t, (&eval_api.Insight{State: state}).Terminal(), "%q is terminal", state) - } - require.True(t, (&eval_api.Insight{State: "Succeeded"}).Succeeded()) - require.False(t, (&eval_api.Insight{State: "Failed"}).Succeeded()) -} - -// The rendered table is how a reader decides whether a change helped, so the -// delta carries its sign and the effect classification is not dropped. -func TestRenderComparisonShowsSignedDeltaAndEffect(t *testing.T) { - insight := &eval_api.Insight{ - State: "Succeeded", - Result: &eval_api.InsightResult{ - Method: "PairedTTest", - Comparisons: []eval_api.MetricComparison{{ - Metric: "task_adherence", - BaselineRunSummary: &eval_api.RunSummary{RunID: "base", Average: 0.75}, - CompareItems: []eval_api.CompareItem{{ - TreatmentRunSummary: &eval_api.RunSummary{RunID: "treat", Average: 0.5}, - DeltaEstimate: -0.25, - PValue: 0.39, - TreatmentEffect: "TooFewSamples", - }}, - }}, - }, - } - - var buf bytes.Buffer - require.NoError(t, renderComparison(&buf, insight)) - out := buf.String() - - require.Contains(t, out, "PairedTTest", "the statistical method qualifies the result") - require.Contains(t, out, "task_adherence") - require.Contains(t, out, "-0.250", "a regression must read as negative") - require.Contains(t, out, "0.390") - require.Contains(t, out, "TooFewSamples", - "an inconclusive result must not look like a finding") -} - -// A positive delta reads as an improvement rather than an unsigned number. -func TestRenderComparisonSignsImprovements(t *testing.T) { - insight := &eval_api.Insight{ - State: "Succeeded", - Result: &eval_api.InsightResult{ - Comparisons: []eval_api.MetricComparison{{ - Metric: "similarity", - BaselineRunSummary: &eval_api.RunSummary{Average: 0.5}, - CompareItems: []eval_api.CompareItem{{ - TreatmentRunSummary: &eval_api.RunSummary{RunID: "t", Average: 0.8}, - DeltaEstimate: 0.3, - }}, - }}, - }, - } - - var buf bytes.Buffer - require.NoError(t, renderComparison(&buf, insight)) - require.Contains(t, buf.String(), "+0.300") -} - -func TestRenderComparisonHandlesEmptyResult(t *testing.T) { - var buf bytes.Buffer - require.NoError(t, renderComparison(&buf, &eval_api.Insight{State: "Succeeded"})) - require.Contains(t, buf.String(), "no metrics") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 151bc14bf42..72f431536ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -9,7 +9,6 @@ import ( "os" "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" "github.com/spf13/cobra" ) @@ -21,83 +20,74 @@ func newEvaluatorCommand() *cobra.Command { } cmd.AddCommand( newEvaluatorCreateCommand(), + newEvaluatorUpdateCommand(), newEvaluatorListCommand(), newEvaluatorShowCommand(), newEvaluatorDeleteCommand(), + newEvaluatorVersionsCommand(), ) return cmd } -// newEvaluatorCreateCommand builds `evaluator create`, named to match -// `dataset create`: both register an artifact and both publish a new immutable -// version every time, so there is nothing for a separate `update` to do. -// -// An evaluator is either a rubric — a JSON file of weighted dimensions — or -// code — one self-contained Python script. They are different definition types -// on the wire, so exactly one of the two sources has to be named. +// newEvaluatorCreateCommand builds `evaluator create `, which registers +// an evaluator that does not exist yet. func newEvaluatorCreateCommand() *cobra.Command { + return newEvaluatorWriteCommand("create", "Register an evaluator, publishing its first version.") +} + +// newEvaluatorUpdateCommand builds `evaluator update `, which publishes a +// further version of one that does. +func newEvaluatorUpdateCommand() *cobra.Command { + return newEvaluatorWriteCommand("update", "Publish a new version of an evaluator.") +} + +// newEvaluatorWriteCommand builds create and update, which send the same +// request and differ only in which starting state they accept. The service has +// one route for both and assigns the version either way, so the existence check +// is ours: without it, `create` on a name already in use would silently publish +// a further version of someone else's evaluator. +func newEvaluatorWriteCommand(verb, short string) *cobra.Command { var ( - name string - rubric string - file string - imageTag string - initParams string - dataSchema string - metrics string + fromFile string endpointFlg string ) - use := "create" - short := "Register a rubric or code evaluator, publishing a new version." - long := short + "\n\n" + - "A rubric (--rubric) is a JSON file of weighted dimensions.\n\n" + - "A code evaluator (--file) is a single Python script declaring a top-level\n" + - "grade(sample, item) function that returns a float. It runs as a python\n" + - "grader, which is handed the script's source and nothing else: there is no\n" + - "package and no import path, so a helper module beside the script cannot be\n" + - "imported. Dependencies come from the image named by --image-tag." - cmd := &cobra.Command{ - Use: use, + Use: verb + " ", Short: short, - Long: long, + Long: short + "\n\n" + + "An evaluator is a rubric: a JSON file of weighted scoring dimensions.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } - flags := codeEvaluatorFlags{ - imageTag: imageTag, - initParams: initParams, - dataSchema: dataSchema, - metrics: metrics, - endpoint: endpointFlg, - } - if err := validateEvaluatorSource(rubric, file, flags); err != nil { - return err + name := args[0] + if fromFile == "" { + return requireFlag("from-file") } - ctx := cmd.Context() - - if file != "" { - return runEvaluatorCreateFromFile(cmd, name, file, flags) - } - - raw, err := os.ReadFile(rubric) + raw, err := os.ReadFile(fromFile) if err != nil { - return fmt.Errorf("reading rubric %q: %w", rubric, err) + return fmt.Errorf("reading evaluator %q: %w", fromFile, err) } body, err := normalizeRubricBody(name, raw) if err != nil { - return fmt.Errorf("rubric %q: %w", rubric, err) + return fmt.Errorf("evaluator %q: %w", fromFile, err) } + ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err } defer ec.Close() + latest := ec.evalClient.LatestEvaluatorVersionNumber( + ctx, name, ProjectEndpointAPIVersion, + ) + if err := checkAssetExistence(verb, "evaluator", name, latest > 0); err != nil { + return err + } + created, err := ec.evalClient.CreateEvaluatorVersion( ctx, name, body, ProjectEndpointAPIVersion, ) @@ -114,165 +104,24 @@ func newEvaluatorCreateCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") - cmd.Flags().StringVar(&rubric, "rubric", "", "Path to the rubric JSON file.") - cmd.Flags().StringVar(&file, "file", "", - "Path to a single Python script declaring a top-level grade(sample, item) function.") - cmd.Flags().StringVar(&imageTag, "image-tag", "", - "Container image the evaluator runs in. Its packages are the only "+ - "dependencies the script can import beyond the standard library.") - cmd.Flags().StringVar(&initParams, "init-params", "", - "Path to a JSON Schema for the evaluator's initialization parameters.") - cmd.Flags().StringVar(&dataSchema, "data-schema", "", - "Path to a JSON Schema for the evaluator's input data.") - cmd.Flags().StringVar(&metrics, "metrics", "", - "Path to a JSON object describing the metrics the evaluator produces.") + cmd.Flags().StringVar(&fromFile, "from-file", "", "Path to the evaluator JSON file.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// codeEvaluatorFlags are the optional settings for a code evaluator. -type codeEvaluatorFlags struct { - imageTag string - initParams string - dataSchema string - metrics string - endpoint string -} - -// validateEvaluatorSource enforces that exactly one source is named, and that -// the code-only settings are only used with the source they apply to. -// -// Deliberately checked here rather than with MarkFlagsMutuallyExclusive: that -// only rejects the "both" case, and its message names a flag group rather than -// saying what the two flags mean. Both mistakes deserve advice, and this is -// testable without driving cobra. -func validateEvaluatorSource(rubric, file string, flags codeEvaluatorFlags) error { +// checkAssetExistence enforces the one difference between create and update. +func checkAssetExistence(verb, kind, name string, exists bool) error { switch { - case rubric == "" && file == "": + case verb == "create" && exists: return fmt.Errorf( - "one of --rubric or --file is required: --rubric takes a JSON file of " + - "weighted dimensions, --file takes a single Python script") - case rubric != "" && file != "": + "%s %q already exists: use `update` to publish a new version", kind, name) + case verb == "update" && !exists: return fmt.Errorf( - "--rubric and --file cannot be used together: an evaluator is either a " + - "rubric or code, not both") - } - - // A rubric's schemas are fixed by the service and a rubric runs no code, so - // these would be accepted and then quietly dropped — the worst kind of - // no-op, because the author believes the evaluator was published carrying - // them. - if file == "" { - for _, named := range []struct { - flag string - value string - }{ - {"image-tag", flags.imageTag}, - {"init-params", flags.initParams}, - {"data-schema", flags.dataSchema}, - {"metrics", flags.metrics}, - } { - if named.value != "" { - return fmt.Errorf( - "--%s applies to a code evaluator and needs --file; "+ - "a rubric runs no code and its schemas are set by the service", - named.flag) - } - } + "%s %q does not exist: use `create` to register it", kind, name) } return nil } -// runEvaluatorCreateFromFile validates the script, then publishes it. -func runEvaluatorCreateFromFile( - cmd *cobra.Command, - name string, - file string, - flags codeEvaluatorFlags, -) error { - script, err := evalcore.LoadCodeEvaluator(name, file) - if err != nil { - return err - } - - opts, err := codeEvaluatorOptions(flags) - if err != nil { - return err - } - - ctx := cmd.Context() - ec, err := newEvalContext(ctx, flags.endpoint) - if err != nil { - return err - } - defer ec.Close() - - created, err := ec.evalClient.CreateCodeEvaluatorVersion( - ctx, script, opts, ProjectEndpointAPIVersion, - ) - if err != nil { - return fmt.Errorf("publishing evaluator %q: %w", name, err) - } - - if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), created) - } - fmt.Fprintf(cmd.OutOrStdout(), - "Published evaluator %s version %s from %s\n", - created.Name, created.Version, file) - return nil -} - -// codeEvaluatorOptions resolves the evaluator's schemas from the flags. -// -// They are not read from the script and not read from a descriptor beside it: -// the grader is handed one file of source, so anything the service needs that -// is not Python has to be named on the command line or carried in the eval -// config. -func codeEvaluatorOptions(flags codeEvaluatorFlags) (eval_api.CodeEvaluatorOptions, error) { - opts := eval_api.CodeEvaluatorOptions{ImageTag: flags.imageTag} - - for _, declared := range []struct { - path string - flag string - field *json.RawMessage - }{ - {flags.initParams, "init-params", &opts.InitParameters}, - {flags.dataSchema, "data-schema", &opts.DataSchema}, - {flags.metrics, "metrics", &opts.Metrics}, - } { - if declared.path == "" { - continue - } - raw, err := readJSONObject(declared.path) - if err != nil { - return opts, fmt.Errorf("--%s %q: %w", declared.flag, declared.path, err) - } - *declared.field = raw - } - - return opts, nil -} - -// readJSONObject reads a file that must hold a JSON object. -// -// Parsing here rather than letting the service reject it keeps a typo from -// costing an upload and a published version, and names the file that is wrong. -func readJSONObject(path string) (json.RawMessage, error) { - raw, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var probe map[string]json.RawMessage - if err := json.Unmarshal(raw, &probe); err != nil { - return nil, fmt.Errorf("not a JSON object: %w", err) - } - return json.RawMessage(raw), nil -} - -// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or -// a full evaluator document ({name, definition}) and returns the request body. // rubricDefinitionType is the discriminator the service uses to deserialize a // rubric definition. const rubricDefinitionType = "rubric" @@ -296,6 +145,8 @@ func ensureDefinitionType(definition json.RawMessage) (json.RawMessage, error) { return json.Marshal(doc) } +// normalizeRubricBody accepts either a bare definition ({type, dimensions}) or +// a full evaluator document ({name, definition}) and returns the request body. func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { var probe map[string]json.RawMessage if err := json.Unmarshal(raw, &probe); err != nil { @@ -303,7 +154,7 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { } if definition, hasDefinition := probe["definition"]; hasDefinition { - // Already a full document; make sure the name matches the flag. + // Already a full document; make sure the name matches the argument. typed, err := ensureDefinitionType(definition) if err != nil { return nil, err @@ -339,14 +190,14 @@ func normalizeRubricBody(name string, raw []byte) (json.RawMessage, error) { func newEvaluatorListCommand() *cobra.Command { var ( - name string builtin bool endpointFlg string ) cmd := &cobra.Command{ Use: "list", - Short: "List evaluators, the versions of one evaluator, or the built-in evaluators.", + Short: "List the project's evaluators, or the built-in ones.", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -355,18 +206,13 @@ func newEvaluatorListCommand() *cobra.Command { } defer ec.Close() - var list *eval_api.EvaluatorListResponse - switch { - case name != "": - list, err = ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) - case builtin: - // The service filters by type, and asking for nothing returns - // only the project's own evaluators. - list, err = ec.evalClient.ListEvaluators( - ctx, eval_api.EvaluatorTypeBuiltin, ProjectEndpointAPIVersion) - default: - list, err = ec.evalClient.ListEvaluators(ctx, "", ProjectEndpointAPIVersion) + // The service filters by type, and asking for nothing returns only + // the project's own evaluators. + filter := "" + if builtin { + filter = eval_api.EvaluatorTypeBuiltin } + list, err := ec.evalClient.ListEvaluators(ctx, filter, ProjectEndpointAPIVersion) if err != nil { return fmt.Errorf("listing evaluators: %w", err) } @@ -374,9 +220,49 @@ func newEvaluatorListCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this evaluator.") - cmd.Flags().BoolVar(&builtin, "builtin", false, "List the built-in evaluators instead of the project's own.") - cmd.MarkFlagsMutuallyExclusive("name", "builtin") + cmd.Flags().BoolVar(&builtin, "builtin", false, + "List the built-in evaluators instead of the project's own.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newEvaluatorVersionsCommand groups the version listing, so that `list` means +// the same thing for evaluators as it does for datasets: the assets, not their +// history. +func newEvaluatorVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one evaluator.", + } + cmd.AddCommand(newEvaluatorVersionsListCommand()) + return cmd +} + +func newEvaluatorVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of an evaluator.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing versions of evaluator %q: %w", name, err) + } + return renderEvaluators(cmd, list) + }, + } + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -398,18 +284,16 @@ func renderEvaluators(cmd *cobra.Command, list *eval_api.EvaluatorListResponse) func newEvaluatorShowCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "show", + Use: "show ", Short: "Show an evaluator definition.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -437,7 +321,6 @@ func newEvaluatorShowCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -445,18 +328,16 @@ func newEvaluatorShowCommand() *cobra.Command { func newEvaluatorDeleteCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "delete", + Use: "delete ", Short: "Delete an evaluator version.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] if version == "" { return requireFlag("version") } @@ -488,7 +369,6 @@ func newEvaluatorDeleteCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the evaluator.") cmd.Flags().StringVar(&version, "version", "", "Version to delete.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go deleted file mode 100644 index df32ba7cd8d..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_live_test.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -//go:build live - -// This file proves the code-evaluator body the extension builds is accepted by -// the real service, and that what comes back is the shape the extension -// expects. -// -// It exists because the wire contract was settled from source rather than from -// a live call: two published documents disagreed on the definition body, and -// only the service can say which one it honours. It asserts the round trip -// field by field so a drift shows up as a named mismatch, not a vague failure. -// -// go test -tags live -v ./internal/cmd/ -run TestLiveCodeEvaluator -// -// Required: AZURE_AI_EVAL_E2E_LIVE=1 and FOUNDRY_PROJECT_ENDPOINT. - -package cmd - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - - "github.com/stretchr/testify/require" -) - -// liveCodeEvaluatorName is unique per run so concurrent runs, and reruns after -// a failure that skipped cleanup, do not collide. -func liveCodeEvaluatorName(t *testing.T, suffix string) string { - t.Helper() - return fmt.Sprintf("azdcode_%s_%d", suffix, time.Now().UnixNano()) -} - -// writeLiveEvaluator writes a self-contained evaluator script and returns its -// path. It goes through the production loader afterwards, so the shipping -// validation is exercised rather than bypassed. -func writeLiveEvaluator(t *testing.T, name string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, name+".py") - - source := `def grade(sample, item) -> float: - return float(len((item or {}).get("response", ""))) -` - require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) - return path -} - -// codeDefinitionOnService reads the registered version back and returns its -// definition, so the assertions run against what the service persisted rather -// than against what was sent. -func codeDefinitionOnService( - t *testing.T, - client *eval_api.EvalClient, - name, version string, -) map[string]json.RawMessage { - t.Helper() - - raw, err := client.GetEvaluatorRaw( - context.Background(), name, version, ProjectEndpointAPIVersion) - require.NoError(t, err, "reading back evaluator %s version %s", name, version) - - var doc map[string]json.RawMessage - require.NoError(t, json.Unmarshal(raw, &doc)) - require.Contains(t, doc, "definition", - "the registered evaluator carries no definition: %s", string(raw)) - - var definition map[string]json.RawMessage - require.NoError(t, json.Unmarshal(doc["definition"], &definition)) - return definition -} - -func stringField(t *testing.T, definition map[string]json.RawMessage, key string) string { - t.Helper() - raw, ok := definition[key] - if !ok { - return "" - } - var value string - if err := json.Unmarshal(raw, &value); err != nil { - return "" - } - return value -} - -// TestLiveCodeEvaluatorRoundTrip publishes a script and asserts it comes back -// as a code definition carrying the source inline. -// -// code_text is the only source property that reaches the executor: the -// definition is consumed as an OpenAI python grader, whose contract is a -// single Source string. A version registered with blob_uri instead publishes -// cleanly and then fails every run with "top-level grade() function not found -// in source", so what matters here is that the source itself round-trips. -func TestLiveCodeEvaluatorRoundTrip(t *testing.T) { - client, _ := liveEvalClient(t) - ctx := context.Background() - - name := liveCodeEvaluatorName(t, "roundtrip") - path := writeLiveEvaluator(t, name) - - // The shipping loader, not a hand-built script: this test has to fail if - // the production path stops producing a publishable script. - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err) - require.Contains(t, script.Source, "def grade(") - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) - require.NoError(t, err) - - created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err, "the service rejected the code evaluator body") - require.NotEmpty(t, created.Version) - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, created.Version, ProjectEndpointAPIVersion) - }) - - definition := codeDefinitionOnService(t, client, name, created.Version) - - require.Equal(t, eval_api.CodeDefinitionType, stringField(t, definition, "type"), - "the discriminator must round-trip as the lowercase snake_case value") - require.Contains(t, stringField(t, definition, "code_text"), "def grade(", - "the source must round-trip inline; an empty code_text means the grader "+ - "would be handed nothing to run") - require.Contains(t, definition, "metrics", - "a code definition must carry metrics; the service rejects one without") -} - -// TestLiveCodeEvaluatorCarriesSchemasAndImage proves the settings that only -// reach the service through flags survive the round trip. -// -// They cannot come from anywhere else. The grader is handed one file of -// source, so a descriptor beside the script would never travel with it, and an -// image tag dropped on the way would leave an evaluator whose imports fail at -// run time with no sign of why. -func TestLiveCodeEvaluatorCarriesSchemasAndImage(t *testing.T) { - client, _ := liveEvalClient(t) - ctx := context.Background() - - name := liveCodeEvaluatorName(t, "schemas") - path := writeLiveEvaluator(t, name) - - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err) - - dir := t.TempDir() - schemaPath := filepath.Join(dir, "schema.json") - require.NoError(t, os.WriteFile(schemaPath, []byte( - `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`, - ), 0o600)) - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{dataSchema: schemaPath}) - require.NoError(t, err) - require.NotEmpty(t, opts.DataSchema) - - created, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err, "the service rejected a definition carrying a data schema") - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, created.Version, ProjectEndpointAPIVersion) - }) - - definition := codeDefinitionOnService(t, client, name, created.Version) - require.Contains(t, definition, "data_schema", - "the declared data schema must round-trip; without it the criteria builder "+ - "derives no data_mapping and the eval cannot be created") - require.Contains(t, string(definition["data_schema"]), "response") -} - -// TestLiveCodeEvaluatorPublishesANewVersion proves a second publish does not -// overwrite the first. -// -// Versions are immutable and evals bind to one, so a publish that replaced the -// previous version would silently change what every existing eval evaluates. -// -// The wait between the two publishes is not padding. The service assigns the -// next version from its own listing, and that listing lags the create by about -// a second: two publishes issued back to back were both answered with version -// 1, the second overwriting the first. The reconciler waits for a published -// version to appear in the listing before it moves on, so this waits the same -// way — the assertion is about publishing twice, not about racing the service. -func TestLiveCodeEvaluatorPublishesANewVersion(t *testing.T) { - client, _ := liveEvalClient(t) - ctx := context.Background() - - name := liveCodeEvaluatorName(t, "versions") - path := writeLiveEvaluator(t, name) - - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err) - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{}) - require.NoError(t, err) - - first, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err) - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, first.Version, ProjectEndpointAPIVersion) - }) - require.NotEmpty(t, first.Version, "the service must assign a version") - - awaitVersionListed(t, client, name, first.Version) - - second, err := client.CreateCodeEvaluatorVersion(ctx, script, opts, ProjectEndpointAPIVersion) - require.NoError(t, err) - t.Cleanup(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, second.Version, ProjectEndpointAPIVersion) - }) - require.NotEqual(t, first.Version, second.Version, - "a second publish must create a new version rather than replace the first") -} - -// awaitVersionListed blocks until a published version shows up in the version -// listing, which is the view the service's own version assignment reads. -func awaitVersionListed(t *testing.T, client *eval_api.EvalClient, name, version string) { - t.Helper() - ctx := context.Background() - - start := time.Now() - for { - list, err := client.ListEvaluatorVersions(ctx, name, ProjectEndpointAPIVersion) - if err == nil && list != nil { - for _, entry := range list.Value { - if entry.Version == version { - t.Logf("version %s listed after %s", - version, time.Since(start).Round(time.Millisecond)) - return - } - } - } - if time.Since(start) > 30*time.Second { - t.Fatalf("version %s of evaluator %s never appeared in the listing", version, name) - } - time.Sleep(250 * time.Millisecond) - } -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go deleted file mode 100644 index 0b1ff9205c2..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_code_test.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "encoding/json" - "io" - "os" - "path/filepath" - "testing" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -func writeTestFile(t *testing.T, root, rel, content string) string { - t.Helper() - path := filepath.Join(root, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - return path -} - -// toneEvaluatorSource is the shape the grader requires: one top-level -// grade(sample, item) returning a float. -const toneEvaluatorSource = "def grade(sample, item) -> float:\n" + - " return float(len((item or {}).get(\"response\", \"\")))\n" - -// An evaluator is either a rubric or code. Naming both, or neither, is a -// mistake the command has to name precisely — the two flags take different -// kinds of path and produce different definition types. -func TestValidateEvaluatorSource(t *testing.T) { - err := validateEvaluatorSource("", "", codeEvaluatorFlags{}) - require.Error(t, err) - require.Contains(t, err.Error(), "--rubric") - require.Contains(t, err.Error(), "--file") - require.Contains(t, err.Error(), "required") - - err = validateEvaluatorSource("rubric.json", "evaluator.py", codeEvaluatorFlags{}) - require.Error(t, err) - require.Contains(t, err.Error(), "cannot be used together") - - require.NoError(t, validateEvaluatorSource("rubric.json", "", codeEvaluatorFlags{})) - require.NoError(t, validateEvaluatorSource("", "evaluator.py", codeEvaluatorFlags{})) -} - -// The code-only settings describe a python grader. Accepting them beside a -// rubric and dropping them would leave the author believing the evaluator was -// published carrying an image and schemas it never had. -func TestValidateEvaluatorSource_RejectsCodeFlagsOnARubric(t *testing.T) { - for flag, flags := range map[string]codeEvaluatorFlags{ - "image-tag": {imageTag: "python:3.11"}, - "init-params": {initParams: "init.json"}, - "data-schema": {dataSchema: "schema.json"}, - "metrics": {metrics: "metrics.json"}, - } { - err := validateEvaluatorSource("rubric.json", "", flags) - require.Error(t, err, "for --%s", flag) - require.Contains(t, err.Error(), "--"+flag) - require.Contains(t, err.Error(), "--file") - - require.NoError(t, validateEvaluatorSource("", "evaluator.py", flags), - "--%s is valid with --file", flag) - } -} - -// The same check the command runs must be reachable from the command, so a -// future refactor cannot leave the flags declared but unvalidated. -func TestEvaluatorCreateRejectsBothSources(t *testing.T) { - cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone", "--rubric", "r.json", "--file", "tone.py"}) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SilenceUsage = true - - err := cmd.Execute() - require.Error(t, err) - require.Contains(t, err.Error(), "cannot be used together") -} - -func TestEvaluatorCreateRejectsNeitherSource(t *testing.T) { - cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone"}) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SilenceUsage = true - - err := cmd.Execute() - require.Error(t, err) - require.Contains(t, err.Error(), "one of --rubric or --file") -} - -// A script with no top-level grade() must be refused before a version is -// published, and the refusal must come from the command rather than from a run -// that fails minutes later. -func TestEvaluatorCreateRejectsAScriptWithoutGrade(t *testing.T) { - dir := t.TempDir() - path := writeTestFile(t, dir, "tone.py", - "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") - - cmd := newEvaluatorCreateCommand() - cmd.SetArgs([]string{"--name", "tone", "--file", path}) - cmd.SetOut(io.Discard) - cmd.SetErr(io.Discard) - cmd.SilenceUsage = true - - err := cmd.Execute() - require.Error(t, err) - require.Contains(t, err.Error(), "grade(sample, item)") -} - -// The flags are the only place a code evaluator's schemas can come from: the -// grader is handed one file of source, so nothing that is not Python can -// travel with it. -func TestCodeEvaluatorOptions_ReadsTheFlags(t *testing.T) { - empty, err := codeEvaluatorOptions(codeEvaluatorFlags{}) - require.NoError(t, err) - require.Empty(t, empty.ImageTag) - require.Empty(t, empty.Metrics) - require.Empty(t, empty.DataSchema) - require.Empty(t, empty.InitParameters) - - dir := t.TempDir() - metricsPath := writeTestFile(t, dir, "metrics.json", - `{"result":{"type":"continuous"}}`) - initPath := writeTestFile(t, dir, "init.json", - `{"type":"object","properties":{"deployment_name":{"type":"string"}}}`) - schemaPath := writeTestFile(t, dir, "schema.json", - `{"type":"object","properties":{"response":{"type":"string"}}}`) - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ - imageTag: "mcr.microsoft.com/azureml/evaluator:latest", - metrics: metricsPath, - initParams: initPath, - dataSchema: schemaPath, - }) - require.NoError(t, err) - require.Equal(t, "mcr.microsoft.com/azureml/evaluator:latest", opts.ImageTag) - require.Contains(t, string(opts.Metrics), "continuous") - require.Contains(t, string(opts.InitParameters), "deployment_name") - require.Contains(t, string(opts.DataSchema), "response") -} - -// A typo in a schema file must be reported against the flag that named it, -// not discovered by the service after a version has been published. -func TestCodeEvaluatorOptions_RejectsMalformedInput(t *testing.T) { - dir := t.TempDir() - - bad := writeTestFile(t, dir, "metrics.json", "[1,2,3]") - _, err := codeEvaluatorOptions(codeEvaluatorFlags{metrics: bad}) - require.Error(t, err) - require.Contains(t, err.Error(), "--metrics") - require.Contains(t, err.Error(), "JSON object") - - _, err = codeEvaluatorOptions(codeEvaluatorFlags{ - dataSchema: filepath.Join(dir, "absent.json"), - }) - require.Error(t, err) - require.Contains(t, err.Error(), "--data-schema") -} - -// The service rejects a code definition carrying no metrics, so a script -// published without any still has to publish with one. -func TestDefaultCodeMetricsIsAJSONObject(t *testing.T) { - var metrics map[string]map[string]any - require.NoError(t, json.Unmarshal(eval_api.DefaultCodeMetrics, &metrics)) - require.Len(t, metrics, 1) - require.Contains(t, metrics, "result") - require.Equal(t, "continuous", metrics["result"]["type"]) -} - -// Both kinds of evaluator source are one file, and the reconciler tells them -// apart by extension rather than by stat-ing the path. -func TestEvaluatorSourceClassificationAndFingerprint(t *testing.T) { - root := t.TempDir() - - rubric := writeTestFile(t, root, "rubric.json", `{"dimensions":[]}`) - script := writeTestFile(t, root, "tone.py", toneEvaluatorSource) - - require.False(t, evalcore.IsCodeEvaluatorSource(rubric)) - require.True(t, evalcore.IsCodeEvaluatorSource(script)) - - rubricDigest, err := project.Fingerprint(rubric) - require.NoError(t, err) - scriptDigest, err := project.Fingerprint(script) - require.NoError(t, err) - require.NotEqual(t, rubricDigest, scriptDigest) - - // Editing the script must be noticed, or a deploy would reuse a version - // holding the old source. - writeTestFile(t, root, "tone.py", toneEvaluatorSource+"\n# tweak\n") - changed, err := project.Fingerprint(script) - require.NoError(t, err) - require.NotEqual(t, scriptDigest, changed) - - _, err = project.Fingerprint(filepath.Join(root, "absent.py")) - require.Error(t, err) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go deleted file mode 100644 index c202a87a4c7..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/model_target_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "encoding/json" - "testing" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// A model answers as plain text and calls no tools. Binding an agent's richer -// output would leave the evaluator waiting on fields the run never produces, -// which the service reports as a missing input rather than a mapping mistake. -func TestSampleBindingsFor(t *testing.T) { - agent := sampleBindingsFor(project.TargetTypeAgent) - assert.Equal(t, "{{sample.output_items}}", agent["response"]) - assert.Contains(t, agent, "tool_calls") - assert.Contains(t, agent, "tool_definitions") - - model := sampleBindingsFor(project.TargetTypeModel) - assert.Equal(t, "{{sample.output_text}}", model["response"]) - assert.NotContains(t, model, "tool_calls", "a model calls no tools") - assert.NotContains(t, model, "tool_definitions") - - assert.Nil(t, sampleBindingsFor(""), "with nothing invoked, nothing is bound") -} - -// The criteria a group sends depend on what it targets. -func TestBuildEvalRequest_BindsByTargetKind(t *testing.T) { - schemas := map[string]*eval_api.EvaluatorSummary{ - "builtin.coherence": { - Name: "builtin.coherence", - Definition: &eval_api.EvaluatorContract{ - DataSchema: &eval_api.JSONSchema{ - Required: []string{"query", "response"}, - Properties: map[string]any{ - "query": map[string]any{"type": "string"}, - "response": map[string]any{"type": "string"}, - }, - }, - }, - }, - } - - for _, tc := range []struct { - targetType string - want string - }{ - {project.TargetTypeAgent, "{{sample.output_items}}"}, - {project.TargetTypeModel, "{{sample.output_text}}"}, - } { - t.Run(tc.targetType, func(t *testing.T) { - group := &project.Eval{ - Name: "quality", - Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.coherence"}}, - Target: &project.Target{Type: tc.targetType, Name: "thing"}, - } - req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) - require.NoError(t, err) - require.Len(t, req.TestingCriteria, 1) - assert.Equal(t, tc.want, req.TestingCriteria[0].DataMapping["response"]) - assert.Equal(t, "{{item.query}}", req.TestingCriteria[0].DataMapping["query"]) - }) - } -} - -// The target the run posts has to match what the group's criteria expect. -func TestNewModelTargetDataSource(t *testing.T) { - ds := eval_api.NewModelTargetDataSource("gpt-4.1-nano") - require.NotNil(t, ds.Target) - assert.Equal(t, "azure_ai_model", ds.Target.Type) - assert.Equal(t, "gpt-4.1-nano", ds.Target.Model) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.Contains(t, body, `"model":"gpt-4.1-nano"`) - assert.NotContains(t, body, `"name"`, "a model target is addressed by deployment, not name") - assert.NotContains(t, body, "tool_descriptions", "a model calls no tools") -} - -func TestNewAgentTargetDataSource_StillSendsAgentFields(t *testing.T) { - ds := eval_api.NewAgentTargetDataSource("support-agent", nil) - require.NotNil(t, ds.Target) - assert.Equal(t, "azure_ai_agent", ds.Target.Type) - assert.Equal(t, "support-agent", ds.Target.Name) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - assert.NotContains(t, string(raw), `"model"`) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 8a1e5eb8138..44df054a40b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -6,8 +6,6 @@ package cmd import ( "bufio" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "os" @@ -18,7 +16,6 @@ import ( "time" "azureaieval/internal/pkg/dataset_api" - "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" ) @@ -256,9 +253,6 @@ func (r *evalReconciler) EnsureEvaluator( if _, err := os.Stat(localPath); err != nil { return "", false, fmt.Errorf("evaluator source %q: %w", localPath, err) } - if evalcore.IsCodeEvaluatorSource(localPath) { - return r.ensureCodeEvaluator(ctx, decl, localPath) - } raw, err := os.ReadFile(localPath) if err != nil { @@ -289,102 +283,6 @@ func (r *evalReconciler) EnsureEvaluator( return created.Version, true, nil } -// codeEvaluatorDigest fingerprints everything a published version depends on. -// -// The script alone is not enough: changing only metrics or the image tag -// changes what gets published, and hashing just the source would leave that -// edit undeployed with `azd up` reporting no change. -func codeEvaluatorDigest(decl project.EvaluatorDecl, path string) (string, error) { - sum := sha256.New() - - script, err := project.Fingerprint(path) - if err != nil { - return "", err - } - fmt.Fprintf(sum, "script:%s\nimage:%s\n", script, decl.ImageTag) - - for _, settings := range []struct { - label string - path string - }{ - {"metrics", decl.Metrics}, - {"data_schema", decl.DataSchema}, - {"init_parameters", decl.InitParameters}, - } { - if settings.path == "" { - continue - } - digest, err := project.Fingerprint(settings.path) - if err != nil { - return "", fmt.Errorf("evaluator %q %s: %w", decl.Name, settings.label, err) - } - fmt.Fprintf(sum, "%s:%s\n", settings.label, digest) - } - - return hex.EncodeToString(sum.Sum(nil)), nil -} - -// ensureCodeEvaluator publishes a Python script only when its content changed -// since the last deploy. -// -// Every publish is a new immutable version, so without this a repeated -// `azd up` would leave a trail of identical versions and force every eval -// bound to the evaluator to be recreated along with them. -func (r *evalReconciler) ensureCodeEvaluator( - ctx context.Context, - decl project.EvaluatorDecl, - path string, -) (string, bool, error) { - // Validated before anything is published: a script with no top-level - // grade() is only rejected when a run executes, long after a version has - // been published and an eval bound to it. - script, err := evalcore.LoadCodeEvaluator(decl.Name, path) - if err != nil { - return "", false, err - } - - digest, err := codeEvaluatorDigest(decl, path) - if err != nil { - return "", false, err - } - - key := project.FingerprintKey("evaluator", decl.Name) - recordedVersion := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)) - if r.ec.getEnvValue(ctx, key) == digest && recordedVersion != "" { - // Unchanged since the last deploy, but that alone does not make the - // recorded version safe to reuse: someone may have published a newer - // one outside the repo, and binding the eval to the older one would - // quietly evaluate with superseded code. - if err := r.checkEvaluatorDrift(ctx, decl.Name, recordedVersion); err != nil { - return "", false, err - } - return recordedVersion, false, nil - } - - opts, err := codeEvaluatorOptions(codeEvaluatorFlags{ - imageTag: decl.ImageTag, - metrics: decl.Metrics, - dataSchema: decl.DataSchema, - initParams: decl.InitParameters, - }) - if err != nil { - return "", false, err - } - - created, err := r.ec.evalClient.CreateCodeEvaluatorVersion( - ctx, script, opts, ProjectEndpointAPIVersion, - ) - if err != nil { - return "", false, err - } - - r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) - - _ = r.ec.setEnvValue(ctx, key, digest) - _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) - return created.Version, true, nil -} - // evaluatorPropagation bounds the wait for a freshly published evaluator to // become usable. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go deleted file mode 100644 index 5fd6d7521c7..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_digest_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "os" - "path/filepath" - "testing" - - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -// A code evaluator's published version depends on more than its script, so the -// digest that decides whether to republish has to cover the rest of it. -// Hashing only the source would leave a changed metric or image tag sitting in -// the config while `azd up` reported nothing to do. -func TestCodeEvaluatorDigest(t *testing.T) { - dir := t.TempDir() - - script := filepath.Join(dir, "grader.py") - require.NoError(t, os.WriteFile(script, - []byte("def grade(sample, item) -> float:\n return 1.0\n"), 0o600)) - - metrics := filepath.Join(dir, "metrics.json") - require.NoError(t, os.WriteFile(metrics, - []byte(`{"result":{"type":"continuous"}}`), 0o600)) - - base := project.EvaluatorDecl{Name: "answer_length"} - baseline, err := codeEvaluatorDigest(base, script) - require.NoError(t, err) - - t.Run("stable across calls", func(t *testing.T) { - again, err := codeEvaluatorDigest(base, script) - require.NoError(t, err) - require.Equal(t, baseline, again, - "an unchanged evaluator must not republish on every deploy") - }) - - t.Run("notices the script", func(t *testing.T) { - require.NoError(t, os.WriteFile(script, - []byte("def grade(sample, item) -> float:\n return 2.0\n"), 0o600)) - changed, err := codeEvaluatorDigest(base, script) - require.NoError(t, err) - require.NotEqual(t, baseline, changed) - }) - - t.Run("notices the image tag", func(t *testing.T) { - withImage := base - withImage.ImageTag = "python:3.12-slim" - changed, err := codeEvaluatorDigest(withImage, script) - require.NoError(t, err) - require.NotEqual(t, baseline, changed, - "changing the image changes what runs, so it must republish") - }) - - t.Run("notices the metrics file", func(t *testing.T) { - withMetrics := base - withMetrics.Metrics = metrics - before, err := codeEvaluatorDigest(withMetrics, script) - require.NoError(t, err) - - require.NoError(t, os.WriteFile(metrics, - []byte(`{"result":{"type":"ordinal","min_value":0,"max_value":1}}`), 0o600)) - after, err := codeEvaluatorDigest(withMetrics, script) - require.NoError(t, err) - require.NotEqual(t, before, after, - "editing metrics alone must republish, or the edit never deploys") - }) - - t.Run("reports a missing settings file", func(t *testing.T) { - missing := base - missing.DataSchema = filepath.Join(dir, "nope.json") - _, err := codeEvaluatorDigest(missing, script) - require.Error(t, err) - require.Contains(t, err.Error(), "data_schema", - "the error must name which setting could not be read") - }) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go index 8b7728bb7bc..d2377f34671 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go @@ -21,10 +21,20 @@ func newResultsCommand() *cobra.Command { Use: "results", Short: "Inspect evaluation results.", } - cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand(), newResultsCompareCommand()) + cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) return cmd } +// formatStat renders a statistic, showing an undefined one as a dash. A score +// the service left unset has no value, and printing the literal "NaN" in a +// results table reads like a failure rather than the arithmetic it is. +func formatStat(verb string, v eval_api.LenientFloat) string { + if !v.Defined() { + return "-" + } + return fmt.Sprintf(verb, float64(v)) +} + func newResultsShowCommand() *cobra.Command { var ( runID string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 8063f09253b..05f2e32b773 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -77,7 +77,7 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { for _, name := range []string{ "run list", "run show", "run cancel", - "results show", "results export", "results compare", + "results show", "results export", } { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go deleted file mode 100644 index 078a62435b9..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/traces_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "encoding/json" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// A window is only sent when one was asked for; the service defaults it -// otherwise. -func TestNewTracesDataSource_OmitsAnUnsetWindow(t *testing.T) { - ds := eval_api.NewTracesDataSource("support-agent", 0, time.Time{}, 0) - assert.Equal(t, eval_api.EvalRunDataSourceTypeTraces, ds.Type) - assert.Equal(t, "support-agent", ds.AgentName) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.NotContains(t, body, "lookback_hours") - assert.NotContains(t, body, "end_time") - assert.NotContains(t, body, "max_traces") - assert.NotContains(t, body, "input_messages", "traces carry no template") -} - -// The service reads `lookback_hours` and has no start bound. Sending a -// start_time is accepted and dropped, which silently leaves the default seven -// days in place, so the window has to travel as hours. -func TestNewTracesDataSource_SendsAWindowTheServiceReads(t *testing.T) { - ds := eval_api.NewTracesDataSource("support-agent", 30*24, time.Time{}, 25) - assert.Equal(t, 720, ds.LookbackHours) - assert.Equal(t, 25, ds.MaxTraces) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.Contains(t, body, `"lookback_hours":720`) - assert.NotContains(t, body, "start_time", - "the service drops start_time and falls back to its default window") -} - -// The reason a run failed is the only actionable part of the response, so it -// has to survive into the output. -func TestRunFailureMessage(t *testing.T) { - var run eval_api.OpenAIEvalRun - require.NoError(t, json.Unmarshal([]byte(`{ - "id": "evalrun_x", "status": "failed", - "error": { "code": "UserError", "message": " No trace data found for agent_name 'a'. " } - }`), &run)) - assert.Equal(t, "No trace data found for agent_name 'a'.", run.Failure()) - - // The field is present and null-valued on success, so presence alone - // must not read as failure. - var ok eval_api.OpenAIEvalRun - require.NoError(t, json.Unmarshal([]byte(`{ - "id": "evalrun_y", "status": "completed", - "error": { "code": null, "message": null } - }`), &ok)) - assert.Empty(t, ok.Failure()) - - var absent eval_api.OpenAIEvalRun - require.NoError(t, json.Unmarshal([]byte(`{"id":"evalrun_z","status":"completed"}`), &absent)) - assert.Empty(t, absent.Failure()) - - var nilRun *eval_api.OpenAIEvalRun - assert.Empty(t, nilRun.Failure()) -} - -// The ids travel as ordinary JSONL rows with a mapping pointing at the field -// that holds each one; that is how the service finds the chat history. -func TestNewResponsesDataSource(t *testing.T) { - ds := eval_api.NewResponsesDataSource([]string{"resp_a", "resp_b"}, 10) - assert.Equal(t, eval_api.EvalRunDataSourceTypeResponses, ds.Type) - require.NotNil(t, ds.ItemGenerationParams) - assert.Equal(t, "response_retrieval", ds.ItemGenerationParams.Type) - assert.Equal(t, 10, ds.ItemGenerationParams.MaxNumTurns) - assert.Equal(t, - map[string]string{"response_id": "{{item.response_id}}"}, - ds.ItemGenerationParams.DataMapping) - - raw, err := json.Marshal(ds) - require.NoError(t, err) - body := string(raw) - assert.Contains(t, body, `"response_id":"resp_a"`) - assert.Contains(t, body, `"response_id":"resp_b"`) - assert.NotContains(t, body, "agent_name", "responses carry no agent") -} - -// An unset turn limit is left to the service rather than sent as zero. -func TestNewResponsesDataSource_OmitsAnUnsetTurnLimit(t *testing.T) { - ds := eval_api.NewResponsesDataSource([]string{"resp_a"}, 0) - raw, err := json.Marshal(ds) - require.NoError(t, err) - assert.NotContains(t, string(raw), "max_num_turns") -} - -func TestRenderRun_ShowsTheFailureReason(t *testing.T) { - var buf bytes.Buffer - run := &eval_api.OpenAIEvalRun{ - ID: "evalrun_x", - Status: "failed", - Error: &eval_api.JobError{Code: "UserError", Message: "No trace data found."}, - } - require.NoError(t, renderRun(&buf, run)) - assert.Contains(t, buf.String(), "failed") - assert.Contains(t, buf.String(), "No trace data found.") - - var clean bytes.Buffer - require.NoError(t, renderRun(&clean, &eval_api.OpenAIEvalRun{ID: "evalrun_y", Status: "completed"})) - assert.NotContains(t, clean.String(), " \n", "a successful run gains no blank reason line") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go deleted file mode 100644 index f9e1305eea6..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/code_evaluator.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package eval_api - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/url" - "strconv" - - "azureaieval/internal/pkg/evalcore" -) - -// CodeDefinitionType is the discriminator the service uses to deserialize a -// code evaluator definition. -// -// The wire shape is snake_case with a lowercase discriminator, matching -// CodeBasedEvaluatorDefinition in the Foundry data-plane OpenAPI document -// (`type` enum ["code"], plus code_text, image_tag, init_parameters, -// data_schema and metrics). An earlier draft documented a camelCase body with -// `type: "CodeBased"`; that shape is not what the deployed service accepts. -const CodeDefinitionType = "code" - -// evaluatorTypeCustom marks an evaluator as authored by the project rather -// than shipped by the platform. -const evaluatorTypeCustom = "custom" - -// foundryFeaturesHeader opts a request in to preview behaviour. The code -// definition's properties are declared as preview, so the header is sent with -// every call that sets one. -const ( - foundryFeaturesHeader = "Foundry-Features" - foundryFeatureEvalsV1 = "Evaluations=V1Preview" - defaultCodeMetricName = "result" - defaultCodeMetricType = "continuous" - defaultMetricDirection = "increase" -) - -// DefaultCodeMetrics is used when the caller declares none. -// -// The service rejects a code definition carrying no metrics, and the documented -// evaluator output is a JSON object whose `result` field holds the score, so -// this describes exactly that. It is a default, not a constraint: any declared -// metrics replace it wholesale. -var DefaultCodeMetrics = json.RawMessage(fmt.Sprintf( - `{%q:{"type":%q,"desirable_direction":%q,"is_primary":true}}`, - defaultCodeMetricName, defaultCodeMetricType, defaultMetricDirection, -)) - -// CodeEvaluatorOptions carries the parts of an evaluator version that do not -// come from the Python source itself. -type CodeEvaluatorOptions struct { - DisplayName string - Description string - Categories []string - ImageTag string - InitParameters json.RawMessage - DataSchema json.RawMessage - Metrics json.RawMessage -} - -// codeDefinition is the wire body of a code evaluator definition. -// -// code_text carries the whole evaluator. The contract's other source property, -// blob_uri, is deliberately absent: the definition is consumed as an OpenAI -// python grader, whose contract (GraderPython) is a single `Source` string -// with no notion of a folder, archive, file list or entry point. A definition -// published with blob_uri alone registers cleanly and then fails the run with -// "Invalid grader source: top-level grade() function not found in source", -// because nothing reads the blob back into Source. -// -// image_tag is how a grader gets dependencies. Only one file is ever sent, so -// a helper module cannot travel with it and anything beyond the standard -// library has to already be in the image. -type codeDefinition struct { - Type string `json:"type"` - CodeText string `json:"code_text,omitempty"` - ImageTag string `json:"image_tag,omitempty"` - InitParameters json.RawMessage `json:"init_parameters,omitempty"` - DataSchema json.RawMessage `json:"data_schema,omitempty"` - Metrics json.RawMessage `json:"metrics,omitempty"` -} - -// createEvaluatorVersionRequest is the POST body for a new evaluator version. -// The service assigns the version; it is not carried here. -type createEvaluatorVersionRequest struct { - Name string `json:"name,omitempty"` - DisplayName string `json:"display_name,omitempty"` - Description string `json:"description,omitempty"` - EvaluatorType string `json:"evaluator_type,omitempty"` - Categories []string `json:"categories,omitempty"` - Definition *codeDefinition `json:"definition"` -} - -// CreateCodeEvaluatorVersion publishes a Python script as a new version of a -// code evaluator. -// -// The source is sent inline. There is no upload step and no storage to -// reserve: the executor is handed a string of source, so a blob it would never -// read adds a round trip, a SAS write, and a failure mode in exchange for -// nothing that reaches the grader. -func (c *EvalClient) CreateCodeEvaluatorVersion( - ctx context.Context, - script *evalcore.CodeEvaluatorScript, - opts CodeEvaluatorOptions, - apiVersion string, -) (*EvaluatorVersion, error) { - if script == nil { - return nil, fmt.Errorf("no evaluator script to publish") - } - - definition := &codeDefinition{ - Type: CodeDefinitionType, - CodeText: script.Source, - ImageTag: opts.ImageTag, - InitParameters: opts.InitParameters, - DataSchema: opts.DataSchema, - Metrics: opts.Metrics, - } - if len(definition.Metrics) == 0 { - definition.Metrics = DefaultCodeMetrics - } - - body := &createEvaluatorVersionRequest{ - Name: script.Name, - DisplayName: opts.DisplayName, - Description: opts.Description, - EvaluatorType: evaluatorTypeCustom, - Categories: opts.Categories, - Definition: definition, - } - - path := pathEvaluators + "/" + url.PathEscape(script.Name) + "/versions" - respBody, err := c.doRequestWithHeaders( - ctx, http.MethodPost, path, nil, body, apiVersion, previewHeaders(), - ) - if err != nil { - return nil, err - } - - var created EvaluatorVersion - if len(respBody) > 0 { - if err := json.Unmarshal(respBody, &created); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - } - if created.Name == "" { - created.Name = script.Name - } - return &created, nil -} - -// LatestEvaluatorVersionNumber reports the newest registered version as an -// integer, or 0 when the evaluator is unknown or its versions are not numeric. -func (c *EvalClient) LatestEvaluatorVersionNumber( - ctx context.Context, - name string, - apiVersion string, -) int { - list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) - if err != nil || list == nil || len(list.Value) == 0 { - return 0 - } - number, err := strconv.Atoi(pickLatestVersion(list.Value)) - if err != nil { - return 0 - } - return number -} - -// previewHeaders returns the opt-in header for the preview properties the code -// definition relies on. -func previewHeaders() map[string]string { - return map[string]string{foundryFeaturesHeader: foundryFeatureEvalsV1} -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 84ad2ac811f..7b3181b6ade 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -158,6 +158,24 @@ func (c *EvalClient) ListEvaluatorVersions( ) } +// LatestEvaluatorVersionNumber reports the newest registered version as an +// integer, or 0 when the evaluator is unknown or its versions are not numeric. +func (c *EvalClient) LatestEvaluatorVersionNumber( + ctx context.Context, + name string, + apiVersion string, +) int { + list, err := c.ListEvaluatorVersions(ctx, name, apiVersion) + if err != nil || list == nil || len(list.Value) == 0 { + return 0 + } + number, err := strconv.Atoi(pickLatestVersion(list.Value)) + if err != nil { + return 0 + } + return number +} + // DeleteEvaluatorVersion removes a single evaluator version. func (c *EvalClient) DeleteEvaluatorVersion( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go deleted file mode 100644 index 2c01e791f09..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" -) - -// pythonExt is the extension a code evaluator's source carries. It is also -// what tells a code evaluator apart from a rubric, which is `.json`. -const pythonExt = ".py" - -// GradeFunctionName is the function the executor calls. -// -// A code evaluator runs as an OpenAI python grader, and the grader contract is -// a single string of source with one entry point: a top-level -// `grade(sample, item)` returning a float. There is no module, package, or -// import path, so nothing else in the file can be reached. -const GradeFunctionName = "grade" - -// CodeEvaluatorScript is a validated evaluator script ready to publish. -type CodeEvaluatorScript struct { - // Name is the evaluator name the script is published under. - Name string - // Path is where the script was read from, kept for error messages. - Path string - // Source is the whole file, which is what the service is sent. The grader - // takes source, not a location: there is nowhere for a second file to go. - Source string -} - -// IsCodeEvaluatorSource reports whether a declared `source:` names a code -// evaluator rather than a rubric. -// -// The decision is made from the extension alone and never touches the -// filesystem, so it answers the same for a path that has not been created yet -// — a config can be validated before the file it names exists. -func IsCodeEvaluatorSource(path string) bool { - return strings.EqualFold(filepath.Ext(path), pythonExt) -} - -// gradeDeclaration matches a top-level `def grade(` — optionally async, and -// anchored at column zero. -// -// Indentation is what makes this specific rather than a substring search: a -// `grade` nested inside a class is a method, and the grader only ever calls a -// module-level function, so an indented match would pass validation here and -// then fail at run time with "top-level grade() function not found". -var gradeDeclaration = regexp.MustCompile( - `(?m)^(?:async[ \t]+)?def[ \t]+` + regexp.QuoteMeta(GradeFunctionName) + `[ \t]*\(`) - -// LoadCodeEvaluator reads an evaluator script and checks it against the grader -// contract. -// -// The check is done here rather than left to the service because the service -// only discovers a missing entry point when a run executes — long after a -// version has been published and an eval bound to it. The failure it reports -// then is "Invalid grader source: top-level grade() function not found in -// source", which names neither the file nor the evaluator. -func LoadCodeEvaluator(name, path string) (*CodeEvaluatorScript, error) { - if name == "" { - return nil, fmt.Errorf("an evaluator name is required to publish %q", path) - } - if path == "" { - return nil, fmt.Errorf("evaluator %q has no source file to publish", name) - } - - info, err := os.Stat(path) - if err != nil { - return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) - } - if info.IsDir() { - return nil, fmt.Errorf( - "evaluator source %q is a directory. A code evaluator is a single %s file: "+ - "it is published as the source of a python grader, which takes one script "+ - "and cannot import a helper module beside it", path, pythonExt) - } - if !IsCodeEvaluatorSource(path) { - return nil, fmt.Errorf( - "evaluator source %q must be a %s file", path, pythonExt) - } - - source, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("reading evaluator source %q: %w", path, err) - } - if len(strings.TrimSpace(string(source))) == 0 { - return nil, fmt.Errorf("evaluator source %q is empty", path) - } - if !gradeDeclaration.Match(source) { - return nil, fmt.Errorf( - "%s does not declare a top-level %s(sample, item) function. A code evaluator "+ - "runs as a python grader, which calls exactly that and nothing else — a "+ - "class, a differently named function, or one nested inside another will "+ - "not be found. For example:\n\ndef %s(sample, item) -> float:\n"+ - " return float(len(item.get(\"response\", \"\")))\n\n"+ - "The script must also be self-contained: only the standard library and "+ - "whatever the image named by --image-tag provides are importable, so a "+ - "helper file next to it cannot be imported", - path, GradeFunctionName, GradeFunctionName) - } - - return &CodeEvaluatorScript{ - Name: name, - Path: path, - Source: string(source), - }, nil -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go deleted file mode 100644 index 7cb75014828..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/codescript_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package evalcore - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// gradeSource is the minimal script that satisfies the grader contract. -const gradeSource = `def grade(sample, item) -> float: - return float(len((item or {}).get("response", ""))) -` - -func writeFile(t *testing.T, root, rel, content string) string { - t.Helper() - path := filepath.Join(root, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - return path -} - -// The extension decides which definition type to publish from the source's -// extension alone, and it has to answer for a path that does not exist yet so -// a config can be validated before its files are written. -func TestIsCodeEvaluatorSource(t *testing.T) { - require.True(t, IsCodeEvaluatorSource("tone.py")) - require.True(t, IsCodeEvaluatorSource("evaluators/tone.py")) - require.True(t, IsCodeEvaluatorSource(`evaluators\tone.PY`), - "the extension is matched case-insensitively") - require.True(t, IsCodeEvaluatorSource("/absent/never/created.py"), - "classification must not touch the filesystem") - - require.False(t, IsCodeEvaluatorSource("rubric.json")) - require.False(t, IsCodeEvaluatorSource("evaluators/rubric.json")) - require.False(t, IsCodeEvaluatorSource("evaluator"), - "a folder is no longer a code evaluator; the grader takes one script") - require.False(t, IsCodeEvaluatorSource("tone.python")) - require.False(t, IsCodeEvaluatorSource("")) -} - -func TestLoadCodeEvaluator_AcceptsATopLevelGrade(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", gradeSource) - - script, err := LoadCodeEvaluator("tone", path) - require.NoError(t, err) - require.Equal(t, "tone", script.Name) - require.Equal(t, path, script.Path) - require.Equal(t, gradeSource, script.Source, - "the whole file is what the grader is sent") -} - -// The grader is handed source and calls grade(); an async definition is still -// a top-level grade(). -func TestLoadCodeEvaluator_AcceptsAsyncAndAnnotatedForms(t *testing.T) { - for label, source := range map[string]string{ - "async": "async def grade(sample, item) -> float:\n return 1.0\n", - "spaced": "def grade (sample, item):\n return 1.0\n", - "no-annot": "def grade(sample, item):\n return 1.0\n", - "after-code": "import json\n\n\ndef grade(sample, item):\n return 1.0\n", - } { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", source) - _, err := LoadCodeEvaluator("tone", path) - require.NoError(t, err, "for %s", label) - } -} - -// Without this the failure surfaces only when a run executes, as "Invalid -// grader source: top-level grade() function not found in source" — long after -// a version has been published and an eval bound to it. -func TestLoadCodeEvaluator_ReportsAMissingGrade(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", - "class ToneEvaluator:\n def __call__(self, **kwargs):\n return {\"result\": 1}\n") - - _, err := LoadCodeEvaluator("tone", path) - require.Error(t, err) - require.Contains(t, err.Error(), "grade(sample, item)") - require.Contains(t, err.Error(), path) -} - -// A grade() nested inside a class is a method. The grader only ever calls a -// module-level function, so an indented match must not pass validation. -func TestLoadCodeEvaluator_RejectsANestedGrade(t *testing.T) { - dir := t.TempDir() - path := writeFile(t, dir, "tone.py", - "class ToneEvaluator:\n def grade(self, sample, item):\n return 1.0\n") - - _, err := LoadCodeEvaluator("tone", path) - require.Error(t, err) - require.Contains(t, err.Error(), "top-level") -} - -func TestLoadCodeEvaluator_RejectsAFolder(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "tone.py", gradeSource) - - _, err := LoadCodeEvaluator("tone", dir) - require.Error(t, err) - require.Contains(t, err.Error(), "single") -} - -func TestLoadCodeEvaluator_RejectsBadInput(t *testing.T) { - dir := t.TempDir() - - _, err := LoadCodeEvaluator("", writeFile(t, dir, "tone.py", gradeSource)) - require.Error(t, err, "a name is required to publish under") - - _, err = LoadCodeEvaluator("tone", "") - require.Error(t, err) - - _, err = LoadCodeEvaluator("tone", filepath.Join(dir, "absent.py")) - require.Error(t, err) - - _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "rubric.json", "{}")) - require.Error(t, err, "a rubric is not a code evaluator") - require.Contains(t, err.Error(), ".py") - - _, err = LoadCodeEvaluator("tone", writeFile(t, dir, "empty.py", " \n\n")) - require.Error(t, err) - require.Contains(t, err.Error(), "empty") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go index 79eabdd69eb..19112c2c95c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -7,7 +7,6 @@ package cli import ( "os" - "path/filepath" "strings" "testing" @@ -60,83 +59,13 @@ func TestCLIJSONListsAreBareArrays(t *testing.T) { } } -// TestCLICodeEvaluatorRoundTrip drives the whole custom evaluator lifecycle -// through the command surface: create from a script, read it back, list its -// versions, then delete it. -func TestCLICodeEvaluatorRoundTrip(t *testing.T) { - name := uniqueName("azdcli_code") - script := writeGrader(t, lengthGrader) - - requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--file", script)) - t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") - }) - - shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "-o", "json")) - var def struct { - Name string `json:"name"` - Definition struct { - Type string `json:"type"` - CodeText string `json:"code_text"` - } `json:"definition"` - } - shown.JSON(t, &def) - require.Equal(t, "code", def.Definition.Type, - "a script must register as a code definition") - require.Contains(t, def.Definition.CodeText, "def grade", - "the script's source must round-trip in code_text") - - listed := requireSuccess(t, run(t, "evaluator", "list", "--name", name, "-o", "json")) - var versions []struct { - Name string `json:"name"` - Version string `json:"version"` - } - listed.JSON(t, &versions) - require.NotEmpty(t, versions, "the evaluator must list its own versions") -} - -// TestCLIEvaluatorSourcesAreMutuallyExclusive covers the validation a user is -// most likely to trip, and asserts it costs nothing to find out — no version -// is published on the way to the error. -func TestCLIEvaluatorSourcesAreMutuallyExclusive(t *testing.T) { - script := writeGrader(t, lengthGrader) - rubric := filepath.Join(t.TempDir(), "rubric.json") - require.NoError(t, os.WriteFile(rubric, - []byte(`{"dimensions":[{"id":"tone","description":"polite","weight":5}]}`), 0o600)) - - both := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_both"), "--file", script, "--rubric", rubric)) - require.Contains(t, strings.ToLower(both.Combined()), "rubric", - "the error must name the flags in conflict") - - neither := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_neither"))) - require.NotEmpty(t, strings.TrimSpace(neither.Combined()), - "refusing without a source must explain itself") -} - -// TestCLIGraderIsValidatedBeforePublishing asserts the check that saves a user -// from a late failure: a script with no top-level grade() is refused locally, -// because the executor would otherwise accept the publish and fail the run. -func TestCLIGraderIsValidatedBeforePublishing(t *testing.T) { - script := writeGrader(t, `class AnswerLengthEvaluator: - def __call__(self, **kwargs): - return {"result": 1.0} -`) - - r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_noglade"), "--file", script)) - require.Contains(t, strings.ToLower(r.Combined()), "grade", - "the refusal must name the function the executor looks for") -} - // TestCLIUnknownEvaluatorIsBrief covers the failure a user hits by typo. // // The service answers with a long JSON body. Printing it verbatim buries the // one useful sentence, so the CLI shortens it, and a regression here is the // kind that only shows up in someone's terminal. func TestCLIUnknownEvaluatorIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "evaluator", "show", "--name", "azdcli-does-not-exist-9999")) + r := requireFailure(t, run(t, "evaluator", "show", "azdcli-does-not-exist-9999")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go index af745cae250..e159f28ff4a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/fixture_test.go @@ -9,16 +9,15 @@ import ( "context" "encoding/json" "fmt" + "net/http" "os" "os/exec" - "path/filepath" "strings" "sync" "testing" "time" "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -26,34 +25,47 @@ import ( // The command tests need an eval that has already been run, and building one // through the CLI is not possible: there is no command that creates an eval -// from flags, only `run`, which needs a config file and a deployed target. -// So the fixture is built with the client and every assertion is made against -// the binary. What is under test is the command surface; the eval is scenery. +// from flags, only `run start`, which needs a config file and a deployed +// target. So the fixture is built with the client and every assertion is made +// against the binary. What is under test is the command surface; the eval is +// scenery. // // It is built once for the whole package because two completed runs cost // minutes, and torn down in TestMain rather than t.Cleanup so that whichever // test happened to trigger the build does not take the fixture away from the // rest. +// +// It evaluates an agent with a built-in evaluator, because that is all M1 can +// run: a deterministic code grader over a target-less dataset would score the +// rows predictably, but code evaluators and no-target runs are both M2. The +// cost is that pass and fail are decided by a judge, so no test may assert how +// many rows failed — only that filtering by verdict is self-consistent. const fixtureAPIVersion = "2025-11-15-preview" -// scoringGrader splits the rows deterministically. A grader that scores every -// row the same way makes --failed-only and a comparison indistinguishable from -// a no-op, so "good" is the difference between a pass and a failure. -const scoringGrader = `def grade(sample, item) -> float: - response = (item or {}).get("response", "") - return 1.0 if "good" in response else 0.0 -` +const defaultFixtureModel = "gpt-4o-mini" + +// fixtureQueries are answered by the agent under evaluation. They are ordinary +// support questions: the fixture proves the command surface, not the agent. +var fixtureQueries = []string{ + "How do I reset my password?", + "How do I change my billing address?", + "What are your support hours?", +} -// evalFixture is one eval with two completed runs of the same criterion. +// evalFixture is one eval with two completed runs. type evalFixture struct { EvaluatorName string EvalID string - // Baseline scores worse than Treatment, so a comparison between them has - // a delta to report rather than zero. - BaselineRunID string - TreatmentRunID string + // The agent the runs evaluate, so that a test needing a further run does + // not have to resolve one again. + AgentName string + + // Two runs of the same eval, so that listing, limiting and defaulting to + // the most recent all have something to distinguish. + FirstRunID string + SecondRunID string } var ( @@ -162,6 +174,72 @@ func sharedEval(t *testing.T) *evalFixture { return fixture } +func fixtureModel() string { + if model := os.Getenv("AZURE_AI_EVAL_MODEL"); model != "" { + return model + } + return defaultFixtureModel +} + +// resolveFixtureAgent names the agent the fixture evaluates. +// +// It reads /agents, not /assistants: they are different collections, and an +// eval target resolves against the former. Naming an assistant is accepted by +// the create and then fails the run with "resources not found". +func resolveFixtureAgent(ctx context.Context) (string, error) { + if name := os.Getenv("AZURE_AI_EVAL_AGENT"); name != "" { + return name, nil + } + + // Builds the shared credential if it does not exist yet; the token below + // comes from it. + if _, err := liveClient(); err != nil { + return "", err + } + + token, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://ai.azure.com/.default"}, + }) + if err != nil { + return "", fmt.Errorf("acquiring a token to list agents: %w", err) + } + + req, err := http.NewRequestWithContext( + ctx, http.MethodGet, endpoint+"/agents?api-version="+fixtureAPIVersion, nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Bearer "+token.Token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("listing the project's agents: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf( + "listing the project's agents returned %d; set AZURE_AI_EVAL_AGENT to name one", + resp.StatusCode) + } + + var listing struct { + Data []struct { + Name string `json:"name"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&listing); err != nil { + return "", err + } + for _, a := range listing.Data { + if a.Name != "" { + return a.Name, nil + } + } + return "", fmt.Errorf( + "this project has no agent in /agents, so an agent-target run cannot be built; " + + "deploy an agent or set AZURE_AI_EVAL_AGENT") +} + func buildFixture(logf func(string, ...any)) (*evalFixture, error) { ctx := context.Background() @@ -170,40 +248,33 @@ func buildFixture(logf func(string, ...any)) (*evalFixture, error) { return nil, fmt.Errorf("acquiring an azd credential: %w", err) } - name := strings.ReplaceAll(uniqueName("azdclifx"), "-", "_") - script, err := publishScoringEvaluator(ctx, client, name) + agent, err := resolveFixtureAgent(ctx) if err != nil { return nil, err } - logf("published code evaluator %s version %s", name, script.Version) - - if err := awaitEvaluatorListed(ctx, client, name, script.Version); err != nil { - return nil, err - } + logf("evaluating agent %q", agent) - evalID, err := createFixtureEval(ctx, client, name) + evaluatorName := "builtin.task_adherence" + evalID, err := createFixtureEval(ctx, client, evaluatorName) if err != nil { return nil, err } logf("created eval %s", evalID) - // Different pass rates so the comparison has something to measure. - baseline, err := startFixtureRun(ctx, client, evalID, "baseline", - []string{"a bad answer", "another bad answer", "a good answer"}) + first, err := startFixtureRun(ctx, client, evalID, agent, "first") if err != nil { return nil, err } - treatment, err := startFixtureRun(ctx, client, evalID, "treatment", - []string{"a good answer", "another good answer", "a third good answer"}) + second, err := startFixtureRun(ctx, client, evalID, agent, "second") if err != nil { return nil, err } - logf("started runs %s and %s", baseline, treatment) + logf("started runs %s and %s", first, second) // Polled together: they are independent, and serialising them doubles the // slowest part of the suite for nothing. errs := make(chan error, 2) - for _, runID := range []string{baseline, treatment} { + for _, runID := range []string{first, second} { go func(id string) { errs <- awaitCompleted(ctx, client, evalID, id, logf) }(runID) } for range 2 { @@ -213,109 +284,50 @@ func buildFixture(logf func(string, ...any)) (*evalFixture, error) { } return &evalFixture{ - EvaluatorName: name, - EvalID: evalID, - BaselineRunID: baseline, - TreatmentRunID: treatment, + // The criterion is named without the builtin. prefix, and that is the + // name results are reported under. + EvaluatorName: strings.TrimPrefix(evaluatorName, "builtin."), + EvalID: evalID, + AgentName: agent, + FirstRunID: first, + SecondRunID: second, }, nil } -func publishScoringEvaluator( - ctx context.Context, - client *eval_api.EvalClient, - name string, -) (*eval_api.EvaluatorVersion, error) { - dir, err := os.MkdirTemp("", "azdcli-grader") - if err != nil { - return nil, err - } - defer os.RemoveAll(dir) - - path := filepath.Join(dir, name+".py") - if err := os.WriteFile(path, []byte(scoringGrader), 0o600); err != nil { - return nil, err - } - script, err := evalcore.LoadCodeEvaluator(name, path) - if err != nil { - return nil, fmt.Errorf("loading the grader: %w", err) - } - - // Without a data_schema the criteria builder has no mapping to derive, so - // the schema is what makes the evaluator usable rather than merely - // publishable. - opts := eval_api.CodeEvaluatorOptions{ - DataSchema: json.RawMessage( - `{"type":"object","properties":{"response":{"type":"string"}},"required":["response"]}`), - Metrics: json.RawMessage( - `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`), - } - var version *eval_api.EvaluatorVersion - if err := retryCredentialFlake(func() error { - var err error - version, err = client.CreateCodeEvaluatorVersion(ctx, script, opts, fixtureAPIVersion) - return err - }); err != nil { - return nil, fmt.Errorf("publishing the code evaluator: %w", err) - } - deferTeardown(func() { - _ = client.DeleteEvaluatorVersion( - context.Background(), name, version.Version, fixtureAPIVersion) - }) - return version, nil -} - -// awaitEvaluatorListed waits for the version listing to catch up, which is the -// view eval creation resolves against. The direct read goes consistent first, -// so waiting on that alone still leaves the create failing with "was not -// found". -func awaitEvaluatorListed( - ctx context.Context, - client *eval_api.EvalClient, - name, version string, -) error { - deadline := time.Now().Add(2 * time.Minute) - for { - list, err := client.ListEvaluatorVersions(ctx, name, fixtureAPIVersion) - if err == nil && list != nil { - for _, entry := range list.Value { - if entry.Version == version { - return nil - } - } - } - if time.Now().After(deadline) { - return fmt.Errorf("evaluator %s version %s never appeared in the listing", name, version) - } - time.Sleep(500 * time.Millisecond) - } -} - func createFixtureEval( ctx context.Context, client *eval_api.EvalClient, evaluatorName string, ) (string, error) { - // Hand-written rather than built with buildEvalRequest, which is - // unexported. That is safe only because the evaluator is one published - // here whose schema is a single `response` column; a built-in would need - // the shipping builder, since their input contracts differ per evaluator. + criterionName := strings.TrimPrefix(evaluatorName, "builtin.") + var group *eval_api.OpenAIEval if err := retryCredentialFlake(func() error { var err error group, err = client.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ Name: uniqueName("azdcli-fixture"), DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", + Type: "custom", + IncludeSampleSchema: true, ItemSchema: map[string]any{ "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, + "properties": map[string]any{"query": map[string]any{"type": "string"}}, }, }, TestingCriteria: []eval_api.TestingCriterion{{ Type: "azure_ai_evaluator", - Name: evaluatorName, + Name: criterionName, EvaluatorName: evaluatorName, - DataMapping: map[string]string{"response": "{{item.response}}"}, + DataMapping: map[string]string{ + "query": "{{item.query}}", + "response": "{{sample.output_items}}", + "tool_calls": "{{sample.tool_calls}}", + "tool_definitions": "{{sample.tool_definitions}}", + }, + InitializationParameters: map[string]any{ + "model": fixtureModel(), + "deployment_name": fixtureModel(), + }, }}, }) return err @@ -331,15 +343,14 @@ func createFixtureEval( func startFixtureRun( ctx context.Context, client *eval_api.EvalClient, - evalID, label string, - responses []string, + evalID, agentName, label string, ) (string, error) { - rows := make([]map[string]any, 0, len(responses)) - for _, r := range responses { - rows = append(rows, map[string]any{"response": r}) + rows := make([]map[string]any, 0, len(fixtureQueries)) + for _, q := range fixtureQueries { + rows = append(rows, map[string]any{"query": q}) } - ds := eval_api.NewDatasetOnlyDataSource() + ds := eval_api.NewAgentTargetDataSource(agentName, nil) ds.SetFileContent(rows) var run *eval_api.OpenAIEvalRun diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go index dec143ce139..1557ea4aa0d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/harness_test.go @@ -204,16 +204,3 @@ func requireFailure(t *testing.T, r result) result { func uniqueName(prefix string) string { return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano()) } - -// writeGrader lays down a code evaluator script and returns its path. -func writeGrader(t *testing.T, body string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, "grader.py") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - return path -} - -const lengthGrader = `def grade(sample, item) -> float: - return float(len((item or {}).get("response", ""))) -` diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go index 8660b5eda26..77313c677dc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go @@ -15,6 +15,12 @@ import ( "github.com/stretchr/testify/require" ) +// The fixture is judged by a model, so no test here may assert how many rows +// passed. What is under test is the command, and the properties that hold +// whatever the judge decided: every dataset row comes back, every row carries a +// verdict and a score, and filtering by verdict returns a subset that agrees +// with the totals. + // resultsPayload is what `results show -o json` emits: the run and the rows. type resultsPayload struct { Run struct { @@ -46,88 +52,95 @@ type resultsPayload struct { // TestCLIResultsShowRendersTheRows is the difference between `results show` and // `run show`: the totals say how many failed, these say which. -// -// The fixture's baseline run scores one row and fails two, so the rendering is -// checked against known verdicts rather than against whatever came back. func TestCLIResultsShowRendersTheRows(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.BaselineRunID)) + r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.FirstRunID)) - require.Contains(t, r.Stdout, f.BaselineRunID) - require.Contains(t, r.Stdout, "Totals: 1 passed, 2 failed, 0 errored") + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, "Totals:") require.Contains(t, r.Stdout, "CRITERION") require.Contains(t, r.Stdout, "ITEM") require.Contains(t, r.Stdout, "EVALUATOR") require.Contains(t, r.Stdout, "SCORE") require.Contains(t, r.Stdout, f.EvaluatorName) - // Both verdicts, and the row's own input: a table that showed only the - // counts would satisfy every assertion above. - require.Contains(t, r.Stdout, "FAIL") - require.Contains(t, r.Stdout, "pass") - require.Contains(t, r.Stdout, "response=a good answer") - require.Contains(t, r.Stdout, "response=a bad answer") + // The row's own input, so that a table printing only counts would not + // satisfy every assertion above. + require.Contains(t, r.Stdout, "query=") require.Contains(t, r.Stdout, "Report:") } func TestCLIResultsShowJSON(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.BaselineRunID, "-o", "json")) - - var payload resultsPayload - r.JSON(t, &payload) + payload := resultsFor(t, f.EvalID, f.FirstRunID) - require.Equal(t, f.BaselineRunID, payload.Run.ID) + require.Equal(t, f.FirstRunID, payload.Run.ID) require.Equal(t, "completed", payload.Run.Status) - require.Equal(t, 3, payload.Run.ResultCounts.Total) - require.Equal(t, 1, payload.Run.ResultCounts.Passed) - require.Equal(t, 2, payload.Run.ResultCounts.Failed) - require.Zero(t, payload.Run.ResultCounts.Errored) + require.Equal(t, len(fixtureQueries), payload.Run.ResultCounts.Total) + require.Zero(t, payload.Run.ResultCounts.Errored, + "an errored row means the fixture measured nothing") require.Len(t, payload.Run.PerTestingCriteria, 1) require.Equal(t, f.EvaluatorName, payload.Run.PerTestingCriteria[0].TestingCriteria) // The rows are the reason this command exists, and a run reporting counts // while returning none would still satisfy everything above. - require.Len(t, payload.OutputItems, 3, "every dataset row must come back as an item") + require.Len(t, payload.OutputItems, len(fixtureQueries), + "every dataset row must come back as an item") passed := 0 for _, item := range payload.OutputItems { - require.NotEmpty(t, item.DataSourceItem["response"], + require.NotEmpty(t, item.DataSourceItem["query"], "each row must carry the column it was evaluated on") require.Len(t, item.Results, 1) require.Equal(t, f.EvaluatorName, item.Results[0].Name) require.NotNil(t, item.Results[0].Score, "a scored row must report its score") if item.Results[0].Passed { passed++ - require.Equal(t, 1.0, *item.Results[0].Score) - } else { - require.Equal(t, 0.0, *item.Results[0].Score) } } - require.Equal(t, 1, passed, "the per-row verdicts must agree with the totals") + require.Equal(t, payload.Run.ResultCounts.Passed, passed, + "the per-row verdicts must agree with the totals") } // TestCLIResultsShowFailedOnly asserts the filter removes rows rather than // merely relabelling them. +// +// The service has no verdict filter — its `status` selects on execution status, +// so `status=failed` returns errored rows, not failing ones — which makes this +// entirely the CLI's own work and worth testing directly. func TestCLIResultsShowFailedOnly(t *testing.T) { f := sharedEval(t) + payload := resultsFor(t, f.EvalID, f.FirstRunID) + failed := payload.Run.ResultCounts.Failed + r := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.BaselineRunID, "--failed-only")) + "--run-id", f.FirstRunID, "--failed-only")) + + if failed == 0 { + // Saying so is not the same as printing an empty table. + require.Contains(t, r.Stdout, "No failing rows.") + return + } - require.Contains(t, r.Stdout, "response=a bad answer") - require.NotContains(t, r.Stdout, "response=a good answer", + require.NotContains(t, r.Stdout, " pass ", "--failed-only must drop the rows that passed") + require.Contains(t, r.Stdout, "FAIL") + require.Equal(t, failed, strings.Count(r.Stdout, "FAIL"), + "every failing row must appear exactly once") +} - // The passing run has nothing to show, and saying so is not the same as - // printing an empty table. - empty := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.TreatmentRunID, "--failed-only")) - require.Contains(t, empty.Stdout, "No failing rows.") +// resultsFor reads a run's results as JSON, which several tests need before +// they can decide what the rendered output should say. +func resultsFor(t *testing.T, evalID, runID string) resultsPayload { + t.Helper() + r := requireSuccess(t, run(t, "results", "show", evalID, "--run-id", runID, "-o", "json")) + var payload resultsPayload + r.JSON(t, &payload) + return payload } func TestCLIResultsExport(t *testing.T) { @@ -135,34 +148,35 @@ func TestCLIResultsExport(t *testing.T) { t.Run("json to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "json")) + "--run-id", f.FirstRunID, "--format", "json")) var exported struct { ID string `json:"id"` Status string `json:"status"` ResultCounts struct { + Total int `json:"total"` Passed int `json:"passed"` Failed int `json:"failed"` } `json:"result_counts"` } r.JSON(t, &exported) - require.Equal(t, f.BaselineRunID, exported.ID) + require.Equal(t, f.FirstRunID, exported.ID) require.Equal(t, "completed", exported.Status) - require.Equal(t, 1, exported.ResultCounts.Passed) - require.Equal(t, 2, exported.ResultCounts.Failed) + require.Equal(t, len(fixtureQueries), exported.ResultCounts.Total) }) t.Run("csv to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "csv")) + "--run-id", f.FirstRunID, "--format", "csv")) rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) require.Len(t, rows, 2, "a header and one row per criterion") require.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) - require.Equal(t, - []string{f.BaselineRunID, "completed", f.EvaluatorName, "1", "2"}, rows[1]) + require.Equal(t, f.FirstRunID, rows[1][0]) + require.Equal(t, "completed", rows[1][1]) + require.Equal(t, f.EvaluatorName, rows[1][2]) }) t.Run("out-file writes the path instead of stdout", func(t *testing.T) { @@ -170,148 +184,25 @@ func TestCLIResultsExport(t *testing.T) { path := filepath.Join(dir, "results.csv") r := requireSuccess(t, runIn(t, dir, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "csv", "-O", path)) + "--run-id", f.FirstRunID, "--format", "csv", "-O", path)) require.Empty(t, strings.TrimSpace(r.Stdout), "-O redirects the payload; leaving it on stdout too would double it") body, err := os.ReadFile(path) require.NoError(t, err) require.Contains(t, string(body), "run_id,status,criterion,passed,failed") - require.Contains(t, string(body), f.BaselineRunID) + require.Contains(t, string(body), f.FirstRunID) }) t.Run("an unknown format is refused", func(t *testing.T) { r := requireFailure(t, run(t, "results", "export", f.EvalID, - "--run-id", f.BaselineRunID, "--format", "xml")) + "--run-id", f.FirstRunID, "--format", "xml")) require.Contains(t, r.Combined(), "json or csv") }) } -// comparison is the shape `results compare -o json` emits. -type comparison struct { - State string `json:"state"` - Request struct { - EvalID string `json:"evalId"` - BaselineRunID string `json:"baselineRunId"` - TreatmentRunIDs []string `json:"treatmentRunIds"` - } `json:"request"` - Result struct { - Method string `json:"method"` - Comparisons []struct { - TestingCriteria string `json:"testingCriteria"` - Metric string `json:"metric"` - BaselineRunSummary struct { - RunID string `json:"runId"` - SampleCount int `json:"sampleCount"` - Average float64 `json:"average"` - } `json:"baselineRunSummary"` - CompareItems []struct { - TreatmentRunSummary struct { - RunID string `json:"runId"` - SampleCount int `json:"sampleCount"` - Average float64 `json:"average"` - } `json:"treatmentRunSummary"` - DeltaEstimate float64 `json:"deltaEstimate"` - TreatmentEffect string `json:"treatmentEffect"` - } `json:"compareItems"` - } `json:"comparisons"` - } `json:"result"` -} - -// TestCLIResultsCompare needs two completed runs of the same eval that scored -// differently, which is why the fixture seeds one run to fail two of three -// rows: comparing two identical runs reports a zero delta, and a comparison -// that computed nothing would look the same. -func TestCLIResultsCompare(t *testing.T) { - f := sharedEval(t) - - t.Run("rendered columns", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "compare", f.EvalID, - "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID)) - - for _, header := range []string{ - "METRIC", "TREATMENT RUN", "BASELINE", "TREATMENT", "DELTA", "P-VALUE", "EFFECT", - } { - require.Containsf(t, r.Stdout, header, "the comparison table lost its %s column", header) - } - require.Contains(t, r.Stdout, "Method:") - require.Contains(t, r.Stdout, f.TreatmentRunID) - require.Contains(t, r.Stdout, f.EvaluatorName) - - // One in three against three in three. The delta is signed, which is - // the whole point of naming a baseline. - require.Contains(t, r.Stdout, "0.333") - require.Contains(t, r.Stdout, "1.000") - require.Contains(t, r.Stdout, "+0.667") - }) - - t.Run("json shape", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "compare", f.EvalID, - "--baseline", f.BaselineRunID, "--treatment", f.TreatmentRunID, "-o", "json")) - - var got comparison - r.JSON(t, &got) - - require.Equal(t, "Succeeded", got.State) - require.Equal(t, f.EvalID, got.Request.EvalID) - require.Equal(t, f.BaselineRunID, got.Request.BaselineRunID) - require.Equal(t, []string{f.TreatmentRunID}, got.Request.TreatmentRunIDs) - - require.NotEmpty(t, got.Result.Method) - require.Len(t, got.Result.Comparisons, 1) - c := got.Result.Comparisons[0] - require.Equal(t, f.EvaluatorName, c.TestingCriteria) - require.Equal(t, f.BaselineRunID, c.BaselineRunSummary.RunID) - require.Equal(t, 3, c.BaselineRunSummary.SampleCount) - require.InDelta(t, 1.0/3.0, c.BaselineRunSummary.Average, 0.001) - - require.Len(t, c.CompareItems, 1) - item := c.CompareItems[0] - require.Equal(t, f.TreatmentRunID, item.TreatmentRunSummary.RunID) - require.Equal(t, 1.0, item.TreatmentRunSummary.Average) - require.InDelta(t, 2.0/3.0, item.DeltaEstimate, 0.001) - require.NotEmpty(t, item.TreatmentEffect, - "the effect classifies the result, including when there are too few samples") - }) - - // Naming neither run is the common case — "did my last change help?" — so - // the defaults are asserted against the listing rather than against the - // fixture's own ids, which is what the command itself resolves from. - t.Run("defaults to the two most recent completed runs", func(t *testing.T) { - listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) - var runs []struct { - ID string `json:"id"` - Status string `json:"status"` - } - listed.JSON(t, &runs) - - completed := make([]string, 0, len(runs)) - for _, r := range runs { - if r.Status == "completed" { - completed = append(completed, r.ID) - } - } - require.GreaterOrEqual(t, len(completed), 2, - "comparing needs two completed runs of the same eval") - - r := requireSuccess(t, run(t, "results", "compare", f.EvalID, "-o", "json")) - var got comparison - r.JSON(t, &got) - - require.Equal(t, completed[0], got.Request.TreatmentRunIDs[0], - "the treatment defaults to the most recent completed run") - require.Equal(t, completed[1], got.Request.BaselineRunID, - "the baseline defaults to the one before it") - require.Equal(t, "Succeeded", got.State) - }) -} - -// TestCLIResultsUnknownEvalIsBrief covers the failure a user hits by typo. The -// service answers with a long JSON body; printing it verbatim buries the one -// useful sentence. func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "results", "show", "eval_azdcli_does_not_exist")) - require.Less(t, len(r.Combined()), 600, - "a not-found must stay short, not dump the service body:\n%s", r.Combined()) - require.Contains(t, r.Combined(), "eval_azdcli_does_not_exist") + r := requireFailure(t, run(t, "results", "show", "eval_does_not_exist")) + require.Contains(t, r.Combined(), "eval_does_not_exist") + require.NotContains(t, r.Combined(), "RESPONSE 404") } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go index ebe5b49bd24..c104f369e46 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/rubric_test.go @@ -56,13 +56,13 @@ func TestCLIRubricRoundTrip(t *testing.T) { {"id":"accuracy","description":"Is the answer correct?","weight":10} ]`) - created := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + created := requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", rubric)) require.Contains(t, created.Stdout, "version 1") t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") + run(t, "evaluator", "delete", name, "--version", "1") }) - shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + shown := requireSuccess(t, run(t, "evaluator", "show", name)) var doc evaluatorDocument shown.JSON(t, &doc) @@ -91,18 +91,18 @@ func TestCLIRubricRoundTrip(t *testing.T) { require.NotEmpty(t, doc.Definition.Metrics) require.NotEmpty(t, doc.SupportedEvaluationLevels) - // Every registration publishes a new immutable version, which is why there - // is no separate update command. - republished := requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", rubric)) + // Every registration publishes a new immutable version, which is what + // `update` means for an evaluator. + republished := requireSuccess(t, run(t, "evaluator", "update", name, "--from-file", rubric)) require.Contains(t, republished.Stdout, "version 2", - "re-registering must advance the version rather than overwrite") + "updating must advance the version rather than overwrite") t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "2") + run(t, "evaluator", "delete", name, "--version", "2") }) // The earlier version stays reachable, which is what makes a published // version safe to reference from a config. - pinned := requireSuccess(t, run(t, "evaluator", "show", "--name", name, "--version", "1")) + pinned := requireSuccess(t, run(t, "evaluator", "show", name, "--version", "1")) var first evaluatorDocument pinned.JSON(t, &first) require.Equal(t, "1", first.Version) @@ -131,7 +131,7 @@ func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { `[{"id":"tone","description":"Is the answer polite?","weight":`+tc.weight+`}]`) r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_badweight"), "--rubric", rubric)) + uniqueName("azdcli_badweight"), "--from-file", rubric)) require.Contains(t, r.Combined(), "between 1 and 10", "the refusal must say what a legal weight is") }) @@ -141,9 +141,9 @@ func TestCLIRubricWeightMustBeAnIntegerFromOneToTen(t *testing.T) { // weight rather than on the rubric shape they share. name := uniqueName("azdcli_goodweight") ok := writeRubric(t, `[{"id":"tone","description":"Is the answer polite?","weight":1}]`) - requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", ok)) + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", ok)) t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") + run(t, "evaluator", "delete", name, "--version", "1") }) } @@ -154,25 +154,10 @@ func TestCLIRubricNeedsDimensions(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(`{"criteria":[]}`), 0o600)) r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_nodims"), "--rubric", path)) + uniqueName("azdcli_nodims"), "--from-file", path)) require.Contains(t, r.Combined(), "dimensions") } -// TestCLIRubricRejectsCodeOnlyFlags asserts the settings that would be -// accepted and then dropped are refused instead. A rubric runs no code and its -// schemas come from the service, so carrying them would be a silent no-op. -func TestCLIRubricRejectsCodeOnlyFlags(t *testing.T) { - rubric := writeRubric(t, `[{"id":"tone","description":"polite","weight":5}]`) - schema := filepath.Join(t.TempDir(), "schema.json") - require.NoError(t, os.WriteFile(schema, []byte(`{"type":"object"}`), 0o600)) - - r := requireFailure(t, run(t, "evaluator", "create", - "--name", uniqueName("azdcli_rubricflags"), "--rubric", rubric, - "--data-schema", schema)) - require.Contains(t, r.Combined(), "--data-schema") - require.Contains(t, r.Combined(), "--file") -} - // TestCLIEvaluatorShowAcceptsAFullDocument proves `evaluator show` emits JSON a // script can consume, whatever the definition kind. It renders the service's // body rather than a typed struct, so nothing else pins that it stays parseable. @@ -186,12 +171,12 @@ func TestCLIEvaluatorShowAcceptsAFullDocument(t *testing.T) { `{"name":"ignored","definition":{"dimensions":[{"id":"tone","description":"polite","weight":3}]}}`, ), 0o600)) - requireSuccess(t, run(t, "evaluator", "create", "--name", name, "--rubric", path)) + requireSuccess(t, run(t, "evaluator", "create", name, "--from-file", path)) t.Cleanup(func() { - run(t, "evaluator", "delete", "--name", name, "--version", "1") + run(t, "evaluator", "delete", name, "--version", "1") }) - shown := requireSuccess(t, run(t, "evaluator", "show", "--name", name)) + shown := requireSuccess(t, run(t, "evaluator", "show", name)) var raw map[string]any require.NoError(t, json.Unmarshal([]byte(shown.Stdout), &raw), "evaluator show must emit parseable JSON:\n%s", shown.Stdout) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 77628b90270..3f37b32462f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -32,9 +32,9 @@ func TestCLIRunList(t *testing.T) { for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } - require.Contains(t, r.Stdout, f.BaselineRunID) - require.Contains(t, r.Stdout, f.TreatmentRunID) - require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored", + require.Contains(t, r.Stdout, f.FirstRunID) + require.Contains(t, r.Stdout, f.SecondRunID) + require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout, "the listing must summarise each run's counts, not just its status") }) @@ -51,11 +51,13 @@ func TestCLIRunList(t *testing.T) { for _, entry := range runs { byID[entry.ID] = entry } - baseline, ok := byID[f.BaselineRunID] + first, ok := byID[f.FirstRunID] require.True(t, ok, "the eval's own run is missing from its listing") - require.Equal(t, "completed", baseline.Status) - require.NotNil(t, baseline.ResultCounts) - require.Equal(t, 2, baseline.ResultCounts.Failed) + require.Equal(t, "completed", first.Status) + require.NotNil(t, first.ResultCounts) + require.Equal(t, len(fixtureQueries), + first.ResultCounts.Passed+first.ResultCounts.Failed, + "every dataset row must be accounted for by a verdict") }) // The client has always taken a limit; until recently the command did not @@ -79,11 +81,11 @@ func TestCLIRunShow(t *testing.T) { f := sharedEval(t) t.Run("by run id", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.BaselineRunID)) - require.Contains(t, r.Stdout, f.BaselineRunID) + r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.FirstRunID)) + require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "status") require.Contains(t, r.Stdout, "completed") - require.Contains(t, r.Stdout, "1 passed, 2 failed, 0 errored") + require.Regexp(t, `\d+ passed, \d+ failed, \d+ errored`, r.Stdout) require.Contains(t, r.Stdout, "report") }) @@ -115,7 +117,7 @@ func TestCLIRunShow(t *testing.T) { r := requireFailure(t, run(t, "run", "show", f.EvalID, "--run-id", "evalrun_azdcli_nope")) require.Contains(t, r.Combined(), "evalrun_azdcli_nope", "the failure must name the run that was asked for") - require.NotContains(t, r.Combined(), f.BaselineRunID, + require.NotContains(t, r.Combined(), f.FirstRunID, "an explicit --run-id must not fall back to another run") }) } @@ -130,7 +132,7 @@ func TestCLIRunCancelAndDelete(t *testing.T) { f := sharedEval(t) t.Run("a finished run is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.BaselineRunID)) + r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.FirstRunID)) require.Contains(t, r.Combined(), "already finished") require.Contains(t, r.Combined(), "completed") }) @@ -185,21 +187,17 @@ func TestCLIRunCancelAndDelete(t *testing.T) { // startCancellableRun adds a run to the fixture's eval and returns it before it // can finish. // -// The rows are padded so the run cannot complete inside the second it takes to -// issue the cancel; a run that finished first would turn the cancel test into -// an assertion about the guard it is not testing. +// An agent-target run invokes the agent once per row and is judged after that, +// which takes far longer than the second it takes to issue the cancel; a run +// that finished first would turn the cancel test into an assertion about the +// guard it is not testing. func startCancellableRun(t *testing.T, f *evalFixture) string { t.Helper() client, err := liveClient() require.NoError(t, err) - responses := make([]string, 0, 40) - for i := range 40 { - responses = append(responses, strings.Repeat("a good answer ", i%5+1)) - } - - runID, err := startFixtureRun(context.Background(), client, f.EvalID, "cancelme", responses) + runID, err := startFixtureRun(context.Background(), client, f.EvalID, f.AgentName, "cancelme") require.NoError(t, err, "starting a run to cancel") t.Cleanup(func() { _ = client.DeleteOpenAIEvalRun(context.Background(), f.EvalID, runID) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go deleted file mode 100644 index fbdfe1b2b87..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/tests/live/code_evaluator_run_test.go +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -//go:build live - -package live - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/pkg/evalcore" - - "github.com/stretchr/testify/require" -) - -// These tests close the gap between publishing a code evaluator and using one. -// -// The publish path was verified on its own first, and passing that proves less -// than it appears to: a script publishes cleanly while carrying no -// data_schema, because only metrics are defaulted. Nothing then tells the -// caller that the evaluator cannot be wired into an eval. The criteria builder -// derives data_mapping from the schema the evaluator publishes, so no schema -// means no mapping, and the service refuses a criterion with none. That -// failure would surface at run time, long after the publish that caused it. -// -// So there are two tests. The first runs a code evaluator end to end and -// requires it to score a sample. The second publishes without a schema and -// records what the service actually does about it, rather than leaving the -// consequence to inference. - -// writeCodeEvaluator writes an evaluator that scores the length of a response. -// -// It is one self-contained script. A code evaluator runs as an OpenAI python -// grader, whose contract is a single Source string with one entry point: a -// top-level grade(sample, item). There is no package and no import path, so a -// helper module beside it could not be reached even if it were published. -func writeCodeEvaluator(t *testing.T, name string) string { - t.Helper() - dir := t.TempDir() - path := filepath.Join(dir, name+".py") - - source := `def grade(sample, item) -> float: - response = (item or {}).get("response", "") - return float(len(response)) -` - require.NoError(t, os.WriteFile(path, []byte(source), 0o600)) - return path -} - -// dataSchemaForResponse is the schema the criteria builder needs to derive a -// data_mapping. Only the caller can supply it: nothing about it is inferable -// from Python source. -func dataSchemaForResponse(t *testing.T) json.RawMessage { - t.Helper() - raw, err := json.Marshal(map[string]any{ - "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, - "required": []string{"response"}, - }) - require.NoError(t, err) - return raw -} - -// publishCodeEvaluator registers the script and returns the version, after -// confirming it is readable. -func publishCodeEvaluator( - t *testing.T, - env *liveEnv, - name string, - withSchema bool, -) *eval_api.EvaluatorVersion { - t.Helper() - ctx := context.Background() - - path := writeCodeEvaluator(t, name) - script, err := evalcore.LoadCodeEvaluator(name, path) - require.NoError(t, err, "loading the evaluator script") - - var opts eval_api.CodeEvaluatorOptions - if withSchema { - opts.DataSchema = dataSchemaForResponse(t) - opts.Metrics = json.RawMessage( - `{"result":{"type":"continuous","desirable_direction":"increase","is_primary":true}}`) - } - - version, err := env.evalClient.CreateCodeEvaluatorVersion(ctx, script, opts, projectAPIVersion) - require.NoError(t, err, "publishing the code evaluator") - t.Cleanup(func() { - _ = env.evalClient.DeleteEvaluatorVersion( - context.Background(), name, version.Version, projectAPIVersion) - }) - t.Logf("published code evaluator %s version %s", name, version.Version) - - awaitEvaluatorResolvable(t, env, name, version.Version) - return version -} - -// awaitEvaluatorResolvable waits for a published version the way the -// reconciler does, and reports how long each of the two views took. -// -// The numbers are the point. The direct read goes consistent almost at once -// while the listing lags it, and the eval-create resolver follows the slower -// one: a publish was observed reading back at 03:06:58 and still failing eval -// creation at 03:06:59. Logging both is what keeps the reconciler's tolerance -// honest instead of guessed, and asserting on the listing here is what proves -// the gate it waits on is the right one. -func awaitEvaluatorResolvable(t *testing.T, env *liveEnv, name, version string) { - t.Helper() - ctx := context.Background() - - start := time.Now() - var readable time.Duration - - for { - if readable == 0 { - if _, err := env.evalClient.GetEvaluatorRaw( - ctx, name, version, projectAPIVersion, - ); err == nil { - readable = time.Since(start) - t.Logf("evaluator %s readable after %s", - name, readable.Round(time.Millisecond)) - } - } - if readable != 0 && evaluatorVersionListed(ctx, env, name, version) { - t.Logf("evaluator %s listed after %s", - name, time.Since(start).Round(time.Millisecond)) - return - } - if time.Since(start) > 2*time.Minute { - t.Fatalf("evaluator %s never became resolvable", name) - } - time.Sleep(250 * time.Millisecond) - } -} - -func evaluatorVersionListed( - ctx context.Context, - env *liveEnv, - name, version string, -) bool { - list, err := env.evalClient.ListEvaluatorVersions(ctx, name, projectAPIVersion) - if err != nil || list == nil { - return false - } - for _, entry := range list.Value { - if entry.Version == version { - return true - } - } - return false -} - -// createEvalReferencing creates an eval naming a custom evaluator, tolerating -// the window in which the evaluator is published but not yet resolvable. -// -// The delay is reported so a run that hits it leaves evidence of how long it -// took, which is the number the reconciler's own tolerance has to be built on. -func createEvalReferencing( - t *testing.T, - env *liveEnv, - req *eval_api.CreateOpenAIEvalRequest, - within time.Duration, -) (*eval_api.OpenAIEval, error) { - t.Helper() - ctx := context.Background() - - start := time.Now() - for { - group, err := env.evalClient.CreateOpenAIEval(ctx, req) - if err == nil { - t.Logf("eval accepted the evaluator after %s", time.Since(start).Round(time.Millisecond)) - return group, nil - } - if !strings.Contains(strings.ToLower(err.Error()), "was not found") { - return nil, err - } - if time.Since(start) > within { - t.Logf("the evaluator was still unresolvable after %s", within) - return nil, err - } - time.Sleep(5 * time.Second) - } -} - -// TestLiveCodeEvaluatorScoresARun is the test the publish tests could not be: -// it requires the evaluator to actually run and return a score. -// -// No agent is involved. A code evaluator reads item fields, so the run uses a -// dataset-only source and the rows are supplied inline. That keeps the test -// about the evaluator rather than about a target being reachable. -func TestLiveCodeEvaluatorScoresARun(t *testing.T) { - env := setup(t) - ctx := context.Background() - - name := strings.ReplaceAll(uniqueName("azdcoderun"), "-", "_") - publishCodeEvaluator(t, env, name, true) - - group, err := createEvalReferencing(t, env, &eval_api.CreateOpenAIEvalRequest{ - Name: uniqueName("azd-code-eval"), - DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", - ItemSchema: map[string]any{ - "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, - }, - }, - TestingCriteria: []eval_api.TestingCriterion{{ - Type: "azure_ai_evaluator", - Name: name, - EvaluatorName: name, - DataMapping: map[string]string{"response": "{{item.response}}"}, - }}, - }, 3*time.Minute) - require.NoError(t, err, "creating an eval that references the code evaluator") - t.Logf("created eval %s", group.ID) - - ds := eval_api.NewDatasetOnlyDataSource() - ds.SetFileContent([]map[string]any{ - {"response": "a short answer"}, - {"response": "a considerably longer answer than the first one"}, - }) - - run, err := env.evalClient.CreateOpenAIEvalRun(ctx, group.ID, &eval_api.CreateOpenAIEvalRunRequest{ - Name: uniqueName("code-run"), - DataSource: ds, - }) - require.NoError(t, err, "starting the run") - t.Cleanup(func() { - _, _ = env.evalClient.CancelOpenAIEvalRun(context.Background(), group.ID, run.ID) - }) - t.Logf("started run %s", run.ID) - - final := awaitRun(t, env, group.ID, run.ID, 10*time.Minute) - - // Reaching a terminal state is not the same as having evaluated anything: - // a run whose every sample errors still reports completed. - require.Equal(t, "completed", strings.ToLower(final.Status), - "the run must complete rather than fail or cancel") - require.NotNil(t, final.ResultCounts, "a completed run must report counts") - require.Zero(t, final.ResultCounts.Errored, - "an errored sample means the code evaluator did not run") - require.Positive(t, final.ResultCounts.Passed+final.ResultCounts.Failed, - "the run must score at least one sample; scoring nothing means the rows "+ - "never reached the evaluator") -} - -// TestLiveCodeEvaluatorWithoutSchemaIsAccepted pins down what happens to a -// script published with no data_schema, which is the shape most people's -// their first evaluator will produce. -// -// It was expected to be refused. The reasoning was that the criteria builder -// derives data_mapping from the evaluator's schema, so no schema means no -// mapping, and the service rejects a criterion with none. An earlier run -// appeared to confirm it. That was wrong: the refusal was the propagation 404 -// in disguise, read as a mapping error because it arrived at the same call. -// With the publish properly gated, a schema-less evaluator is accepted and an -// empty data_mapping is allowed, so this is not the trap it looked like. -// -// The create is deliberately not retried. publishCodeEvaluator has already -// waited on the same condition the reconciler waits on, so a "was not found" -// here would mean that gate is the wrong one — which is worth failing on, -// because a retry would hide it. -func TestLiveCodeEvaluatorWithoutSchemaIsAccepted(t *testing.T) { - env := setup(t) - ctx := context.Background() - - name := strings.ReplaceAll(uniqueName("azdcodenoschema"), "-", "_") - published := publishCodeEvaluator(t, env, name, false) - require.NotEmpty(t, published.Version, - "a script with no schema still publishes; the schema is not required to register") - - _, err := env.evalClient.CreateOpenAIEval(ctx, &eval_api.CreateOpenAIEvalRequest{ - Name: uniqueName("azd-code-eval-noschema"), - DataSourceConfig: &eval_api.DataSourceConfig{ - Type: "custom", - ItemSchema: map[string]any{ - "type": "object", - "properties": map[string]any{"response": map[string]any{"type": "string"}}, - }, - }, - TestingCriteria: []eval_api.TestingCriterion{{ - Type: "azure_ai_evaluator", - Name: name, - EvaluatorName: name, - // Deliberately empty: this is what the criteria builder produces - // for an evaluator that publishes no data_schema. - DataMapping: map[string]string{}, - }}, - }) - - if err == nil { - t.Log("an evaluator with no data_schema was accepted with an empty data_mapping; " + - "publishing without a schema is not by itself a blocker") - return - } - require.NotContains(t, strings.ToLower(err.Error()), "was not found", - "the evaluator was published and waited for, so a not-found here means the "+ - "propagation gate the reconciler uses does not cover eval creation") - t.Logf("an evaluator with no data_schema was refused: %v", err) - require.Contains(t, strings.ToLower(err.Error()), "mapping", - "the refusal should name the mapping, so the CLI can explain it at publish time") -} - -// awaitRun polls until the run reaches a terminal state or the deadline passes. -func awaitRun( - t *testing.T, - env *liveEnv, - evalID string, - runID string, - within time.Duration, -) *eval_api.OpenAIEvalRun { - t.Helper() - ctx := context.Background() - - terminal := map[string]bool{ - "completed": true, "failed": true, "canceled": true, "cancelled": true, "error": true, - } - deadline := time.Now().Add(within) - for { - current, err := env.evalClient.GetOpenAIEvalRun(ctx, evalID, runID) - require.NoError(t, err, "polling the run") - if terminal[strings.ToLower(current.Status)] { - if current.ResultCounts != nil { - t.Logf("run %s reached %s: passed=%d failed=%d errored=%d", - runID, current.Status, - current.ResultCounts.Passed, - current.ResultCounts.Failed, - current.ResultCounts.Errored) - } - body, _ := json.MarshalIndent(current.PerTestingCriteria, "", " ") - t.Logf("per-criteria results: %s", string(body)) - return current - } - if time.Now().After(deadline) { - t.Fatalf("run %s did not finish within %s (last status %q)", - runID, within, current.Status) - } - time.Sleep(10 * time.Second) - } -} From 34f9c0c3405d9c9d848ea2b83ae6c52cfb24f324 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:15:15 +0300 Subject: [PATCH 074/114] Leave M1 with the one target it ships Trace-backed, stored-response, model-target and target-less runs are all deferred, so their data sources, the flags that reached them, and the config that declared them come out. What remains is an agent target over a dataset, which is what the four hero scenarios use. An eval that names no target is now an error rather than a target-less run. It used to mean the rows already held both sides of the exchange, and that is exactly the deferred case, so accepting it silently would ship the feature through the back door. The code evaluator settings leave EvaluatorDecl for the same reason: image_tag, metrics, data_schema and init_parameters only ever fed a python grader. --- .../internal/cmd/build.go | 9 +-- .../azure.ai.evaluations/internal/cmd/run.go | 76 +----------------- .../internal/pkg/eval_api/models.go | 79 ------------------- .../internal/project/eval_config.go | 27 ++----- .../internal/project/service_target_eval.go | 3 - 5 files changed, 10 insertions(+), 184 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 6c938f32a70..d5008b57b9d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -56,18 +56,11 @@ var sampleBindings = map[string]string{ // modelSampleBindings are what a model target produces. A model answers as // plain text and calls no tools, so binding an agent's richer output would // leave the evaluator waiting on fields the run never produces. -var modelSampleBindings = map[string]string{ - "response": "{{sample.output_text}}", -} - // sampleBindingsFor returns the run-time bindings a target of this kind can // satisfy. An empty target kind means nothing is invoked, so nothing is bound. func sampleBindingsFor(targetType string) map[string]string { - switch targetType { - case project.TargetTypeAgent: + if targetType == project.TargetTypeAgent { return sampleBindings - case project.TargetTypeModel: - return modelSampleBindings } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e6087ae0e38..b9ccd7d398a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -54,11 +54,6 @@ func buildRunCommand(use, short string) *cobra.Command { runName string level string maxSamples int - fromTraces bool - traceWindow string - maxTraces int - responseIDs []string - maxTurns int wait bool endpointFlg string ) @@ -107,10 +102,6 @@ func buildRunCommand(use, short string) *cobra.Command { // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource switch { - case len(responseIDs) > 0: - dataSource = eval_api.NewResponsesDataSource(responseIDs, maxTurns) - case fromTraces: - dataSource, err = buildTracesDataSource(ctx, ec, group, evalID, traceWindow, maxTraces) case group == nil: dataSource, err = ec.reuseDataSourceFromLastRun(ctx, evalID) default: @@ -188,16 +179,6 @@ func buildRunCommand(use, short string) *cobra.Command { "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from the dataset.") - cmd.Flags().BoolVar(&fromTraces, "from-traces", false, - "Evaluate the agent's recorded traces instead of the dataset.") - cmd.Flags().StringVar(&traceWindow, "trace-window", "", - "How far back to read traces, for example 7d. Defaults to the service's window.") - cmd.Flags().IntVar(&maxTraces, "max-traces", 0, "Cap the traces evaluated.") - cmd.Flags().StringSliceVar(&responseIDs, "response-id", nil, - "Evaluate stored responses by id instead of the dataset; repeatable.") - cmd.Flags().IntVar(&maxTurns, "max-turns", 0, - "Turns of chat history to pull back per response. Defaults to the service's limit.") - cmd.MarkFlagsMutuallyExclusive("from-traces", "response-id") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool @@ -367,47 +348,6 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return list.Data[0].DataSource, nil } -// buildTracesDataSource evaluates what the agent has already done, rather than -// asking it fresh questions from a dataset. -// -// The service reads the traces from Application Insights, so the agent has to -// be emitting gen_ai.input.messages / gen_ai.output.messages for anything to be -// found; when it is not, the run fails with the service saying so. -func buildTracesDataSource( - ctx context.Context, - ec *evalContext, - group *project.Eval, - evalID, window string, - maxTraces int, -) (*eval_api.EvalRunDataSource, error) { - agent := "" - switch { - case group != nil && group.Target != nil: - agent = group.Target.Name - default: - // With --eval-id there is no config, so the agent comes from whatever - // the group ran against last. - last, err := ec.reuseDataSourceFromLastRun(ctx, evalID) - if err != nil { - return nil, err - } - if last != nil && last.Target != nil { - agent = last.Target.Name - } - } - if agent == "" { - return nil, fmt.Errorf( - "--from-traces needs to know whose traces to read, and the eval does not " + - "name an agent. Declare target.type: agent on the eval") - } - - var lookbackHours int - if days := parseWindowDays(window); days > 0 { - lookbackHours = days * 24 - } - return eval_api.NewTracesDataSource(agent, lookbackHours, time.Time{}, maxTraces), nil -} - // buildRunDataSource binds the dataset to the run. The eval carries no // dataset today, so it is supplied here. func (ec *evalContext) buildRunDataSource( @@ -419,20 +359,12 @@ func (ec *evalContext) buildRunDataSource( if group == nil { return nil, fmt.Errorf("no eval to run") } - - // A group with no target scores a dataset that already holds the exchange, - // so there is nothing to invoke. That is how recorded conversations are - // evaluated. - var ds *eval_api.EvalRunDataSource - switch { - case group.Target == nil || group.Target.Name == "": - ds = eval_api.NewDatasetOnlyDataSource() - case group.Target.Type == project.TargetTypeModel: - ds = eval_api.NewModelTargetDataSource(group.Target.Name) - default: - ds = eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + if group.Target == nil || group.Target.Name == "" { + return nil, fmt.Errorf("eval %q does not name a target agent", group.Name) } + ds := eval_api.NewAgentTargetDataSource(group.Target.Name, nil) + if group.Dataset == "" { return nil, fmt.Errorf("eval %q does not reference a dataset", group.Name) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 2b9fa2ca9d1..6441568db72 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -8,7 +8,6 @@ import ( "fmt" "sort" "strings" - "time" ) // --------------------------------------------------------------------------- @@ -422,84 +421,6 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } } -// NewTracesDataSource evaluates an agent's recorded traces instead of a dataset. -// -// The window is a lookback in hours. The service's own field is -// `lookback_hours` and it has no start bound: a `start_time` is accepted and -// dropped, leaving the default seven days in place, so the conversion happens -// here rather than being left to look like it worked. -func NewTracesDataSource(agentName string, lookbackHours int, end time.Time, maxTraces int) *EvalRunDataSource { - ds := &EvalRunDataSource{ - Type: EvalRunDataSourceTypeTraces, - AgentName: agentName, - LookbackHours: lookbackHours, - MaxTraces: maxTraces, - } - if !end.IsZero() { - ds.EndTime = end.Unix() - } - return ds -} - -// NewDatasetOnlyDataSource scores the dataset as it stands, invoking nothing. -// -// Used when a group declares no target: the rows already hold both sides of -// the exchange, which is how a recorded conversation is evaluated. -func NewDatasetOnlyDataSource() *EvalRunDataSource { - return &EvalRunDataSource{Type: EvalRunDataSourceTypeJSONL} -} - -// NewModelTargetDataSource sends the dataset's questions straight to a model -// deployment, with no agent in front of it. -// -// The model answers as plain text, so a group evaluating one has to bind its -// response to {{sample.output_text}} rather than the richer output an agent -// produces. -func NewModelTargetDataSource(model string) *EvalRunDataSource { - return &EvalRunDataSource{ - Type: EvalRunDataSourceTypeAgentTarget, - InputMessages: &EvalRunInputMessages{ - Type: "template", - Template: []EvalRunMessageTemplate{ - { - Role: "user", - Content: "{{item.query}}", - Type: "message", - }, - }, - }, - Target: &EvalRunTarget{ - Type: "azure_ai_model", - Model: model, - }, - } -} - -// NewResponsesDataSource evaluates responses the project already stored. -// -// The ids travel as ordinary JSONL rows and a data_mapping points the service -// at the field holding each one, which is how it retrieves the chat history -// behind the response. -func NewResponsesDataSource(responseIDs []string, maxTurns int) *EvalRunDataSource { - rows := make([]map[string]any, 0, len(responseIDs)) - for _, id := range responseIDs { - rows = append(rows, map[string]any{"item": map[string]any{"response_id": id}}) - } - - return &EvalRunDataSource{ - Type: EvalRunDataSourceTypeResponses, - ItemGenerationParams: &ItemGenerationParams{ - Type: "response_retrieval", - MaxNumTurns: maxTurns, - DataMapping: map[string]string{"response_id": "{{item.response_id}}"}, - Source: &EvalRunDataContent{ - Type: EvalRunDataContentTypeFileContent, - Content: rows, - }, - }, - } -} - // SetFileContent sets the data source to use inline file content. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 76b0d4bf7cb..0c75f38fdf0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -34,23 +34,12 @@ type DatasetDecl struct { // EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly // from an eval and never declared here. // -// Source decides which kind of evaluator this is, by extension: a `.py` file is -// a single self-contained Python script and publishes a code evaluator, a -// `.json` file holds a rubric. A code evaluator cannot name a folder — it runs -// as a python grader, which is handed one script's source and cannot import a -// helper module beside it. +// Source names a `.json` file holding a rubric: a list of weighted scoring +// dimensions. type EvaluatorDecl struct { Name string `yaml:"name" json:"name"` Source string `yaml:"source,omitempty" json:"source,omitempty"` Version string `yaml:"version,omitempty" json:"version,omitempty"` - - // Code evaluators only. The three schema fields name JSON files beside the - // script, resolved like Source, because they are edited as files rather - // than written inline in YAML. - ImageTag string `yaml:"image_tag,omitempty" json:"image_tag,omitempty"` - Metrics string `yaml:"metrics,omitempty" json:"metrics,omitempty"` - DataSchema string `yaml:"data_schema,omitempty" json:"data_schema,omitempty"` - InitParameters string `yaml:"init_parameters,omitempty" json:"init_parameters,omitempty"` } // Eval is a run definition: evaluators plus options, bound to a dataset. @@ -72,11 +61,6 @@ type Target struct { const TargetTypeAgent = "agent" -// TargetTypeModel evaluates a model deployment directly, with no agent in -// front of it. A model answers as plain text, so a group targeting one binds -// its response differently from a group targeting an agent. -const TargetTypeModel = "model" - // Options are run settings carried on the group. type Options struct { EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` @@ -175,11 +159,10 @@ func (c *EvalConfig) Validate() error { i, g.Name, ref.Name, evalcore.BuiltinPrefix) } } - if g.Target != nil && g.Target.Type != "" && - g.Target.Type != TargetTypeAgent && g.Target.Type != TargetTypeModel { + if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { return fmt.Errorf( - "evals[%d] (%s): target.type %q is not supported; use %q or %q", - i, g.Name, g.Target.Type, TargetTypeAgent, TargetTypeModel) + "evals[%d] (%s): target.type %q is not supported; use %q", + i, g.Name, g.Target.Type, TargetTypeAgent) } if g.Options != nil { switch g.Options.EvaluationLevel { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 5f6786e954e..f4902f236a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -162,9 +162,6 @@ func (p *EvalServiceTargetProvider) Deploy( for _, decl := range cfg.Evaluators { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) - decl.Metrics = resolveSource(baseDir, decl.Metrics) - decl.DataSchema = resolveSource(baseDir, decl.DataSchema) - decl.InitParameters = resolveSource(baseDir, decl.InitParameters) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) if err != nil { return nil, fmt.Errorf("evaluator %q: %w", decl.Name, err) From 4bbcd923198641dca6a137cbd0a78375066f0751 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:28:52 +0300 Subject: [PATCH 075/114] Give datasets the same surface as evaluators Name positionally, content through --from-file, and versions under their own subcommand so that list means the project's datasets rather than one dataset's history. create and update split the same upload the way they do for evaluators. Also drops a stray copy of eval_api/schedules.go from the working tree. It was left behind by the checkout that built the M2 branch, was never tracked here, and had been compiling into M1 ever since - dead code that M1 does not call and that its own history does not contain. --- .../internal/cmd/dataset.go | 178 +++++++++++------- .../tests/cli/dataset_test.go | 62 +++--- 2 files changed, 149 insertions(+), 91 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index 9ea7c5b3a48..daecc8c3d05 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -23,52 +23,51 @@ func newDatasetCommand() *cobra.Command { } cmd.AddCommand( newDatasetCreateCommand(), + newDatasetUpdateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), + newDatasetVersionsCommand(), ) return cmd } -// newDatasetCreateCommand builds `dataset create`. -// -// There is no separate `update`: every registration publishes a new immutable -// version and the server auto-increments, so `create` covers both the first -// version and every later one. +// newDatasetCreateCommand builds `dataset create `, which registers a +// dataset that does not exist yet. func newDatasetCreateCommand() *cobra.Command { + return newDatasetWriteCommand("create", "Register a dataset, publishing its first version.") +} + +// newDatasetUpdateCommand builds `dataset update `, which publishes a +// further version of one that does. +func newDatasetUpdateCommand() *cobra.Command { + return newDatasetWriteCommand("update", "Publish a new version of a dataset.") +} + +// newDatasetWriteCommand builds create and update. Both run the same upload, +// and the existence check is the only thing that separates them: a version is +// brought into being by startPendingUpload, which neither knows nor cares +// whether the name was already in use. +func newDatasetWriteCommand(verb, short string) *cobra.Command { var ( - name string - file string + fromFile string version string endpointFlg string ) - use := "create" - short := "Register a dataset, publishing a new version." - cmd := &cobra.Command{ - Use: use, + Use: verb + " ", Short: short, + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } - if file == "" { - return requireFlag("file") + name := args[0] + if fromFile == "" { + return requireFlag("from-file") } - info, err := os.Stat(file) + localDir, err := datasetUploadDir(fromFile) if err != nil { - return fmt.Errorf("reading --file %q: %w", file, err) - } - // The upload helper scans a directory for the first .jsonl, so pass - // the containing directory when given a file path. - localDir := file - if !info.IsDir() { - if !strings.EqualFold(filepath.Ext(file), ".jsonl") { - return fmt.Errorf("--file must be a .jsonl file or a directory containing one, got %q", file) - } - localDir = filepath.Dir(file) + return err } ctx := cmd.Context() @@ -78,6 +77,14 @@ func newDatasetCreateCommand() *cobra.Command { } defer ec.Close() + existing, err := ec.datasetClient.ListDatasetVersions( + ctx, name, ProjectEndpointAPIVersion, + ) + exists := err == nil && existing != nil && len(existing.Value) > 0 + if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { + return err + } + ds, err := ec.datasetClient.UploadNextVersion( ctx, name, version, localDir, ProjectEndpointAPIVersion, ) @@ -103,23 +110,37 @@ func newDatasetCreateCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") - cmd.Flags().StringVar(&file, "file", "", "Path to a .jsonl file, or a directory containing one.") + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Path to a .jsonl file, or a directory containing one.") cmd.Flags().StringVar(&version, "version", "", "Current version to increment from. Omit to increment from the latest registered version.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +// datasetUploadDir resolves what was named into the directory the upload scans. +func datasetUploadDir(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("reading --from-file %q: %w", path, err) + } + if info.IsDir() { + return path, nil + } + if !strings.EqualFold(filepath.Ext(path), ".jsonl") { + return "", fmt.Errorf( + "--from-file must be a .jsonl file or a directory containing one, got %q", path) + } + return filepath.Dir(path), nil +} + func newDatasetListCommand() *cobra.Command { - var ( - name string - endpointFlg string - ) + var endpointFlg string cmd := &cobra.Command{ Use: "list", - Short: "List registered datasets, or the versions of one dataset.", + Short: "List the project's datasets.", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -128,50 +149,85 @@ func newDatasetListCommand() *cobra.Command { } defer ec.Close() - var list *dataset_api.DatasetList - if name != "" { - list, err = ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) - } else { - list, err = ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) - } + list, err := ec.datasetClient.ListDatasets(ctx, ProjectEndpointAPIVersion) if err != nil { return fmt.Errorf("listing datasets: %w", err) } + return renderDatasets(cmd, list) + }, + } - if isJSON(cmd) { - return emitJSONList(cmd.OutOrStdout(), list.Value) - } - rows := make([][]string, 0, len(list.Value)) - for _, d := range list.Value { - rows = append(rows, []string{d.Name, d.Version, d.Format}) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// newDatasetVersionsCommand groups the version listing, so that `list` means +// the assets rather than the history of one of them. +func newDatasetVersionsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "versions", + Short: "Inspect the versions of one dataset.", + } + cmd.AddCommand(newDatasetVersionsListCommand()) + return cmd +} + +func newDatasetVersionsListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list ", + Short: "List the versions of a dataset.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err } - if len(rows) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") - return nil + defer ec.Close() + + list, err := ec.datasetClient.ListDatasetVersions(ctx, name, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing versions of dataset %q: %w", name, err) } - return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) + return renderDatasets(cmd, list) }, } - cmd.Flags().StringVar(&name, "name", "", "Limit the listing to versions of this dataset.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +func renderDatasets(cmd *cobra.Command, list *dataset_api.DatasetList) error { + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Value) + } + rows := make([][]string, 0, len(list.Value)) + for _, d := range list.Value { + rows = append(rows, []string{d.Name, d.Version, d.Format}) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No datasets found.") + return nil + } + return emitTable(cmd.OutOrStdout(), []string{"NAME", "VERSION", "FORMAT"}, rows) +} + func newDatasetShowCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "show", + Use: "show ", Short: "Show a dataset version.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -211,7 +267,6 @@ func newDatasetShowCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") cmd.Flags().StringVar(&version, "version", "", "Version to show. Omit for the latest.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -219,18 +274,16 @@ func newDatasetShowCommand() *cobra.Command { func newDatasetDeleteCommand() *cobra.Command { var ( - name string version string endpointFlg string ) cmd := &cobra.Command{ - Use: "delete", + Use: "delete ", Short: "Delete a dataset version.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - if name == "" { - return requireFlag("name") - } + name := args[0] if version == "" { return requireFlag("version") } @@ -262,7 +315,6 @@ func newDatasetDeleteCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&name, "name", "", "Name of the dataset.") cmd.Flags().StringVar(&version, "version", "", "Version to delete.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go index 3b0dbeb1856..d82415517fc 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/dataset_test.go @@ -59,9 +59,15 @@ func registerDataset(t *testing.T, versions int) *registeredDataset { require.NoError(t, os.WriteFile(path, []byte(datasetRows), 0o600)) ds := ®isteredDataset{Name: uniqueName("azdcli_ds")} - for range versions { - r := requireSuccess(t, run(t, "dataset", "create", - "--name", ds.Name, "--file", path, "-o", "json")) + for i := range versions { + // The first publish is a create; every later one is an update, which is + // the only difference between them. + verb := "update" + if i == 0 { + verb = "create" + } + r := requireSuccess(t, run(t, "dataset", verb, + ds.Name, "--from-file", path, "-o", "json")) var created datasetSummary r.JSON(t, &created) @@ -70,12 +76,12 @@ func registerDataset(t *testing.T, versions int) *registeredDataset { version := created.Version deferTeardown(func() { - runQuietly("dataset", "delete", "--name", ds.Name, "--version", version) + runQuietly("dataset", "delete", ds.Name, "--version", version) }) } require.Len(t, ds.Versions, versions) require.NotEqual(t, ds.Versions[0], ds.Versions[len(ds.Versions)-1], - "re-registering must advance the version rather than overwrite") + "updating must advance the version rather than overwrite") return ds } @@ -83,17 +89,17 @@ func TestCLIDatasetList(t *testing.T) { ds := sharedDataset(t) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name)) + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name)) for _, header := range []string{"NAME", "VERSION", "FORMAT"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } require.Contains(t, r.Stdout, ds.Name) }) - // --name is what makes the listing usable once a project holds more than a - // screenful: it narrows to one dataset's versions. - t.Run("name scopes the listing to one dataset's versions", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "list", "--name", ds.Name, "-o", "json")) + // `versions list` is what makes the listing usable once a project holds more + // than a screenful: it narrows to one dataset's versions. + t.Run("versions list scopes to one dataset's versions", func(t *testing.T) { + r := requireSuccess(t, run(t, "dataset", "versions", "list", ds.Name, "-o", "json")) var listed []datasetSummary r.JSON(t, &listed) require.NotEmpty(t, listed) @@ -101,7 +107,7 @@ func TestCLIDatasetList(t *testing.T) { seen := map[string]bool{} for _, v := range listed { require.Equalf(t, ds.Name, v.Name, - "--name must return only that dataset's versions; got %q", v.Name) + "the listing must return only that dataset's versions; got %q", v.Name) seen[v.Version] = true } for _, want := range ds.Versions { @@ -127,8 +133,8 @@ func TestCLIDatasetList(t *testing.T) { }) t.Run("an unknown name lists nothing rather than failing", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "list", - "--name", "azdcli-no-such-dataset", "-o", "json")) + r := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) var listed []datasetSummary r.JSON(t, &listed) require.Empty(t, listed) @@ -142,7 +148,7 @@ func TestCLIDatasetShow(t *testing.T) { // Omitting the version means the latest, which is the only sensible // default for a name that gains a version on every registration. t.Run("defaults to the latest version", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name, "-o", "json")) + r := requireSuccess(t, run(t, "dataset", "show", ds.Name, "-o", "json")) var shown datasetSummary r.JSON(t, &shown) require.Equal(t, ds.Name, shown.Name) @@ -151,7 +157,7 @@ func TestCLIDatasetShow(t *testing.T) { t.Run("version pins an earlier one", func(t *testing.T) { r := requireSuccess(t, run(t, "dataset", "show", - "--name", ds.Name, "--version", ds.Versions[0], "-o", "json")) + ds.Name, "--version", ds.Versions[0], "-o", "json")) var shown datasetSummary r.JSON(t, &shown) require.Equal(t, ds.Versions[0], shown.Version) @@ -159,7 +165,7 @@ func TestCLIDatasetShow(t *testing.T) { }) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "dataset", "show", "--name", ds.Name)) + r := requireSuccess(t, run(t, "dataset", "show", ds.Name)) for _, header := range []string{"NAME", "VERSION", "FORMAT", "URI"} { require.Containsf(t, r.Stdout, header, "the table lost its %s column", header) } @@ -168,11 +174,11 @@ func TestCLIDatasetShow(t *testing.T) { t.Run("the name is required", func(t *testing.T) { r := requireFailure(t, run(t, "dataset", "show")) - require.Contains(t, r.Combined(), "--name is required") + require.Contains(t, r.Combined(), "accepts 1 arg") }) t.Run("an unknown dataset is brief", func(t *testing.T) { - r := requireFailure(t, run(t, "dataset", "show", "--name", "azdcli-no-such-dataset")) + r := requireFailure(t, run(t, "dataset", "show", "azdcli-no-such-dataset")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) require.Contains(t, r.Combined(), "azdcli-no-such-dataset") @@ -180,19 +186,19 @@ func TestCLIDatasetShow(t *testing.T) { t.Run("an unknown version of a real dataset is refused", func(t *testing.T) { r := requireFailure(t, run(t, "dataset", "show", - "--name", ds.Name, "--version", "9999")) + ds.Name, "--version", "9999")) require.Contains(t, r.Combined(), "9999") require.Less(t, len(r.Combined()), 600, r.Combined()) }) } func TestCLIDatasetDelete(t *testing.T) { - t.Run("both flags are required", func(t *testing.T) { + t.Run("the name and version are both required", func(t *testing.T) { require.Contains(t, requireFailure(t, run(t, "dataset", "delete", "--version", "1")).Combined(), - "--name is required") + "accepts 1 arg") require.Contains(t, - requireFailure(t, run(t, "dataset", "delete", "--name", "whatever")).Combined(), + requireFailure(t, run(t, "dataset", "delete", "whatever")).Combined(), "--version is required") }) @@ -204,11 +210,11 @@ func TestCLIDatasetDelete(t *testing.T) { // entitled to know it means "gone", not "was there and is now gone". t.Run("deleting an unregistered dataset is idempotent, not an error", func(t *testing.T) { r := requireSuccess(t, run(t, "dataset", "delete", - "--name", "azdcli-no-such-dataset", "--version", "1")) + "azdcli-no-such-dataset", "--version", "1")) require.Contains(t, r.Stdout, "Deleted dataset") - listed := requireSuccess(t, run(t, "dataset", "list", - "--name", "azdcli-no-such-dataset", "-o", "json")) + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + "azdcli-no-such-dataset", "-o", "json")) var remaining []datasetSummary listed.JSON(t, &remaining) require.Empty(t, remaining, "nothing was there to delete in the first place") @@ -222,12 +228,12 @@ func TestCLIDatasetDelete(t *testing.T) { gone, kept := ds.Versions[0], ds.Versions[1] r := requireSuccess(t, run(t, "dataset", "delete", - "--name", ds.Name, "--version", gone)) + ds.Name, "--version", gone)) require.Contains(t, r.Stdout, "Deleted dataset") require.Contains(t, r.Stdout, ds.Name) - listed := requireSuccess(t, run(t, "dataset", "list", - "--name", ds.Name, "-o", "json")) + listed := requireSuccess(t, run(t, "dataset", "versions", "list", + ds.Name, "-o", "json")) var remaining []datasetSummary listed.JSON(t, &remaining) From f7ae5632a9df953d216b545924f8620c2b75f5bd Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 04:50:31 +0300 Subject: [PATCH 076/114] Add the eval and job groups Evals gain list, show and delete at the root; creation stays with azd up so that reconciliation has one owner. Jobs gain list, show and cancel, which is the resume path for a generate started with --no-wait. Both job routes turned out to carry more than the spec's table records: list and cancel exist for datasets and evaluators alike, verified against the live project. Cancel is the colon form - {id}:cancel reaches the action while {id}/cancel is a 404 - and needs a body, because without a content type the route answers 415. A job id does not say which kind started it, so show and cancel try both rather than making the caller remember. --- .../internal/cmd/eval_group.go | 134 ++++++++++++ .../azure.ai.evaluations/internal/cmd/job.go | 201 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 4 + .../internal/pkg/eval_api/operations.go | 56 +++++ 4 files changed, 395 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go new file mode 100644 index 00000000000..8f7566c00cf --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// The eval group is read and delete only. Creation belongs to `azd up`, which +// owns reconciliation: a second creation path would drift from the declared +// config, and reconciliation could not then tell whether to adopt an eval it +// found or replace it. + +func newEvalListCommand() *cobra.Command { + var ( + limit int + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's evals.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + list, err := ec.evalClient.ListOpenAIEvals(ctx, limit) + if err != nil { + return fmt.Errorf("listing evals: %w", err) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), list.Data) + } + if len(list.Data) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No evals found.") + return nil + } + rows := make([][]string, 0, len(list.Data)) + for _, e := range list.Data { + rows = append(rows, []string{e.ID, e.Name}) + } + return emitTable(cmd.OutOrStdout(), []string{"EVAL ID", "NAME"}, rows) + }, + } + + cmd.Flags().IntVar(&limit, "limit", 0, "Cap the number of evals returned.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show an eval definition.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + group, err := ec.evalClient.GetOpenAIEval(ctx, evalID) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no eval %q in this project; "+ + "`azd ai eval list` shows the ones there are", evalID) + } + return fmt.Errorf("reading eval %q: %w", evalID, err) + } + return emitJSON(cmd.OutOrStdout(), group) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newEvalDeleteCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete an eval and everything under it.", + Long: "Delete an eval and everything under it.\n\n" + + "An eval owns its runs, so deleting one discards their results too.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + evalID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + if err := ec.evalClient.DeleteOpenAIEval(ctx, evalID); err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf("no eval %q in this project", evalID) + } + return fmt.Errorf("deleting eval %q: %w", evalID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": evalID, "status": "deleted", + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "Deleted eval %s\n", evalID) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go new file mode 100644 index 00000000000..1c094e20c77 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Generation runs as two independent long-running resources — one for datasets, +// one for evaluators — and a job id does not say which it came from. Rather +// than make the caller remember, every command here tries both. + +const ( + jobKindDataset = "dataset" + jobKindEvaluator = "evaluator" +) + +func newJobCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "job", + Short: "Inspect and cancel generation jobs.", + Long: "Inspect and cancel generation jobs.\n\n" + + "This is the resume path for `dataset generate` and `evaluator generate`: " + + "a job started with --no-wait, or one whose client was interrupted, is " + + "reattached to here rather than restarted.", + } + cmd.AddCommand( + newJobListCommand(), + newJobShowCommand(), + newJobCancelCommand(), + ) + return cmd +} + +func newJobListCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "list", + Short: "List the project's generation jobs.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + datasets, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing dataset generation jobs: %w", err) + } + evaluators, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return fmt.Errorf("listing evaluator generation jobs: %w", err) + } + + type jobRow struct { + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + } + rows := make([]jobRow, 0, len(datasets.Data)+len(evaluators.Data)) + for _, j := range datasets.Data { + rows = append(rows, jobRow{ID: j.ID, Kind: jobKindDataset, Status: j.Status}) + } + for _, j := range evaluators.Data { + rows = append(rows, jobRow{ID: j.ID, Kind: jobKindEvaluator, Status: j.Status}) + } + + if isJSON(cmd) { + return emitJSONList(cmd.OutOrStdout(), rows) + } + if len(rows) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No generation jobs found.") + return nil + } + table := make([][]string, 0, len(rows)) + for _, r := range rows { + table = append(table, []string{r.ID, r.Kind, r.Status}) + } + return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "KIND", "STATUS"}, table) + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobShowCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + job, _, err := findGenerationJob(ctx, ec, jobID) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), job) + } + fmt.Fprintf(cmd.OutOrStdout(), "%s %s\n", job.ID, job.Status) + if job.Error != nil && job.Error.Message != "" { + fmt.Fprintf(cmd.OutOrStdout(), "error: %s\n", job.Error.Message) + } + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobCancelCommand() *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "cancel ", + Short: "Cancel an in-flight generation job.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + _, kind, err := findGenerationJob(ctx, ec, jobID) + if err != nil { + return err + } + + var canceled *eval_api.GenerationJob + if kind == jobKindDataset { + canceled, err = ec.evalClient.CancelDataGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion) + } else { + canceled, err = ec.evalClient.CancelEvaluatorGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion) + } + if err != nil { + return fmt.Errorf("cancelling job %s: %w", jobID, err) + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) + } + fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", + kind, jobID, canceled.Status) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +// findGenerationJob resolves an id against both job types and reports which one +// answered, so that a caller never has to know which command started it. +func findGenerationJob( + ctx context.Context, + ec *evalContext, + jobID string, +) (*eval_api.GenerationJob, string, error) { + if job, err := ec.evalClient.GetDataGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion, + ); err == nil { + return job, jobKindDataset, nil + } + if job, err := ec.evalClient.GetEvaluatorGenerationJob( + ctx, jobID, ProjectEndpointAPIVersion, + ); err == nil { + return job, jobKindEvaluator, nil + } + return nil, "", fmt.Errorf( + "no generation job %s in this project; "+ + "`azd ai eval job list` shows the ones there are", jobID) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 40724c5a0a7..af9f5f58156 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -51,6 +51,10 @@ func NewRootCommand() *cobra.Command { newRunCommand(), newEvaluatorCommand(), newResultsCommand(), + newJobCommand(), + newEvalListCommand(), + newEvalShowCommand(), + newEvalDeleteCommand(), newListenCommand(), ) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 4ba2d628994..cf61480d16b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -115,6 +115,62 @@ func (c *EvalClient) GetEvaluatorGenerationJob( return doRequestTyped[GenerationJob](c, ctx, http.MethodGet, path, nil, nil, apiVersion) } +// GenerationJobList is the listing envelope both job types answer with. It is +// `data`, not the `value` the dataset and evaluator routes use. +type GenerationJobList struct { + Data []GenerationJob `json:"data"` +} + +// ListDataGenerationJobs returns the project's dataset generation jobs. +func (c *EvalClient) ListDataGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, pathDataGenerationJobs, nil, nil, apiVersion) +} + +// ListEvaluatorGenerationJobs returns the project's evaluator generation jobs. +func (c *EvalClient) ListEvaluatorGenerationJobs( + ctx context.Context, + apiVersion string, +) (*GenerationJobList, error) { + return doRequestTyped[GenerationJobList]( + c, ctx, http.MethodGet, pathEvaluatorGenerationJobs, nil, nil, apiVersion) +} + +// CancelDataGenerationJob stops a dataset generation job. +func (c *EvalClient) CancelDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// CancelEvaluatorGenerationJob stops an evaluator generation job. +func (c *EvalClient) CancelEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) (*GenerationJob, error) { + return c.cancelGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// cancelGenerationJob posts to the colon form of the route. +// +// The separator is a colon, not a path segment: `{id}/cancel` is a 404 while +// `{id}:cancel` reaches the action. The empty object is what carries a content +// type, without which the route answers 415. +func (c *EvalClient) cancelGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) (*GenerationJob, error) { + path := basePath + "/" + url.PathEscape(operationID) + ":cancel" + return doRequestTyped[GenerationJob]( + c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) +} + // GetAgent reads an agent from the project's catalog. // // Only the newest version is returned, which is the one generation is seeded From 975e1a559979b7e916545a6dec44aad95fc1684a Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 05:13:08 +0300 Subject: [PATCH 077/114] Turn results into run output, and make run a group The per-sample views move under the run they belong to: run output list, show and export. The positional argument is now the run rather than the eval, since the eval is what --eval names and the API path nests runs under it. run output show is new, and reads one row whole. The listing truncates the input and the reason to keep a table readable, so there was no way to see either in full. run stops being executable. Once run output exists, a bare run would make azd ai eval run list read as run the thing called list. --- .../azure.ai.evaluations/internal/cmd/root.go | 1 - .../azure.ai.evaluations/internal/cmd/run.go | 13 ++- .../internal/cmd/run_ops.go | 53 ++++----- .../internal/cmd/run_ops_test.go | 24 +++-- .../cmd/{results.go => run_output.go} | 102 ++++++++++++++---- .../internal/pkg/eval_api/evaluators.go | 12 +++ .../tests/cli/run_ops_test.go | 32 +++--- .../{results_test.go => run_output_test.go} | 26 ++--- 8 files changed, 177 insertions(+), 86 deletions(-) rename cli/azd/extensions/azure.ai.evaluations/internal/cmd/{results.go => run_output.go} (78%) rename cli/azd/extensions/azure.ai.evaluations/tests/cli/{results_test.go => run_output_test.go} (87%) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index af9f5f58156..24323e839b9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -50,7 +50,6 @@ func NewRootCommand() *cobra.Command { newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), - newResultsCommand(), newJobCommand(), newEvalListCommand(), newEvalShowCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index b9ccd7d398a..73b963960b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -35,17 +35,22 @@ var terminalRunStates = map[string]bool{ // newRunCommand builds the composite `azd ai eval run` and attaches the atomic // run operations, including `run start` which the spec lists as the atomic form // of this same command. +// newRunCommand builds the run group. +// +// `run` is a group, not an executable verb: once `run output` exists, a bare +// `run` would make `azd ai eval run list` read as "run the thing called list". func newRunCommand() *cobra.Command { - cmd := buildRunCommand( - "run", "Run an evaluation, creating the eval if it does not exist yet.") + cmd := &cobra.Command{ + Use: "run", + Short: "Start and inspect evaluation runs.", + } addRunSubcommands(cmd) cmd.AddCommand(buildRunCommand( "start", "Start a run, creating the eval if it does not exist yet.")) return cmd } -// buildRunCommand is shared by `run` and `run start` so the two forms cannot -// drift apart. +// buildRunCommand builds `run start`. func buildRunCommand(use, short string) *cobra.Command { var ( configPath string diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index fbfdb2c8271..71af58eccab 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -22,6 +22,7 @@ func addRunSubcommands(cmd *cobra.Command) { newRunShowCommand(), newRunCancelCommand(), newRunDeleteCommand(), + newRunOutputCommand(), ) } @@ -33,9 +34,9 @@ func newRunListCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "list [eval-id]", + Use: "list", Short: "List runs for an eval.", - Args: cobra.MaximumNArgs(1), + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() ec, err := newEvalContext(ctx, endpointFlg) @@ -44,7 +45,7 @@ func newRunListCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } @@ -87,13 +88,12 @@ func newRunListCommand() *cobra.Command { func newRunShowCommand() *cobra.Command { var ( - runID string endpointFlg string groupName string ) cmd := &cobra.Command{ - Use: "show [eval-id]", + Use: "show [run]", Short: "Show a single run.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -104,12 +104,13 @@ func newRunShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - run, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -130,21 +131,27 @@ func newRunShowCommand() *cobra.Command { return nil }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } +// firstArg returns the positional argument, or empty when none was given. +func firstArg(args []string) string { + if len(args) > 0 { + return args[0] + } + return "" +} + func newRunCancelCommand() *cobra.Command { var ( - runID string endpointFlg string groupName string ) cmd := &cobra.Command{ - Use: "cancel [eval-id]", + Use: "cancel [run]", Short: "Cancel an in-flight run.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -155,12 +162,13 @@ func newRunCancelCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - target, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + target, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -186,7 +194,6 @@ func newRunCancelCommand() *cobra.Command { return nil }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to cancel. Defaults to the most recent run.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -194,33 +201,30 @@ func newRunCancelCommand() *cobra.Command { // newRunDeleteCommand removes a run. // -// Runs accumulate — every `azd ai eval run` adds one — and a run that evaluated -// the wrong dataset or target is noise in every later listing and comparison. -// The id is required rather than defaulted to the most recent run, because -// deleting is not undoable and "the latest one" is a poor thing to guess at. +// Runs accumulate — every `run start` adds one — and a run that evaluated the +// wrong dataset or target is noise in every later listing. The run is required +// rather than defaulted to the most recent, because deleting is not undoable +// and "the latest one" is a poor thing to guess at. func newRunDeleteCommand() *cobra.Command { var ( - runID string endpointFlg string groupName string ) cmd := &cobra.Command{ - Use: "delete [eval-id]", + Use: "delete ", Short: "Delete a run.", - Args: cobra.MaximumNArgs(1), + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - if runID == "" { - return requireFlag("run-id") - } + runID := args[0] ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } @@ -241,7 +245,6 @@ func newRunDeleteCommand() *cobra.Command { return nil }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to delete.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 05f2e32b773..1a5bd5ac978 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -70,14 +70,16 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { subs := map[string]*cobra.Command{} for _, sub := range newRunCommand().Commands() { subs["run "+sub.Name()] = sub - } - for _, sub := range newResultsCommand().Commands() { - subs["results "+sub.Name()] = sub + if sub.Name() == "output" { + for _, leaf := range sub.Commands() { + subs["run output "+leaf.Name()] = leaf + } + } } for _, name := range []string{ "run list", "run show", "run cancel", - "results show", "results export", + "run output list", "run output show", "run output export", } { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) @@ -87,9 +89,15 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { } // --no-wait is documented in the spec, and cobra does not derive it from the -// --wait bool. +// --wait bool. It belongs to `run start`: `run` itself is a group. func TestRunCommandAcceptsNoWait(t *testing.T) { - cmd := newRunCommand() - require.NotNil(t, cmd.Flags().Lookup("no-wait"), "run should accept --no-wait") - require.NotNil(t, cmd.Flags().Lookup("wait"), "run should keep --wait") + var start *cobra.Command + for _, sub := range newRunCommand().Commands() { + if sub.Name() == "start" { + start = sub + } + } + require.NotNil(t, start) + require.NotNil(t, start.Flags().Lookup("no-wait"), "run start should accept --no-wait") + require.NotNil(t, start.Flags().Lookup("wait"), "run start should keep --wait") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go similarity index 78% rename from cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go rename to cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index d2377f34671..4a30bcd200d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/results.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -16,12 +16,20 @@ import ( "github.com/spf13/cobra" ) -func newResultsCommand() *cobra.Command { +// newRunOutputCommand groups the per-sample views of a run. +// +// `run show` is the summary - how many passed. These are the rows: which ones +// failed, and why. +func newRunOutputCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "results", - Short: "Inspect evaluation results.", + Use: "output", + Short: "Inspect the per-sample results of a run.", } - cmd.AddCommand(newResultsShowCommand(), newResultsExportCommand()) + cmd.AddCommand( + newRunOutputListCommand(), + newRunOutputShowCommand(), + newRunOutputExportCommand(), + ) return cmd } @@ -35,9 +43,8 @@ func formatStat(verb string, v eval_api.LenientFloat) string { return fmt.Sprintf(verb, float64(v)) } -func newResultsShowCommand() *cobra.Command { +func newRunOutputListCommand() *cobra.Command { var ( - runID string failedOnly bool outFile string endpointFlg string @@ -45,8 +52,8 @@ func newResultsShowCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "show ", - Short: "Show per-sample results for a run.", + Use: "list [run]", + Short: "List the per-sample results of a run.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -56,12 +63,13 @@ func newResultsShowCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - run, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -101,7 +109,6 @@ func newResultsShowCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to show. Defaults to the most recent run.") cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") addEvalFlags(cmd, &groupName) @@ -109,9 +116,63 @@ func newResultsShowCommand() *cobra.Command { return cmd } -func newResultsExportCommand() *cobra.Command { +// newRunOutputShowCommand reads one evaluated row by its id. +// +// The listing truncates the input and the reason to keep a table readable, so +// this is how the whole of either is seen. +func newRunOutputShowCommand() *cobra.Command { var ( runID string + endpointFlg string + groupName string + ) + + cmd := &cobra.Command{ + Use: "show ", + Short: "Show a single evaluated row.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + itemID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + evalID, err := resolveEvalID(cmd, ec, nil, groupName) + if err != nil { + return err + } + + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") + if err != nil { + return err + } + + item, err := ec.evalClient.GetOutputItem(ctx, evalID, run.ID, itemID) + if err != nil { + if eval_api.IsNotFound(err) { + return fmt.Errorf( + "no output item %q on run %s; "+ + "`azd ai eval run output list` shows the ones there are", + itemID, run.ID) + } + return fmt.Errorf("reading output item %q: %w", itemID, err) + } + return emitJSON(cmd.OutOrStdout(), item) + }, + } + + cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") + addEvalFlags(cmd, &groupName) + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newRunOutputExportCommand() *cobra.Command { + var ( format string outFile string endpointFlg string @@ -119,7 +180,7 @@ func newResultsExportCommand() *cobra.Command { ) cmd := &cobra.Command{ - Use: "export ", + Use: "export [run]", Short: "Export run results as JSON or CSV.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -135,12 +196,13 @@ func newResultsExportCommand() *cobra.Command { } defer ec.Close() - evalID, err := resolveEvalID(cmd, ec, args, groupName) + evalID, err := resolveEvalID(cmd, ec, nil, groupName) if err != nil { return err } - run, err := ec.latestOrNamedRun(cmd, evalID, runID) + runID := firstArg(args) + run, err := ec.latestOrNamedRun(cmd, evalID, runID, runID != "") if err != nil { return err } @@ -162,7 +224,6 @@ func newResultsExportCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&runID, "run-id", "", "Run to export. Defaults to the most recent run.") cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") addEvalFlags(cmd, &groupName) @@ -226,9 +287,14 @@ func addEvalFlags(cmd *cobra.Command, target *string) { } // latestOrNamedRun returns the named run, or the most recent one for the eval. +// +// explicit says whether the caller named the run rather than leaving it to +// default. A remembered run that no longer resolves is worth falling through +// on; one that was asked for by name is not. func (ec *evalContext) latestOrNamedRun( cmd *cobra.Command, evalID, runID string, + explicit bool, ) (*eval_api.OpenAIEvalRun, error) { ctx := cmd.Context() @@ -243,9 +309,7 @@ func (ec *evalContext) latestOrNamedRun( if err == nil { return run, nil } - // A remembered run that no longer resolves is not worth failing on: - // fall through to whatever the group has now. - if cmd.Flag("run-id") != nil && cmd.Flag("run-id").Changed { + if explicit { return nil, fmt.Errorf("reading run %s: %w", runID, err) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 7b3181b6ade..9f0a6f17b35 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -239,3 +239,15 @@ func (c *EvalClient) ListOutputItems( ) return doRequestTyped[OutputItemList](c, ctx, http.MethodGet, path, query, nil, "") } + +// GetOutputItem reads a single evaluated row. +func (c *EvalClient) GetOutputItem( + ctx context.Context, + evalID, runID, itemID string, +) (*OutputItem, error) { + path := fmt.Sprintf( + "%s/%s/runs/%s/output_items/%s", + pathOpenAIEvals, url.PathEscape(evalID), url.PathEscape(runID), url.PathEscape(itemID), + ) + return doRequestTyped[OutputItem](c, ctx, http.MethodGet, path, nil, nil, "") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 3f37b32462f..626ab9dbccd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -28,7 +28,7 @@ func TestCLIRunList(t *testing.T) { f := sharedEval(t) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", f.EvalID)) + r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID)) for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } @@ -39,7 +39,7 @@ func TestCLIRunList(t *testing.T) { }) t.Run("json", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "-o", "json")) require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), "a list must be a bare array, not the service's envelope") @@ -63,14 +63,14 @@ func TestCLIRunList(t *testing.T) { // The client has always taken a limit; until recently the command did not // expose one, so a service-side truncation would have passed unnoticed. t.Run("limit", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) var runs []runSummary r.JSON(t, &runs) require.Len(t, runs, 1, "--limit must reach the service") }) t.Run("unknown eval is brief", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "list", "eval_azdcli_no_such_eval")) + r := requireFailure(t, run(t, "run", "list", "--eval-id", "eval_azdcli_no_such_eval")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") @@ -81,7 +81,7 @@ func TestCLIRunShow(t *testing.T) { f := sharedEval(t) t.Run("by run id", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", f.FirstRunID)) + r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval-id", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "status") require.Contains(t, r.Stdout, "completed") @@ -93,12 +93,12 @@ func TestCLIRunShow(t *testing.T) { // environment there is no remembered id to fall back on, so what is // exercised is the listing path. t.Run("defaults to the most recent run", func(t *testing.T) { - listed := requireSuccess(t, run(t, "run", "list", f.EvalID, "--limit", "1", "-o", "json")) + listed := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) var newest []runSummary listed.JSON(t, &newest) require.Len(t, newest, 1) - r := requireSuccess(t, run(t, "run", "show", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "show", "--eval-id", f.EvalID, "-o", "json")) var shown runSummary r.JSON(t, &shown) require.Equal(t, newest[0].ID, shown.ID, @@ -114,7 +114,7 @@ func TestCLIRunShow(t *testing.T) { // about 1700 characters of raw JSON — recorded in the report rather than // pinned here, since pinning it would make the length a requirement. t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "show", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope", "the failure must name the run that was asked for") require.NotContains(t, r.Combined(), f.FirstRunID, @@ -132,7 +132,7 @@ func TestCLIRunCancelAndDelete(t *testing.T) { f := sharedEval(t) t.Run("a finished run is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "cancel", f.EvalID, "--run-id", f.FirstRunID)) + r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval-id", f.EvalID)) require.Contains(t, r.Combined(), "already finished") require.Contains(t, r.Combined(), "completed") }) @@ -148,21 +148,21 @@ func TestCLIRunCancelAndDelete(t *testing.T) { t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { runID := startCancellableRun(t, f) - cancelled := requireSuccess(t, run(t, "run", "cancel", f.EvalID, "--run-id", runID)) + cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval-id", f.EvalID)) require.Contains(t, cancelled.Stdout, runID) require.Contains(t, cancelled.Stdout, "is now") - shown := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + shown := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) var after runSummary shown.JSON(t, &after) require.NotEqual(t, "completed", after.Status, "a cancelled run must not go on to complete") - deleted := requireSuccess(t, run(t, "run", "delete", f.EvalID, "--run-id", runID)) + deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval-id", f.EvalID)) require.Contains(t, deleted.Stdout, "Deleted run") require.Contains(t, deleted.Stdout, runID) - still := requireSuccess(t, run(t, "run", "show", f.EvalID, "--run-id", runID, "-o", "json")) + still := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) var survivor runSummary still.JSON(t, &survivor) t.Logf("the run is still readable after a successful delete (status %q); "+ @@ -172,12 +172,12 @@ func TestCLIRunCancelAndDelete(t *testing.T) { // Deleting is not undoable, so the id is required rather than defaulted to // whichever run happens to be newest. t.Run("delete requires the run id", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", f.EvalID)) - require.Contains(t, r.Combined(), "--run-id is required") + r := requireFailure(t, run(t, "run", "delete", "--eval-id", f.EvalID)) + require.Contains(t, r.Combined(), "accepts 1 arg") }) t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", f.EvalID, "--run-id", "evalrun_azdcli_nope")) + r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope") require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go similarity index 87% rename from cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go rename to cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 77313c677dc..9b9d71e9c91 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/results_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -55,7 +55,7 @@ type resultsPayload struct { func TestCLIResultsShowRendersTheRows(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "results", "show", f.EvalID, "--run-id", f.FirstRunID)) + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval-id", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "Totals:") @@ -117,8 +117,8 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { payload := resultsFor(t, f.EvalID, f.FirstRunID) failed := payload.Run.ResultCounts.Failed - r := requireSuccess(t, run(t, "results", "show", f.EvalID, - "--run-id", f.FirstRunID, "--failed-only")) + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, + "--eval-id", f.EvalID, "--failed-only")) if failed == 0 { // Saying so is not the same as printing an empty table. @@ -137,7 +137,7 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { // they can decide what the rendered output should say. func resultsFor(t *testing.T, evalID, runID string) resultsPayload { t.Helper() - r := requireSuccess(t, run(t, "results", "show", evalID, "--run-id", runID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval-id", evalID, "-o", "json")) var payload resultsPayload r.JSON(t, &payload) return payload @@ -147,8 +147,8 @@ func TestCLIResultsExport(t *testing.T) { f := sharedEval(t) t.Run("json to stdout", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "json")) + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "json")) var exported struct { ID string `json:"id"` @@ -166,8 +166,8 @@ func TestCLIResultsExport(t *testing.T) { }) t.Run("csv to stdout", func(t *testing.T) { - r := requireSuccess(t, run(t, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "csv")) + r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "csv")) rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) @@ -183,8 +183,8 @@ func TestCLIResultsExport(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "results.csv") - r := requireSuccess(t, runIn(t, dir, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "csv", "-O", path)) + r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "csv", "-O", path)) require.Empty(t, strings.TrimSpace(r.Stdout), "-O redirects the payload; leaving it on stdout too would double it") @@ -195,14 +195,14 @@ func TestCLIResultsExport(t *testing.T) { }) t.Run("an unknown format is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "results", "export", f.EvalID, - "--run-id", f.FirstRunID, "--format", "xml")) + r := requireFailure(t, run(t, "run", "output", "export", f.FirstRunID, + "--eval-id", f.EvalID, "--format", "xml")) require.Contains(t, r.Combined(), "json or csv") }) } func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "results", "show", "eval_does_not_exist")) + r := requireFailure(t, run(t, "run", "output", "list", "--eval-id", "eval_does_not_exist")) require.Contains(t, r.Combined(), "eval_does_not_exist") require.NotContains(t, r.Combined(), "RESPONSE 404") } From 6dc159425f98169d75911f479f3310aa4e6309f5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 16:43:19 +0300 Subject: [PATCH 078/114] Split generation per artifact, and add CI gating Generation becomes dataset generate and evaluator generate, because the service splits it the same way: two long-running resources, one per artifact. A single composite verb left partial failure undefined, could not regenerate one artifact after the other was hand-edited, and gave --no-wait nothing to reattach to. Neither command edits azure.yaml any more - init declares where the artifacts live, these fill them in, so a generation run produces a data-file-only diff. --fail-on gates a build on quality: any-failure, or pass-rate=<0..1>. It is opt-in, because failing samples are the expected output of a working evaluation and a default that returned non-zero would break a build the first time a grader disagreed. Errored rows count against the rate - they are in the total and did not pass - and a run that scored nothing breaches rather than dividing by zero, so a broken evaluation cannot hold a gate open. The exit code matters more than the flag: 2 says the evaluation regressed, 1 says it could not run. Those have different owners. It exits from applyGate because the extension SDK collapses every returned error to 1. --- .../internal/cmd/dataset.go | 1 + .../internal/cmd/evaluator.go | 1 + .../internal/cmd/gating.go | 116 ++++++++++ .../internal/cmd/gating_test.go | 70 ++++++ .../internal/cmd/generate.go | 146 ------------- .../internal/cmd/generate_commands.go | 199 ++++++++++++++++++ .../azure.ai.evaluations/internal/cmd/root.go | 1 - .../azure.ai.evaluations/internal/cmd/run.go | 22 +- .../internal/cmd/run_ops.go | 28 ++- .../tests/cli/generate_test.go | 122 +++++------ 10 files changed, 482 insertions(+), 224 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index daecc8c3d05..ef4361b8cc8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -24,6 +24,7 @@ func newDatasetCommand() *cobra.Command { cmd.AddCommand( newDatasetCreateCommand(), newDatasetUpdateCommand(), + newDatasetGenerateCommand(), newDatasetListCommand(), newDatasetShowCommand(), newDatasetDeleteCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 72f431536ed..a4750af5a00 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -21,6 +21,7 @@ func newEvaluatorCommand() *cobra.Command { cmd.AddCommand( newEvaluatorCreateCommand(), newEvaluatorUpdateCommand(), + newEvaluatorGenerateCommand(), newEvaluatorListCommand(), newEvaluatorShowCommand(), newEvaluatorDeleteCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go new file mode 100644 index 00000000000..c7754fa5064 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "os" + "strconv" + "strings" + + "azureaieval/internal/pkg/eval_api" + + "github.com/spf13/cobra" +) + +// Gating is opt-in. A completed run with failing samples exits 0 without +// --fail-on: failing samples are the expected output of a working evaluation, +// not a tool error, and `run start` is used constantly in the inner loop. A +// default that returned non-zero on any failure would break a build the first +// time a noisy grader disagreed. +// +// The separate exit code matters more than the flag. It lets a pipeline tell +// "the evaluation regressed" from "the evaluation could not run", which are +// different failures with different owners. + +// exitCodeGateBreached is returned when a run completed but missed its +// threshold. +const exitCodeGateBreached = 2 + +// gate is a parsed --fail-on threshold. +type gate struct { + set bool + anyFailure bool + passRate float64 +} + +// parseGate reads the --fail-on value. An empty value means no gating. +func parseGate(spec string) (gate, error) { + spec = strings.TrimSpace(spec) + if spec == "" { + return gate{}, nil + } + if spec == "any-failure" { + return gate{set: true, anyFailure: true}, nil + } + + rate, ok := strings.CutPrefix(spec, "pass-rate=") + if !ok { + return gate{}, fmt.Errorf( + "--fail-on must be any-failure or pass-rate=<0..1>, got %q", spec) + } + value, err := strconv.ParseFloat(rate, 64) + if err != nil { + return gate{}, fmt.Errorf("--fail-on pass-rate must be a number, got %q", rate) + } + if value < 0 || value > 1 { + return gate{}, fmt.Errorf("--fail-on pass-rate must be between 0 and 1, got %v", value) + } + return gate{set: true, passRate: value}, nil +} + +// breach reports why the run missed the threshold, or empty when it met it. +// +// Errored and skipped rows count against the pass rate: they are in the total +// and they did not pass. A run that scored nothing at all breaches every +// threshold rather than dividing by zero — "no rows passed" is the honest +// reading of an empty result, and treating it as 100% would let a broken +// evaluation hold a gate open. +func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { + if !g.set { + return "" + } + if counts == nil { + return "the run reported no result counts, so the threshold cannot be checked" + } + if g.anyFailure { + unpassed := counts.Total - counts.Passed + if unpassed > 0 { + return fmt.Sprintf("%d of %d samples did not pass", unpassed, counts.Total) + } + return "" + } + if counts.Total == 0 { + return "the run scored no rows, so its pass rate is below any threshold" + } + actual := float64(counts.Passed) / float64(counts.Total) + if actual < g.passRate { + return fmt.Sprintf("pass rate %.3f is below the required %.3f (%d of %d passed)", + actual, g.passRate, counts.Passed, counts.Total) + } + return "" +} + +// applyGate ends the process with exit code 2 when the run missed its +// threshold. +// +// It exits here rather than returning an error because the extension SDK's +// Run collapses every error to exit 1, and the whole point of the flag is a +// code a pipeline can tell apart from an operational failure. +func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { + if run == nil { + return + } + reason := g.breach(run.ResultCounts) + if reason == "" { + return + } + fmt.Fprintf(os.Stderr, "Evaluation gate failed: %s\n", reason) + os.Exit(exitCodeGateBreached) +} + +func addFailOnFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "fail-on", "", + "Exit 2 when the run misses this threshold: any-failure, or pass-rate=<0..1>.") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go new file mode 100644 index 00000000000..fb612cd6aca --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +func TestParseGate(t *testing.T) { + t.Run("empty means no gating", func(t *testing.T) { + g, err := parseGate("") + require.NoError(t, err) + require.False(t, g.set) + require.Empty(t, g.breach(&eval_api.EvalRunResultCounts{Total: 3}), + "an unset gate must never breach") + }) + + t.Run("any-failure", func(t *testing.T) { + g, err := parseGate("any-failure") + require.NoError(t, err) + require.True(t, g.anyFailure) + }) + + t.Run("pass-rate", func(t *testing.T) { + g, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + require.InDelta(t, 0.8, g.passRate, 1e-9) + }) + + for _, bad := range []string{"passrate=0.8", "pass-rate=abc", "pass-rate=1.5", "pass-rate=-1", "sometimes"} { + t.Run("refuses "+bad, func(t *testing.T) { + _, err := parseGate(bad) + require.Error(t, err) + }) + } +} + +func TestGateBreach(t *testing.T) { + anyFailure, err := parseGate("any-failure") + require.NoError(t, err) + eighty, err := parseGate("pass-rate=0.8") + require.NoError(t, err) + + t.Run("any-failure passes only when every row passed", func(t *testing.T) { + require.Empty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 2})) + require.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 1, Failed: 1})) + }) + + // Errored rows are in the total and did not pass, so they count against the + // threshold the same way a failing row does. + t.Run("errored rows count against the rate", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} + require.Empty(t, eighty.breach(counts), "0.8 exactly meets a 0.8 threshold") + + counts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Errored: 3} + require.NotEmpty(t, eighty.breach(counts)) + }) + + // A run that scored nothing has no defensible pass rate, and treating it as + // 100% would let a broken evaluation hold a gate open. + t.Run("a run that scored nothing breaches", func(t *testing.T) { + require.NotEmpty(t, eighty.breach(&eval_api.EvalRunResultCounts{Total: 0})) + require.NotEmpty(t, eighty.breach(nil)) + }) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 1b33daeba1d..86cd9c7727c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -17,8 +17,6 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" - - "github.com/spf13/cobra" ) // generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. @@ -29,150 +27,6 @@ var generatePollBudget = eval_api.PollerOptions{ MaxAttempts: 720, // one hour } -func newGenerateCommand() *cobra.Command { - var ( - configPath string - deployPath string - target string - instruction string - instructionFile string - datasetFlag string - evaluators []string - maxSamples int - traceDays int - evalModel string - noWait bool - endpointFlg string - ) - - cmd := &cobra.Command{ - Use: "generate", - Short: "Generate a rubric and dataset, download them, and reference them from the deployment spec.", - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - out := cmd.OutOrStdout() - - instruction, err := resolveInstruction(instruction, instructionFile) - if err != nil { - return err - } - - cfg, err := resolveGenerateConfig( - configPath, target, evalModel, datasetFlag, maxSamples, traceDays, - ) - if err != nil { - return err - } - if err := cfg.Validate(); err != nil { - return err - } - // Written to stdout because azd does not surface an extension's - // stderr, and guarded so `-o json` stays parseable. - if !isJSON(cmd) { - warnIgnoredTraceFields(cfg, out) - } - - // Both jobs are billed against a model deployment. Checking before - // any network work keeps the failure at the flag the caller can act - // on, instead of a service rejection partway through the command. - generatingRubric := len(evaluators) == 0 && cfg.Generate.Rubric != nil - generatingDataset := datasetFlag == "" && cfg.Generate.Dataset != nil - if (generatingRubric || generatingDataset) && generationModel(cfg) == "" { - return fmt.Errorf( - "a model deployment is required to generate: pass --generation-model, " + - "or set generate.rubric.model in the generation spec") - } - - ec, err := newEvalContext(ctx, endpointFlg) - if err != nil { - return err - } - defer ec.Close() - - instruction, err = ec.resolveGenerationInstruction( - ctx, cfg, instruction, configPath, out, isJSON(cmd), - ) - if err != nil { - return err - } - - baseDir := filepath.Dir(deployPath) - var datasetRefs, evaluatorRefs []project.ArtifactRef - - // Supplied evaluators are honored: their generation is skipped. - if len(evaluators) > 0 { - fmt.Fprintf(out, "Using the supplied evaluators; skipping rubric generation.\n") - } else if cfg.Generate.Rubric != nil { - ref, err := ec.generateRubric(ctx, cfg, instruction, baseDir, out, noWait) - if err != nil { - return err - } - if ref != nil { - evaluatorRefs = append(evaluatorRefs, *ref) - } - } - - // --dataset means use this one, whether it names a local file or a - // dataset already registered on the project. Either way there is - // nothing to generate, which is how --evaluator behaves too. - if datasetFlag != "" { - fmt.Fprintf(out, "Using the supplied dataset; skipping data generation.\n") - } else if cfg.Generate.Dataset != nil { - ref, err := ec.generateDataset(ctx, cfg, instruction, baseDir, out, noWait) - if err != nil { - return err - } - if ref != nil { - datasetRefs = append(datasetRefs, *ref) - } - } - - if len(datasetRefs) == 0 && len(evaluatorRefs) == 0 { - // With --no-wait the jobs were submitted and nothing was - // downloaded, which is success, not an empty result. - if noWait { - fmt.Fprintln(out, - "\nJobs submitted. Re-run without --no-wait to download the artifacts "+ - "and reference them from the deployment spec.") - return nil - } - fmt.Fprintln(out, "Nothing was generated.") - return nil - } - - if err := project.MergeArtifactRefs(deployPath, datasetRefs, evaluatorRefs); err != nil { - return err - } - fmt.Fprintf(out, "\nUpdated %s\n", deployPath) - fmt.Fprintln(out, "Review the generated artifacts, then run: azd up && azd ai eval run") - return nil - }, - } - - cmd.Flags().StringVar(&configPath, "config", project.DefaultGenerateConfig, - "Path to the generation spec. Optional; flags alone are sufficient.") - cmd.Flags().StringVar(&deployPath, "deploy-config", project.DefaultDeployConfig, - "Deployment spec to write source references into.") - cmd.Flags().StringVar(&target, "target", "", "Agent whose context seeds generation.") - cmd.Flags().StringVar(&instruction, "agent-instruction", "", - "What the agent does and what to test.") - cmd.Flags().StringVar(&instructionFile, "agent-instruction-file", "", - "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") - cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") - cmd.Flags().StringVar(&datasetFlag, "dataset", "", - "Use this dataset instead of generating one.") - cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, - "Use these evaluators instead of generating a rubric; repeatable.") - cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) - cmd.Flags().IntVar(&traceDays, "trace-days", 0, - "Days of traces to seed rubric generation. 0 disables.") - cmd.Flags().StringVar(&evalModel, "generation-model", "", "Model deployment that generates the dataset and rubric.") - cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the jobs and return without polling.") - cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") - return cmd -} - // warnIgnoredTraceFields reports generation settings that are accepted but have // no effect yet. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go new file mode 100644 index 00000000000..b3c7b86c374 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -0,0 +1,199 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation is split per artifact because the service splits it: datasets and +// evaluators are separate long-running resources. One composite verb leaves +// partial failure undefined, cannot regenerate one artifact after the other has +// been hand-edited, and gives --no-wait nothing to reattach to. +// +// Neither command edits azure.yaml. `init` declares where the artifacts live; +// these fill them in, so a generation run produces a data-file-only diff. + +// generateFlags are the settings both generate commands share. +type generateFlags struct { + configPath string + target string + instruction string + instructionFile string + model string + outputDir string + noWait bool + endpoint string +} + +func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { + cmd.Flags().StringVar(&f.configPath, "config", project.DefaultGenerateConfig, + "Path to the generation spec. Optional; flags alone are sufficient.") + cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") + cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", + "What the agent does and what to test.") + cmd.Flags().StringVar(&f.instructionFile, "agent-instruction-file", "", + "Read the agent instruction from this file. Mutually exclusive with --agent-instruction.") + cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") + cmd.Flags().StringVar(&f.model, "generation-model", "", + "Model deployment that generates the artifact.") + cmd.Flags().StringVar(&f.outputDir, "output-dir", project.DefaultEvalDir, + "Directory the generated artifact is written under.") + cmd.Flags().BoolVar(&f.noWait, "no-wait", false, + "Submit the job and return its id without polling.") + cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") +} + +// prepareGeneration resolves everything both commands need before they diverge. +// +// The model check happens here rather than at the service, because a generation +// job is billed against a deployment and a rejection partway through the +// command says less than a refusal at the flag that caused it. +func prepareGeneration( + cmd *cobra.Command, + f *generateFlags, + maxSamples, traceDays int, +) (*evalContext, *project.GenerateConfig, string, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return nil, nil, "", err + } + + cfg, err := resolveGenerateConfig( + f.configPath, f.target, f.model, "", maxSamples, traceDays, + ) + if err != nil { + return nil, nil, "", err + } + if err := cfg.Validate(); err != nil { + return nil, nil, "", err + } + if !isJSON(cmd) { + warnIgnoredTraceFields(cfg, cmd.OutOrStdout()) + } + if generationModel(cfg) == "" { + return nil, nil, "", fmt.Errorf( + "a model deployment is required to generate: pass --generation-model, " + + "or set it in the generation spec") + } + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, f.endpoint) + if err != nil { + return nil, nil, "", err + } + + instruction, err = ec.resolveGenerationInstruction( + ctx, cfg, instruction, f.configPath, cmd.OutOrStdout(), isJSON(cmd), + ) + if err != nil { + ec.Close() + return nil, nil, "", err + } + return ec, cfg, instruction, nil +} + +func newDatasetGenerateCommand() *cobra.Command { + var ( + flags generateFlags + maxSamples int + ) + + cmd := &cobra.Command{ + Use: "generate ", + Short: "Generate a dataset and download it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ec, cfg, instruction, err := prepareGeneration(cmd, &flags, maxSamples, 0) + if err != nil { + return err + } + defer ec.Close() + + if cfg.Generate.Dataset == nil { + return fmt.Errorf("the generation spec declares no dataset to generate") + } + cfg.Generate.Dataset.Name = name + + ref, err := ec.generateDataset( + cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + if err != nil { + return err + } + return reportGenerated(cmd, ref, flags.noWait) + }, + } + + cmd.Flags().IntVar(&maxSamples, "max-samples", 0, + fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) + addGenerateFlags(cmd, &flags) + return cmd +} + +func newEvaluatorGenerateCommand() *cobra.Command { + var ( + flags generateFlags + traceDays int + ) + + cmd := &cobra.Command{ + Use: "generate ", + Short: "Generate a rubric evaluator and download it.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + + ec, cfg, instruction, err := prepareGeneration(cmd, &flags, 0, traceDays) + if err != nil { + return err + } + defer ec.Close() + + if cfg.Generate.Rubric == nil { + return fmt.Errorf("the generation spec declares no rubric to generate") + } + cfg.Generate.Rubric.Name = name + + ref, err := ec.generateRubric( + cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + if err != nil { + return err + } + return reportGenerated(cmd, ref, flags.noWait) + }, + } + + cmd.Flags().IntVar(&traceDays, "trace-days", 0, + "Days of traces to seed generation. 0 disables.") + addGenerateFlags(cmd, &flags) + return cmd +} + +// reportGenerated closes out either command. +// +// With --no-wait nothing was downloaded and there is no ref, which is success: +// the job id was printed and `job show` reattaches to it. +func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { + out := cmd.OutOrStdout() + if ref == nil { + if noWait { + fmt.Fprintln(out, + "\nSubmitted. `azd ai eval job show ` reports its progress.") + return nil + } + fmt.Fprintln(out, "Nothing was generated.") + return nil + } + if isJSON(cmd) { + return emitJSON(out, ref) + } + fmt.Fprintf(out, "\nReference it from your eval config as: %s\n", ref.Source) + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 24323e839b9..5be5fcc9e46 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -46,7 +46,6 @@ func NewRootCommand() *cobra.Command { rootCmd.AddCommand( newInitCommand(), - newGenerateCommand(), newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 73b963960b4..7c2ab84e458 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -60,6 +60,7 @@ func buildRunCommand(use, short string) *cobra.Command { level string maxSamples int wait bool + failOn string endpointFlg string ) @@ -70,6 +71,13 @@ func buildRunCommand(use, short string) *cobra.Command { ctx := cmd.Context() out := cmd.OutOrStdout() + // Parsed before any network work, so a malformed threshold costs + // nothing to find out about. + threshold, err := parseGate(failOn) + if err != nil { + return err + } + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err @@ -167,9 +175,18 @@ func buildRunCommand(use, short string) *cobra.Command { } if isJSON(cmd) { - return emitJSON(out, final) + if err := emitJSON(out, final); err != nil { + return err + } + } else if err := renderRun(out, final); err != nil { + return err } - return renderRun(out, final) + + // Last, so that the results are reported whether or not the gate + // holds: a pipeline that only learns it failed is worse off than + // one that can see by how much. + applyGate(cmd, threshold, final) + return nil }, } @@ -185,6 +202,7 @@ func buildRunCommand(use, short string) *cobra.Command { cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from the dataset.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") + addFailOnFlag(cmd, &failOn) // The spec documents --no-wait, and cobra does not derive it from a bool. var noWait bool cmd.Flags().BoolVar(&noWait, "no-wait", false, "Submit the run and return immediately.") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index 71af58eccab..e7e537e544c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -90,6 +90,8 @@ func newRunShowCommand() *cobra.Command { var ( endpointFlg string groupName string + wait bool + failOn string ) cmd := &cobra.Command{ @@ -98,6 +100,12 @@ func newRunShowCommand() *cobra.Command { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + + threshold, err := parseGate(failOn) + if err != nil { + return err + } + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { return err @@ -114,8 +122,22 @@ func newRunShowCommand() *cobra.Command { if err != nil { return err } + + // Reattaching to a run started asynchronously: the pipeline that + // gates on it is often not the one that started it. + if wait { + run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) + if err != nil { + return err + } + } + if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), run) + if err := emitJSON(cmd.OutOrStdout(), run); err != nil { + return err + } + applyGate(cmd, threshold, run) + return nil } out := cmd.OutOrStdout() @@ -128,9 +150,13 @@ func newRunShowCommand() *cobra.Command { if run.ReportURL != "" { fmt.Fprintf(out, " report : %s\n", run.ReportURL) } + applyGate(cmd, threshold, run) return nil }, } + cmd.Flags().BoolVar(&wait, "wait", false, + "Block until the run reaches a terminal state before reporting.") + addFailOnFlag(cmd, &failOn) addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index ff3f37b0a31..cc4eed3edac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -8,21 +8,21 @@ package cli import ( "os" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/require" ) -// `generate` submits two jobs that cost model time and take minutes, so what -// is exercised here is everything up to that point: the flag combinations it -// refuses, the spec it parses, and the two flags that mean "I already have -// this one, do not make another". None of these tests submits a job — the last -// one reaches the service and deliberately generates nothing. +// Generation submits a job that costs model time and takes minutes, so what is +// exercised here is everything up to that point: the flag combinations each +// command refuses and the spec it parses. No test here submits a job. +// +// There is one command per artifact, so nothing suppresses anything: a caller +// who already has a dataset simply does not run `dataset generate`. // TestCLIGenerateRefusesBadFlagCombinations covers the mistakes that must cost -// nothing to make. Each of these is decided locally, so a user finds out -// before a job is billed. +// nothing to make. Each is decided locally, so a user finds out before a job is +// billed. func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { dir := t.TempDir() instruction := filepath.Join(dir, "instruction.md") @@ -34,40 +34,56 @@ func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { want string }{{ name: "the two instruction sources are mutually exclusive", - args: []string{"--target", "a", "--agent-instruction", "inline", - "--agent-instruction-file", instruction}, + args: []string{"dataset", "generate", "d", "--target", "a", + "--agent-instruction", "inline", "--agent-instruction-file", instruction}, want: "agent-instruction-file", }, { name: "below the minimum sample size", - args: []string{"--target", "a", "--max-samples", "14"}, + args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "14"}, want: "between 15 and 1000", }, { name: "above the maximum sample size", - args: []string{"--target", "a", "--max-samples", "1001"}, + args: []string{"dataset", "generate", "d", "--target", "a", "--max-samples", "1001"}, want: "between 15 and 1000", }, { name: "a missing instruction file names the flag", - args: []string{"--target", "a", "--agent-instruction-file", - filepath.Join(dir, "absent.md")}, + args: []string{"dataset", "generate", "d", "--target", "a", + "--agent-instruction-file", filepath.Join(dir, "absent.md")}, want: "--agent-instruction-file", }, { - name: "generating needs a model deployment", - args: []string{"--target", "a", "--agent-instruction", "inline"}, + name: "generating a dataset needs a model deployment", + args: []string{"dataset", "generate", "d", "--target", "a", "--agent-instruction", "inline"}, + want: "--generation-model", + }, { + name: "generating an evaluator needs a model deployment", + args: []string{"evaluator", "generate", "e", "--target", "a", "--agent-instruction", "inline"}, want: "--generation-model", }} for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - r := requireFailure(t, runIn(t, dir, append([]string{"generate"}, tc.args...)...)) + r := requireFailure(t, runIn(t, dir, tc.args...)) require.Contains(t, r.Combined(), tc.want) }) } } +// TestCLIGenerateNamesTheArtifact pins the positional argument. Without it the +// name would come from the spec, and two runs would quietly overwrite the same +// artifact. +func TestCLIGenerateNamesTheArtifact(t *testing.T) { + for _, group := range []string{"dataset", "evaluator"} { + t.Run(group, func(t *testing.T) { + r := requireFailure(t, runIn(t, t.TempDir(), group, "generate")) + require.Contains(t, r.Combined(), "accepts 1 arg") + }) + } +} + // TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with no target and // nothing to prompt with, the process has to end saying which flag to pass. func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { - r := requireFailure(t, runIn(t, t.TempDir(), "generate", "--no-prompt")) + r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) require.Contains(t, r.Combined(), "--target is required") require.Contains(t, r.Combined(), "--no-prompt", "the message must say why it could not be resolved") @@ -91,68 +107,26 @@ generate: strategy: from-traces `), 0o600)) - r := requireFailure(t, runIn(t, dir, "generate", "--config", spec)) + r := requireFailure(t, runIn(t, dir, "dataset", "generate", "d", "--config", spec)) require.Contains(t, r.Combined(), "from-traces") require.Contains(t, r.Combined(), "agent.context.traces.window", "the refusal must point at the field that does seed generation from traces") } -// TestCLIGenerateSkipsWhatWasSupplied is the one generate test that reaches the -// service, and it is here because the skip decision is made in the command -// body rather than in the config resolver. -// -// With both artifacts supplied there is nothing left to generate, so the whole -// command runs without submitting a job — which is what makes it affordable to -// assert on. A regression that stopped honouring either flag would show up as -// a generation job starting instead of this returning. -func TestCLIGenerateSkipsWhatWasSupplied(t *testing.T) { +// TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not +// share settings that only one of them can honour. A sample count means nothing +// to a rubric, and a trace window means nothing to a synthetic dataset; either +// would be accepted and dropped. +func TestCLIGenerateFlagsAreScopedToTheirArtifact(t *testing.T) { dir := t.TempDir() - r := requireSuccess(t, runIn(t, dir, "generate", - "--target", "azd-eval-probe-agent", - "--agent-instruction", "answer questions about orders", - "--evaluator", "already-published", - "--dataset", "already-registered")) - - require.Contains(t, r.Stdout, "skipping rubric generation") - require.Contains(t, r.Stdout, "skipping data generation") - require.Contains(t, r.Stdout, "Nothing was generated.") - - // The deployment spec is only rewritten when something was produced, and - // writing an empty reference into it would be worse than not writing. - entries, err := os.ReadDir(dir) - require.NoError(t, err) - require.Empty(t, entries, "a generate that produced nothing must write nothing") -} - -// TestCLIGenerateSuppressionIsPerArtifact pins the two flags apart: neither -// may suppress the artifact it does not name. -// -// Each case supplies one artifact and leaves the other to be generated, and is -// stopped at the model check that precedes submission. Reaching that error is -// the proof: it is only raised when something is still going to be generated, -// so it says the unsupplied artifact survived the other flag. -func TestCLIGenerateSuppressionIsPerArtifact(t *testing.T) { - cases := []struct { - name string - supplied []string - survives string - }{ - {"a supplied evaluator leaves the dataset", []string{"--evaluator", "already-published"}, "dataset"}, - {"a supplied dataset leaves the rubric", []string{"--dataset", "already-registered"}, "rubric"}, - } + r := requireFailure(t, runIn(t, dir, "evaluator", "generate", "e", + "--target", "a", "--max-samples", "20")) + require.Contains(t, r.Combined(), "max-samples", + "--max-samples belongs to dataset generate") - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - args := append([]string{"generate", - "--target", "azd-eval-probe-agent", - "--agent-instruction", "answer questions about orders"}, tc.supplied...) - - r := requireFailure(t, runIn(t, t.TempDir(), args...)) - require.Contains(t, r.Combined(), "--generation-model", - "the %s was suppressed by a flag that does not name it", tc.survives) - require.NotContains(t, strings.ToLower(r.Combined()), "generating ", - "the run must stop at the model check, before any job is submitted") - }) - } + r = requireFailure(t, runIn(t, dir, "dataset", "generate", "d", + "--target", "a", "--trace-days", "7")) + require.Contains(t, r.Combined(), "trace-days", + "--trace-days belongs to evaluator generate") } From 4199cd283dac3736f1f17f85e568cc54d39188e1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 17:03:22 +0300 Subject: [PATCH 079/114] Match the gate wording the scenario documents The pass rate reads as a percentage and the failure prints the two lines the CI scenario shows, so a pipeline log looks like the spec rather than close to it. Also records what the counts mean, which the contract left ambiguous: errored rows sit inside total. Verified on a run built to mix them - total=3 passed=2 errored=1. Had they been outside it, two passes and one error would have reported total=2 and scored a perfect rate, which is the broken evaluation a gate exists to catch. --- .../internal/cmd/gating.go | 20 +++++++++++-------- .../internal/cmd/gating_test.go | 12 +++++++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go index c7754fa5064..0b6940a28d5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating.go @@ -62,11 +62,14 @@ func parseGate(spec string) (gate, error) { // breach reports why the run missed the threshold, or empty when it met it. // -// Errored and skipped rows count against the pass rate: they are in the total -// and they did not pass. A run that scored nothing at all breaches every -// threshold rather than dividing by zero — "no rows passed" is the honest -// reading of an empty result, and treating it as 100% would let a broken -// evaluation hold a gate open. +// Errored and skipped rows count against the pass rate, and they can: the +// service puts them inside `total`, verified live on a run that reported +// total=3 passed=2 errored=1. Were they outside it, a run with two passes and +// one error would report total=2 and score a perfect rate, which is precisely +// the broken evaluation a gate exists to catch. +// +// A run that scored nothing at all breaches every threshold rather than +// dividing by zero — "no rows passed" is the honest reading of an empty result. func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { if !g.set { return "" @@ -86,8 +89,8 @@ func (g gate) breach(counts *eval_api.EvalRunResultCounts) string { } actual := float64(counts.Passed) / float64(counts.Total) if actual < g.passRate { - return fmt.Sprintf("pass rate %.3f is below the required %.3f (%d of %d passed)", - actual, g.passRate, counts.Passed, counts.Total) + return fmt.Sprintf("pass rate %.1f%% is below the required %.1f%%", + actual*100, g.passRate*100) } return "" } @@ -106,7 +109,8 @@ func applyGate(cmd *cobra.Command, g gate, run *eval_api.OpenAIEvalRun) { if reason == "" { return } - fmt.Fprintf(os.Stderr, "Evaluation gate failed: %s\n", reason) + fmt.Fprintf(os.Stderr, "(x) Failed: Evaluation gate: %s\n\n", reason) + fmt.Fprintln(os.Stderr, "ERROR: evaluation quality gate not met.") os.Exit(exitCodeGateBreached) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go index fb612cd6aca..5952e53d694 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/gating_test.go @@ -51,8 +51,8 @@ func TestGateBreach(t *testing.T) { require.NotEmpty(t, anyFailure.breach(&eval_api.EvalRunResultCounts{Total: 2, Passed: 1, Failed: 1})) }) - // Errored rows are in the total and did not pass, so they count against the - // threshold the same way a failing row does. + // Errored rows are inside the total, verified live, so they count against + // the threshold the same way a failing row does. t.Run("errored rows count against the rate", func(t *testing.T) { counts := &eval_api.EvalRunResultCounts{Total: 10, Passed: 8, Errored: 2} require.Empty(t, eighty.breach(counts), "0.8 exactly meets a 0.8 threshold") @@ -61,6 +61,14 @@ func TestGateBreach(t *testing.T) { require.NotEmpty(t, eighty.breach(counts)) }) + // The wording is pinned because the hero scenario shows it verbatim. + t.Run("reads as a percentage", func(t *testing.T) { + counts := &eval_api.EvalRunResultCounts{Total: 1000, Passed: 764} + require.Equal(t, + "pass rate 76.4% is below the required 80.0%", + eighty.breach(counts)) + }) + // A run that scored nothing has no defensible pass rate, and treating it as // 100% would let a broken evaluation hold a gate open. t.Run("a run that scored nothing breaches", func(t *testing.T) { From a36717658e4c094b92436b7f37a69fb2ceca275e Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 17:18:44 +0300 Subject: [PATCH 080/114] Ask the direct read whether an evaluator exists create and update differ only in which starting state they accept, and the check was asking the version listing. That listing lags a publish by up to a second and a half, so an update issued straight after a create was told the evaluator it had just made did not exist - which the live suite caught, with show succeeding between the two because the direct read had already converged. The same lag is why reconciliation waits on both views before creating an eval that references a fresh evaluator. Here only one view is needed, and it is the faster one. --- .../internal/cmd/evaluator.go | 13 ++++++++++--- .../internal/pkg/evalcore/evaluator.go | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index a4750af5a00..2407f4434ee 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -82,10 +82,17 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { } defer ec.Close() - latest := ec.evalClient.LatestEvaluatorVersionNumber( - ctx, name, ProjectEndpointAPIVersion, + // Asked of the direct read, not the version listing. The listing + // lags a publish by up to a second and a half, so an update + // issued straight after a create would be told the evaluator it + // just made does not exist. + _, readErr := ec.evalClient.GetEvaluatorRaw( + ctx, name, "", ProjectEndpointAPIVersion, ) - if err := checkAssetExistence(verb, "evaluator", name, latest > 0); err != nil { + if readErr != nil && !eval_api.IsNotFound(readErr) { + return fmt.Errorf("checking whether evaluator %q exists: %w", name, readErr) + } + if err := checkAssetExistence(verb, "evaluator", name, readErr == nil); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index f31584fd48a..88538768b37 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -17,14 +17,26 @@ import ( const BuiltinPrefix = "builtin." // EvaluatorRef references an evaluator from an eval. It accepts either a -// bare string or a mapping carrying a pass threshold: +// bare string or a mapping carrying the rest of the declaration: // // evaluators: // - builtin.task_adherence -// - { name: support-quality, threshold: 4.0 } +// - name: support-quality +// source: ./evaluators/support-quality.json +// initialization_parameters: +// deployment_name: gpt-5.6-luna +// +// A built-in needs nothing but its name. One with a Source is the project's +// own, and is published before the eval that references it is created. type EvaluatorRef struct { Name string `yaml:"name" json:"name"` Version string `yaml:"version,omitempty" json:"version,omitempty"` + // Source names a local rubric JSON file. Empty means the evaluator is + // already registered, or is a built-in. + Source string `yaml:"source,omitempty" json:"source,omitempty"` + // InitializationParameters are passed through to the testing criterion, + // which is where a judge deployment is named. + InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"initialization_parameters,omitempty"` // Threshold maps to testing_criteria[].initialization_parameters.threshold. Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"` } From 60239952f44a37658b6a698f572c10944e594d26 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 19:26:49 +0300 Subject: [PATCH 081/114] One service per eval: restructure the configuration model The spec's configuration model is one `azure.ai.eval` service per eval, with the eval's body in `evals/.yaml` and the service key supplying its name. The extension implemented an older shape: a single `evals/azure.yaml` carrying parallel `datasets:`, `evaluators:` and `evals:` lists. That made the service key meaningless, gave azd no way to order one eval after the agent it evaluates, and forced a name to be repeated in three places. - `EvalConfig` is now one eval: `description`, `dataset`, `evaluators`, `target`, `options`. `Eval(name)` resolves it against the service key. - Evaluators are declared once, in the list the eval references. A built-in needs nothing; one carrying a `source` is the config's to publish. `CustomEvaluators()` is that subset. - `ResolveEvalConfigPath` resolves `--eval ` to a file, and names the candidates rather than guessing when there is more than one. `options.eval_model` is gone. A judge deployment is a testing criterion's `initialization_parameters.deployment_name`, which differs per evaluator, so it is declared on the evaluator reference; `--judge-model` goes with it. Verified against the live schemas: every built-in that judges declares `deployment_name` required, so `init` stamps the detected deployment onto each evaluator it writes rather than leaving behind an eval that cannot run. `EvaluatorList.MarshalJSON` dropped `source` and `initialization_parameters`. The eval fingerprint is taken over that encoding, so a judge-model change was invisible to reconciliation and would not have recreated the eval. Caught by the fingerprint test. `evals/generate.yaml` takes its documented shape: `generationModel` plus `dataset:`/`evaluator:` maps keyed by artifact name, so `dataset generate ` reads exactly the entry it was asked for. The resolution order -- flags, spec, then the eval's own target -- is settled in `resolvePlan` before the client is built, so a missing model or an out-of-range sample count costs no authentication round trip. `MergeArtifactRefs` is deleted. The spec is explicit that `generate` writes artifacts only and never edits `azure.yaml`, which is what keeps a generation run a reviewable data-file-only diff; the code was already unreachable from any command. `init` follows: `--name`, `--output-dir` (was `--out-dir`), `--generation-model`, a `-smoke` default eval name, the built-in plus generated rubric default evaluator set, and a service entry keyed by the eval name that `uses` both the Foundry project and the target agent. --- .../internal/cmd/agent_context_test.go | 99 +---- .../internal/cmd/build.go | 32 +- .../internal/cmd/build_live_test.go | 25 +- .../internal/cmd/build_test.go | 60 +-- .../internal/cmd/description_test.go | 7 +- .../internal/cmd/envkeys_test.go | 33 +- .../internal/cmd/generate.go | 214 +++------- .../internal/cmd/generate_commands.go | 185 +++++++-- .../internal/cmd/generate_defaults_test.go | 47 --- .../internal/cmd/generate_plan_test.go | 291 +++++++------- .../azure.ai.evaluations/internal/cmd/init.go | 379 +++++++++++------- .../internal/cmd/init_test.go | 132 ++++-- .../internal/cmd/init_wiring_test.go | 66 ++- .../internal/cmd/output.go | 8 + .../azure.ai.evaluations/internal/cmd/run.go | 50 ++- .../internal/cmd/run_output.go | 15 +- .../internal/cmd/schemas_live_test.go | 6 +- .../internal/cmd/trace_warning_test.go | 46 --- .../internal/pkg/evalcore/evaluator.go | 29 +- .../internal/project/eval_config.go | 248 ++++++------ .../internal/project/eval_config_test.go | 263 +++++++----- .../internal/project/generate_config.go | 180 +++++---- .../internal/project/generate_config_test.go | 74 ++-- .../internal/project/merge.go | 171 -------- .../internal/project/merge_test.go | 142 ------- .../internal/project/service_target_eval.go | 44 +- .../project/service_target_eval_test.go | 48 +-- .../tests/cli/evaluator_test.go | 2 +- .../tests/cli/generate_test.go | 52 ++- 29 files changed, 1408 insertions(+), 1540 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go index 7ba9e5ce23a..e6fa74be253 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -4,7 +4,6 @@ package cmd import ( - "bytes" "encoding/json" "os" "path/filepath" @@ -20,7 +19,7 @@ import ( // The generation spec names an instructions file relative to itself, not to the // working directory, so `generate --config ` reads the same file the // author sees next to the spec. -func TestAgentContextInstructions_ResolvesRelativeToTheSpec(t *testing.T) { +func TestDeclaredInstructions_ResolvesRelativeToTheSpec(t *testing.T) { dir := t.TempDir() specDir := filepath.Join(dir, "evals") require.NoError(t, os.MkdirAll(filepath.Join(specDir, "agent"), 0o755)) @@ -29,28 +28,23 @@ func TestAgentContextInstructions_ResolvesRelativeToTheSpec(t *testing.T) { require.NoError(t, os.WriteFile( filepath.Join(specDir, "agent", "instructions.md"), []byte(" "+body+"\n"), 0o600)) - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Instructions = "./agent/instructions.md" - - got, err := agentContextInstructions(cfg, filepath.Join(specDir, "eval_generate.yaml")) + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(specDir, "generate.yaml")) require.NoError(t, err) assert.Equal(t, body, got, "the file's contents should be used, trimmed") } -// `init` writes the instructions path before that file exists. Treating the -// gap as an error would break the flow init itself scaffolds. -func TestAgentContextInstructions_MissingFileIsNotAnError(t *testing.T) { - dir := t.TempDir() - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Instructions = "./agent/instructions.md" - - got, err := agentContextInstructions(cfg, filepath.Join(dir, "eval_generate.yaml")) +// A path can be declared before that file exists. Treating the gap as an error +// would break the flow `init` itself scaffolds. +func TestDeclaredInstructions_MissingFileIsNotAnError(t *testing.T) { + got, err := declaredInstructions( + "./agent/instructions.md", filepath.Join(t.TempDir(), "generate.yaml")) require.NoError(t, err) assert.Empty(t, got) } -func TestAgentContextInstructions_UnsetIsEmpty(t *testing.T) { - got, err := agentContextInstructions(&project.GenerateConfig{}, "eval_generate.yaml") +func TestDeclaredInstructions_UnsetIsEmpty(t *testing.T) { + got, err := declaredInstructions("", "generate.yaml") require.NoError(t, err) assert.Empty(t, got) } @@ -75,73 +69,10 @@ func TestAgentInstructions(t *testing.T) { assert.Empty(t, nilAgent.Instructions()) } -// Dataset generation has no model of its own; it runs against the judge model -// the spec declares. +// Dataset generation has no model of its own; both jobs run against the one +// generation model the spec declares. func TestGenerationModel(t *testing.T) { - cfg := &project.GenerateConfig{} - assert.Empty(t, generationModel(cfg), "no rubric means no model to borrow") - - cfg.Generate.Rubric = &project.RubricSpec{Model: "gpt-4.1-nano"} - assert.Equal(t, "gpt-4.1-nano", generationModel(cfg)) -} - -// Trace selection is accepted and ignored, so it has to be called out. -func TestWarnIgnoredFields_CoversTraceSelection(t *testing.T) { - cases := []struct { - name string - build func(*project.GenerateConfig) - want []string - quiet bool - }{ - { - name: "nothing set stays silent", - build: func(*project.GenerateConfig) {}, - quiet: true, - }, - { - name: "a source alone", - build: func(c *project.GenerateConfig) { - c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights"} - }, - want: []string{"agent.context.traces.source", "has no effect"}, - }, - { - name: "source and sample agree in number", - build: func(c *project.GenerateConfig) { - c.Agent.Context.Traces = &project.TraceSpec{Source: "app-insights", Sample: 100} - }, - want: []string{"agent.context.traces.source", "agent.context.traces.sample", "have no effect"}, - }, - { - name: "a window alone is honored, so no warning", - build: func(c *project.GenerateConfig) { - c.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} - }, - quiet: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - cfg := &project.GenerateConfig{} - tc.build(cfg) - - var buf bytes.Buffer - warnIgnoredTraceFields(cfg, &buf) - - if tc.quiet { - assert.Empty(t, buf.String()) - return - } - for _, want := range tc.want { - assert.Contains(t, buf.String(), want) - } - }) - } -} - -// init scaffolds only the context fields that are read. -func TestInitScaffold_KeepsInstructions(t *testing.T) { - cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") - assert.Equal(t, "./agent/instructions.md", cfg.Agent.Context.Instructions) + assert.Empty(t, generationModel(&project.GenerateConfig{})) + assert.Equal(t, "gpt-4.1-nano", + generationModel(&project.GenerateConfig{GenerationModel: "gpt-4.1-nano"})) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index d5008b57b9d..37f220dcab9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -148,7 +148,6 @@ func planCriterion( schema *eval_api.EvaluatorSummary, targetBindings map[string]string, datasetColumns map[string]bool, - evalModel string, level string, ) (*criterionPlan, error) { accepted := legacyInputs @@ -214,13 +213,16 @@ func planCriterion( } // Evaluators disagree on what the judge model is called: built-ins declare - // deployment_name, custom rubrics declare model. Bind whichever the - // evaluator actually accepts rather than guessing one spelling. - if evalModel != "" { - for _, alias := range []string{"deployment_name", "model"} { - if accepts(alias) { - plan.initParams[alias] = evalModel - } + // deployment_name, custom rubrics declare model. The declaration names one + // of them; bind whichever the evaluator actually accepts rather than + // forwarding a spelling it will reject. + for name, value := range ref.InitializationParameters { + if accepts(name) { + plan.initParams[name] = value + continue + } + if alias, ok := judgeModelAliases[name]; ok && accepts(alias) { + plan.initParams[alias] = value } } if ref.Threshold != nil && accepts("threshold") { @@ -239,7 +241,8 @@ func planCriterion( } if len(missingInit) > 0 { return nil, fmt.Errorf( - "evaluator %q requires %s; set the judge model on the eval", + "evaluator %q requires %s; set it under the evaluator's "+ + "`initialization_parameters` in the eval config", ref.Name, quoteList(missingInit), ) } @@ -248,6 +251,13 @@ func planCriterion( return plan, nil } +// judgeModelAliases maps the two spellings of the judge deployment onto each +// other, so one declaration works whichever the evaluator publishes. +var judgeModelAliases = map[string]string{ + "deployment_name": "model", + "model": "deployment_name", +} + // buildEvalRequest converts an eval declaration into the create // request. Each evaluator becomes a testing criterion bound to its own // contract, and the item schema declares every dataset column those bindings @@ -279,10 +289,8 @@ func buildEvalRequest( metadata["azd_description"] = group.Description } - evalModel := "" level := "" if group.Options != nil { - evalModel = group.Options.EvalModel level = group.Options.EvaluationLevel } @@ -299,7 +307,7 @@ func buildEvalRequest( schema = &eval_api.EvaluatorSummary{Name: ref.Name} } - plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, evalModel, level) + plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, level) if err != nil { return nil, err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 1018e9cb4e4..64bcf50b2c2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -114,11 +114,14 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { } group := &project.Eval{ - Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), - Dataset: "inline", - Target: &project.Target{Type: "agent", Name: "probe-agent"}, - Evaluators: []evalcore.EvaluatorRef{{Name: summary.Name}}, - Options: &project.Options{EvalModel: judge, EvaluationLevel: level}, + Name: fmt.Sprintf("azd-live-%d", time.Now().UTC().UnixNano()), + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Name: summary.Name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, + Options: &project.Options{EvaluationLevel: level}, } req, err := buildEvalRequest(group, schemas, columns) @@ -156,11 +159,13 @@ func TestLiveBuildRejectsMissingColumnsLocally(t *testing.T) { "this test relies on ifeval declaring required inputs") group := &project.Eval{ - Name: "azd-live-negative", - Dataset: "inline", - Target: &project.Target{Type: "agent", Name: "probe-agent"}, - Evaluators: []evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, - Options: &project.Options{EvalModel: judge}, + Name: "azd-live-negative", + Dataset: "inline", + Target: &project.Target{Type: "agent", Name: "probe-agent"}, + Evaluators: []evalcore.EvaluatorRef{{ + Name: "builtin.ifeval", + InitializationParameters: map[string]any{"deployment_name": judge}, + }}, } // A dataset with only `query` cannot satisfy ifeval. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index f6c80e602a8..9a17598b680 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -45,6 +45,15 @@ func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *proje } } +// withJudge declares the judge deployment where the service reads it from: an +// evaluator's initialization parameters, not a setting on the eval. +func withJudge(model string, refs ...evalcore.EvaluatorRef) []evalcore.EvaluatorRef { + for i := range refs { + refs[i].InitializationParameters = map[string]any{"deployment_name": model} + } + return refs +} + // An agent evaluator takes its response from the sample and its query from the // dataset. func TestBuildBindsAgentFieldsFromSample(t *testing.T) { @@ -55,8 +64,8 @@ func TestBuildBindsAgentFieldsFromSample(t *testing.T) { "turn"), } group := groupWith( - []evalcore.EvaluatorRef{{Name: "builtin.task_adherence"}}, - &project.Options{EvalModel: "gpt-4.1-nano"}, + withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Name: "builtin.task_adherence"}), + nil, ) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -126,10 +135,10 @@ func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { nil, []string{"query", "response", "ground_truth"}, []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{ - {Name: "builtin.ifeval", Threshold: &threshold}, - {Name: "builtin.similarity", Threshold: &threshold}, - }, &project.Options{EvalModel: "gpt-4.1-nano"}) + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Name: "builtin.ifeval", Threshold: &threshold}, + evalcore.EvaluatorRef{Name: "builtin.similarity", Threshold: &threshold}, + ), nil) req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "ground_truth": true, @@ -158,10 +167,10 @@ func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{ - {Name: "builtin.task_completion"}, - {Name: "builtin.similarity"}, - }, &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + group := groupWith(withJudge("m", + evalcore.EvaluatorRef{Name: "builtin.task_completion"}, + evalcore.EvaluatorRef{Name: "builtin.similarity"}, + ), &project.Options{EvaluationLevel: "turn"}) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) @@ -178,8 +187,8 @@ func TestBuildRejectsUnsupportedLevel(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, - &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), + &project.Options{EvaluationLevel: "conversation"}) _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -204,8 +213,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { // An evaluator with no published contract keeps the historical agent-target // shape, so custom evaluators still deploy. func TestBuildFallsBackWithoutSchema(t *testing.T) { - group := groupWith([]evalcore.EvaluatorRef{{Name: "my-custom-evaluator"}}, - &project.Options{EvalModel: "m"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "my-custom-evaluator"}), nil) req, err := buildEvalRequest(group, nil, nil) require.NoError(t, err) @@ -230,8 +238,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { columns := map[string]bool{"query": true, "messages": true, "response": true} // Turn level keeps query/response and drops messages. - turn := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, - &project.Options{EvalModel: "m", EvaluationLevel: "turn"}) + turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), + &project.Options{EvaluationLevel: "turn"}) req, err := buildEvalRequest(turn, schemas, columns) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping @@ -239,8 +247,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "messages") // Conversation level keeps messages and drops query/response. - conv := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, - &project.Options{EvalModel: "m", EvaluationLevel: "conversation"}) + conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), + &project.Options{EvaluationLevel: "conversation"}) req, err = buildEvalRequest(conv, schemas, columns) require.NoError(t, err) mapping = req.TestingCriteria[0].DataMapping @@ -249,16 +257,16 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "response") // An unset level behaves as turn, matching the service default. - dflt := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.task_completion"}}, - &project.Options{EvalModel: "m"}) + dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), nil) req, err = buildEvalRequest(dflt, schemas, columns) require.NoError(t, err) require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") } // Evaluators disagree on what the judge model is called. Built-ins declare -// deployment_name; a custom rubric declares model, and rejects the group with -// "requires model" if only deployment_name is sent. +// deployment_name; a custom rubric declares model, and rejects the eval with +// "requires model" if only deployment_name is sent. One declaration binds +// whichever the evaluator actually accepts. func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { schemas := map[string]*eval_api.EvaluatorSummary{ "builtin.similarity": schema("builtin.similarity", @@ -268,10 +276,10 @@ func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { nil, []string{"query", "response"}, []string{"model"}, []string{"model"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{ - {Name: "builtin.similarity"}, - {Name: "my-rubric"}, - }, &project.Options{EvalModel: "gpt-4.1-nano"}) + group := groupWith(withJudge("gpt-4.1-nano", + evalcore.EvaluatorRef{Name: "builtin.similarity"}, + evalcore.EvaluatorRef{Name: "my-rubric"}, + ), nil) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go index 6d14b672856..a9fb31e194f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -8,7 +8,6 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/pkg/evalcore" - "azureaieval/internal/project" "github.com/stretchr/testify/require" ) @@ -21,8 +20,7 @@ func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, - &project.Options{EvalModel: "m"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) group.Description = "Quality gate for the support agent" req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -37,8 +35,7 @@ func TestBuildOmitsEmptyDescription(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, - &project.Options{EvalModel: "m"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index e7c404211c7..b1df7509305 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -4,9 +4,12 @@ package cmd import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // Ids are per declaration. A shared key works only while a config has one @@ -36,18 +39,28 @@ func TestIDKey_DoesNotCollideWithVersionKey(t *testing.T) { assert.NotEqual(t, idKey("dataset", "golden"), versionKey("dataset", "golden")) } -// Setting EVAL_ID by hand is the documented way to point a config at a -// group that already exists. It is also the key the extension writes itself, -// which is what let a second group adopt the first one's id — so it stays -// readable only where it cannot be ambiguous. Fixing the aliasing dropped this -// fallback entirely once, silently breaking the documented behaviour. +// Setting EVAL_ID by hand is the documented way to point a config at an eval +// that already exists. It is also the key the extension writes itself, which is +// what let a second eval adopt the first one's id — so it stays readable only +// where it cannot be ambiguous. Fixing the aliasing dropped this fallback +// entirely once, silently breaking the documented behaviour. func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { - sole := evalIDKeys("quality", true) + write := func(t *testing.T, names ...string) string { + t.Helper() + dir := t.TempDir() + for _, n := range names { + require.NoError(t, os.WriteFile(filepath.Join(dir, n+".yaml"), []byte("{}\n"), 0o600)) + } + return dir + } + + sole := evalIDKeys("quality", write(t, "quality")) assert.Equal(t, idKey("eval", "quality"), sole[0], - "a group's own entry is preferred over the shared one") + "an eval's own entry is preferred over the shared one") assert.Contains(t, sole, envKeyEvalID, - "a single-group config honours an id set by hand") + "a project with one eval honours an id set by hand") - assert.Equal(t, []string{idKey("eval", "quality")}, evalIDKeys("quality", false), - "with several groups the shared entry cannot say which group it means") + assert.Equal(t, []string{idKey("eval", "quality")}, + evalIDKeys("quality", write(t, "quality", "nightly")), + "with several evals the shared entry cannot say which one it means") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 86cd9c7727c..823ed4e251b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -11,7 +11,6 @@ import ( "io" "os" "path/filepath" - "strconv" "strings" "time" @@ -27,36 +26,35 @@ var generatePollBudget = eval_api.PollerOptions{ MaxAttempts: 720, // one hour } -// warnIgnoredTraceFields reports generation settings that are accepted but have -// no effect yet. -// -// The generation API takes a day window and nothing else, so `source` and -// `sample` are parsed and dropped. Silently discarding them is worse than not -// accepting them: the author believes they narrowed the trace selection when -// nothing changed. -func warnIgnoredTraceFields(cfg *project.GenerateConfig, out io.Writer) { - var fields []string - - if traces := cfg.Agent.Context.Traces; traces != nil { - if traces.Source != "" { - fields = append(fields, "agent.context.traces.source") - } - if traces.Sample > 0 { - fields = append(fields, "agent.context.traces.sample") - } - } - if len(fields) == 0 { - return - } +// generationPlan is everything one generation job needs, after the flags, the +// generation spec, and the eval's own target have been reconciled. +type generationPlan struct { + // Name of the artifact being generated — the positional argument. + Name string + // Agent whose context seeds generation. May be empty, in which case + // generation runs from the instruction alone. + Agent string + // Model deployment the generation job runs against. + Model string + // Instruction describing what the agent does and what to test. + Instruction string + // BaseDir is the directory OutputDir resolves against. + BaseDir string + // OutputDir is where the artifact is written. + OutputDir string + // SampleSize applies to dataset generation only. + SampleSize int + // TraceDays seeds generation from that many days of recent traces. + TraceDays int +} - verb := "has" - if len(fields) > 1 { - verb = "have" +// traceOptions converts the plan's trace window into the generation client's +// day count. Traces seed generation only; they are never a run's data source. +func (p generationPlan) traceOptions() *eval_api.TraceOptions { + if p.TraceDays <= 0 { + return nil } - fmt.Fprintf(out, - "warning: %s %s no effect yet; generation is seeded from the agent's "+ - "instructions and, when a window is set, its traces.\n", - strings.Join(fields, " and "), verb) + return &eval_api.TraceOptions{Days: p.TraceDays} } // resolveInstruction returns the generation instruction, reading it from a @@ -82,23 +80,19 @@ func resolveInstruction(inline, path string) (string, error) { // generationModel returns the deployment both generation jobs run against. // -// Dataset generation has no model of its own: the spec carries one judge model -// and both jobs use it. +// Dataset generation has no model of its own: the spec carries one generation +// model and both jobs use it. func generationModel(cfg *project.GenerateConfig) string { - if cfg.Generate.Rubric == nil { - return "" - } - return cfg.Generate.Rubric.Model + return cfg.GenerationModel } -// agentContextInstructions reads the instructions named by -// `agent.context.instructions`, relative to the spec that declared them. +// declaredInstructions reads the file named by a generation entry's +// `instructions`, relative to the spec that declared it. // -// A missing file is not an error. `init` writes the field pointing at a -// conventional path before that file exists, so treating its absence as a -// failure would break the flow it scaffolds. -func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (string, error) { - named := cfg.Agent.Context.Instructions +// A missing file is not an error. The path can be written before the file +// exists, so treating its absence as a failure would break the flow `init` +// scaffolds. +func declaredInstructions(named, configPath string) (string, error) { if named == "" { return "", nil } @@ -112,7 +106,7 @@ func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (s return "", nil } if err != nil { - return "", fmt.Errorf("reading agent.context.instructions %q: %w", named, err) + return "", fmt.Errorf("reading instructions %q: %w", named, err) } return strings.TrimSpace(string(raw)), nil } @@ -128,8 +122,7 @@ func agentContextInstructions(cfg *project.GenerateConfig, configPath string) (s // which is the flow `init` sets up. func (ec *evalContext) resolveGenerationInstruction( ctx context.Context, - cfg *project.GenerateConfig, - explicit, configPath string, + explicit, declared, configPath, agentName string, out io.Writer, quiet bool, ) (string, error) { @@ -137,7 +130,7 @@ func (ec *evalContext) resolveGenerationInstruction( return explicit, nil } - fromFile, err := agentContextInstructions(cfg, configPath) + fromFile, err := declaredInstructions(declared, configPath) if err != nil { return "", err } @@ -145,98 +138,39 @@ func (ec *evalContext) resolveGenerationInstruction( return fromFile, nil } - if cfg.Agent.Name == "" { + if agentName == "" { return "", nil } - agent, err := ec.evalClient.GetAgent(ctx, cfg.Agent.Name, ProjectEndpointAPIVersion) + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) if err != nil { // Generation can still proceed from the agent source alone, so a // failure to read the agent is reported without stopping. if !quiet { fmt.Fprintf(out, " warning: could not read agent %q for generation context: %v\n", - cfg.Agent.Name, err) + agentName, err) } return "", nil } instructions := agent.Instructions() if instructions != "" && !quiet { - fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", cfg.Agent.Name) + fmt.Fprintf(out, " Seeding generation from the instructions of agent %q.\n", agentName) } return instructions, nil } -// resolveGenerateConfig loads the spec when present, then layers flags on top. -// A missing file is not an error: flags alone are sufficient. -func resolveGenerateConfig( - path, target, evalModel, datasetFlag string, - maxSamples, traceDays int, -) (*project.GenerateConfig, error) { - cfg := &project.GenerateConfig{} - - if _, err := os.Stat(path); err == nil { - loaded, err := project.LoadGenerateConfig(path) - if err != nil { - return nil, err - } - cfg = loaded - } - - if target != "" { - cfg.Agent.Name = target - } - if cfg.Agent.Name == "" { - return nil, requireFlag("target") - } - - if cfg.Generate.Rubric == nil { - cfg.Generate.Rubric = &project.RubricSpec{ - Name: cfg.Agent.Name + "-quality", - LocalDir: "./" + project.DefaultEvaluatorsDir, - } - } - if cfg.Generate.Dataset == nil && datasetFlag == "" { - cfg.Generate.Dataset = &project.DatasetSpec{ - Name: cfg.Agent.Name + "-golden", - Strategy: project.StrategySynthetic, - SampleSize: project.DefaultSampleSize, - LocalDir: "./" + project.DefaultDatasetsDir, - } - } - - if evalModel != "" { - cfg.Generate.Rubric.Model = evalModel - } - if maxSamples > 0 && cfg.Generate.Dataset != nil { - cfg.Generate.Dataset.SampleSize = maxSamples - } - if cfg.Generate.Dataset != nil && cfg.Generate.Dataset.SampleSize == 0 { - cfg.Generate.Dataset.SampleSize = project.DefaultSampleSize - } - if traceDays > 0 { - if cfg.Agent.Context.Traces == nil { - cfg.Agent.Context.Traces = &project.TraceSpec{} - } - cfg.Agent.Context.Traces.Window = fmt.Sprintf("%dd", traceDays) - } - - return cfg, nil -} - // generateRubric submits the evaluator generation job and saves the rubric. func (ec *evalContext) generateRubric( ctx context.Context, - cfg *project.GenerateConfig, - instruction, baseDir string, + plan generationPlan, out io.Writer, noWait bool, ) (*project.ArtifactRef, error) { - spec := cfg.Generate.Rubric - fmt.Fprintf(out, "Generating rubric %s...\n", spec.Name) + fmt.Fprintf(out, "Generating rubric %s...\n", plan.Name) sources := eval_api.BuildGenerationSources( - "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), ) - req := eval_api.NewEvaluatorGenerationJobRequest(spec.Name, spec.Model, sources) + req := eval_api.NewEvaluatorGenerationJobRequest(plan.Name, plan.Model, sources) job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) if err != nil { @@ -253,31 +187,28 @@ func (ec *evalContext) generateRubric( return nil, fmt.Errorf("rubric generation: %w", err) } - path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".json") + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".json") if err := writeRubric(path, completed.Result); err != nil { return nil, err } fmt.Fprintf(out, " wrote %s\n", path) - return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil + return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } // generateDataset submits the data generation job and downloads the result. func (ec *evalContext) generateDataset( ctx context.Context, - cfg *project.GenerateConfig, - instruction, baseDir string, + plan generationPlan, out io.Writer, noWait bool, ) (*project.ArtifactRef, error) { - spec := cfg.Generate.Dataset - fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", spec.Name, spec.SampleSize) + fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) sources := eval_api.BuildGenerationSources( - "agent", cfg.Agent.Name, "", instruction, traceOptions(cfg), + "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), ) - model := generationModel(cfg) - req := eval_api.NewDataGenerationJobRequest(spec.Name, model, spec.SampleSize, sources) + req := eval_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) if err != nil { @@ -299,10 +230,10 @@ func (ec *evalContext) generateDataset( if eval_api.HasPromptSource(promptOnly) { fmt.Fprintf(out, " warning: generating from agent %q failed in the service; "+ - "retrying from the instruction alone.\n", cfg.Agent.Name) + "retrying from the instruction alone.\n", plan.Agent) req = eval_api.NewDataGenerationJobRequest( - spec.Name, model, spec.SampleSize, promptOnly) + plan.Name, plan.Model, plan.SampleSize, promptOnly) job, err = ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) if err != nil { return nil, fmt.Errorf("submitting the data generation job: %w", err) @@ -312,7 +243,7 @@ func (ec *evalContext) generateDataset( } } if err != nil { - return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, cfg.Agent.Name)) + return nil, fmt.Errorf("data generation: %w", explainDataGenerationFailure(err, plan.Agent)) } name, version := completed.ResolvedNameVersion() @@ -332,7 +263,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("downloading the generated dataset %q: %w", name, err) } - path := project.ArtifactPath(baseDir, spec.LocalDir, spec.Name, ".jsonl") + path := project.ArtifactPath(plan.BaseDir, plan.OutputDir, plan.Name, ".jsonl") if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return nil, fmt.Errorf("creating %q: %w", filepath.Dir(path), err) } @@ -341,7 +272,7 @@ func (ec *evalContext) generateDataset( } fmt.Fprintf(out, " wrote %s\n", path) - return &project.ArtifactRef{Name: spec.Name, Source: relativeSource(baseDir, path)}, nil + return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } // isAgentSeededGenerationFailure recognises the service-side failure that hits @@ -393,35 +324,6 @@ func (ec *evalContext) pollGeneration( return poller.Poll(ctx) } -// traceOptions converts the config's trace window into the generation client's -// day count. Traces seed rubric generation only; they are never a run's data -// source. -func traceOptions(cfg *project.GenerateConfig) *eval_api.TraceOptions { - t := cfg.Agent.Context.Traces - if t == nil { - return nil - } - days := parseWindowDays(t.Window) - if days <= 0 { - return nil - } - return &eval_api.TraceOptions{Days: days} -} - -// parseWindowDays reads a window such as "30d" or a bare day count. -func parseWindowDays(window string) int { - w := strings.TrimSpace(strings.ToLower(window)) - if w == "" { - return 0 - } - w = strings.TrimSuffix(w, "d") - days, err := strconv.Atoi(w) - if err != nil { - return 0 - } - return days -} - // writeRubric persists only the rubric dimensions so the developer can edit // weights and descriptions and publish a new version. func writeRubric(path string, result json.RawMessage) error { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index b3c7b86c374..6a22d4fcbed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -5,6 +5,7 @@ package cmd import ( "fmt" + "path/filepath" "azureaieval/internal/project" @@ -22,6 +23,7 @@ import ( // generateFlags are the settings both generate commands share. type generateFlags struct { configPath string + evalName string target string instruction string instructionFile string @@ -34,6 +36,7 @@ type generateFlags struct { func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.Flags().StringVar(&f.configPath, "config", project.DefaultGenerateConfig, "Path to the generation spec. Optional; flags alone are sufficient.") + addEvalFlag(cmd, &f.evalName) cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", "What the agent does and what to test.") @@ -42,8 +45,8 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.MarkFlagsMutuallyExclusive("agent-instruction", "agent-instruction-file") cmd.Flags().StringVar(&f.model, "generation-model", "", "Model deployment that generates the artifact.") - cmd.Flags().StringVar(&f.outputDir, "output-dir", project.DefaultEvalDir, - "Directory the generated artifact is written under.") + cmd.Flags().StringVar(&f.outputDir, "output-dir", "", + "Directory the generated artifact is written to. Overrides the generation spec.") cmd.Flags().BoolVar(&f.noWait, "no-wait", false, "Submit the job and return its id without polling.") cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") @@ -57,45 +60,130 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { func prepareGeneration( cmd *cobra.Command, f *generateFlags, - maxSamples, traceDays int, -) (*evalContext, *project.GenerateConfig, string, error) { + plan generationPlan, + declared genEntry, +) (*evalContext, generationPlan, error) { instruction, err := resolveInstruction(f.instruction, f.instructionFile) if err != nil { - return nil, nil, "", err + return nil, plan, err } - cfg, err := resolveGenerateConfig( - f.configPath, f.target, f.model, "", maxSamples, traceDays, - ) + ctx := cmd.Context() + ec, err := newEvalContext(ctx, f.endpoint) if err != nil { - return nil, nil, "", err + return nil, plan, err } - if err := cfg.Validate(); err != nil { - return nil, nil, "", err + + plan.Instruction, err = ec.resolveGenerationInstruction( + ctx, instruction, declared.instructions, f.configPath, plan.Agent, + cmd.OutOrStdout(), isJSON(cmd), + ) + if err != nil { + ec.Close() + return nil, plan, err } - if !isJSON(cmd) { - warnIgnoredTraceFields(cfg, cmd.OutOrStdout()) + return ec, plan, nil +} + +// resolvePlan settles every input that does not need the network. +// +// Resolution order is the one the spec fixes for every input: flags, then the +// generation spec, then what can be detected from the eval configuration. +// Doing it before the client is built means a missing model or an out-of-range +// sample count is refused without an authentication round trip. +func resolvePlan( + f *generateFlags, + cfg *project.GenerateConfig, + name string, + declared genEntry, +) (generationPlan, error) { + plan := generationPlan{ + Name: name, + Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), + Model: firstNonEmpty(f.model, cfg.GenerationModel), + BaseDir: filepath.Dir(f.configPath), + OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), + SampleSize: declared.sampleSize, + TraceDays: declared.traceDays, } - if generationModel(cfg) == "" { - return nil, nil, "", fmt.Errorf( + if plan.Model == "" { + return plan, fmt.Errorf( "a model deployment is required to generate: pass --generation-model, " + - "or set it in the generation spec") + "or set `generationModel` in the generation spec") } + return plan, nil +} - ctx := cmd.Context() - ec, err := newEvalContext(ctx, f.endpoint) +// genEntry is the subset of a generation spec entry both commands share, so +// resolvePlan does not need to know which one it is serving. +type genEntry struct { + outputDir string + deriveFrom string + instructions string + sampleSize int + traceDays int +} + +// evalTarget reads the agent from the eval configuration, which is where the +// target is already declared, so `generate` does not need it repeated. +// +// Best effort: generation runs from the instruction alone when there is no +// eval config to read, which is the case in a bare directory. +func evalTarget(f *generateFlags) string { + path, err := project.ResolveEvalConfigPath(filepath.Dir(f.configPath), f.evalName) if err != nil { - return nil, nil, "", err + return "" + } + cfg, err := project.LoadEvalConfig(path) + if err != nil || cfg.Target == nil { + return "" } + return cfg.Target.Name +} - instruction, err = ec.resolveGenerationInstruction( - ctx, cfg, instruction, f.configPath, cmd.OutOrStdout(), isJSON(cmd), - ) - if err != nil { - ec.Close() - return nil, nil, "", err +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } } - return ec, cfg, instruction, nil + return "" +} + +// datasetGenEntry reads one dataset's settings out of the generation spec, +// applying the flag override and the default row count. +func datasetGenEntry(cfg *project.GenerateConfig, name string, maxSamples int) genEntry { + spec, _ := cfg.DatasetSpec(name) + entry := genEntry{ + outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultDatasetsDir), + deriveFrom: spec.DeriveFrom, + instructions: spec.Instructions, + sampleSize: spec.SampleSize, + traceDays: spec.TraceDays, + } + if maxSamples > 0 { + entry.sampleSize = maxSamples + } + if entry.sampleSize == 0 { + entry.sampleSize = project.DefaultSampleSize + } + return entry +} + +// evaluatorGenEntry reads one evaluator's settings out of the generation spec, +// applying the flag override. +func evaluatorGenEntry(cfg *project.GenerateConfig, name string, traceDays int) genEntry { + spec, _ := cfg.EvaluatorSpec(name) + entry := genEntry{ + outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultEvaluatorsDir), + deriveFrom: spec.DeriveFrom, + instructions: spec.Instructions, + traceDays: spec.TraceDays, + } + if traceDays > 0 { + entry.traceDays = traceDays + } + return entry } func newDatasetGenerateCommand() *cobra.Command { @@ -111,19 +199,31 @@ func newDatasetGenerateCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - ec, cfg, instruction, err := prepareGeneration(cmd, &flags, maxSamples, 0) + if err := project.ValidateSampleSize(maxSamples); err != nil { + return err + } + + cfg, err := project.LoadGenerateConfig(flags.configPath) if err != nil { return err } - defer ec.Close() + declared := datasetGenEntry(cfg, name, maxSamples) + + plan, err := resolvePlan(&flags, cfg, name, declared) + if err != nil { + return err + } + if err := project.ValidateSampleSize(plan.SampleSize); err != nil { + return err + } - if cfg.Generate.Dataset == nil { - return fmt.Errorf("the generation spec declares no dataset to generate") + ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + if err != nil { + return err } - cfg.Generate.Dataset.Name = name + defer ec.Close() - ref, err := ec.generateDataset( - cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + ref, err := ec.generateDataset(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) if err != nil { return err } @@ -150,19 +250,24 @@ func newEvaluatorGenerateCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - ec, cfg, instruction, err := prepareGeneration(cmd, &flags, 0, traceDays) + cfg, err := project.LoadGenerateConfig(flags.configPath) if err != nil { return err } - defer ec.Close() + declared := evaluatorGenEntry(cfg, name, traceDays) - if cfg.Generate.Rubric == nil { - return fmt.Errorf("the generation spec declares no rubric to generate") + plan, err := resolvePlan(&flags, cfg, name, declared) + if err != nil { + return err + } + + ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + if err != nil { + return err } - cfg.Generate.Rubric.Name = name + defer ec.Close() - ref, err := ec.generateRubric( - cmd.Context(), cfg, instruction, flags.outputDir, cmd.OutOrStdout(), flags.noWait) + ref, err := ec.generateRubric(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) if err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go deleted file mode 100644 index 97f141a2f53..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_defaults_test.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// --dataset is documented as taking a path or the name of a registered -// dataset, and means "use this one instead of generating". Only a local path -// used to suppress generation, so passing a registered name still submitted a -// generation job. -func TestGenerateScaffoldSkipsDatasetWhenSupplied(t *testing.T) { - cases := []struct { - name string - datasetFlag string - wantSpec bool - }{ - {"registered name", "prod-sample", false}, - {"relative path", "./data/golden.jsonl", false}, - {"bare filename", "golden.jsonl", false}, - {"not supplied", "", true}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - // A path that does not exist means flags alone drive the config. - cfg, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), - "my-agent", "gpt-4.1-nano", tc.datasetFlag, 0, 0) - require.NoError(t, err) - - if tc.wantSpec { - require.NotNil(t, cfg.Generate.Dataset, - "a dataset spec is needed when none was supplied") - } else { - require.Nil(t, cfg.Generate.Dataset, - "a supplied dataset must not produce a generation spec") - } - require.NotNil(t, cfg.Generate.Rubric, "the rubric spec is independent") - }) - } -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go index 584a19e7547..c018ddda8ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -6,7 +6,6 @@ package cmd import ( "os" "path/filepath" - "strings" "testing" "azureaieval/internal/project" @@ -15,133 +14,157 @@ import ( ) // `generate` decides what to submit before it touches the network, so the plan -// it builds — which artifacts, from what instruction, at what sample size — is -// checkable without paying for a generation job. These are the parts that -// cannot be observed afterwards: once the jobs are submitted, a wrong default -// is indistinguishable from an intended one. +// it builds — which agent, what model, where the artifact lands, at what sample +// size — is checkable without paying for a generation job. These are the parts +// that cannot be observed afterwards: once the job is submitted, a wrong +// default is indistinguishable from an intended one. -func writeConfig(t *testing.T, body string) string { +// evalsDir writes a generation spec and returns the flags pointing at it. +func evalsDir(t *testing.T, generateBody string, files map[string]string) *generateFlags { t.Helper() - path := filepath.Join(t.TempDir(), "eval_generate.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - return path + dir := t.TempDir() + configPath := filepath.Join(dir, "generate.yaml") + if generateBody != "" { + require.NoError(t, os.WriteFile(configPath, []byte(generateBody), 0o600)) + } + for name, body := range files { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) + } + return &generateFlags{configPath: configPath} +} + +func loadSpec(t *testing.T, f *generateFlags) *project.GenerateConfig { + t.Helper() + cfg, err := project.LoadGenerateConfig(f.configPath) + require.NoError(t, err) + return cfg } -// A spec is optional, so the defaults are what most callers actually run with. -func TestResolveGenerateConfigDefaultsFromFlagsAlone(t *testing.T) { - absent := filepath.Join(t.TempDir(), "absent.yaml") +// A spec is optional, so flags alone are what most callers actually run with. +func TestResolvePlan_FromFlagsAlone(t *testing.T) { + f := evalsDir(t, "", nil) + f.target = "shop-agent" + f.model = "gpt-4o-mini" - cfg, err := resolveGenerateConfig(absent, "shop-agent", "gpt-4o-mini", "", 0, 0) + plan, err := resolvePlan(f, loadSpec(t, f), "shop-golden", + datasetGenEntry(loadSpec(t, f), "shop-golden", 0)) require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - require.Equal(t, "shop-agent", cfg.Agent.Name) - require.NotNil(t, cfg.Generate.Rubric) - require.Equal(t, "shop-agent-quality", cfg.Generate.Rubric.Name) - require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) - require.Equal(t, "./"+project.DefaultEvaluatorsDir, cfg.Generate.Rubric.LocalDir) - - require.NotNil(t, cfg.Generate.Dataset) - require.Equal(t, "shop-agent-golden", cfg.Generate.Dataset.Name) - require.Equal(t, project.StrategySynthetic, cfg.Generate.Dataset.Strategy) - require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) - require.Equal(t, "./"+project.DefaultDatasetsDir, cfg.Generate.Dataset.LocalDir) + + require.Equal(t, "shop-golden", plan.Name) + require.Equal(t, "shop-agent", plan.Agent) + require.Equal(t, "gpt-4o-mini", plan.Model) + require.Equal(t, "./"+project.DefaultDatasetsDir, plan.OutputDir) + require.Equal(t, project.DefaultSampleSize, plan.SampleSize) } -// Without a target there is nothing to generate from, and the refusal has to -// name the flag rather than a config field the caller may not have. -func TestResolveGenerateConfigRequiresATarget(t *testing.T) { - _, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), "", "gpt-4o-mini", "", 0, 0) +// Without a model there is nothing to bill the job against, and the refusal has +// to name both ways of supplying one. +func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { + f := evalsDir(t, "", nil) + f.target = "shop-agent" + + _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) require.Error(t, err) - require.Contains(t, err.Error(), "--target") + require.Contains(t, err.Error(), "--generation-model") + require.Contains(t, err.Error(), "generationModel") } -func TestResolveGenerateConfigReadsTheSpec(t *testing.T) { - path := writeConfig(t, ` -agent: - name: from-spec - context: - instructions: ./instructions.md - traces: - window: 7d - source: ignored-today - sample: 5 -generate: - rubric: - name: spec-rubric - model: gpt-4o - local_dir: ./custom-evaluators - dataset: - name: spec-dataset - strategy: synthetic +// The spec is read per artifact name, so generating one artifact never picks up +// the other's settings. +func TestResolvePlan_ReadsTheNamedSpecEntry(t *testing.T) { + f := evalsDir(t, ` +generationModel: gpt-4o +dataset: + spec-dataset: sampleSize: 200 - local_dir: ./custom-datasets -`) + outputDir: ./custom-datasets + deriveFrom: from-spec +evaluator: + spec-rubric: + outputDir: ./custom-evaluators + deriveFrom: rubric-agent +`, nil) + + cfg := loadSpec(t, f) + + ds, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 0)) + require.NoError(t, err) + require.Equal(t, "gpt-4o", ds.Model) + require.Equal(t, 200, ds.SampleSize) + require.Equal(t, "./custom-datasets", ds.OutputDir) + require.Equal(t, "from-spec", ds.Agent) - cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) + ev, err := resolvePlan(f, cfg, "spec-rubric", evaluatorGenEntry(cfg, "spec-rubric", 0)) require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - require.Equal(t, "from-spec", cfg.Agent.Name) - require.Equal(t, "./instructions.md", cfg.Agent.Context.Instructions) - require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) - require.Equal(t, "gpt-4o", cfg.Generate.Rubric.Model) - require.Equal(t, "./custom-evaluators", cfg.Generate.Rubric.LocalDir) - require.Equal(t, "spec-dataset", cfg.Generate.Dataset.Name) - require.Equal(t, 200, cfg.Generate.Dataset.SampleSize) - - require.NotNil(t, cfg.Agent.Context.Traces) - require.Equal(t, "7d", cfg.Agent.Context.Traces.Window) - require.Equal(t, 5, cfg.Agent.Context.Traces.Sample) + require.Equal(t, "./custom-evaluators", ev.OutputDir) + require.Equal(t, "rubric-agent", ev.Agent) + + // An artifact the spec says nothing about still generates, on the defaults. + other, err := resolvePlan(f, cfg, "unlisted", datasetGenEntry(cfg, "unlisted", 0)) + require.NoError(t, err) + require.Equal(t, project.DefaultSampleSize, other.SampleSize) + require.Equal(t, "./"+project.DefaultDatasetsDir, other.OutputDir) } // Flags win over the spec, which is what makes a one-off run possible without // editing a file that is checked in. -func TestResolveGenerateConfigLayersFlagsOverTheSpec(t *testing.T) { - path := writeConfig(t, ` -agent: - name: from-spec -generate: - rubric: - name: spec-rubric - model: gpt-4o - dataset: - name: spec-dataset +func TestResolvePlan_LayersFlagsOverTheSpec(t *testing.T) { + f := evalsDir(t, ` +generationModel: gpt-4o +dataset: + spec-dataset: sampleSize: 200 -`) + outputDir: ./custom-datasets + deriveFrom: from-spec +`, nil) + f.target = "from-flag" + f.model = "gpt-4o-mini" + f.outputDir = "./from-flag-dir" + + cfg := loadSpec(t, f) + plan, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 500)) + require.NoError(t, err) + + require.Equal(t, "from-flag", plan.Agent) + require.Equal(t, "gpt-4o-mini", plan.Model) + require.Equal(t, "./from-flag-dir", plan.OutputDir) + require.Equal(t, 500, plan.SampleSize) +} + +// The target is already declared on the eval, so `generate` does not need it +// repeated on every invocation. +func TestResolvePlan_FallsBackToTheEvalTarget(t *testing.T) { + f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ + "support-agent-smoke.yaml": "evaluators: [builtin.relevance]\n" + + "target:\n type: agent\n name: support-agent\n", + }) - cfg, err := resolveGenerateConfig(path, "from-flag", "gpt-4o-mini", "", 500, 14) + cfg := loadSpec(t, f) + plan, err := resolvePlan(f, cfg, "support-agent-smoke", + datasetGenEntry(cfg, "support-agent-smoke", 0)) require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - require.Equal(t, "from-flag", cfg.Agent.Name) - require.Equal(t, "gpt-4o-mini", cfg.Generate.Rubric.Model) - require.Equal(t, 500, cfg.Generate.Dataset.SampleSize) - require.Equal(t, "14d", cfg.Agent.Context.Traces.Window, - "--trace-days must reach the spec as a window, since that is the only "+ - "trace field the generation API takes") - - // The rubric name is not derived when the spec named one, so a --target - // override must not silently rename an artifact the spec author declared. - require.Equal(t, "spec-rubric", cfg.Generate.Rubric.Name) + require.Equal(t, "support-agent", plan.Agent, + "the eval's declared target is the agent to generate from") } -// A spec that declares a dataset without a size still has to submit a legal -// job, so the default is applied rather than left at zero. -func TestResolveGenerateConfigFillsAMissingSampleSize(t *testing.T) { - path := writeConfig(t, ` -agent: - name: sized -generate: - dataset: - name: no-size -`) - - cfg, err := resolveGenerateConfig(path, "", "", "", 0, 0) +// With more than one eval the target is ambiguous, so nothing is guessed: +// generation falls back to the instruction alone rather than picking one. +func TestResolvePlan_AmbiguousEvalTargetIsNotGuessed(t *testing.T) { + f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ + "a.yaml": "target:\n type: agent\n name: agent-a\n", + "b.yaml": "target:\n type: agent\n name: agent-b\n", + }) + + cfg := loadSpec(t, f) + plan, err := resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) + require.NoError(t, err) + require.Empty(t, plan.Agent) + + // Naming one resolves it. + f.evalName = "b" + plan, err = resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) require.NoError(t, err) - require.Equal(t, project.DefaultSampleSize, cfg.Generate.Dataset.SampleSize) - require.NoError(t, cfg.Validate()) + require.Equal(t, "agent-b", plan.Agent) } // The bounds are the service's, and the boundaries themselves have to be @@ -158,43 +181,24 @@ func TestGenerateSampleSizeBounds(t *testing.T) { {project.MaxSampleSize, true}, {project.MaxSampleSize + 1, false}, } { - cfg, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), - "bounded", "gpt-4o-mini", "", tc.size, 0) - require.NoError(t, err) - require.Equal(t, tc.size, cfg.Generate.Dataset.SampleSize) - - err = cfg.Validate() + err := project.ValidateSampleSize(tc.size) if tc.allowed { require.NoErrorf(t, err, "%d is inside the service's range", tc.size) continue } require.Errorf(t, err, "%d is outside the service's range", tc.size) - require.Contains(t, err.Error(), "sampleSize") + require.Contains(t, err.Error(), "must be between") } } -// --dataset and --evaluator both mean "use this one". Only the dataset side is -// resolved here; the evaluator side is decided in the command body, so it is -// covered by the CLI test that watches for the skip message. -func TestResolveGenerateConfigSkipsTheDatasetWhenOneIsSupplied(t *testing.T) { - cfg, err := resolveGenerateConfig( - filepath.Join(t.TempDir(), "absent.yaml"), - "supplied", "gpt-4o-mini", "prod-sample", 0, 0) - require.NoError(t, err) - require.Nil(t, cfg.Generate.Dataset, "a supplied dataset must not be generated") - require.NotNil(t, cfg.Generate.Rubric) - require.NoError(t, cfg.Validate()) -} - -// Both jobs bill against one deployment, so a spec with no rubric has no model -// to run either of them. -func TestGenerationModelComesFromTheRubricSpec(t *testing.T) { - require.Equal(t, "", generationModel(&project.GenerateConfig{})) - - cfg := &project.GenerateConfig{} - cfg.Generate.Rubric = &project.RubricSpec{Name: "r", Model: "gpt-4o"} - require.Equal(t, "gpt-4o", generationModel(cfg)) +// Trace days come from the spec, and the flag overrides them. +func TestEvaluatorGenEntry_TraceDays(t *testing.T) { + cfg := &project.GenerateConfig{ + Evaluator: map[string]project.EvaluatorGenSpec{"r": {TraceDays: 7}}, + } + require.Equal(t, 7, evaluatorGenEntry(cfg, "r", 0).traceDays) + require.Equal(t, 30, evaluatorGenEntry(cfg, "r", 30).traceDays) + require.Zero(t, evaluatorGenEntry(cfg, "absent", 0).traceDays) } func TestResolveInstruction(t *testing.T) { @@ -230,24 +234,3 @@ func TestResolveInstruction(t *testing.T) { require.Contains(t, err.Error(), "--agent-instruction-file") }) } - -// The generation API takes a day window and nothing else, so the two fields it -// drops are reported rather than silently discarded. -func TestWarnIgnoredTraceFields(t *testing.T) { - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Traces = &project.TraceSpec{Window: "7d", Source: "some-source", Sample: 5} - - var out strings.Builder - warnIgnoredTraceFields(cfg, &out) - require.Contains(t, out.String(), "agent.context.traces.source") - require.Contains(t, out.String(), "agent.context.traces.sample") - require.NotContains(t, out.String(), "agent.context.traces.window", - "the window is the one trace field the API takes, so it is not a no-op") - - // A window on its own is fully supported and must not produce a warning. - quiet := &project.GenerateConfig{} - quiet.Agent.Context.Traces = &project.TraceSpec{Window: "7d"} - out.Reset() - warnIgnoredTraceFields(quiet, &out) - require.Empty(t, out.String()) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index fba0d533682..f9cbbbddd86 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -23,11 +23,12 @@ import ( // all, so it works offline and unauthenticated. func newInitCommand() *cobra.Command { var ( + evalName string target string dataset string evaluators []string - evalModel string - outDir string + genModel string + outputDir string force bool ) @@ -40,14 +41,17 @@ func newInitCommand() *cobra.Command { if target == "" { return requireFlag("target") } - if outDir == "" { - outDir = project.DefaultEvalDir + if outputDir == "" { + outputDir = project.DefaultEvalDir + } + if evalName == "" { + evalName = target + "-smoke" } - genPath := filepath.Join(outDir, "eval_generate.yaml") - depPath := filepath.Join(outDir, "azure.yaml") + evalPath := project.EvalConfigPath(outputDir, evalName) + genPath := filepath.Join(outputDir, "generate.yaml") - for _, p := range []string{genPath, depPath} { + for _, p := range []string{evalPath, genPath} { if _, err := os.Stat(p); err == nil && !force { return fmt.Errorf("%s already exists; pass --force to overwrite", p) } @@ -56,26 +60,29 @@ func newInitCommand() *cobra.Command { // Asked before anything is written: the project is the one thing init // cannot supply for itself, and failing after creating directories // leaves a half-scaffolded tree behind for the user to clean up. - if err := confirmAzdProject(cmd.Context()); err != nil { + azdProject, err := readAzdProject(cmd.Context()) + if err != nil { return err } + if genModel == "" { + genModel = detectModelDeployment(azdProject) + } - if err := os.MkdirAll(filepath.Join(outDir, project.DefaultDatasetsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultDatasetsDir), 0o750); err != nil { return fmt.Errorf("creating the datasets directory: %w", err) } - if err := os.MkdirAll(filepath.Join(outDir, project.DefaultEvaluatorsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultEvaluatorsDir), 0o750); err != nil { return fmt.Errorf("creating the evaluators directory: %w", err) } - rubricName := fmt.Sprintf("%s-quality", target) + rubricName := target + "-quality" + + plan := planScaffold(evalName, target, rubricName, dataset, evaluators, genModel, outputDir) - genCfg := buildGenerateScaffold(target, rubricName, evalModel) - if err := writeYAML(genPath, genCfg); err != nil { + if err := writeYAML(evalPath, plan.eval); err != nil { return err } - - depCfg := buildDeployScaffold(target, rubricName, dataset, evaluators, evalModel, outDir) - if err := writeYAML(depPath, depCfg); err != nil { + if err := writeYAML(genPath, plan.generate); err != nil { return err } @@ -85,44 +92,59 @@ func newInitCommand() *cobra.Command { // Printing the block and leaving the edit to the reader was enough // to make the documented flow stop working between `init` and // `azd up`. - rootWiring, err := ensureRootEvalService(cmd.Context(), depPath) + rootWiring, err := ensureRootEvalService(cmd.Context(), evalName, target, evalPath) if err != nil { return err } if isJSON(cmd) { return emitJSON(out, map[string]any{ - "generateConfig": genPath, - "deployConfig": depPath, - "datasetsDir": filepath.Join(outDir, project.DefaultDatasetsDir), - "evaluatorsDir": filepath.Join(outDir, project.DefaultEvaluatorsDir), - "rootConfig": rootWiring, + "eval": evalName, + "evalConfig": evalPath, + "generateConfig": genPath, + "datasetsDir": filepath.Join(outputDir, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(outputDir, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, + "target": target, + "generationModel": genModel, + "evaluators": plan.evaluatorNames(), }) } - fmt.Fprintf(out, "Wrote %s\n", genPath) - fmt.Fprintf(out, "Wrote %s\n", depPath) + fmt.Fprintf(out, "%s Detected agent target: %s\n", doneMark, target) + if genModel != "" { + fmt.Fprintf(out, "%s Detected model deployment: %s\n", doneMark, genModel) + } + fmt.Fprintf(out, "%s Planned evaluators: %s\n", doneMark, plan.evaluatorSummary()) + + fmt.Fprintln(out, "\nCreated") + fmt.Fprintf(out, " %-33s eval definition\n", filepath.ToSlash(evalPath)) + fmt.Fprintf(out, " %-33s generation settings (%d samples, %d rubric)\n", + filepath.ToSlash(genPath), project.DefaultSampleSize, plan.rubricCount()) switch rootWiring { case wiringAdded: - fmt.Fprintf(out, "Added the evals service to %s\n", rootConfigName) + fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, evalName) case wiringPresent: - fmt.Fprintf(out, "%s already declares an eval service\n", rootConfigName) + fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, evalName) } - fmt.Fprintln(out, "\nNext:") - fmt.Fprintln(out, " 1. azd ai eval generate (or supply your own dataset)") - fmt.Fprintln(out, " 2. azd up") - fmt.Fprintln(out, " 3. azd ai eval run") + fmt.Fprintf(out, "\nNext: azd ai eval dataset generate %s\n", plan.datasetName) + if plan.rubricCount() > 0 { + fmt.Fprintf(out, " azd ai eval evaluator generate %s\n", rubricName) + } return nil }, } + cmd.Flags().StringVar(&evalName, "name", "", "Name of the eval. Defaults to -smoke.") cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, - "Evaluator reference, repeatable. Use builtin. for a built-in.") - cmd.Flags().StringVar(&evalModel, "judge-model", "", "Model deployment that scores the results.") - cmd.Flags().StringVar(&outDir, "out-dir", project.DefaultEvalDir, + "Evaluator reference, repeatable. Use builtin. for a built-in. "+ + "Passing this replaces the defaults, so it also opts out of rubric generation.") + cmd.Flags().StringVar(&genModel, "generation-model", "", + "Model deployment that generates and judges. Detected from the project when omitted.") + cmd.Flags().StringVar(&outputDir, "output-dir", project.DefaultEvalDir, "Directory to write the config into. Used verbatim, never re-rooted.") cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") return cmd @@ -145,19 +167,51 @@ const ( const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + "or run this from the root of an existing one; the eval service is " -// confirmAzdProject reports whether a project exists, without changing it. -func confirmAzdProject(ctx context.Context) error { +// readAzdProject returns the project, without changing it. +// +// It is read before anything is written: the project is the one thing init +// cannot supply for itself, and it also carries the agent and model detection +// that `init` reports. +func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { azdClient, err := azdext.NewAzdClient() if err != nil { - return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) } defer azdClient.Close() resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - return fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) } - return nil + return resp.GetProject(), nil +} + +// aiModelHost is the model-deployment service the sibling Foundry extensions +// declare, which is where a judge deployment can be read without a service +// call. +const aiModelHost = "azure.ai.model" + +// detectModelDeployment finds the deployment generation and judging run +// against, from what the project already declares. +// +// `init` makes no service calls, so detection is limited to the project file. +// Coming back empty is not a failure: --generation-model supplies it, and the +// generate commands say so when it is missing. +func detectModelDeployment(proj *azdext.ProjectConfig) string { + for name, svc := range proj.GetServices() { + if svc.GetHost() != aiModelHost { + continue + } + if props := svc.GetAdditionalProperties().AsMap(); props != nil { + for _, key := range []string{"deployment", "deploymentName", "name", "model"} { + if v, ok := props[key].(string); ok && v != "" { + return v + } + } + } + return name + } + return "" } // ensureRootEvalService declares the eval service in azd's project file. @@ -167,10 +221,11 @@ func confirmAzdProject(ctx context.Context) error { // the agents extension uses, so azd owns the edit and the project file keeps // whatever shape azd gives it. // -// The eval config itself stays in evals/azure.yaml and is referenced with -// `$ref`. azd carries unknown keys through AdditionalProperties untouched, -// which is how the extension gets it back at deploy time. -func ensureRootEvalService(ctx context.Context, depPath string) (string, error) { +// The service key is the eval's name — one `azure.ai.eval` service per eval — +// and the eval body stays in evals/.yaml, referenced with `$ref`. +// azd carries unknown keys through AdditionalProperties untouched, which is how +// the extension gets it back at deploy time. +func ensureRootEvalService(ctx context.Context, evalName, target, evalPath string) (string, error) { azdClient, err := azdext.NewAzdClient() if err != nil { return "", fmt.Errorf("connecting to azd: %w", err) @@ -187,16 +242,15 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) "added to its %s", rootConfigName) } - // A service already pointing at an eval config is left alone, whatever it - // is called: a second one would deploy the same evals twice. - for _, svc := range resp.GetProject().GetServices() { - if svc.GetHost() == project.EvalHost { - return wiringPresent, nil - } + // A service already declaring this eval is left alone: re-adding it would + // deploy the same eval twice. A differently-named eval service is not a + // conflict, because one service is one eval. + if svc, ok := resp.GetProject().GetServices()[evalName]; ok && svc.GetHost() == project.EvalHost { + return wiringPresent, nil } props, err := structpb.NewStruct(map[string]any{ - "$ref": "./" + filepath.ToSlash(depPath), + "$ref": "./" + filepath.ToSlash(evalPath), }) if err != nil { return "", fmt.Errorf("building the eval service entry: %w", err) @@ -204,9 +258,9 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ Service: &azdext.ServiceConfig{ - Name: evalServiceName(resp.GetProject()), + Name: evalName, Host: project.EvalHost, - Uses: projectServiceUses(resp.GetProject()), + Uses: evalServiceUses(resp.GetProject(), target), AdditionalProperties: props, }, }) @@ -216,74 +270,76 @@ func ensureRootEvalService(ctx context.Context, depPath string) (string, error) return wiringAdded, nil } -// projectServiceUses points the eval service at the Foundry project service -// when the repo declares one, so azd provisions it first. +// evalServiceUses orders the eval after the things it reads. // // It is conditional for the same reason the agents extension makes it // conditional: naming a service the project does not declare is a broken // reference, and an eval config can perfectly well sit in a repo that reaches -// an existing Foundry project by endpoint instead. -func projectServiceUses(proj *azdext.ProjectConfig) []string { +// an existing Foundry project by endpoint and an agent that is deployed +// elsewhere. +func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { + var uses []string for name, svc := range proj.GetServices() { if svc.GetHost() == aiProjectHost { - return []string{name} + uses = append(uses, name) + break } } - return nil + if _, ok := proj.GetServices()[target]; ok { + uses = append(uses, target) + } + return uses +} + +// scaffold is what `init` writes: one eval body and the generation settings +// that fill in the artifacts it references. +type scaffold struct { + eval *project.EvalConfig + generate *project.GenerateConfig + datasetName string + rubricName string +} + +// evaluatorNames lists the evaluators the eval will run, in declaration order. +func (s scaffold) evaluatorNames() []string { + names := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + names = append(names, ref.Name) + } + return names } -// evalServiceName avoids colliding with a service the project already has. -// azd keys services by name, so the map key is the name to avoid. -func evalServiceName(proj *azdext.ProjectConfig) string { - taken := proj.GetServices() - if _, exists := taken["evals"]; !exists { - return "evals" - } - for i := 2; ; i++ { - candidate := fmt.Sprintf("evals%d", i) - if _, exists := taken[candidate]; !exists { - return candidate +// evaluatorSummary is the one-line form `init` reports, marking the evaluator +// that still has to be generated. +func (s scaffold) evaluatorSummary() string { + parts := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + if ref.Name == s.rubricName && s.rubricCount() > 0 { + parts = append(parts, ref.Name+" (rubric)") + continue } + parts = append(parts, ref.Name) } + return strings.Join(parts, ", ") } -func buildGenerateScaffold(target, rubricName, evalModel string) *project.GenerateConfig { - return &project.GenerateConfig{ - Agent: project.AgentSpec{ - Name: target, - Context: project.AgentContext{ - // Scaffolded even though the file does not exist yet: writing - // it overrides the agent's published instructions, which is the - // usual way to narrow what gets generated. `tools` is left out - // because nothing reads it yet. - Instructions: "./agent/instructions.md", - }, - }, - Generate: project.GenerateSpec{ - Rubric: &project.RubricSpec{ - Name: rubricName, - Model: evalModel, - LocalDir: "./" + project.DefaultEvaluatorsDir, - }, - Dataset: &project.DatasetSpec{ - Name: fmt.Sprintf("%s-golden", target), - Strategy: project.StrategySynthetic, - SampleSize: project.DefaultSampleSize, - LocalDir: "./" + project.DefaultDatasetsDir, - }, - }, +// rubricCount is the number of evaluators `init` expects to be generated. +func (s scaffold) rubricCount() int { + if s.generate == nil { + return 0 } + return len(s.generate.Evaluator) } // relativeToConfig rewrites a path given relative to the working directory so -// it resolves from the directory holding the deploy spec. +// it resolves from the directory holding the eval config. // -// `--dataset ./tests/golden.jsonl` means "relative to where I am", but the -// deploy spec's `source:` is resolved relative to that file, so writing the -// path through unchanged sends the deploy looking inside evals/. An absolute -// path is left alone, and forward slashes are kept so the config reads the same -// on every platform. -func relativeToConfig(path, outDir string) string { +// `--dataset ./tests/golden.jsonl` means "relative to where I am", but +// `source:` is resolved relative to the config file, so writing the path +// through unchanged sends the deploy looking inside evals/. An absolute path is +// left alone, and forward slashes are kept so the config reads the same on +// every platform. +func relativeToConfig(path, outputDir string) string { if filepath.IsAbs(path) { return path } @@ -292,7 +348,7 @@ func relativeToConfig(path, outDir string) string { if err != nil { return path } - absOut, err := filepath.Abs(outDir) + absOut, err := filepath.Abs(outputDir) if err != nil { return path } @@ -309,73 +365,120 @@ func relativeToConfig(path, outDir string) string { return rel } -func buildDeployScaffold( - target, rubricName, dataset string, +// planScaffold builds both files `init` writes. +// +// The default evaluator set is a built-in plus a generated rubric: the built-in +// alone would be generic, and the rubric is what makes the baseline about this +// agent. Passing --evaluator replaces both, which is how a caller opts out of +// rubric generation. +func planScaffold( + evalName, target, rubricName, dataset string, evaluators []string, - evalModel string, - outDir string, -) *project.EvalConfig { - cfg := &project.EvalConfig{} + genModel string, + outputDir string, +) scaffold { + cfg := &project.EvalConfig{ + Description: fmt.Sprintf("Basic quality evaluation for %s", target), + } - datasetName := fmt.Sprintf("%s-golden", target) + datasetName := evalName datasetSource := "" + generateDataset := true if dataset != "" { if looksLikeLocalDataset(dataset) { // --dataset is given relative to where the user is standing, but - // source: is resolved relative to the deploy spec, so the path has - // to be rebased or the deploy looks for it inside evals/. - datasetSource = relativeToConfig(dataset, outDir) + // source: is resolved relative to the config, so the path has to be + // rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(dataset, outputDir) datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) } else { // A bare name references an already-registered dataset. datasetName = dataset } + generateDataset = false } else { datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) } - cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + cfg.Dataset = &project.DatasetDecl{ Name: datasetName, Source: datasetSource, - }) + } + + // Every evaluator carries the judge deployment, because that is where the + // service reads it from: built-ins declare `deployment_name` as required, + // so an eval that leaves it off is rejected before it runs. + initParams := map[string]any{} + if genModel != "" { + initParams["deployment_name"] = genModel + } + withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { + if len(initParams) == 0 { + return ref + } + params := make(map[string]any, len(initParams)) + for k, v := range initParams { + params[k] = v + } + ref.InitializationParameters = params + return ref + } - // Evaluators supplied on the command line win; otherwise scaffold the - // generated rubric so `generate` has somewhere to write its reference. refs := evalcore.EvaluatorList{} + generateRubric := false if len(evaluators) == 0 { - cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ - Name: rubricName, - Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), - }) - refs = append(refs, evalcore.EvaluatorRef{Name: rubricName}) + refs = append(refs, + withModel(evalcore.EvaluatorRef{Name: evalcore.BuiltinPrefix + "task_adherence"}), + withModel(evalcore.EvaluatorRef{ + Name: rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), + }), + ) + generateRubric = true } else { for _, e := range evaluators { ref := evalcore.EvaluatorRef{Name: e} if !ref.IsBuiltin() { - cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ - Name: e, - Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), - }) + ref.Source = fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e) } - refs = append(refs, ref) + refs = append(refs, withModel(ref)) } } + cfg.Evaluators = refs - group := project.Eval{ - Name: fmt.Sprintf("%s-quality", target), - Description: fmt.Sprintf("Quality gate for %s", target), - Dataset: datasetName, - Evaluators: refs, - Target: &project.Target{ - Type: project.TargetTypeAgent, - Name: target, - }, + cfg.Target = &project.Target{ + Type: project.TargetTypeAgent, + Name: target, } - if evalModel != "" { - group.Options = &project.Options{EvalModel: evalModel} + cfg.Options = &project.Options{ + MaxSamples: project.DefaultSampleSize, + EvaluationLevel: project.EvaluationLevelTurn, } - cfg.Evals = append(cfg.Evals, group) - return cfg + gen := &project.GenerateConfig{GenerationModel: genModel} + if generateDataset { + gen.Dataset = map[string]project.DatasetGenSpec{ + datasetName: { + SampleSize: project.DefaultSampleSize, + OutputDir: "./" + project.DefaultDatasetsDir, + DeriveFrom: target, + }, + } + } + if generateRubric { + gen.Evaluator = map[string]project.EvaluatorGenSpec{ + rubricName: { + OutputDir: "./" + project.DefaultEvaluatorsDir, + DeriveFrom: target, + }, + } + } + + return scaffold{ + eval: cfg, + generate: gen, + datasetName: datasetName, + rubricName: rubricName, + } } // looksLikeLocalDataset distinguishes a path from a registered dataset name. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index a4e22c85f87..40c92cf0855 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -17,54 +17,93 @@ import ( // config the tool itself produced. func TestScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() - depPath := filepath.Join(dir, "azure.yaml") + evalPath := filepath.Join(dir, "support-agent-smoke.yaml") - cfg := buildDeployScaffold("support-agent", "support-agent-quality", "", nil, "gpt-4.1-nano", project.DefaultEvalDir) - require.NoError(t, writeYAML(depPath, cfg)) + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "gpt-4.1-nano", project.DefaultEvalDir) + require.NoError(t, writeYAML(evalPath, plan.eval)) - loaded, err := project.LoadEvalConfig(depPath) + loaded, err := project.LoadEvalConfig(evalPath) require.NoError(t, err) require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") - g, err := loaded.ResolveGroup("") - require.NoError(t, err) - require.Equal(t, project.TargetTypeAgent, g.Target.Type) - require.Equal(t, "support-agent", g.Target.Name) - require.Equal(t, "gpt-4.1-nano", g.Options.EvalModel) - require.Len(t, g.Evaluators, 1) + require.Equal(t, project.TargetTypeAgent, loaded.Target.Type) + require.Equal(t, "support-agent", loaded.Target.Name) + require.Equal(t, project.DefaultSampleSize, loaded.Options.MaxSamples) + + // The eval takes its name from the file, which is the azure.yaml service key. + require.Equal(t, "support-agent-smoke", loaded.Eval("support-agent-smoke").Name) +} + +// The default set is a built-in plus a generated rubric: the built-in alone +// would be generic, and the rubric is what makes the baseline about this agent. +func TestScaffold_DefaultEvaluators(t *testing.T) { + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "gpt-5.6-luna", project.DefaultEvalDir) + + require.Equal(t, + []string{"builtin.task_adherence", "support-agent-quality"}, + plan.evaluatorNames()) + require.Contains(t, plan.evaluatorSummary(), "support-agent-quality (rubric)") + + // Every evaluator carries the judge deployment, because built-ins declare + // deployment_name as required and an eval that leaves it off is rejected. + for _, ref := range plan.eval.Evaluators { + require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["deployment_name"], + "%s must name a judge deployment", ref.Name) + } +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation. +func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "support-agent-quality", "", + []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + + require.Equal(t, []string{"builtin.task_adherence"}, plan.evaluatorNames()) + require.Zero(t, plan.rubricCount(), "no rubric is generated when evaluators are given") + require.Empty(t, plan.generate.Evaluator) } func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() - genPath := filepath.Join(dir, "eval_generate.yaml") + genPath := filepath.Join(dir, "generate.yaml") - cfg := buildGenerateScaffold("support-agent", "support-agent-quality", "gpt-4.1-nano") - require.NoError(t, writeYAML(genPath, cfg)) + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "gpt-4.1-nano", project.DefaultEvalDir) + require.NoError(t, writeYAML(genPath, plan.generate)) loaded, err := project.LoadGenerateConfig(genPath) require.NoError(t, err) - require.NoError(t, loaded.Validate()) - require.Equal(t, "support-agent", loaded.Agent.Name) - require.Equal(t, project.StrategySynthetic, loaded.Generate.Dataset.Strategy) - require.Equal(t, project.DefaultSampleSize, loaded.Generate.Dataset.SampleSize) + require.Equal(t, "gpt-4.1-nano", loaded.GenerationModel) + + ds, ok := loaded.DatasetSpec("support-agent-smoke") + require.True(t, ok, "the generation spec is keyed by artifact name") + require.Equal(t, project.DefaultSampleSize, ds.SampleSize) + require.Equal(t, "support-agent", ds.DeriveFrom) + + ev, ok := loaded.EvaluatorSpec("support-agent-quality") + require.True(t, ok) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) } -// Built-ins are referenced from the group but never declared as custom -// evaluators; declaring one is a validation error. -func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { - cfg := buildDeployScaffold( - "support-agent", "unused", "", - []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir, - ) +// Built-ins are referenced but never published, so the scaffold must not give +// one a local source to upload. +func TestScaffold_BuiltinEvaluatorsHaveNoSource(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "unused", "", + []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir) + cfg := plan.eval - require.Len(t, cfg.Evaluators, 1, "only the custom evaluator should be declared") - require.Equal(t, "my-custom", cfg.Evaluators[0].Name) + require.Len(t, cfg.Evaluators, 2) + require.True(t, cfg.Evaluators[0].IsBuiltin()) + require.Empty(t, cfg.Evaluators[0].Source) + require.False(t, cfg.Evaluators[1].IsBuiltin()) + require.NotEmpty(t, cfg.Evaluators[1].Source) - require.Len(t, cfg.Evals[0].Evaluators, 2) - require.True(t, cfg.Evals[0].Evaluators[0].IsBuiltin()) - require.False(t, cfg.Evals[0].Evaluators[1].IsBuiltin()) + require.Len(t, cfg.CustomEvaluators(), 1, + "only the custom evaluator is this config's to publish") - path := filepath.Join(t.TempDir(), "azure.yaml") + path := filepath.Join(t.TempDir(), "smoke.yaml") require.NoError(t, writeYAML(path, cfg)) loaded, err := project.LoadEvalConfig(path) require.NoError(t, err) @@ -72,26 +111,35 @@ func TestScaffold_BuiltinEvaluatorsAreNotDeclared(t *testing.T) { } // A bare name means an already-registered dataset; a path means a local file. +// Either way the dataset was supplied, so nothing is scheduled to generate it — +// only a missing --dataset produces a generation entry. func TestScaffold_DatasetReferenceForms(t *testing.T) { t.Run("local path becomes a source", func(t *testing.T) { // --dataset is relative to the working directory, but source: is - // resolved relative to the deploy spec, so it has to be rebased. - cfg := buildDeployScaffold("a", "r", "./tests/golden.jsonl", nil, "", "evals") - require.Equal(t, "../tests/golden.jsonl", cfg.Datasets[0].Source, + // resolved relative to the eval config, so it has to be rebased. + plan := planScaffold("smoke", "a", "r", "./tests/golden.jsonl", nil, "", "evals") + require.Equal(t, "../tests/golden.jsonl", plan.eval.Dataset.Source, "a dataset outside the eval dir must be reached with ..") - require.Equal(t, "golden", cfg.Datasets[0].Name) + require.Equal(t, "golden", plan.eval.Dataset.Name) + require.Empty(t, plan.generate.Dataset, + "a supplied dataset must not be scheduled for generation") }) t.Run("bare name references a registered dataset", func(t *testing.T) { - cfg := buildDeployScaffold("a", "r", "prod-sample", nil, "", project.DefaultEvalDir) - require.Equal(t, "prod-sample", cfg.Datasets[0].Name) - require.Empty(t, cfg.Datasets[0].Source, + plan := planScaffold("smoke", "a", "r", "prod-sample", nil, "", project.DefaultEvalDir) + require.Equal(t, "prod-sample", plan.eval.Dataset.Name) + require.Empty(t, plan.eval.Dataset.Source, "a registered dataset must not get a local source") + require.Empty(t, plan.generate.Dataset) }) - t.Run("no dataset flag scaffolds a local path", func(t *testing.T) { - cfg := buildDeployScaffold("support-agent", "r", "", nil, "", project.DefaultEvalDir) - require.Contains(t, cfg.Datasets[0].Source, "support-agent-golden.jsonl") + t.Run("no dataset flag scaffolds a local path and a generation entry", func(t *testing.T) { + plan := planScaffold("support-agent-smoke", "support-agent", "r", "", + nil, "", project.DefaultEvalDir) + require.Equal(t, "support-agent-smoke", plan.eval.Dataset.Name, + "the dataset is named after the eval") + require.Contains(t, plan.eval.Dataset.Source, "support-agent-smoke.jsonl") + require.Contains(t, plan.generate.Dataset, "support-agent-smoke") }) } @@ -106,13 +154,13 @@ func TestLooksLikeLocalDataset(t *testing.T) { // in the agent-scoped command must not reappear. func TestWriteYAML_UsesPathVerbatim(t *testing.T) { dir := t.TempDir() - nested := filepath.Join(dir, "evals", "azure.yaml") + nested := filepath.Join(dir, "evals", "smoke.yaml") require.NoError(t, writeYAML(nested, &project.EvalConfig{})) _, err := os.Stat(nested) require.NoError(t, err, "the file must land exactly at the requested path") - doubled := filepath.Join(dir, "evals", "evals", "azure.yaml") + doubled := filepath.Join(dir, "evals", "evals", "smoke.yaml") _, err = os.Stat(doubled) require.Error(t, err, "the path must not be re-rooted under itself") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go index 18b37e8e969..03745d01e75 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_wiring_test.go @@ -8,6 +8,8 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azdext" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" ) func projectWith(names ...string) *azdext.ProjectConfig { @@ -18,31 +20,51 @@ func projectWith(names ...string) *azdext.ProjectConfig { return proj } -// The eval service is added through azd's own AddService, so it has to pick a -// name azd will accept. Reusing one already in the project would overwrite -// somebody else's service. -func TestEvalServiceName_PrefersEvals(t *testing.T) { - assert.Equal(t, "evals", evalServiceName(projectWith())) - assert.Equal(t, "evals", evalServiceName(projectWith("api", "web"))) -} +// The eval service is ordered after everything it reads, but only names +// services the project actually declares. Naming one it does not have is a +// broken reference, and an eval config can sit in a repo that reaches an +// existing Foundry project by endpoint and an agent deployed elsewhere. +func TestEvalServiceUses_OnlyWhatTheProjectDeclares(t *testing.T) { + assert.Nil(t, evalServiceUses(projectWith("api", "web"), "support-agent"), + "neither the project service nor the agent is declared, so there is nothing to order after") + + withProject := projectWith("api", "support-agent") + withProject.Services["ai-project"] = &azdext.ServiceConfig{ + Name: "ai-project", Host: aiProjectHost, + } + assert.Equal(t, []string{"ai-project", "support-agent"}, + evalServiceUses(withProject, "support-agent"), + "the eval runs after the project it evaluates against and the agent it evaluates") -func TestEvalServiceName_StepsAsideForAnExistingName(t *testing.T) { - assert.Equal(t, "evals2", evalServiceName(projectWith("evals"))) - assert.Equal(t, "evals3", evalServiceName(projectWith("evals", "evals2"))) + assert.Equal(t, []string{"support-agent"}, + evalServiceUses(projectWith("support-agent"), "support-agent"), + "an agent alone is still worth ordering after") } -// The agents extension wires uses: only to services the project actually -// declares. Naming one it does not have is a broken reference, and an eval -// config can sit in a repo that reaches an existing Foundry project by -// endpoint instead of declaring one. -func TestProjectServiceUses_OnlyWhenTheProjectDeclaresOne(t *testing.T) { - assert.Nil(t, projectServiceUses(projectWith("api", "web")), - "no Foundry project service means no uses entry") +// `init` detects the judge deployment from the project, because it makes no +// service calls and this is the only place it can read one. +func TestDetectModelDeployment(t *testing.T) { + assert.Empty(t, detectModelDeployment(projectWith("api", "web"))) - withProject := projectWith("api") - withProject.Services["ai-project"] = &azdext.ServiceConfig{ - Name: "ai-project", Host: aiProjectHost, + proj := projectWith("api") + proj.Services["chat"] = &azdext.ServiceConfig{Name: "chat", Host: aiModelHost} + assert.Equal(t, "chat", detectModelDeployment(proj), + "the service name is the deployment name when nothing more specific is declared") + + named := projectWith() + named.Services["chat"] = &azdext.ServiceConfig{ + Name: "chat", + Host: aiModelHost, + AdditionalProperties: mustStruct(t, map[string]any{ + "deployment": "gpt-5.6-luna", + }), } - assert.Equal(t, []string{"ai-project"}, projectServiceUses(withProject), - "the eval service should be ordered after the project it evaluates against") + assert.Equal(t, "gpt-5.6-luna", detectModelDeployment(named)) +} + +func mustStruct(t *testing.T, m map[string]any) *structpb.Struct { + t.Helper() + s, err := structpb.NewStruct(m) + require.NoError(t, err) + return s } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 319ed2565f0..62f9c2eade1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -15,6 +15,14 @@ import ( const outputJSON = "json" +// Progress markers from the azd style guide, so the extension's lines sit +// alongside core's without a second vocabulary. +const ( + doneMark = "(✓) Done:" // finished successfully + skippedMark = "(-) Skipped:" // intentionally not done, not a failure + failedMark = "(x) Failed:" // the step did not complete +) + // outputFormat reads the inherited -o/--output flag. func outputFormat(cmd *cobra.Command) string { if cmd == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 7c2ab84e458..af31fb489c1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -53,7 +53,6 @@ func newRunCommand() *cobra.Command { // buildRunCommand builds `run start`. func buildRunCommand(use, short string) *cobra.Command { var ( - configPath string groupName string evalID string runName string @@ -86,18 +85,21 @@ func buildRunCommand(use, short string) *cobra.Command { // --eval-id bypasses the config entirely. var group *project.Eval + configPath := "" if evalID == "" { - cfg, err := project.LoadEvalConfig(configPath) + configPath, err = project.ResolveEvalConfigPath(project.DefaultEvalDir, groupName) if err != nil { return err } - if err := cfg.Validate(); err != nil { + cfg, err := project.LoadEvalConfig(configPath) + if err != nil { return err } - group, err = cfg.ResolveGroup(groupName) - if err != nil { + if err := cfg.Validate(); err != nil { return err } + resolved := cfg.Eval(evalNameFromPath(configPath)) + group = &resolved if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { return err @@ -105,7 +107,7 @@ func buildRunCommand(use, short string) *cobra.Command { evalID, err = ec.resolveEvalIDFromConfig( ctx, group, configPath, resolveLevel(level, group), - len(cfg.Evals) == 1, out, isJSON(cmd)) + out, isJSON(cmd)) if err != nil { return err } @@ -190,10 +192,8 @@ func buildRunCommand(use, short string) *cobra.Command { }, } - cmd.Flags().StringVar(&configPath, "config", project.DefaultDeployConfig, - "Path to the eval deployment config.") cmd.Flags().StringVar(&groupName, "eval", "", - "Which evals entry to run. Defaults to the only one.") + "Name of the eval to run. Defaults to the only one declared.") cmd.Flags().StringVar(&evalID, "eval-id", "", "Run against an existing eval by id, ignoring the config.") cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") @@ -225,7 +225,6 @@ func (ec *evalContext) resolveEvalIDFromConfig( group *project.Eval, configPath string, level string, - soleGroup bool, out interface{ Write([]byte) (int, error) }, jsonMode bool, ) (string, error) { @@ -233,7 +232,7 @@ func (ec *evalContext) resolveEvalIDFromConfig( return group.ID, nil } - for _, key := range evalIDKeys(group.Name, soleGroup) { + for _, key := range evalIDKeys(group.Name, filepath.Dir(configPath)) { cached := ec.getEnvValue(ctx, key) if cached == "" { continue @@ -279,18 +278,18 @@ func (ec *evalContext) resolveEvalIDFromConfig( return created.ID, nil } -// evalIDKeys lists the env entries that may hold this group's id, most +// evalIDKeys lists the env entries that may hold this eval's id, most // specific first. // // The per-name entry is what the extension writes. EVAL_ID is also the -// documented way to point a config at a group that already exists, created in +// documented way to point a config at an eval that already exists, created in // the portal or by another tool, so it stays readable — but only when the -// config declares a single group. With more than one there is no way to tell -// which group a shared entry refers to, and reading it anyway is what let a -// second group adopt the first one's id. -func evalIDKeys(name string, soleGroup bool) []string { +// project declares a single eval. With more than one there is no way to tell +// which eval a shared entry refers to, and reading it anyway is what let a +// second eval adopt the first one's id. +func evalIDKeys(name, evalDir string) []string { keys := []string{idKey("eval", name)} - if soleGroup { + if names, err := project.EvalNamesIn(evalDir); err == nil && len(names) == 1 { keys = append(keys, envKeyEvalID) } return keys @@ -318,8 +317,8 @@ func (ec *evalContext) checkDatasetRegistered( return nil } - decl, ok := cfg.Dataset(group.Dataset) - if !ok { + decl := cfg.Dataset + if decl == nil { return nil } @@ -496,8 +495,8 @@ func localDatasetPath(configPath string, group *project.Eval) string { if err != nil { return "" } - decl, ok := cfg.Dataset(group.Dataset) - if !ok || decl.Source == "" { + decl := cfg.Dataset + if decl == nil || decl.Source == "" { return "" } if filepath.IsAbs(decl.Source) { @@ -506,6 +505,13 @@ func localDatasetPath(configPath string, group *project.Eval) string { return filepath.Join(filepath.Dir(configPath), decl.Source) } +// evalNameFromPath is the eval's name: one config file is one eval, and the +// file is named after it, matching the azure.yaml service key that $refs it. +func evalNameFromPath(configPath string) string { + base := filepath.Base(configPath) + return strings.TrimSuffix(base, filepath.Ext(base)) +} + // readJSONL reads newline-delimited JSON, optionally truncating to limit rows. func readJSONL(path string, limit int) ([]map[string]any, error) { f, err := os.Open(path) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 4a30bcd200d..0f7545fffd6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -269,10 +269,14 @@ func resolveEvalID( envKeyEvalID) } -// addEvalFlag registers the flag that names a group from the config, so -// every command taking an eval-id can reach a group by the name its author -// used. -// addEvalFlags registers the two ways to say which group a command acts +// addEvalFlag registers the flag that names an eval from the config, for +// commands that never take a raw service id. +func addEvalFlag(cmd *cobra.Command, target *string) { + cmd.Flags().StringVar(target, "eval", "", + "Name of the eval declared in azure.yaml.") +} + +// addEvalFlags registers the two ways to say which eval a command acts // on: --eval names one from the config, --eval-id gives its service id. // // The id is also accepted as a positional argument. The flag exists because @@ -280,8 +284,7 @@ func resolveEvalID( // it there should not have to find out that the sibling commands take only a // positional. func addEvalFlags(cmd *cobra.Command, target *string) { - cmd.Flags().StringVar(target, "eval", "", - "Name an evals entry from the config instead of passing its id.") + addEvalFlag(cmd, target) cmd.Flags().String("eval-id", "", "Id of the eval. Same as passing the id as an argument.") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go index dd852e0a778..55329762df9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/schemas_live_test.go @@ -82,11 +82,13 @@ func TestLiveConversationEvaluatorBindsMessages(t *testing.T) { } plan, err := planCriterion( - evalcore.EvaluatorRef{Name: name}, + evalcore.EvaluatorRef{ + Name: name, + InitializationParameters: map[string]any{"deployment_name": judge}, + }, schemas[name], nil, // no target: the dataset holds both sides of the exchange map[string]bool{conversationField: true}, - judge, "conversation", ) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go deleted file mode 100644 index 6c952569aa4..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/trace_warning_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package cmd - -import ( - "bytes" - "testing" - - "azureaieval/internal/project" - - "github.com/stretchr/testify/require" -) - -func warnFor(t *testing.T, traces *project.TraceSpec) string { - t.Helper() - cfg := &project.GenerateConfig{} - cfg.Agent.Context.Traces = traces - - var buf bytes.Buffer - warnIgnoredTraceFields(cfg, &buf) - return buf.String() -} - -// source and sample are accepted by the config model but the generation API -// takes only a day window, so they are dropped. Dropping them silently lets an -// author believe they narrowed the trace selection when nothing changed. -func TestWarnsAboutTraceFieldsWithNoEffect(t *testing.T) { - out := warnFor(t, &project.TraceSpec{Source: "production", Window: "30d", Sample: 500}) - require.Contains(t, out, "agent.context.traces.source") - require.Contains(t, out, "agent.context.traces.sample") - require.Contains(t, out, "window") - require.Contains(t, out, "have no effect", "two fields take a plural verb") - - out = warnFor(t, &project.TraceSpec{Source: "production", Window: "30d"}) - require.Contains(t, out, "agent.context.traces.source") - require.NotContains(t, out, "sample") - require.Contains(t, out, "has no effect", "one field takes a singular verb") -} - -// The field that does work draws no warning, and neither does an absent block. -func TestNoWarningWhenOnlyWindowIsSet(t *testing.T) { - require.Empty(t, warnFor(t, &project.TraceSpec{Window: "30d"})) - require.Empty(t, warnFor(t, nil)) - require.Empty(t, warnFor(t, &project.TraceSpec{})) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index 88538768b37..df6aa791b7d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -93,7 +93,7 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { func (el EvaluatorList) MarshalYAML() (any, error) { out := make([]any, 0, len(el)) for _, ref := range el { - if ref.Threshold == nil && ref.Version == "" { + if ref.isBareName() { out = append(out, ref.Name) continue } @@ -141,18 +141,29 @@ func (el *EvaluatorList) UnmarshalJSON(data []byte) error { } // MarshalJSON mirrors MarshalYAML's compact form. +// +// Everything the reference carries has to survive the round trip, including +// the source and the initialization parameters: the eval fingerprint is taken +// over this encoding, so a field dropped here is a change the reconciler +// cannot see. func (el EvaluatorList) MarshalJSON() ([]byte, error) { + // Aliased so the element encoder does not recurse through this method. + type ref = EvaluatorRef + out := make([]any, 0, len(el)) - for _, ref := range el { - if ref.Threshold == nil && ref.Version == "" { - out = append(out, ref.Name) + for _, r := range el { + if r.isBareName() { + out = append(out, r.Name) continue } - out = append(out, struct { - Name string `json:"name"` - Version string `json:"version,omitempty"` - Threshold *float64 `json:"threshold,omitempty"` - }{ref.Name, ref.Version, ref.Threshold}) + out = append(out, ref(r)) } return json.Marshal(out) } + +// isBareName reports whether the reference carries nothing but its name, in +// which case both encoders emit the compact string form. +func (e EvaluatorRef) isBareName() bool { + return e.Threshold == nil && e.Version == "" && e.Source == "" && + len(e.InitializationParameters) == 0 +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index 0c75f38fdf0..e2ff2e74772 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -8,6 +8,8 @@ package project import ( "fmt" "os" + "path/filepath" + "sort" "strings" "azureaieval/internal/pkg/evalcore" @@ -15,12 +17,18 @@ import ( "go.yaml.in/yaml/v3" ) -// EvalConfig is the deployment spec — the body of the azure.ai.eval service -// entry, normally kept in evals/azure.yaml and pulled in with $ref. +// EvalConfig is one eval — the body of a single `azure.ai.eval` service entry, +// kept in evals/.yaml and pulled in with $ref. +// +// The eval's name is the service key in azure.yaml and is not repeated here. +// One service per eval is what lets azd's own dependency graph order an eval +// after the agent it evaluates. type EvalConfig struct { - Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` - Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` - Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset *DatasetDecl `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` + Options *Options `yaml:"options,omitempty" json:"options,omitempty"` } // DatasetDecl declares a dataset. A local Source is uploaded on deploy; without @@ -61,9 +69,12 @@ type Target struct { const TargetTypeAgent = "agent" -// Options are run settings carried on the group. +// Options are run settings carried on the eval. +// +// There is deliberately no judge-model option. A judge deployment is a testing +// criterion's `initialization_parameters.deployment_name`, which differs per +// evaluator, so it is declared on the evaluator reference instead. type Options struct { - EvalModel string `yaml:"eval_model,omitempty" json:"eval_model,omitempty"` MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` } @@ -74,7 +85,7 @@ const ( EvaluationLevelConversation = "conversation" ) -// LoadEvalConfig reads a deployment spec from disk. The path is used verbatim, +// LoadEvalConfig reads an eval body from disk. The path is used verbatim, // relative to the process working directory — never re-rooted. func LoadEvalConfig(path string) (*EvalConfig, error) { data, err := os.ReadFile(path) @@ -89,18 +100,77 @@ func LoadEvalConfig(path string) (*EvalConfig, error) { return &cfg, nil } +// EvalNamesIn lists the evals declared under evalDir, in sorted order. +// +// One file is one eval, named after it. The generation spec shares the +// directory and is not one, so it is excluded by name. +func EvalNamesIn(evalDir string) ([]string, error) { + entries, err := os.ReadDir(evalDir) + if err != nil { + return nil, err + } + + var names []string + for _, e := range entries { + if e.IsDir() { + continue + } + ext := strings.ToLower(filepath.Ext(e.Name())) + if ext != ".yaml" && ext != ".yml" { + continue + } + name := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())) + if name == generateConfigBase { + continue + } + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// generateConfigBase is the reserved file name in the evals directory. +const generateConfigBase = "generate" + +// ResolveEvalConfigPath finds the config file holding one eval's body. +// +// A named eval is evals/.yaml. With no name the directory must hold +// exactly one eval, and anything else names the candidates rather than +// picking one, because guessing which eval a command meant is the kind of +// mistake that is only noticed after it has run. +func ResolveEvalConfigPath(evalDir, evalName string) (string, error) { + if evalName != "" { + path := EvalConfigPath(evalDir, evalName) + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("eval %q is not declared in %s", evalName, evalDir) + } + return path, nil + } + + names, err := EvalNamesIn(evalDir) + if err != nil { + return "", fmt.Errorf("reading %s: %w", evalDir, err) + } + switch len(names) { + case 0: + return "", fmt.Errorf("no evals are declared in %s", evalDir) + case 1: + return EvalConfigPath(evalDir, names[0]), nil + default: + return "", fmt.Errorf( + "%s declares %d evals (%s); choose one with --eval", + evalDir, len(names), strings.Join(names, ", ")) + } +} + // Validate checks the invariants the provider relies on before it calls the // service, so failures surface as config errors rather than opaque 4xx. func (c *EvalConfig) Validate() error { - datasets := map[string]bool{} - for i, d := range c.Datasets { - if d.Name == "" { - return fmt.Errorf("datasets[%d]: 'name' is required", i) - } - if datasets[d.Name] { - return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) - } - datasets[d.Name] = true + if c.Dataset != nil && c.Dataset.Name == "" { + return fmt.Errorf("dataset: 'name' is required") + } + if len(c.Evaluators) == 0 { + return fmt.Errorf("at least one evaluator is required") } evaluators := map[string]bool{} @@ -108,17 +178,22 @@ func (c *EvalConfig) Validate() error { if e.Name == "" { return fmt.Errorf("evaluators[%d]: 'name' is required", i) } - if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { - return fmt.Errorf( - "evaluators[%d]: built-in evaluator %q must not be declared; "+ - "reference it directly from an eval", i, e.Name) - } if evaluators[e.Name] { return fmt.Errorf("evaluators[%d]: duplicate evaluator name %q", i, e.Name) } + evaluators[e.Name] = true + + if e.IsBuiltin() { + if e.Source != "" { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in evaluator has no source to publish", + i, e.Name) + } + continue + } // The service assigns an evaluator's version on publish, so a declared // one cannot be honoured alongside a source: the upload lands on - // whatever comes next and the group binds that, leaving the pin + // whatever comes next and the eval binds that, leaving the pin // describing a version nothing uses. if e.Source != "" && e.Version != "" { return fmt.Errorf( @@ -127,110 +202,55 @@ func (c *EvalConfig) Validate() error { "publish this file, or drop `source` to reference a version already "+ "on the project", i, e.Name) } - evaluators[e.Name] = true } - groups := map[string]bool{} - for i, g := range c.Evals { - if g.Name == "" { - return fmt.Errorf("evals[%d]: 'name' is required", i) - } - if groups[g.Name] { - return fmt.Errorf("evals[%d]: duplicate eval name %q", i, g.Name) - } - groups[g.Name] = true - - if g.Dataset != "" && !datasets[g.Dataset] { - return fmt.Errorf( - "evals[%d] (%s): dataset %q is not declared in datasets", - i, g.Name, g.Dataset) - } - if len(g.Evaluators) == 0 { - return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, g.Name) - } - for _, ref := range g.Evaluators { - if ref.IsBuiltin() { - continue - } - if !evaluators[ref.Name] { - return fmt.Errorf( - "evals[%d] (%s): evaluator %q is not declared in evaluators "+ - "(built-ins need the %q prefix)", - i, g.Name, ref.Name, evalcore.BuiltinPrefix) - } - } - if g.Target != nil && g.Target.Type != "" && g.Target.Type != TargetTypeAgent { + if c.Target != nil && c.Target.Type != "" && c.Target.Type != TargetTypeAgent { + return fmt.Errorf( + "target.type %q is not supported; use %q", c.Target.Type, TargetTypeAgent) + } + if c.Options != nil { + switch c.Options.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: return fmt.Errorf( - "evals[%d] (%s): target.type %q is not supported; use %q", - i, g.Name, g.Target.Type, TargetTypeAgent) - } - if g.Options != nil { - switch g.Options.EvaluationLevel { - case "", EvaluationLevelTurn, EvaluationLevelConversation: - default: - return fmt.Errorf( - "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", - i, g.Name, g.Options.EvaluationLevel, - EvaluationLevelTurn, EvaluationLevelConversation) - } + "options.evaluation_level %q is invalid; expected %q or %q", + c.Options.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) } } return nil } -// Dataset returns the declaration with the given name. -func (c *EvalConfig) Dataset(name string) (*DatasetDecl, bool) { - for i := range c.Datasets { - if c.Datasets[i].Name == name { - return &c.Datasets[i], true - } - } - return nil, false -} - -// Evaluator returns the declaration with the given name. -func (c *EvalConfig) Evaluator(name string) (*EvaluatorDecl, bool) { - for i := range c.Evaluators { - if c.Evaluators[i].Name == name { - return &c.Evaluators[i], true - } +// Eval resolves the config into the eval the reconciler publishes, taking its +// name from the service entry that pulled the file in. +func (c *EvalConfig) Eval(name string) Eval { + resolved := Eval{ + Name: name, + Description: c.Description, + Evaluators: c.Evaluators, + Target: c.Target, + Options: c.Options, } - return nil, false -} - -// Group returns the eval with the given name. -func (c *EvalConfig) Group(name string) (*Eval, bool) { - for i := range c.Evals { - if c.Evals[i].Name == name { - return &c.Evals[i], true - } + if c.Dataset != nil { + resolved.Dataset = c.Dataset.Name } - return nil, false + return resolved } -// ResolveGroup picks the group to act on: the named one, or the only one when -// the config declares exactly one. -func (c *EvalConfig) ResolveGroup(name string) (*Eval, error) { - if name != "" { - g, ok := c.Group(name) - if !ok { - return nil, fmt.Errorf("eval %q is not declared in the config", name) - } - return g, nil - } - switch len(c.Evals) { - case 0: - return nil, fmt.Errorf("no evals are declared in the config") - case 1: - return &c.Evals[0], nil - default: - names := make([]string, 0, len(c.Evals)) - for _, g := range c.Evals { - names = append(names, g.Name) +// CustomEvaluators are the evaluators this config owns — the referenced ones +// carrying a local source, which are published before the eval that names them. +// A built-in needs nothing, and one without a source is already registered. +func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { + var owned []EvaluatorDecl + for _, ref := range c.Evaluators { + if ref.IsBuiltin() || ref.Source == "" { + continue } - return nil, fmt.Errorf( - "the config declares %d evals (%s); choose one with --eval", - len(c.Evals), strings.Join(names, ", ")) + owned = append(owned, EvaluatorDecl{ + Name: ref.Name, + Source: ref.Source, + Version: ref.Version, + }) } + return owned } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 76a5a0cc96b..03e58c5cf35 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -14,38 +14,36 @@ import ( "go.yaml.in/yaml/v3" ) -const sampleDeployConfig = ` +// sampleEvalConfig is the shape the spec documents for evals/.yaml. +const sampleEvalConfig = ` +description: Quality gate for the support agent + +dataset: + name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + evaluators: + - builtin.task_adherence - name: support-quality - source: ./evaluators/support-quality/rubric_dimensions.json - - name: safety-check - source: ./evaluators/safety-check.json - -datasets: - - name: support-golden - source: ./datasets/support-golden.jsonl - version: "1" - -evals: - - name: pr-gate - description: Quality gate for the support agent - dataset: support-golden - evaluators: - - builtin.task_adherence - - { name: support-quality, threshold: 4.0 } - - safety-check - target: - type: agent - name: support-agent - options: - eval_model: gpt-4.1-nano - max_samples: 100 - evaluation_level: conversation + source: ./evaluators/support-quality.json + threshold: 4.0 + initialization_parameters: + deployment_name: gpt-4.1-nano + - safety-check + +target: + type: agent + name: support-agent + +options: + max_samples: 100 + evaluation_level: conversation ` func loadFromString(t *testing.T, body string) *EvalConfig { t.Helper() - path := filepath.Join(t.TempDir(), "azure.yaml") + path := filepath.Join(t.TempDir(), "support-agent-smoke.yaml") require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) cfg, err := LoadEvalConfig(path) require.NoError(t, err) @@ -53,45 +51,66 @@ func loadFromString(t *testing.T, body string) *EvalConfig { } func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { - cfg := loadFromString(t, sampleDeployConfig) - - require.Len(t, cfg.Evaluators, 2) - require.Len(t, cfg.Datasets, 1) - require.Len(t, cfg.Evals, 1) - - ds, ok := cfg.Dataset("support-golden") - require.True(t, ok) - require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) - require.Equal(t, "1", ds.Version) - - g, ok := cfg.Group("pr-gate") - require.True(t, ok) - require.Equal(t, "support-golden", g.Dataset) - require.Equal(t, TargetTypeAgent, g.Target.Type) - require.Equal(t, "support-agent", g.Target.Name) - require.Equal(t, EvaluationLevelConversation, g.Options.EvaluationLevel) + cfg := loadFromString(t, sampleEvalConfig) + + require.NotNil(t, cfg.Dataset) + require.Equal(t, "support-golden", cfg.Dataset.Name) + require.Equal(t, "./datasets/support-golden.jsonl", cfg.Dataset.Source) + require.Equal(t, "1", cfg.Dataset.Version) + + require.Len(t, cfg.Evaluators, 3) + require.Equal(t, TargetTypeAgent, cfg.Target.Type) + require.Equal(t, "support-agent", cfg.Target.Name) + require.Equal(t, EvaluationLevelConversation, cfg.Options.EvaluationLevel) + require.Equal(t, 100, cfg.Options.MaxSamples) +} + +// The eval takes its name from the service entry that pulled the file in, so +// the body never repeats it. +func TestEval_TakesNameFromTheService(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval := cfg.Eval("support-agent-smoke") + require.Equal(t, "support-agent-smoke", eval.Name) + require.Equal(t, "support-golden", eval.Dataset) + require.Equal(t, "Quality gate for the support agent", eval.Description) + require.Len(t, eval.Evaluators, 3) + require.Same(t, cfg.Target, eval.Target) } -// Evaluator entries accept a bare string or a mapping carrying a threshold. +// Only the referenced evaluators carrying a local source are this config's to +// publish. A built-in needs nothing, and one without a source already exists. +func TestCustomEvaluators_OnlyOwnsLocalSources(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + owned := cfg.CustomEvaluators() + require.Len(t, owned, 1) + require.Equal(t, "support-quality", owned[0].Name) + require.Equal(t, "./evaluators/support-quality.json", owned[0].Source) +} + +// Evaluator entries accept a bare string or a mapping carrying the rest of the +// declaration. func TestEvaluatorList_MixedForms(t *testing.T) { - cfg := loadFromString(t, sampleDeployConfig) - g, ok := cfg.Group("pr-gate") - require.True(t, ok) - require.Len(t, g.Evaluators, 3) - - require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) - require.True(t, g.Evaluators[0].IsBuiltin()) - require.Equal(t, "task_adherence", g.Evaluators[0].APIName(), + cfg := loadFromString(t, sampleEvalConfig) + require.Len(t, cfg.Evaluators, 3) + + require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) + require.True(t, cfg.Evaluators[0].IsBuiltin()) + require.Equal(t, "task_adherence", cfg.Evaluators[0].APIName(), "the builtin prefix must be stripped before it reaches the service") - require.Nil(t, g.Evaluators[0].Threshold) + require.Nil(t, cfg.Evaluators[0].Threshold) - require.Equal(t, "support-quality", g.Evaluators[1].Name) - require.False(t, g.Evaluators[1].IsBuiltin()) - require.NotNil(t, g.Evaluators[1].Threshold) - require.InDelta(t, 4.0, *g.Evaluators[1].Threshold, 0.0001) + require.Equal(t, "support-quality", cfg.Evaluators[1].Name) + require.False(t, cfg.Evaluators[1].IsBuiltin()) + require.NotNil(t, cfg.Evaluators[1].Threshold) + require.InDelta(t, 4.0, *cfg.Evaluators[1].Threshold, 0.0001) + require.Equal(t, "gpt-4.1-nano", + cfg.Evaluators[1].InitializationParameters["deployment_name"], + "the judge deployment is declared per evaluator, not once per eval") - require.Equal(t, "safety-check", g.Evaluators[2].Name) - require.Nil(t, g.Evaluators[2].Threshold) + require.Equal(t, "safety-check", cfg.Evaluators[2].Name) + require.Nil(t, cfg.Evaluators[2].Threshold) } // Round-tripping must not rewrite bare names into mappings. @@ -115,8 +134,27 @@ func TestEvaluatorList_RoundTripKeepsCompactForm(t *testing.T) { "an evaluator with only a name should stay a plain string") } +// An evaluator carrying a source must not be flattened to its name, or the +// declaration that says what to publish is lost on the next write. +func TestEvaluatorList_RoundTripKeepsSource(t *testing.T) { + list := evalcore.EvaluatorList{ + {Name: "support-quality", Source: "./evaluators/support-quality.json"}, + {Name: "builtin.task_adherence", + InitializationParameters: map[string]any{"deployment_name": "gpt-4.1-nano"}}, + } + + out, err := yaml.Marshal(list) + require.NoError(t, err) + + var back evalcore.EvaluatorList + require.NoError(t, yaml.Unmarshal(out, &back)) + require.Len(t, back, 2) + require.Equal(t, "./evaluators/support-quality.json", back[0].Source) + require.Equal(t, "gpt-4.1-nano", back[1].InitializationParameters["deployment_name"]) +} + func TestValidate_Accepts(t *testing.T) { - require.NoError(t, loadFromString(t, sampleDeployConfig).Validate()) + require.NoError(t, loadFromString(t, sampleEvalConfig).Validate()) } func TestValidate_Rejects(t *testing.T) { @@ -126,42 +164,41 @@ func TestValidate_Rejects(t *testing.T) { wantErr string }{ { - name: "dataset referenced but not declared", - body: "evals:\n - name: g\n dataset: missing\n evaluators: [builtin.relevance]\n", - wantErr: "is not declared in datasets", + name: "dataset without a name", + body: "dataset:\n source: ./d.jsonl\nevaluators: [builtin.relevance]\n", + wantErr: "'name' is required", }, { - name: "custom evaluator referenced but not declared", - body: "datasets:\n - name: d\n" + - "evals:\n - name: g\n dataset: d\n evaluators: [not-declared]\n", - wantErr: "is not declared in evaluators", + name: "no evaluators", + body: "evaluators: []\n", + wantErr: "at least one evaluator is required", }, { - name: "built-in declared as a custom evaluator", - body: "evaluators:\n - name: builtin.relevance\n", - wantErr: "must not be declared", + name: "built-in with a source to publish", + body: "evaluators:\n - name: builtin.relevance\n source: ./x.json\n", + wantErr: "has no source to publish", }, { - name: "group without evaluators", - body: "evals:\n - name: g\n evaluators: []\n", - wantErr: "at least one evaluator is required", + name: "duplicate evaluator", + body: "evaluators: [builtin.relevance, builtin.relevance]\n", + wantErr: "duplicate evaluator name", + }, + { + name: "version pinned alongside a source", + body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n", + wantErr: "cannot be set with `source`", }, { name: "unsupported target type", - body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n target:\n type: prompt\n", + body: "evaluators: [builtin.relevance]\ntarget:\n type: prompt\n", wantErr: "is not supported", }, { name: "invalid evaluation level", - body: "evals:\n - name: g\n evaluators: [builtin.relevance]\n" + - " options:\n evaluation_level: sentence\n", + body: "evaluators: [builtin.relevance]\n" + + "options:\n evaluation_level: sentence\n", wantErr: "evaluation_level", }, - { - name: "duplicate dataset", - body: "datasets:\n - name: d\n - name: d\n", - wantErr: "duplicate dataset name", - }, } for _, tc := range cases { @@ -173,55 +210,73 @@ func TestValidate_Rejects(t *testing.T) { } } -func TestResolveGroup(t *testing.T) { - single := loadFromString(t, sampleDeployConfig) +// One file is one eval, named after the file, so the directory listing is the +// list of evals a project declares. +func TestResolveEvalConfigPath(t *testing.T) { + write := func(t *testing.T, dir string, names ...string) { + t.Helper() + for _, n := range names { + require.NoError(t, os.WriteFile( + filepath.Join(dir, n), []byte("evaluators: [builtin.relevance]\n"), 0o600)) + } + } + + t.Run("the only eval is used when unnamed", func(t *testing.T) { + dir := t.TempDir() + write(t, dir, "pr-gate.yaml", "generate.yaml") - t.Run("only group is used when unnamed", func(t *testing.T) { - g, err := single.ResolveGroup("") + path, err := ResolveEvalConfigPath(dir, "") require.NoError(t, err) - require.Equal(t, "pr-gate", g.Name) + require.Equal(t, filepath.Join(dir, "pr-gate.yaml"), path, + "the generation spec shares the directory and is not an eval") }) - t.Run("named group", func(t *testing.T) { - g, err := single.ResolveGroup("pr-gate") + t.Run("named eval", func(t *testing.T) { + dir := t.TempDir() + write(t, dir, "pr-gate.yaml", "nightly.yaml") + + path, err := ResolveEvalConfigPath(dir, "nightly") require.NoError(t, err) - require.Equal(t, "pr-gate", g.Name) + require.Equal(t, filepath.Join(dir, "nightly.yaml"), path) }) t.Run("unknown name is an error", func(t *testing.T) { - _, err := single.ResolveGroup("nope") + dir := t.TempDir() + write(t, dir, "pr-gate.yaml") + + _, err := ResolveEvalConfigPath(dir, "nope") require.ErrorContains(t, err, "is not declared") }) t.Run("ambiguous without a name", func(t *testing.T) { - multi := loadFromString(t, - "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n"+ - " - name: nightly\n evaluators: [builtin.relevance]\n") - _, err := multi.ResolveGroup("") + dir := t.TempDir() + write(t, dir, "pr-gate.yaml", "nightly.yaml") + + _, err := ResolveEvalConfigPath(dir, "") require.ErrorContains(t, err, "--eval") require.ErrorContains(t, err, "nightly") }) - t.Run("empty config", func(t *testing.T) { - _, err := (&EvalConfig{}).ResolveGroup("") + t.Run("empty directory", func(t *testing.T) { + _, err := ResolveEvalConfigPath(t.TempDir(), "") require.ErrorContains(t, err, "no evals") }) } -// local_dir accepts a directory or an explicit file path. +// outputDir accepts a directory or an explicit file path. func TestArtifactPath(t *testing.T) { cases := []struct { - name string - localDir string - resource string - ext string - want string + name string + outputDir string + resource string + ext string + want string }{ {"directory derives the file name", "datasets", "support-golden", ".jsonl", filepath.Join("base", "datasets", "support-golden.jsonl")}, {"explicit file path is used as-is", "generated/datasets/support-golden.jsonl", "ignored", ".jsonl", filepath.Join("base", "generated", "datasets", "support-golden.jsonl")}, - {"empty local_dir falls back to the base", "", "support-quality", ".json", + {"empty outputDir falls back to the base", "", "support-quality", ".json", filepath.Join("base", "support-quality.json")}, {"yaml rubric file path", "generated/rubrics/quality.yaml", "ignored", ".json", filepath.Join("base", "generated", "rubrics", "quality.yaml")}, @@ -229,7 +284,7 @@ func TestArtifactPath(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, ArtifactPath("base", tc.localDir, tc.resource, tc.ext)) + require.Equal(t, tc.want, ArtifactPath("base", tc.outputDir, tc.resource, tc.ext)) }) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 1c41c92f9cf..7ab28e5d7ac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -12,70 +12,82 @@ import ( "go.yaml.in/yaml/v3" ) -// Conventional locations. Both are relative to the working directory and are +// Conventional locations. All are relative to the working directory and are // used verbatim — never re-rooted under the agent or project directory. const ( DefaultEvalDir = "evals" - DefaultGenerateConfig = "evals/eval_generate.yaml" - DefaultDeployConfig = "evals/azure.yaml" + DefaultGenerateConfig = "evals/generate.yaml" DefaultDatasetsDir = "datasets" DefaultEvaluatorsDir = "evaluators" ) -// GenerateConfig is the generation spec — input to `azd ai eval generate`. It -// is never deployed. -type GenerateConfig struct { - Agent AgentSpec `yaml:"agent" json:"agent"` - Generate GenerateSpec `yaml:"generate" json:"generate"` -} - -// AgentSpec identifies the agent and the context the generator reads. -type AgentSpec struct { - Name string `yaml:"name" json:"name"` - Context AgentContext `yaml:"context,omitempty" json:"context,omitempty"` +// EvalConfigPath is where the body of the eval named by a service entry lives. +func EvalConfigPath(evalDir, evalName string) string { + return filepath.Join(evalDir, evalName+".yaml") } -// AgentContext points at the material used to synthesize a rubric and dataset. -type AgentContext struct { - Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - Traces *TraceSpec `yaml:"traces,omitempty" json:"traces,omitempty"` -} - -// TraceSpec seeds rubric generation from recent traces. Traces are a generation -// input only; they cannot be a run's data source. -type TraceSpec struct { - Source string `yaml:"source,omitempty" json:"source,omitempty"` - Window string `yaml:"window,omitempty" json:"window,omitempty"` - Sample int `yaml:"sample,omitempty" json:"sample,omitempty"` +// GenerateConfig says how the local dataset and evaluator artifacts referenced +// by an eval are produced. It is never deployed. +// +// generationModel: gpt-5.6-luna +// dataset: +// support-agent-smoke: +// sampleSize: 15 +// outputDir: ./datasets +// evaluator: +// support-quality: +// outputDir: ./evaluators +// deriveFrom: support-agent +// +// The maps are keyed by artifact name so `dataset generate ` and +// `evaluator generate ` each look up exactly the entry they were asked +// for, and generating one artifact never reads the other's settings. +type GenerateConfig struct { + GenerationModel string `yaml:"generationModel,omitempty" json:"generationModel,omitempty"` + Dataset map[string]DatasetGenSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Evaluator map[string]EvaluatorGenSpec `yaml:"evaluator,omitempty" json:"evaluator,omitempty"` } -// GenerateSpec configures what gets produced. -type GenerateSpec struct { - Rubric *RubricSpec `yaml:"rubric,omitempty" json:"rubric,omitempty"` - Dataset *DatasetSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` +// DatasetGenSpec configures synthetic dataset generation for one dataset. +type DatasetGenSpec struct { + SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` + OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` + // DeriveFrom names the agent whose context seeds generation. Optional: the + // eval's target supplies it, and --target overrides both. + DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` + // Instructions points at a local file whose contents stand in for the + // agent's published instructions for this generation only. + Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` + // TraceDays seeds generation from that many days of recent traces. Zero + // disables it. Traces are a generation input only; they cannot be a run's + // data source. + TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` } -// RubricSpec configures rubric (LLM-graded evaluator) generation. -type RubricSpec struct { - Name string `yaml:"name" json:"name"` - Model string `yaml:"model,omitempty" json:"model,omitempty"` - LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +// EvaluatorGenSpec configures rubric generation for one evaluator. +type EvaluatorGenSpec struct { + OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` + // DeriveFrom names the agent the rubric is written against. + DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` + // Instructions points at a local file whose contents stand in for the + // agent's published instructions for this generation only. + Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` + // TraceDays seeds generation from that many days of recent traces. Zero + // disables it. + TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` } -// DatasetSpec configures synthetic dataset generation. -type DatasetSpec struct { - Name string `yaml:"name" json:"name"` - Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"` - SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` - LocalDir string `yaml:"local_dir,omitempty" json:"local_dir,omitempty"` +// ArtifactRef is the name/source pair a generation run produces, so the +// command can tell the developer how to reference it. +// +// Generation writes artifacts only and never edits azure.yaml or the eval +// config: `init` declares the paths and `generate` fills them in, which is what +// keeps a generation run a data-file-only diff. +type ArtifactRef struct { + Name string `json:"name"` + Source string `json:"source"` } -// Generation strategies. -const ( - StrategySynthetic = "synthetic" - StrategyFromTraces = "from-traces" -) - // Sample-count bounds enforced by the generation service. const ( MinSampleSize = 15 @@ -84,9 +96,16 @@ const ( ) // LoadGenerateConfig reads a generation spec from disk. +// +// A missing file is not an error. Generation is optional — a developer with +// hand-authored data and evaluators never writes one — and every setting it +// carries can be given on the command line instead. func LoadGenerateConfig(path string) (*GenerateConfig, error) { data, err := os.ReadFile(path) if err != nil { + if os.IsNotExist(err) { + return &GenerateConfig{}, nil + } return nil, fmt.Errorf("reading generation config %q: %w", path, err) } @@ -97,58 +116,43 @@ func LoadGenerateConfig(path string) (*GenerateConfig, error) { return &cfg, nil } -// Validate reports configuration errors before any generation job is submitted. -func (c *GenerateConfig) Validate() error { - if c.Agent.Name == "" { - return fmt.Errorf("agent.name is required") - } - if c.Generate.Rubric == nil && c.Generate.Dataset == nil { - return fmt.Errorf("generate must declare a rubric, a dataset, or both") - } - if r := c.Generate.Rubric; r != nil && r.Name == "" { - return fmt.Errorf("generate.rubric.name is required") - } - if d := c.Generate.Dataset; d != nil { - if d.Name == "" { - return fmt.Errorf("generate.dataset.name is required") - } - switch d.Strategy { - case "", StrategySynthetic: - case StrategyFromTraces: - // Accepting this and generating synthetic rows anyway would hand back - // data that looks nothing like what was asked for. The generation API - // takes one dataset strategy today; traces seed generation through - // the agent's context instead. - return fmt.Errorf( - "generate.dataset.strategy %q is not supported yet; "+ - "use %q, and set agent.context.traces.window to seed generation from traces", - StrategyFromTraces, StrategySynthetic) - default: - return fmt.Errorf( - "generate.dataset.strategy %q is invalid; expected %q", - d.Strategy, StrategySynthetic) - } - if d.SampleSize != 0 && (d.SampleSize < MinSampleSize || d.SampleSize > MaxSampleSize) { - return fmt.Errorf( - "generate.dataset.sampleSize must be between %d and %d, got %d", - MinSampleSize, MaxSampleSize, d.SampleSize) - } +// DatasetSpec returns the settings for one dataset, and whether the config +// declared them. +func (c *GenerateConfig) DatasetSpec(name string) (DatasetGenSpec, bool) { + spec, ok := c.Dataset[name] + return spec, ok +} + +// EvaluatorSpec returns the settings for one evaluator, and whether the config +// declared them. +func (c *GenerateConfig) EvaluatorSpec(name string) (EvaluatorGenSpec, bool) { + spec, ok := c.Evaluator[name] + return spec, ok +} + +// ValidateSampleSize rejects a row count the service would reject, before a +// generation job is submitted and billed. +func ValidateSampleSize(n int) error { + if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { + return fmt.Errorf( + "sample size must be between %d and %d, got %d", + MinSampleSize, MaxSampleSize, n) } return nil } -// ArtifactPath resolves a local_dir value against baseDir. The value may be a +// ArtifactPath resolves an outputDir value against baseDir. The value may be a // directory, in which case the file name is derived from resourceName and ext, // or an explicit file path, which is used as-is. -func ArtifactPath(baseDir, localDir, resourceName, ext string) string { - if localDir == "" { +func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { + if outputDir == "" { return filepath.Join(baseDir, resourceName+ext) } - candidate := localDir + candidate := outputDir if !filepath.IsAbs(candidate) { candidate = filepath.Join(baseDir, candidate) } - if looksLikeFile(localDir, ext) { + if looksLikeFile(outputDir, ext) { return candidate } return filepath.Join(candidate, resourceName+ext) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go index 227674b0fc4..5746b2286b6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go @@ -4,37 +4,65 @@ package project import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" ) -func generateCfgWithStrategy(strategy string) *GenerateConfig { - cfg := &GenerateConfig{} - cfg.Agent.Name = "my-agent" - cfg.Generate.Rubric = &RubricSpec{Name: "r"} - cfg.Generate.Dataset = &DatasetSpec{Name: "d", Strategy: strategy} - return cfg -} +// The generation spec is keyed by artifact name, which is what makes +// `dataset generate ` and `evaluator generate ` able to look up +// exactly the entry they were asked for. +func TestLoadGenerateConfig_ParsesTheDocumentedShape(t *testing.T) { + body := ` +generationModel: gpt-5.6-luna +dataset: + support-agent-smoke: + sampleSize: 15 + outputDir: ./datasets +evaluator: + support-quality: + outputDir: ./evaluators + deriveFrom: support-agent +` + path := filepath.Join(t.TempDir(), "generate.yaml") + require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) + + cfg, err := LoadGenerateConfig(path) + require.NoError(t, err) + require.Equal(t, "gpt-5.6-luna", cfg.GenerationModel) + + ds, ok := cfg.DatasetSpec("support-agent-smoke") + require.True(t, ok) + require.Equal(t, 15, ds.SampleSize) + require.Equal(t, "./datasets", ds.OutputDir) -// from-traces used to pass validation and then generate synthetic rows anyway, -// handing back data that looked nothing like what was asked for. Rejecting it -// is better than answering the wrong question. -func TestValidateRejectsUnsupportedDatasetStrategy(t *testing.T) { - err := generateCfgWithStrategy(StrategyFromTraces).Validate() - require.Error(t, err) - require.Contains(t, err.Error(), "not supported yet") - require.Contains(t, err.Error(), "agent.context.traces.window", - "the error should point at the way traces are actually used") + ev, ok := cfg.EvaluatorSpec("support-quality") + require.True(t, ok) + require.Equal(t, "./evaluators", ev.OutputDir) + require.Equal(t, "support-agent", ev.DeriveFrom) } -func TestValidateAcceptsSupportedDatasetStrategies(t *testing.T) { - require.NoError(t, generateCfgWithStrategy("").Validate()) - require.NoError(t, generateCfgWithStrategy(StrategySynthetic).Validate()) +// Generation is optional: a developer with hand-authored data and evaluators +// never writes a spec, and the generate commands still run from flags alone. +func TestLoadGenerateConfig_MissingFileIsNotAnError(t *testing.T) { + cfg, err := LoadGenerateConfig(filepath.Join(t.TempDir(), "generate.yaml")) + require.NoError(t, err) + require.Empty(t, cfg.GenerationModel) + require.Empty(t, cfg.Dataset) + + _, ok := cfg.DatasetSpec("anything") + require.False(t, ok) } -func TestValidateRejectsUnknownDatasetStrategy(t *testing.T) { - err := generateCfgWithStrategy("made-up").Validate() - require.Error(t, err) - require.Contains(t, err.Error(), "invalid") +// A row count the service would reject costs a billed job to find out about, +// so it is refused at the flag that carried it. +func TestValidateSampleSize(t *testing.T) { + require.NoError(t, ValidateSampleSize(0), "unset means the default applies") + require.NoError(t, ValidateSampleSize(MinSampleSize)) + require.NoError(t, ValidateSampleSize(MaxSampleSize)) + + require.ErrorContains(t, ValidateSampleSize(MinSampleSize-1), "must be between") + require.ErrorContains(t, ValidateSampleSize(MaxSampleSize+1), "must be between") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go deleted file mode 100644 index c8329356c23..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "fmt" - "os" - - "go.yaml.in/yaml/v3" -) - -// ArtifactRef is a name/source pair written back into the deployment spec after -// generation. -type ArtifactRef struct { - Name string - Source string -} - -// MergeArtifactRefs writes `source:` references for generated artifacts into -// the deployment spec, matching entries by name and appending when absent. -// -// It edits the document through the yaml Node API rather than round-tripping -// through structs, so comments, key order, and formatting survive. Only the -// `source` key of a matched entry is touched; anything the developer hand-edited -// is left alone. -func MergeArtifactRefs(path string, datasets, evaluators []ArtifactRef) error { - data, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("reading %q: %w", path, err) - } - - var doc yaml.Node - if err := yaml.Unmarshal(data, &doc); err != nil { - return fmt.Errorf("parsing %q: %w", path, err) - } - - root := documentRoot(&doc) - if root == nil { - return fmt.Errorf("%q is not a YAML mapping", path) - } - - if err := mergeSection(root, "datasets", datasets); err != nil { - return fmt.Errorf("%q: %w", path, err) - } - if err := mergeSection(root, "evaluators", evaluators); err != nil { - return fmt.Errorf("%q: %w", path, err) - } - - out, err := yaml.Marshal(&doc) - if err != nil { - return fmt.Errorf("serializing %q: %w", path, err) - } - if err := os.WriteFile(path, out, 0o600); err != nil { - return fmt.Errorf("writing %q: %w", path, err) - } - return nil -} - -// documentRoot unwraps the document node to the top-level mapping. -func documentRoot(doc *yaml.Node) *yaml.Node { - if doc.Kind == yaml.DocumentNode && len(doc.Content) > 0 { - doc = doc.Content[0] - } - if doc.Kind != yaml.MappingNode { - return nil - } - return doc -} - -// mergeSection updates or appends entries in a top-level sequence. -func mergeSection(root *yaml.Node, key string, refs []ArtifactRef) error { - if len(refs) == 0 { - return nil - } - - seq := findOrCreateSequence(root, key) - if seq == nil { - return fmt.Errorf("%q is present but is not a sequence", key) - } - - for _, ref := range refs { - if entry := findEntryByName(seq, ref.Name); entry != nil { - setMappingValue(entry, "source", ref.Source) - continue - } - seq.Content = append(seq.Content, newArtifactNode(ref)) - } - return nil -} - -// findOrCreateSequence returns the sequence node for key, creating an empty one -// when the key is absent. -func findOrCreateSequence(root *yaml.Node, key string) *yaml.Node { - for i := 0; i+1 < len(root.Content); i += 2 { - if root.Content[i].Value != key { - continue - } - value := root.Content[i+1] - if value.Kind == yaml.SequenceNode { - return value - } - // An explicit null is treated as an empty sequence. - if value.Tag == "!!null" { - value.Kind = yaml.SequenceNode - value.Tag = "!!seq" - value.Value = "" - return value - } - return nil - } - - keyNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key} - seqNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - root.Content = append(root.Content, keyNode, seqNode) - return seqNode -} - -// findEntryByName locates a mapping entry whose `name` matches. -func findEntryByName(seq *yaml.Node, name string) *yaml.Node { - for _, item := range seq.Content { - if item.Kind != yaml.MappingNode { - continue - } - if mappingValue(item, "name") == name { - return item - } - } - return nil -} - -// mappingValue reads a scalar value from a mapping node. -func mappingValue(node *yaml.Node, key string) string { - for i := 0; i+1 < len(node.Content); i += 2 { - if node.Content[i].Value == key { - return node.Content[i+1].Value - } - } - return "" -} - -// setMappingValue updates a scalar in place, or appends it when absent. Only -// the targeted key is touched. -func setMappingValue(node *yaml.Node, key, value string) { - for i := 0; i+1 < len(node.Content); i += 2 { - if node.Content[i].Value == key { - node.Content[i+1].Kind = yaml.ScalarNode - node.Content[i+1].Tag = "!!str" - node.Content[i+1].Value = value - return - } - } - node.Content = append(node.Content, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, - &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, - ) -} - -// newArtifactNode builds a fresh `{name, source}` entry. -func newArtifactNode(ref ArtifactRef) *yaml.Node { - return &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - Content: []*yaml.Node{ - {Kind: yaml.ScalarNode, Tag: "!!str", Value: "name"}, - {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Name}, - {Kind: yaml.ScalarNode, Tag: "!!str", Value: "source"}, - {Kind: yaml.ScalarNode, Tag: "!!str", Value: ref.Source}, - }, - } -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go deleted file mode 100644 index 56f159102dd..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/merge_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -const handAuthored = `# Eval deployment spec -# Edited by hand - comments must survive generate. -evaluators: - - name: safety-check # hand-authored - source: ./evaluators/safety-check.json - -datasets: - - name: support-golden - source: ./datasets/old.jsonl - version: "3" - -evals: - - name: pr-gate - dataset: support-golden - evaluators: - - builtin.task_adherence - - { name: safety-check, threshold: 4.0 } -` - -func writeTemp(t *testing.T, body string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "azure.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - return path -} - -// Regenerating must not destroy a hand-edited file. -func TestMergeArtifactRefs_PreservesCommentsAndSiblings(t *testing.T) { - path := writeTemp(t, handAuthored) - - require.NoError(t, MergeArtifactRefs(path, - []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}}, - []ArtifactRef{{Name: "support-quality", Source: "./evaluators/support-quality.json"}}, - )) - - out, err := os.ReadFile(path) - require.NoError(t, err) - text := string(out) - - require.Contains(t, text, "# Eval deployment spec", "leading comments must survive") - require.Contains(t, text, "# hand-authored", "inline comments must survive") - require.Contains(t, text, "./datasets/new.jsonl", "the matched source must be updated") - require.NotContains(t, text, "./datasets/old.jsonl", "the old source must be replaced") - require.Contains(t, text, "support-quality", "a new evaluator must be appended") - require.Contains(t, text, "safety-check", "existing entries must be kept") - - cfg, err := LoadEvalConfig(path) - require.NoError(t, err) - require.NoError(t, cfg.Validate()) - - ds, ok := cfg.Dataset("support-golden") - require.True(t, ok) - require.Equal(t, "./datasets/new.jsonl", ds.Source) - require.Equal(t, "3", ds.Version, "sibling keys must not be disturbed") - require.Len(t, cfg.Evaluators, 2) -} - -// The eval's evaluator list must be left exactly as written. -func TestMergeArtifactRefs_DoesNotTouchEvals(t *testing.T) { - path := writeTemp(t, handAuthored) - require.NoError(t, MergeArtifactRefs(path, nil, - []ArtifactRef{{Name: "support-quality", Source: "./evaluators/q.json"}})) - - cfg, err := LoadEvalConfig(path) - require.NoError(t, err) - g, ok := cfg.Group("pr-gate") - require.True(t, ok) - require.Len(t, g.Evaluators, 2) - require.Equal(t, "builtin.task_adherence", g.Evaluators[0].Name) - require.NotNil(t, g.Evaluators[1].Threshold) -} - -// Sections absent from the file are created rather than erroring. -func TestMergeArtifactRefs_CreatesMissingSections(t *testing.T) { - path := writeTemp(t, "evals:\n - name: pr-gate\n evaluators: [builtin.relevance]\n") - - require.NoError(t, MergeArtifactRefs(path, - []ArtifactRef{{Name: "d1", Source: "./datasets/d1.jsonl"}}, - []ArtifactRef{{Name: "e1", Source: "./evaluators/e1.json"}}, - )) - - cfg, err := LoadEvalConfig(path) - require.NoError(t, err) - require.Len(t, cfg.Datasets, 1) - require.Len(t, cfg.Evaluators, 1) - require.Equal(t, "d1", cfg.Datasets[0].Name) -} - -// Running generate twice must be idempotent. -func TestMergeArtifactRefs_IsIdempotent(t *testing.T) { - path := writeTemp(t, handAuthored) - refs := []ArtifactRef{{Name: "support-golden", Source: "./datasets/new.jsonl"}} - - require.NoError(t, MergeArtifactRefs(path, refs, nil)) - first, err := os.ReadFile(path) - require.NoError(t, err) - - require.NoError(t, MergeArtifactRefs(path, refs, nil)) - second, err := os.ReadFile(path) - require.NoError(t, err) - - require.Equal(t, string(first), string(second), - "merging the same references twice must not change the file") -} - -func TestFingerprint_DetectsChange(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "data.jsonl") - - require.NoError(t, os.WriteFile(path, []byte(`{"query":"a"}`), 0o600)) - first, err := Fingerprint(path) - require.NoError(t, err) - - again, err := Fingerprint(path) - require.NoError(t, err) - require.Equal(t, first, again, "unchanged content must hash the same") - - require.NoError(t, os.WriteFile(path, []byte(`{"query":"b"}`), 0o600)) - changed, err := Fingerprint(path) - require.NoError(t, err) - require.NotEqual(t, first, changed, "changed content must hash differently") -} - -func TestFingerprintKey_IsEnvSafe(t *testing.T) { - require.Equal(t, "EVAL_FINGERPRINT_DATASET_SUPPORT_GOLDEN", - FingerprintKey("dataset", "support-golden")) - require.Equal(t, "EVAL_FINGERPRINT_EVALUATOR_MY_EVAL_1", - FingerprintKey("evaluator", "my.eval-1")) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index f4902f236a3..6fd01d06680 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -145,21 +145,27 @@ func (p *EvalServiceTargetProvider) Deploy( baseDir := serviceRelativeDir(serviceConfig) - // 1. Datasets. + // The eval takes its name from the service entry that pulled this config + // in, which is what makes one service per eval work. + eval := cfg.Eval(serviceConfig.Name) + + // 1. Dataset. anyChanged := false - for _, decl := range cfg.Datasets { - report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) - localPath := resolveSource(baseDir, decl.Source) - version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) + datasetPath := "" + if cfg.Dataset != nil { + report(progress, fmt.Sprintf("Reconciling dataset %s", cfg.Dataset.Name)) + datasetPath = resolveSource(baseDir, cfg.Dataset.Source) + version, changed, err := reconciler.EnsureDataset(ctx, *cfg.Dataset, datasetPath) if err != nil { - return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) + return nil, fmt.Errorf("dataset %q: %w", cfg.Dataset.Name, err) } anyChanged = anyChanged || changed - report(progress, describeResult("dataset", decl.Name, version, changed)) + report(progress, describeResult("dataset", cfg.Dataset.Name, version, changed)) } - // 2. Evaluators. - for _, decl := range cfg.Evaluators { + // 2. Evaluators this config owns. Built-ins and already-registered ones + // need no publish. + for _, decl := range cfg.CustomEvaluators() { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) version, changed, err := reconciler.EnsureEvaluator(ctx, decl, localPath) @@ -170,20 +176,14 @@ func (p *EvalServiceTargetProvider) Deploy( report(progress, describeResult("evaluator", decl.Name, version, changed)) } - // 3. Evals. Groups are immutable, so a change upstream means a new - // group must be created and the stored id replaced. - for _, group := range cfg.Evals { - report(progress, fmt.Sprintf("Reconciling eval %s", group.Name)) - datasetPath := "" - if decl, ok := cfg.Dataset(group.Dataset); ok { - datasetPath = resolveSource(baseDir, decl.Source) - } - id, err := reconciler.EnsureEval(ctx, group, datasetPath, anyChanged) - if err != nil { - return nil, fmt.Errorf("eval %q: %w", group.Name, err) - } - report(progress, fmt.Sprintf("Eval %s is %s", group.Name, id)) + // 3. The eval. Evals are immutable, so a change upstream means a new one + // must be created and the stored id replaced. + report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) + id, err := reconciler.EnsureEval(ctx, eval, datasetPath, anyChanged) + if err != nil { + return nil, fmt.Errorf("eval %q: %w", eval.Name, err) } + report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) return &azdext.ServiceDeployResult{}, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index bd24e6c0030..987b2815c44 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -68,29 +68,25 @@ func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { // includes against. func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { svc := &azdext.ServiceConfig{ - Name: "evals", + Name: "support-agent-smoke", AdditionalProperties: propsFrom(t, map[string]any{ - "datasets": []any{ - map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, - }, - "evals": []any{ - map[string]any{ - "name": "quality", - "dataset": "golden", - "evaluators": []any{"builtin.task_adherence"}, - "target": map[string]any{"type": "agent", "name": "my-agent"}, - }, - }, + "dataset": map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + "evaluators": []any{"builtin.task_adherence"}, + "target": map[string]any{"type": "agent", "name": "my-agent"}, }), } cfg, err := EvalConfigFromService(svc, "") require.NoError(t, err) - require.Len(t, cfg.Datasets, 1) - require.Equal(t, "golden", cfg.Datasets[0].Name) - require.Len(t, cfg.Evals, 1) - require.Len(t, cfg.Evals[0].Evaluators, 1) - require.Equal(t, "builtin.task_adherence", cfg.Evals[0].Evaluators[0].Name) + require.NotNil(t, cfg.Dataset) + require.Equal(t, "golden", cfg.Dataset.Name) + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) + require.Equal(t, "my-agent", cfg.Target.Name) + + // The eval's name is the service key, which is what makes one service per + // eval work without the body repeating it. + require.Equal(t, "support-agent-smoke", cfg.Eval(svc.Name).Name) } func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { @@ -99,16 +95,16 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { require.Contains(t, err.Error(), "no eval configuration") } -// Groups are immutable, so a change to the group's own declaration has to be -// detectable. Upstream artifact fingerprints do not cover it: retargeting a -// group at a different agent leaves the dataset and evaluators untouched. +// Evals are immutable, so a change to the eval's own declaration has to be +// detectable. Upstream artifact fingerprints do not cover it: retargeting an +// eval at a different agent leaves the dataset and evaluators untouched. func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { base := Eval{ Name: "quality", Dataset: "golden", Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, Target: &Target{Type: "agent", Name: "agent-a"}, - Options: &Options{EvalModel: "gpt-4.1-nano"}, + Options: &Options{EvaluationLevel: EvaluationLevelTurn}, } original, err := FingerprintGroup(base) @@ -116,14 +112,20 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { same, err := FingerprintGroup(base) require.NoError(t, err) - require.Equal(t, original, same, "an unchanged group must keep its fingerprint") + require.Equal(t, original, same, "an unchanged eval must keep its fingerprint") cases := map[string]func(g *Eval){ "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, "evaluators": func(g *Eval) { g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) }, - "options": func(g *Eval) { g.Options = &Options{EvalModel: "gpt-4o-mini"} }, + "judge deployment": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Name: "builtin.task_adherence", + InitializationParameters: map[string]any{"deployment_name": "gpt-4o-mini"}, + }} + }, + "options": func(g *Eval) { g.Options = &Options{EvaluationLevel: EvaluationLevelConversation} }, "dataset": func(g *Eval) { g.Dataset = "other" }, } for name, mutate := range cases { diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go index 19112c2c95c..e6c97dffd9e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/evaluator_test.go @@ -84,7 +84,7 @@ func TestCLIInitNeedsAnAzdProject(t *testing.T) { r := requireFailure(t, runIn(t, dir, "init", "--target", "probe-agent", - "--judge-model", "gpt-4o-mini", + "--generation-model", "gpt-4o-mini", "--no-prompt")) require.Contains(t, r.Combined(), "azd init", diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index cc4eed3edac..71466a9b203 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -59,7 +59,6 @@ func TestCLIGenerateRefusesBadFlagCombinations(t *testing.T) { args: []string{"evaluator", "generate", "e", "--target", "a", "--agent-instruction", "inline"}, want: "--generation-model", }} - for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { r := requireFailure(t, runIn(t, dir, tc.args...)) @@ -80,37 +79,48 @@ func TestCLIGenerateNamesTheArtifact(t *testing.T) { } } -// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with no target and -// nothing to prompt with, the process has to end saying which flag to pass. +// TestCLIGenerateNoPromptNamesWhatIsMissing is the CI case: with nothing to +// prompt with, the process has to end saying which flag to pass. +// +// The target is no longer among them — it is read from the eval's declaration — +// but the generation model has no other source, so it is the one input a bare +// directory cannot supply. func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) - require.Contains(t, r.Combined(), "--target is required") - require.Contains(t, r.Combined(), "--no-prompt", - "the message must say why it could not be resolved") + require.Contains(t, r.Combined(), "--generation-model") + require.Contains(t, r.Combined(), "generationModel", + "the message must name both ways of supplying it") } -// TestCLIGenerateReadsTheSpec proves the config file is loaded and validated -// rather than only the flags. +// TestCLIGenerateReadsTheSpec proves the config file is loaded and that its +// entries are looked up by artifact name, rather than only the flags being read. // -// The strategy is the clearest evidence: `from-traces` is a value the spec -// accepts syntactically and the generation API cannot honour, so the refusal -// can only come from having parsed the file. +// The sample size is the clearest evidence: a value the service would reject is +// refused before any job is submitted, and it can only have come from the file. func TestCLIGenerateReadsTheSpec(t *testing.T) { dir := t.TempDir() - spec := filepath.Join(dir, "gen.yaml") + spec := filepath.Join(dir, "generate.yaml") require.NoError(t, os.WriteFile(spec, []byte(` -agent: - name: from-spec -generate: - dataset: - name: spec-dataset - strategy: from-traces +generationModel: gpt-4o-mini +dataset: + spec-dataset: + sampleSize: 5 `), 0o600)) + r := requireFailure(t, runIn(t, dir, "dataset", "generate", "spec-dataset", "--config", spec)) + require.Contains(t, r.Combined(), "between 15 and 1000", + "the spec's sampleSize must be validated before a job is billed") +} + +// A spec that cannot be parsed has to name itself, or the caller is left +// guessing which of several YAML files the command choked on. +func TestCLIGenerateReportsAnUnparseableSpec(t *testing.T) { + dir := t.TempDir() + spec := filepath.Join(dir, "generate.yaml") + require.NoError(t, os.WriteFile(spec, []byte("dataset: [not-a-mapping\n"), 0o600)) + r := requireFailure(t, runIn(t, dir, "dataset", "generate", "d", "--config", spec)) - require.Contains(t, r.Combined(), "from-traces") - require.Contains(t, r.Combined(), "agent.context.traces.window", - "the refusal must point at the field that does seed generation from traces") + require.Contains(t, r.Combined(), "generate.yaml") } // TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not From 0aa5ec5ca01a7be172d91fb1874b75a701923d53 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 19:40:43 +0300 Subject: [PATCH 082/114] Refuse a wrong flag before a missing one `dataset generate d --agent-instruction-file ` answered "a model deployment is required", naming a flag the caller had not touched and saying nothing about the one they had. Splitting the offline resolution out of `prepareGeneration` had moved the instruction-file read behind the model check, and both checks are local, so the only thing deciding which error the user sees is the order they run in. The instruction file is read in `resolvePlan` now. An input the caller named and got wrong is a typo they can act on; a missing generation model has a documented default path and is the weaker complaint. Caught by the live CLI suite. Pinned offline now too, since the ordering is exactly the kind of thing a refactor moves without noticing and a 500-second live run is a slow way to find out. --- .../internal/cmd/generate_commands.go | 39 +++++++++++-------- .../internal/cmd/generate_plan_test.go | 15 +++++++ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 6a22d4fcbed..9e8c5a33dac 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -52,22 +52,18 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") } -// prepareGeneration resolves everything both commands need before they diverge. +// prepareGeneration builds the client and settles the one input that needs it. // -// The model check happens here rather than at the service, because a generation -// job is billed against a deployment and a rejection partway through the -// command says less than a refusal at the flag that caused it. +// Everything decidable offline is already on the plan by this point, so a +// mistake in the flags has been reported without an authentication round trip. +// What is left is the generation instruction's last fallback: the agent's +// published instructions, which only the service can supply. func prepareGeneration( cmd *cobra.Command, f *generateFlags, plan generationPlan, declared genEntry, ) (*evalContext, generationPlan, error) { - instruction, err := resolveInstruction(f.instruction, f.instructionFile) - if err != nil { - return nil, plan, err - } - ctx := cmd.Context() ec, err := newEvalContext(ctx, f.endpoint) if err != nil { @@ -75,7 +71,7 @@ func prepareGeneration( } plan.Instruction, err = ec.resolveGenerationInstruction( - ctx, instruction, declared.instructions, f.configPath, plan.Agent, + ctx, plan.Instruction, declared.instructions, f.configPath, plan.Agent, cmd.OutOrStdout(), isJSON(cmd), ) if err != nil { @@ -91,20 +87,29 @@ func prepareGeneration( // generation spec, then what can be detected from the eval configuration. // Doing it before the client is built means a missing model or an out-of-range // sample count is refused without an authentication round trip. +// +// The instruction file is read here rather than later so that an input the +// caller named and got wrong is reported ahead of one they simply left out. func resolvePlan( f *generateFlags, cfg *project.GenerateConfig, name string, declared genEntry, ) (generationPlan, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return generationPlan{}, err + } + plan := generationPlan{ - Name: name, - Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), - Model: firstNonEmpty(f.model, cfg.GenerationModel), - BaseDir: filepath.Dir(f.configPath), - OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), - SampleSize: declared.sampleSize, - TraceDays: declared.traceDays, + Name: name, + Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), + Model: firstNonEmpty(f.model, cfg.GenerationModel), + Instruction: instruction, + BaseDir: filepath.Dir(f.configPath), + OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), + SampleSize: declared.sampleSize, + TraceDays: declared.traceDays, } if plan.Model == "" { return plan, fmt.Errorf( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go index c018ddda8ac..9e4f9bd3e5d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -69,6 +69,21 @@ func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { require.Contains(t, err.Error(), "generationModel") } +// An input the caller named and got wrong is reported ahead of one they simply +// left out. Both checks are local, so the only thing deciding which the user +// sees is the order they run in — and a missing instruction file is a typo the +// caller can act on, while the model has a documented default path. +func TestResolvePlan_ReportsABadExplicitInputFirst(t *testing.T) { + f := evalsDir(t, "", nil) + f.target = "shop-agent" + f.instructionFile = filepath.Join(t.TempDir(), "absent.md") + + _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) + require.Error(t, err) + require.Contains(t, err.Error(), "--agent-instruction-file", + "the flag the caller got wrong must win over the one they omitted") +} + // The spec is read per artifact name, so generating one artifact never picks up // the other's settings. func TestResolvePlan_ReadsTheNamedSpecEntry(t *testing.T) { From 9bf1505e4b1829eec957f77aef0495a8bcb32abc Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 20:02:04 +0300 Subject: [PATCH 083/114] Drive the hero scenarios through real azd `init` is the first command in Scenario 1 and the one that produces the local diff every later step depends on, and nothing was testing it end to end. It resolves the project and edits azure.yaml over azd's gRPC channel, so the CLI suite -- which runs the extension binary directly -- cannot reach it: with no azd on the other end the command refuses before it does anything. What covered it instead was a unit test calling the scaffold function, which cannot see the service entry azd writes, the detection that reads the project, or the terminal output the spec pins line for line. tests/hero runs the extension the way a user installs it. Scenario 1's output is compared whole rather than by keyword: every line is a promise the spec makes to a reader deciding whether to adopt this, and a keyword assertion would pass while their terminal said something else. It found one: with `--dataset prod-golden` -- a dataset that already exists -- `init` still closed with "Next: azd ai eval dataset generate prod-golden", sending the reader to submit a billed generation job for an artifact they had just supplied. Next steps are now built from what was actually scheduled, and when everything is already in place they point at `azd up` instead. Two properties are asserted that only this harness can see: that `init` makes no service calls, by running it behind a proxy pointing nowhere; and that a second `init` refuses without having edited the project, then under --force leaves exactly one eval service rather than a duplicate that would deploy the same eval twice. TestMain fails rather than skips, and rejects a stale install. A suite that silently reports on a binary other than the one under test is worse than no suite, and `azd x pack` leaves the local registry's checksum alone when the version has not changed, so a stale install is the easy mistake to make. The guard builds the working tree and compares help output, which is the cheapest fingerprint that actually moves with the code. Verified by making it fail. --- .../azure.ai.evaluations/internal/cmd/init.go | 31 +- .../internal/cmd/init_test.go | 29 ++ .../tests/hero/init_test.go | 348 ++++++++++++++++++ 3 files changed, 405 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index f9cbbbddd86..6e17b5f2806 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -128,9 +128,13 @@ func newInitCommand() *cobra.Command { fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, evalName) } - fmt.Fprintf(out, "\nNext: azd ai eval dataset generate %s\n", plan.datasetName) - if plan.rubricCount() > 0 { - fmt.Fprintf(out, " azd ai eval evaluator generate %s\n", rubricName) + // Only what was actually scheduled is offered. Suggesting + // `dataset generate` for a dataset the caller supplied sends them + // to submit a billed job for an artifact they already have. + next := plan.nextSteps() + fmt.Fprintf(out, "\nNext: %s\n", next[0]) + for _, step := range next[1:] { + fmt.Fprintf(out, " %s\n", step) } return nil }, @@ -331,6 +335,27 @@ func (s scaffold) rubricCount() int { return len(s.generate.Evaluator) } +// nextSteps are the commands to run after `init`, and only the ones that have +// something to do. +// +// A caller who supplied both a dataset and their evaluators has nothing left to +// generate, and pointing them at a generation command would submit a billed job +// for an artifact they already have. With everything in place the next step is +// to deploy it. +func (s scaffold) nextSteps() []string { + var steps []string + if s.generate != nil && len(s.generate.Dataset) > 0 { + steps = append(steps, "azd ai eval dataset generate "+s.datasetName) + } + if s.rubricCount() > 0 { + steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) + } + if len(steps) == 0 { + steps = append(steps, "azd up", "azd ai eval run start") + } + return steps +} + // relativeToConfig rewrites a path given relative to the working directory so // it resolves from the directory holding the eval config. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 40c92cf0855..b391dd908c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -65,6 +65,35 @@ func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { require.Empty(t, plan.generate.Evaluator) } +// `init` closes by naming what to run next, and only what has something to do. +// Pointing a caller who supplied their own artifacts at a generation command +// would submit a billed job for something they already have. +func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { + t.Run("nothing supplied", func(t *testing.T) { + plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", + "", nil, "m", project.DefaultEvalDir) + require.Equal(t, []string{ + "azd ai eval dataset generate support-agent-smoke", + "azd ai eval evaluator generate support-agent-quality", + }, plan.nextSteps()) + }) + + t.Run("dataset supplied", func(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "support-agent-quality", + "prod-golden", nil, "m", project.DefaultEvalDir) + require.Equal(t, + []string{"azd ai eval evaluator generate support-agent-quality"}, + plan.nextSteps()) + }) + + t.Run("everything supplied", func(t *testing.T) { + plan := planScaffold("smoke", "support-agent", "support-agent-quality", + "prod-golden", []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + require.Equal(t, []string{"azd up", "azd ai eval run start"}, plan.nextSteps(), + "with every artifact in place the next step is to deploy") + }) +} + func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() genPath := filepath.Join(dir, "generate.yaml") diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go new file mode 100644 index 00000000000..0618e49ccd8 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -0,0 +1,348 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build hero + +// Package hero drives the hero scenarios through real azd, with the extension +// installed the way a user installs it. +// +// The CLI suite in ../cli runs the extension binary directly, which covers the +// command surface but cannot reach `init`: `init` resolves the project and +// edits azure.yaml over azd's gRPC channel, so without azd hosting the process +// there is nothing on the other end. That is not a detail — it is the first +// command in Scenario 1 and the one that produces the local diff every later +// step depends on, and until now the only thing asserting its output was a +// unit test calling the scaffold function directly. A unit test cannot see the +// service entry azd writes, the detection that reads the project, or the +// terminal output the spec pins line for line. +// +// azd x pack --rebuild +// azd extension install azure.ai.evaluations --source local +// go test -tags hero -v ./tests/hero/... +package hero + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// reinstall is what to run when the installed extension is not this code. +// +// `azd x pack` rewrites the artifacts but leaves the checksum in the local +// registry alone when the version has not changed, so a plain reinstall then +// fails validation. Bumping the version in extension.yaml is the way through. +const reinstall = " azd x pack --rebuild\n" + + " azd extension uninstall azure.ai.evaluations\n" + + " azd extension install azure.ai.evaluations --source local\n" + +// TestMain refuses to run against an azd that cannot reach the extension, or +// that is hosting a different build of it. +// +// Skipping would be worse than failing here: these tests exist because nothing +// else covers the azd-hosted path, so a silent skip returns the suite to the +// state it was in before they were written. Running against a stale install is +// worse still — it reports on code that is not the code under test, which is +// the one outcome a test must never produce. +func TestMain(m *testing.M) { + if os.Getenv("AZURE_AI_EVAL_HERO") != "1" { + fmt.Fprintf(os.Stderr, + "set AZURE_AI_EVAL_HERO=1 to run the hero scenarios. They need azd "+ + "hosting this extension:\n%s", reinstall) + os.Exit(0) + } + + hosted, err := exec.Command("azd", "ai", "eval", "init", "--help").CombinedOutput() + if err != nil || !strings.Contains(string(hosted), "Scaffold evaluation config") { + fmt.Fprintf(os.Stderr, + "azd cannot reach the evaluations extension. Install it first:\n%s\n%s\n", + reinstall, hosted) + os.Exit(1) + } + + if err := requireCurrentInstall(string(hosted)); err != nil { + fmt.Fprintf(os.Stderr, "%v\n\n%s", err, reinstall) + os.Exit(1) + } + + os.Exit(m.Run()) +} + +// requireCurrentInstall compares the installed extension's help against this +// working tree's, so a stale install fails loudly instead of quietly reporting +// on the wrong binary. +// +// Help text is the cheapest available fingerprint that actually moves: it +// carries every command and flag, which is what these tests assert on, and it +// costs one build rather than a version stamp nobody remembers to bump. +func requireCurrentInstall(hosted string) error { + dir, err := os.MkdirTemp("", "azdeval-hero") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + binary := filepath.Join(dir, "azdeval"+exeSuffix()) + build := exec.Command("go", "build", "-o", binary, ".") + build.Dir = "../.." + if out, err := build.CombinedOutput(); err != nil { + return fmt.Errorf("building this working tree to compare against: %v\n%s", err, out) + } + + local, err := exec.Command(binary, "init", "--help").CombinedOutput() + if err != nil { + return fmt.Errorf("reading this working tree's help: %w", err) + } + + if normalize(string(local)) != normalize(hosted) { + return fmt.Errorf( + "azd is hosting a different build of this extension.\n"+ + "installed:\n%s\nthis working tree:\n%s", + normalize(hosted), normalize(string(local))) + } + return nil +} + +func exeSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +// project writes a minimal azd project for `init` to attach to. +// +// It declares the two services detection reads — the Foundry project and the +// agent — because what `init` writes into azure.yaml depends on which of them +// exist, and a project with neither would exercise only the fallback. +func project(t *testing.T, agent string) string { + t.Helper() + dir := t.TempDir() + body := fmt.Sprintf(`name: support-app +services: + ai-project: + host: azure.ai.project + %s: + host: azure.ai.agent +`, agent) + require.NoError(t, os.WriteFile(filepath.Join(dir, "azure.yaml"), []byte(body), 0o600)) + return dir +} + +// azdEval runs the extension through azd, in dir. +func azdEval(t *testing.T, dir string, args ...string) (string, int) { + t.Helper() + + cmd := exec.Command("azd", append([]string{"ai", "eval"}, args...)...) + cmd.Dir = dir + var out strings.Builder + cmd.Stdout = &out + cmd.Stderr = &out + + code := 0 + if err := cmd.Run(); err != nil { + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("could not run azd ai eval %v: %v", args, err) + } + code = exitErr.ExitCode() + } + + // azd prints its own upgrade notice to stderr, which is not the command's + // output and would break an exact comparison. + text := dropUpgradeNotice(out.String()) + t.Logf("$ azd ai eval %s -> exit %d\n%s", strings.Join(args, " "), code, text) + return text, code +} + +// dropUpgradeNotice removes azd's "Update available" banner and everything +// after it, which azd appends regardless of the command. +func dropUpgradeNotice(s string) string { + if i := strings.Index(s, "Update available:"); i >= 0 { + s = s[:i] + } + return strings.TrimRight(s, " \r\n\t") +} + +// normalize makes terminal output comparable across platforms. +func normalize(s string) string { + return strings.ReplaceAll(dropUpgradeNotice(s), "\r\n", "\n") +} + +// TestHeroScenario1ColdStart is the first half of Scenario 1: the offline +// baseline `init` writes, asserted against the terminal block the spec shows. +// +// The output is compared whole rather than by keyword. Every line of it is a +// promise the spec makes to a reader deciding whether to adopt this — which +// files appear, what was detected, what to run next — and a keyword assertion +// would pass while the reader's terminal said something else. +func TestHeroScenario1ColdStart(t *testing.T) { + const ( + agent = "support-agent" + model = "gpt-5.6-luna" + ) + dir := project(t, agent) + + out, code := azdEval(t, dir, "init", "--target", agent, "--generation-model", model) + require.Zero(t, code, "init makes no service calls, so nothing can fail it here") + + want := `(✓) Done: Detected agent target: support-agent +(✓) Done: Detected model deployment: gpt-5.6-luna +(✓) Done: Planned evaluators: builtin.task_adherence, support-agent-quality (rubric) + +Created + evals/support-agent-smoke.yaml eval definition + evals/generate.yaml generation settings (15 samples, 1 rubric) + azure.yaml added service 'support-agent-smoke' + +Next: azd ai eval dataset generate support-agent-smoke + azd ai eval evaluator generate support-agent-quality` + + require.Equal(t, want, normalize(out)) +} + +// `init` is offline, and being offline is the property that makes its output a +// reviewable local diff. A service call here would also make the command fail +// for a user who has not authenticated yet, which is exactly when they run it. +func TestHeroInitMakesNoServiceCalls(t *testing.T) { + dir := project(t, "support-agent") + + cmd := exec.Command("azd", "ai", "eval", "init", + "--target", "support-agent", "--generation-model", "m") + cmd.Dir = dir + // A proxy pointing nowhere fails any outbound request, so a command that + // stays offline is unaffected and one that does not cannot be mistaken for + // working. + cmd.Env = append(os.Environ(), + "HTTPS_PROXY=http://127.0.0.1:9", + "HTTP_PROXY=http://127.0.0.1:9", + "NO_PROXY=", + ) + + out, err := cmd.CombinedOutput() + require.NoError(t, err, "init must not need the network:\n%s", out) +} + +// The eval service has to be declared in azure.yaml before azd will act on it. +// Printing the block and leaving the edit to the reader was enough, once, to +// make the documented flow stop working between `init` and `azd up`. +func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", "--target", "support-agent", "--generation-model", "m") + require.Zero(t, code) + + root, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + text := string(root) + + require.Contains(t, text, "support-agent-smoke:", + "the service key is the eval's name") + require.Contains(t, text, "host: azure.ai.eval") + require.Contains(t, text, "$ref: ./evals/support-agent-smoke.yaml") + + // azd owns the edit, so everything the project already declared survives it. + require.Contains(t, text, "name: support-app") + require.Contains(t, text, "host: azure.ai.project") + require.Contains(t, text, "host: azure.ai.agent") + + // The eval reads both, so azd has to deploy both first. + require.Regexp(t, `(?s)support-agent-smoke:.*uses:.*ai-project.*support-agent`, text) +} + +// Running `init` twice must not deploy the same eval twice. The service key is +// the eval's name, so the second run recognises its own work. +func TestHeroInitIsIdempotent(t *testing.T) { + dir := project(t, "support-agent") + args := []string{"init", "--target", "support-agent", "--generation-model", "m"} + + _, code := azdEval(t, dir, args...) + require.Zero(t, code) + first, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + + out, code := azdEval(t, dir, args...) + require.NotZero(t, code, "the scaffold already exists, so a second run must refuse") + require.Contains(t, out, "--force", "the refusal has to say how to proceed") + + second, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, string(first), string(second), + "a refused init must not have edited the project") + + // With --force the files are rewritten, and the service is still declared + // exactly once. + out, code = azdEval(t, dir, append(args, "--force")...) + require.Zero(t, code, out) + + third, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) + require.NoError(t, err) + require.Equal(t, 1, strings.Count(string(third), "host: azure.ai.eval"), + "a second eval service would deploy the same eval twice") + require.Contains(t, normalize(out), "already declares service 'support-agent-smoke'") +} + +// Evals attach to a project; they do not create one. Naming the command that +// makes a project is more use than a transport error from the gRPC channel +// that was not there. +func TestHeroInitNeedsAnAzdProject(t *testing.T) { + dir := t.TempDir() + + out, code := azdEval(t, dir, "init", "--target", "support-agent", "--no-prompt") + require.NotZero(t, code) + require.Contains(t, out, "azd init") + require.NotContains(t, strings.ToLower(out), "grpc", + "a missing project must not surface as a transport error") + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Empty(t, entries, "a refused init must leave nothing behind") +} + +// Passing --evaluator replaces the defaults, which is how a caller opts out of +// rubric generation — so the "next" steps must stop offering to generate one. +func TestHeroInitExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--generation-model", "m", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + + text := normalize(out) + require.Contains(t, text, "Planned evaluators: builtin.task_adherence") + require.NotContains(t, text, "(rubric)") + require.NotContains(t, text, "evaluator generate", + "nothing was scheduled to be generated, so nothing should be suggested") + require.Contains(t, text, "generation settings (15 samples, 0 rubric)") +} + +// A supplied dataset is not generated either, so `init` has nothing left to +// suggest and must not send the reader to a command that would submit a job +// for an artifact they already have. +func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { + dir := project(t, "support-agent") + + out, code := azdEval(t, dir, "init", + "--target", "support-agent", "--generation-model", "m", + "--dataset", "prod-golden", + "--evaluator", "builtin.task_adherence") + require.Zero(t, code, out) + require.NotContains(t, normalize(out), "dataset generate") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "generate.yaml")) + require.NoError(t, err) + require.NotContains(t, string(body), "dataset:", + "nothing is left to generate, so the spec declares nothing") + + eval, err := os.ReadFile(filepath.Join(dir, "evals", "support-agent-smoke.yaml")) + require.NoError(t, err) + require.Contains(t, string(eval), "name: prod-golden") + require.NotContains(t, string(eval), "source:", + "a registered dataset has nothing to upload") +} From e24dc7447989ee7a0da48628c74fe10125d1b5e3 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 20:48:39 +0300 Subject: [PATCH 084/114] Pin the command surface, and fix the flag it caught The flag that writes results to a file was `--out-file`. The spec, its Scenario 4, and `azd ai skill download` all say `--output-file`. It took reading the two documents side by side to notice, which is the wrong way to find something a user types from memory. The surface is a contract -- with the spec, and with the sibling Foundry extensions whose vocabulary this shares -- and nothing was checking it. So the tree and the flags are pinned: - Every command in the tree, so one that is renamed, dropped or quietly added has to be acknowledged against the spec's command table. - `init`'s flags whole, because they are a table in the spec: an extra flag there is a promise the spec does not make, a missing one is a promise it does. - The spellings this extension must not invent, including the ones the spec explicitly rules out (`--judge-model`) and the ones belonging to M2 (`--from-traces`, `--folder`, `--cron`), so a deferred surface cannot reappear by accident. - That `init` takes no `--project-endpoint` and every command that does reach the service takes one. Verified by renaming the flag back and watching both guards fail. --- .../internal/cmd/run_output.go | 4 +- .../internal/cmd/surface_test.go | 196 ++++++++++++++++++ .../tests/cli/run_output_test.go | 6 +- 3 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 0f7545fffd6..49a72a2efb7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -110,7 +110,7 @@ func newRunOutputListCommand() *cobra.Command { } cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") - cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write JSON results to this path.") + cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd @@ -225,7 +225,7 @@ func newRunOutputExportCommand() *cobra.Command { } cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") - cmd.Flags().StringVarP(&outFile, "out-file", "O", "", "Write to this path instead of stdout.") + cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") addEvalFlags(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go new file mode 100644 index 00000000000..516ee0d2520 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The command surface is a contract with the spec and with the sibling Foundry +// extensions, and it is the part of this tool users type from memory. Nothing +// was checking it: the flag that writes results to a file was `--out-file` +// while the spec, Scenario 4, and `azd ai skill download` all say +// `--output-file`, and it took reading the two documents side by side to see. +// +// These tests walk the built tree, so a command or flag that is renamed, +// dropped, or quietly added has to be acknowledged here. + +// walk visits every command in the tree, skipping the ones azd contributes. +func walk(t *testing.T, cmd *cobra.Command, path []string, visit func(string, *cobra.Command)) { + t.Helper() + for _, child := range cmd.Commands() { + name := strings.Fields(child.Use)[0] + switch name { + case "help", "completion", "listen", "metadata": + continue + } + full := append(append([]string{}, path...), name) + visit(strings.Join(full, " "), child) + walk(t, child, full, visit) + } +} + +// commandTree is every command the extension exposes, and is the surface the +// spec's command table describes. +func TestCommandTreeMatchesTheSpec(t *testing.T) { + want := []string{ + "dataset", + "dataset create", + "dataset delete", + "dataset generate", + "dataset list", + "dataset show", + "dataset update", + "dataset versions", + "dataset versions list", + "delete", + "evaluator", + "evaluator create", + "evaluator delete", + "evaluator generate", + "evaluator list", + "evaluator show", + "evaluator update", + "evaluator versions", + "evaluator versions list", + "init", + "job", + "job cancel", + "job list", + "job show", + "list", + "run", + "run cancel", + "run delete", + "run list", + "run output", + "run output export", + "run output list", + "run output show", + "run show", + "run start", + "show", + } + + var got []string + walk(t, NewRootCommand(), nil, func(path string, _ *cobra.Command) { + got = append(got, path) + }) + + assert.ElementsMatch(t, want, got, + "the command tree changed; update the spec's command table with it") +} + +// Flag names are shared vocabulary across the Foundry extensions. A command +// that invents its own spelling for something the others already name is the +// kind of difference nobody notices until a user types the one they learned +// somewhere else. +func TestFlagVocabularyIsShared(t *testing.T) { + // Meaning → the one spelling for it, from the spec's vocabulary table. + // A command that means one of these must use exactly this name, and the + // near-misses are listed so a rename back is caught rather than accepted. + forbidden := map[string]string{ + "--out-file": "--output-file", + "--out-dir": "--output-dir", + "--file": "--from-file", + "--rubric": "--from-file", + "--judge-model": "--generation-model, declared per evaluator instead", + "--from-traces": "deferred to M2", + "--response-id": "deferred to M2", + "--no-target": "deferred to M2", + "--out": "--output-file", + "--dir": "--output-dir", + "--baseline": "deferred to M2", + "--cron": "deferred to M2", + "--folder": "deferred to M2", + "--init-params": "deferred to M2", + "--data-schema": "deferred to M2", + "--metrics": "deferred to M2", + "--trace-window": "deferred to M2", + "--max-traces": "deferred to M2", + "--max-turns": "deferred to M2", + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if want, bad := forbidden["--"+f.Name]; bad { + t.Errorf("%s declares --%s; use %s", path, f.Name, want) + } + }) + }) +} + +// The two commands that write a file have to agree on what that flag is +// called, and it has to be the name the sibling extensions use. +func TestOutputFileFlagIsSpelledTheSharedWay(t *testing.T) { + for _, path := range []string{"run output list", "run output export"} { + cmd := find(t, path) + require.NotNil(t, cmd.Flags().Lookup("output-file"), + "%s must write to --output-file, the name `azd ai skill download` uses", path) + assert.Nil(t, cmd.Flags().Lookup("out-file"), + "%s must not keep the old spelling alongside the shared one", path) + } +} + +// `init` is the one command with a documented flag table, so it is pinned +// whole: an extra flag there is a promise the spec does not make, and a +// missing one is a promise it does. +func TestInitFlagsMatchTheSpec(t *testing.T) { + cmd := find(t, "init") + + var got []string + cmd.LocalFlags().VisitAll(func(f *pflag.Flag) { + if f.Name != "help" { + got = append(got, "--"+f.Name) + } + }) + + assert.ElementsMatch(t, []string{ + "--name", "--target", "--dataset", "--evaluator", + "--generation-model", "--output-dir", "--force", + }, got, "init's flags are a table in the spec; change both together") +} + +// `init` makes no service calls, so it must not offer the flag that says where +// to make them. +func TestInitTakesNoProjectEndpoint(t *testing.T) { + assert.Nil(t, find(t, "init").Flags().Lookup("project-endpoint"), + "init is offline; a project endpoint would imply otherwise") +} + +// Every command that does reach the service accepts it, because the shared +// Foundry resolver is how a project is named without an azd environment. +func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { + groups := map[string]bool{ + "dataset": true, "evaluator": true, "run": true, + "job": true, "run output": true, + } + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if cmd.RunE == nil || path == "init" { + return + } + if groups[path] { + return + } + assert.NotNil(t, cmd.Flags().Lookup("project-endpoint"), + "%s reaches the service, so it must accept --project-endpoint", path) + }) +} + +// find resolves a command path, failing the test when it does not exist. +func find(t *testing.T, path string) *cobra.Command { + t.Helper() + cmd, _, err := NewRootCommand().Find(strings.Fields(path)) + require.NoError(t, err, "no such command: %s", path) + require.Equal(t, strings.Fields(path)[len(strings.Fields(path))-1], + strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) + return cmd +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 9b9d71e9c91..4c688dd9976 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -179,14 +179,14 @@ func TestCLIResultsExport(t *testing.T) { require.Equal(t, f.EvaluatorName, rows[1][2]) }) - t.Run("out-file writes the path instead of stdout", func(t *testing.T) { + t.Run("output-file writes the path instead of stdout", func(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "results.csv") r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "csv", "-O", path)) + "--eval-id", f.EvalID, "--format", "csv", "--output-file", path)) require.Empty(t, strings.TrimSpace(r.Stdout), - "-O redirects the payload; leaving it on stdout too would double it") + "--output-file redirects the payload; leaving it on stdout too would double it") body, err := os.ReadFile(path) require.NoError(t, err) From 8dd23d0c3501a4091592b1ac5f567a9ec09d59f5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 21:02:57 +0300 Subject: [PATCH 085/114] Stop pointing users at a command that was renamed away `run start --no-wait` closed with "Check progress with: azd ai eval results show --run-id ". `results` became `run output` when the surface was rebuilt, so the one instruction printed at the exact moment a user needs it was the one thing guaranteed to fail. The no-previous-run error had the same problem, sending the reader to a bare `azd ai eval run` that is now a group. Nothing catches this on its own: the string compiles, the command that prints it succeeds, and only somebody who follows the advice finds out. So every `azd ai eval ...` in a non-comment line is now resolved against the real command tree, with trailing prose trimmed so "run start and summarize" is read as the command it begins with. Verified by restoring the original message and watching the test name it. Also fixes an assertion that the live suite caught being wrong rather than the code: `--failed-only` counted the bare substring "FAIL" against `ResultCounts.Failed`. Two things were conflated. The per-criterion table's FAILED column header contains FAIL, and one rendered line is one evaluator's verdict on one sample -- a sample failing two evaluators is two lines, while `ResultCounts.Failed` counts samples. It matches verdict cells on a word boundary against the failing results in the payload now, so it holds whatever the judge decides. --- .../azure.ai.evaluations/internal/cmd/run.go | 9 +-- .../internal/cmd/surface_test.go | 58 +++++++++++++++++++ .../tests/cli/run_output_test.go | 25 ++++++-- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index af31fb489c1..e93f313a0d7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -32,9 +32,6 @@ var terminalRunStates = map[string]bool{ "error": true, } -// newRunCommand builds the composite `azd ai eval run` and attaches the atomic -// run operations, including `run start` which the spec lists as the atomic form -// of this same command. // newRunCommand builds the run group. // // `run` is a group, not an executable verb: once `run output` exists, a bare @@ -167,7 +164,7 @@ func buildRunCommand(use, short string) *cobra.Command { return emitJSON(out, run) } fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) - fmt.Fprintf(out, "Check progress with: azd ai eval results show %s --run-id %s\n", evalID, run.ID) + fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval-id %s\n", run.ID, evalID) return nil } @@ -363,8 +360,8 @@ func (ec *evalContext) reuseDataSourceFromLastRun( return nil, fmt.Errorf( "eval %s has no previous run to repeat, so there is no target or dataset "+ "to reuse.\n"+ - " Run it from the config once with `azd ai eval run`, or pass a config that "+ - "declares the eval", + " Run it from the config once with `azd ai eval run start`, or name an "+ + "eval that declares one with `--eval`", evalID) } return list.Data[0].DataSource, nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 516ee0d2520..551dd45359c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -4,6 +4,10 @@ package cmd import ( + "io/fs" + "os" + "path/filepath" + "regexp" "strings" "testing" @@ -194,3 +198,57 @@ func find(t *testing.T, path string) *cobra.Command { strings.Fields(cmd.Use)[0], "resolved the wrong command for %s", path) return cmd } + +// Messages that tell a user what to run next have to name a command that +// exists. +// +// Rebuilding the surface left `run start --no-wait` closing with "Check +// progress with: azd ai eval results show", a command that had been renamed +// out of existence — so the one instruction printed at the moment a user needs +// it was the one thing guaranteed to fail. Nothing catches that: the string +// compiles, the command that prints it succeeds, and only someone following +// the advice finds out. +func TestSuggestedCommandsExist(t *testing.T) { + root := "../.." + pattern := regexp.MustCompile("azd ai eval ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) + if err != nil { + return err + } + + for _, line := range strings.Split(string(body), "\n") { + // Comments explain the surface; only what reaches a terminal has + // to resolve. + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range pattern.FindAllStringSubmatch(line, -1) { + words := strings.Fields(m[1]) + // Trim trailing prose: "run start" is a command, "run start + // and summarize" is a sentence that begins with one. + for len(words) > 0 { + if _, _, err := NewRootCommand().Find(words); err == nil { + resolved, _, _ := NewRootCommand().Find(words) + if strings.Fields(resolved.Use)[0] == words[len(words)-1] { + break + } + } + words = words[:len(words)-1] + } + assert.NotEmpty(t, words, + "%s suggests `azd ai eval %s`, which is not a command", path, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 4c688dd9976..84358f2c81e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -9,6 +9,7 @@ import ( "encoding/csv" "os" "path/filepath" + "regexp" "strings" "testing" @@ -115,12 +116,23 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { f := sharedEval(t) payload := resultsFor(t, f.EvalID, f.FirstRunID) - failed := payload.Run.ResultCounts.Failed + + // One rendered row is one evaluator's verdict on one sample, so the count + // to expect is failing *results*, not failing rows: a sample that fails two + // evaluators is two lines. `ResultCounts.Failed` answers the other question. + failing := 0 + for _, item := range payload.OutputItems { + for _, r := range item.Results { + if !r.Passed { + failing++ + } + } + } r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval-id", f.EvalID, "--failed-only")) - if failed == 0 { + if failing == 0 { // Saying so is not the same as printing an empty table. require.Contains(t, r.Stdout, "No failing rows.") return @@ -128,9 +140,12 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { require.NotContains(t, r.Stdout, " pass ", "--failed-only must drop the rows that passed") - require.Contains(t, r.Stdout, "FAIL") - require.Equal(t, failed, strings.Count(r.Stdout, "FAIL"), - "every failing row must appear exactly once") + + // Matched on a word boundary so the per-criterion table's FAILED column + // header is not counted as a verdict. + verdicts := regexp.MustCompile(`\bFAIL\b`).FindAllString(r.Stdout, -1) + require.Equal(t, failing, len(verdicts), + "every failing verdict must appear exactly once:\n%s", r.Stdout) } // resultsFor reads a run's results as JSON, which several tests need before From c176044b81a9174585a94e2beabbee00c3602446 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 23:08:36 +0300 Subject: [PATCH 086/114] Note the platforms nothing has run on yet build.sh cross-compiles six platforms. Every build and test in this extension's history has been windows/amd64, so five of them ship unexecuted. The test suites carry the same assumption rather than guarding against it: both harnesses derive the .exe suffix from the path separator, and the hero suite shells out to \zd\ and to a proxy address, none of which has been exercised off Windows. Recorded next to the note that already says what CI does and does not cover, since that is what a release reads. --- cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 index 314095e75b3..8be21c6bbf4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -8,6 +8,11 @@ # The live integration tests are excluded: they carry the `live` build tag, so # an untagged run does not compile them, and they additionally require # AZURE_AI_EVAL_E2E_LIVE and a project endpoint. +# +# TODO before the first release: build.sh ships six platforms and only +# windows/amd64 has ever been executed. The live and hero suites assume it too +# -- they derive the .exe suffix from the path separator and shell out to `azd` +# -- so run both on linux and darwin before publishing. $gopath = go env GOPATH $gotestsumBinary = "gotestsum" From 35aa6999869e84295814a61cdefba828a9cc2e81 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 23:17:21 +0300 Subject: [PATCH 087/114] Correct what CI already covers The previous note claimed only windows/amd64 had ever been executed. That was true of my desk, not of CI: release-ext-azure-ai-evaluations.yml has a pr: trigger on this directory, and every BuildMatrix job -- windows, linux and darwin amd64 -- runs this script. The untagged tests are covered on all three. What is genuinely uncovered is narrower and worse: ci-test.ps1 runs go test ./... untagged, so the live and hero suites are never compiled by any pipeline on any platform. Both have only ever run on Windows, by hand. Darwin needs no manual pass -- CI has it. Linux does, for the tagged suites only. --- cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 index 8be21c6bbf4..1050a817798 100644 --- a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -9,10 +9,11 @@ # an untagged run does not compile them, and they additionally require # AZURE_AI_EVAL_E2E_LIVE and a project endpoint. # -# TODO before the first release: build.sh ships six platforms and only -# windows/amd64 has ever been executed. The live and hero suites assume it too -# -- they derive the .exe suffix from the path separator and shell out to `azd` -# -- so run both on linux and darwin before publishing. +# TODO before the first release: PR CI runs this script on windows, linux and +# darwin amd64, so the untagged tests are covered on all three. The live and +# hero suites are not -- being tagged, no pipeline has ever compiled them, and +# both have only ever run on Windows by hand. Run them once on linux, where +# they assume a path separator and shell out to `azd` and to a proxy address. $gopath = go env GOPATH $gotestsumBinary = "gotestsum" From 5feb5c5a22678a72b693c7f97636bf9d1aec2b86 Mon Sep 17 00:00:00 2001 From: mohessie Date: Mon, 3 Aug 2026 23:29:00 +0300 Subject: [PATCH 088/114] Type-check the tagged suites in CI ci-test.ps1 runs go test ./... untagged, so nothing in any pipeline compiled the live or hero suites. A change that broke one reached main silently -- the surface tests added earlier would not catch it either, because they are untagged too. go vet -tags live,hero needs no credentials and no endpoint, so it costs a few seconds and runs on every platform the tests already do. It runs after the tests so a genuine test failure still reports first. Verified both ways: breaking a file under either tag fails the script with exit 1 and suppresses the success line; the clean tree exits 0. --- .../azure.ai.evaluations/ci-test.ps1 | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 index 1050a817798..6175585d318 100644 --- a/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 +++ b/cli/azd/extensions/azure.ai.evaluations/ci-test.ps1 @@ -7,13 +7,14 @@ # # The live integration tests are excluded: they carry the `live` build tag, so # an untagged run does not compile them, and they additionally require -# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. +# AZURE_AI_EVAL_E2E_LIVE and a project endpoint. They are still type-checked +# below, so a change that breaks them cannot reach main unnoticed. # # TODO before the first release: PR CI runs this script on windows, linux and # darwin amd64, so the untagged tests are covered on all three. The live and -# hero suites are not -- being tagged, no pipeline has ever compiled them, and -# both have only ever run on Windows by hand. Run them once on linux, where -# they assume a path separator and shell out to `azd` and to a proxy address. +# hero suites are only type-checked, never executed, and both have only ever +# run on Windows by hand. Run them once on linux, where they assume a path +# separator and shell out to `azd` and to a proxy address. $gopath = go env GOPATH $gotestsumBinary = "gotestsum" @@ -37,6 +38,19 @@ if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# The tagged suites are never run here, so without this nothing compiles them +# and a change that breaks one reaches main silently. Type-checking needs no +# credentials, so it costs a few seconds and runs everywhere the tests do. +Write-Host "" +Write-Host "Type-checking the live and hero suites..." +go vet -tags live,hero ./... + +if ($LASTEXITCODE -ne 0) { + Write-Host "" + Write-Host "The tagged test suites do not compile: $LASTEXITCODE" -ForegroundColor Red + exit $LASTEXITCODE +} + Write-Host "" Write-Host "All tests passed!" -ForegroundColor Green exit 0 From 52cedc3d45a1b8f49b8fe3d65e6c552814929138 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 01:16:49 +0300 Subject: [PATCH 089/114] Stop an update from overwriting the version it is updating For a few seconds after a publish the service can answer the next one with the version it just assigned, writing over that version's definition instead of adding one. So \evaluator create\ followed by \evaluator update\ - which is what a first authoring session looks like - reported success and left a single version holding the second rubric, with every eval bound to the first scoring against something nobody chose. It cannot be waited out by watching for state, because there is none to watch: a publish 258ms after the first came back as version 1 and one six seconds later advanced to 2, while the version listing was already answering with 1 - and immediately after a create that same listing 404s, so a guard trusting it would stand down in exactly the case it exists for. What is reliable is what the caller has already read. Both call sites read the evaluator to decide between creating and updating, so that document is passed into the publish: it says which version exists and when it was written. The publish waits until that version has had a few seconds, then checks that what came back is a new one, reissuing the same body if it is not. A collision that happens anyway cannot be undone - the request that collided has already written - so the recheck is bounded and ends in an error naming the version that was replaced, rather than in a reported success. --- .../internal/cmd/evaluator.go | 10 +- .../cmd/evaluator_version_live_test.go | 134 +++++++++++++ .../internal/cmd/reconciler.go | 6 +- .../internal/pkg/eval_api/evaluators.go | 10 + .../internal/pkg/eval_api/operations.go | 116 ++++++++++- .../pkg/eval_api/publish_version_test.go | 184 ++++++++++++++++++ 6 files changed, 455 insertions(+), 5 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index 2407f4434ee..c5f86a19d15 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -86,7 +86,7 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { // lags a publish by up to a second and a half, so an update // issued straight after a create would be told the evaluator it // just made does not exist. - _, readErr := ec.evalClient.GetEvaluatorRaw( + existing, readErr := ec.evalClient.GetEvaluatorRaw( ctx, name, "", ProjectEndpointAPIVersion, ) if readErr != nil && !eval_api.IsNotFound(readErr) { @@ -96,8 +96,14 @@ func newEvaluatorWriteCommand(verb, short string) *cobra.Command { return err } + // What that read saw is what keeps the publish from being + // answered with the same version and replacing it. + if readErr != nil { + existing = nil + } + created, err := ec.evalClient.CreateEvaluatorVersion( - ctx, name, body, ProjectEndpointAPIVersion, + ctx, name, body, existing, ProjectEndpointAPIVersion, ) if err != nil { return fmt.Errorf("registering evaluator %q: %w", name, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go new file mode 100644 index 00000000000..fabc4f222b6 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator_version_live_test.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// Evaluator versions are the unit an eval binds to, and the service assigns +// them. This proves the extension never hands back a version it has quietly +// overwritten. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/require" +) + +// TestLiveEvaluatorUpdateAlwaysPublishesANewVersion covers the shape of a +// first authoring session: create a rubric, look at it, change one weight, +// update. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over it rather than adding one. +// Nothing observable marks the end of that race — the version listing lags a +// publish as well, answering 404 immediately after a create — so the defence +// is the document the caller already read: it says which version exists and +// when it was written. +// +// Without it, `evaluator update` run straight after `evaluator create` reports +// success, leaves a single version holding the second rubric, and every eval +// bound to the first scores against a rubric nobody chose. +func TestLiveEvaluatorUpdateAlwaysPublishesANewVersion(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-version-%d", time.Now().UnixNano()) + + rubric := func(weight int) json.RawMessage { + body, err := normalizeRubricBody(name, []byte(fmt.Sprintf( + `{"dimensions":[{"id":"tone","weight":%d,"description":"polite"}]}`, weight))) + require.NoError(t, err) + return body + } + + first, err := client.CreateEvaluatorVersion(ctx, name, rubric(1), nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEmpty(t, first.Version) + t.Cleanup(func() { + for _, v := range []string{first.Version, "1", "2"} { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, v, ProjectEndpointAPIVersion) + } + }) + + // Deliberately immediate, and passing what the caller holds rather than + // re-reading: this is the window the guard exists for, and a test that + // waited first would pass with the guard removed. + previous, err := json.Marshal(first) + require.NoError(t, err) + + started := time.Now() + second, err := client.CreateEvaluatorVersion( + ctx, name, rubric(2), previous, ProjectEndpointAPIVersion) + require.NoError(t, err) + require.NotEqual(t, first.Version, second.Version, + "an update issued inside the race must still publish a new version") + t.Logf("the second version was assigned after %s", time.Since(started).Round(time.Millisecond)) + + // The new version holds the new rubric, and both versions are readable. + // The earlier one is not asserted on: if the service does collide, the + // attempt that collided has already written the new definition over it, + // and no amount of care on this side can undo that. + require.Equal(t, 2, liveRubricWeight(t, client, name, second.Version)) + require.NotZero(t, liveRubricWeight(t, client, name, first.Version), + "version %s must remain readable", first.Version) +} + +// liveRubricWeight reads back the one weight the fixture rubric carries. +// +// Read as JSON rather than matched as a substring: the service reformats what +// it stores, so `"weight":1` goes in and `"weight": 1` comes back, and a +// substring assertion would fail for a reason that has nothing to do with what +// is being tested. +func liveRubricWeight( + t *testing.T, + client *eval_api.EvalClient, + name, version string, +) int { + t.Helper() + + raw, err := client.GetEvaluatorRaw( + context.Background(), name, version, ProjectEndpointAPIVersion) + require.NoError(t, err) + + var doc struct { + Definition struct { + Dimensions []struct { + ID string `json:"id"` + Weight int `json:"weight"` + } `json:"dimensions"` + } `json:"definition"` + } + require.NoError(t, json.Unmarshal(raw, &doc)) + require.Len(t, doc.Definition.Dimensions, 1) + return doc.Definition.Dimensions[0].Weight +} + +// TestLiveFirstPublishReturnsVersionOne is the other half: the guard must not +// change what a first publish answers. +func TestLiveFirstPublishReturnsVersionOne(t *testing.T) { + client, _ := liveEvalClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive-firstpub-%d", time.Now().UnixNano()) + body, err := normalizeRubricBody(name, []byte( + `{"dimensions":[{"id":"tone","weight":1,"description":"polite"}]}`)) + require.NoError(t, err) + + created, err := client.CreateEvaluatorVersion(ctx, name, body, nil, ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteEvaluatorVersion( + context.Background(), name, created.Version, ProjectEndpointAPIVersion) + }) + + require.Equal(t, "1", created.Version, + "a name the project has never seen must publish as version 1") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 44df054a40b..c2de22db18e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -265,16 +265,20 @@ func (r *evalReconciler) EnsureEvaluator( } // Compare against the definition already on the service. + var known json.RawMessage if existing, err := r.ec.evalClient.GetEvaluatorRaw( ctx, decl.Name, "", ProjectEndpointAPIVersion, ); err == nil { if sameDefinition(existing, body) { return versionFromRaw(existing, decl.Version), false, nil } + // Different, so a version is about to be published. What that read + // saw is what keeps the publish from being answered with it again. + known = existing } created, err := r.ec.evalClient.CreateEvaluatorVersion( - ctx, decl.Name, body, ProjectEndpointAPIVersion, + ctx, decl.Name, body, known, ProjectEndpointAPIVersion, ) if err != nil { return "", false, err diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go index 9f0a6f17b35..dd230ef1f08 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/evaluators.go @@ -176,6 +176,16 @@ func (c *EvalClient) LatestEvaluatorVersionNumber( return number } +// parseVersionNumber reads a version string as an integer, answering 0 for one +// that is not numeric. +func parseVersionNumber(version string) int { + number, err := strconv.Atoi(version) + if err != nil { + return 0 + } + return number +} + // DeleteEvaluatorVersion removes a single evaluator version. func (c *EvalClient) DeleteEvaluatorVersion( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index cf61480d16b..09291317d10 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -13,6 +13,7 @@ import ( "net/http" "net/url" "strconv" + "time" "azureaieval/internal/version" @@ -186,14 +187,125 @@ func (c *EvalClient) GetAgent( // CreateEvaluatorVersion creates a new version of a named evaluator. // The body should be the full evaluator JSON with the definition field updated. +// +// previous is the evaluator document the caller has already read, or nil when +// it read none. It is what keeps the publish from being answered with the +// version that document holds. func (c *EvalClient) CreateEvaluatorVersion( ctx context.Context, name string, body json.RawMessage, + previous json.RawMessage, + apiVersion string, +) (*EvaluatorVersion, error) { + return c.publishEvaluatorVersion(ctx, name, previous, apiVersion, func() (*EvaluatorVersion, error) { + path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" + return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) + }) +} + +// versionSettle bounds the wait for the service to start assigning the next +// version number. +const ( + versionSettleTimeout = 45 * time.Second + versionSettleInterval = 3 * time.Second + versionSettleAge = 8 * time.Second +) + +// publishedVersion is the little of an evaluator document this needs: which +// version it is, and when it was written. +type publishedVersion struct { + Version string `json:"version"` + ModifiedAt time.Time `json:"modified_at"` + CreatedAt time.Time `json:"created_at"` +} + +// writtenAt reports when the version was last written, preferring the +// modification time and falling back to creation. +func (p publishedVersion) writtenAt() time.Time { + if !p.ModifiedAt.IsZero() { + return p.ModifiedAt + } + return p.CreatedAt +} + +// publishEvaluatorVersion publishes and then makes sure a new version is what +// came back. +// +// For a few seconds after a publish the service can answer the next one with +// the version it just assigned, writing over that version's contents instead +// of adding one. It is a race rather than a fixed window — a second publish +// has been seen both colliding a quarter of a second later and succeeding +// immediately — and nothing observable marks its end. +// +// That matters because versions are the unit an eval binds to. `evaluator +// create` followed by `evaluator update`, which is what a first authoring +// session looks like, would otherwise leave one version holding the second +// definition and every eval bound to the first silently scoring against a +// rubric nobody chose. +// +// So there are two defences. The publish is held back until the version the +// caller read has had time to settle, which is what keeps the collision from +// happening at all; and the version that comes back is checked, which is what +// keeps a collision that happens anyway from being reported as success. The +// recheck republishes the same body, so it cannot make a collision worse than +// the first attempt already did. +// +// What the caller reads is used rather than the version listing because the +// listing lags a publish too: asked immediately after a create it answers 404, +// so a guard that trusted it would stand down in exactly the case it exists +// for. Callers that publish an evaluator have already read it to decide +// between creating and updating. +func (c *EvalClient) publishEvaluatorVersion( + ctx context.Context, + name string, + previous json.RawMessage, apiVersion string, + publish func() (*EvaluatorVersion, error), ) (*EvaluatorVersion, error) { - path := pathEvaluators + "/" + url.PathEscape(name) + "/versions" - return doRequestTyped[EvaluatorVersion](c, ctx, http.MethodPost, path, nil, body, apiVersion) + var known publishedVersion + if len(previous) > 0 { + _ = json.Unmarshal(previous, &known) + } + + latest := parseVersionNumber(known.Version) + if listed := c.LatestEvaluatorVersionNumber(ctx, name, apiVersion); listed > latest { + latest = listed + } + + if written := known.writtenAt(); !written.IsZero() { + if wait := versionSettleAge - time.Since(written); wait > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(wait): + } + } + } + + deadline := time.Now().Add(versionSettleTimeout) + for { + created, err := publish() + if err != nil { + return nil, err + } + if latest == 0 || parseVersionNumber(created.Version) > latest { + return created, nil + } + if time.Now().After(deadline) { + return nil, fmt.Errorf( + "publishing evaluator %q kept returning version %s, which already "+ + "existed. The service was still assigning that version after %s, so "+ + "version %s now holds what was just published and any eval bound to "+ + "it is scoring against it", + name, created.Version, versionSettleTimeout, created.Version) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(versionSettleInterval): + } + } } // GetEvaluatorRaw gets an evaluator by name and version as raw JSON. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go new file mode 100644 index 00000000000..e34e1460017 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/publish_version_test.go @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newRecordingClient points a client at a test server, with no credential +// policy in the pipeline. +func newRecordingClient(t *testing.T, handler http.HandlerFunc) *EvalClient { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + return NewEvalClientFromPipeline( + server.URL, runtime.NewPipeline("test", "v1.0.0", runtime.PipelineOptions{}, nil)) +} + +// versionServer answers a version listing and a publish, assigning whatever +// version the caller decides for each attempt. +func versionServer(t *testing.T, existing []string, assign func(attempt int) string) ( + http.HandlerFunc, *atomic.Int32, +) { + t.Helper() + var publishes atomic.Int32 + + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + if r.Method == http.MethodGet { + values := []map[string]any{} + for _, v := range existing { + values = append(values, map[string]any{"name": "tone", "version": v}) + } + if len(existing) == 0 { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"code":"NotFound"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + return + } + + attempt := int(publishes.Add(1)) + w.WriteHeader(http.StatusCreated) + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "tone", "version": assign(attempt), + })) + }, &publishes +} + +// A name the project has never seen has no version to collide with, so it must +// publish once and return. Waiting there would tax every first publish for a +// hazard that cannot apply. +func TestCreateEvaluatorVersion_FirstPublishDoesNotRetry(t *testing.T) { + handler, publishes := versionServer(t, nil, func(int) string { return "1" }) + client := newRecordingClient(t, handler) + + started := time.Now() + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "1", created.Version) + assert.Equal(t, int32(1), publishes.Load(), "a first publish must be issued once") + assert.Less(t, time.Since(started), versionSettleInterval, + "a first publish must not wait on a version that cannot exist") +} + +// For a few seconds after a publish the service answers the next one with the +// version it just assigned, replacing that version rather than adding one. +// Accepting it would leave every eval bound to the earlier version scoring +// against a definition nobody chose, so the publish is reissued until the +// version advances. +func TestCreateEvaluatorVersion_RetriesUntilTheVersionAdvances(t *testing.T) { + handler, publishes := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 3 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(3), publishes.Load(), + "the publish must be reissued until the service assigns a new version") +} + +// A service that never advances must end in an error rather than in a version +// the caller believes is new. Reporting success there is the failure the whole +// guard exists to prevent. +func TestCreateEvaluatorVersion_GivesUpRatherThanReportASharedVersion(t *testing.T) { + handler, _ := versionServer(t, []string{"4"}, func(int) string { return "4" }) + client := newRecordingClient(t, handler) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + _, err := client.CreateEvaluatorVersion( + ctx, "tone", json.RawMessage(`{}`), nil, "2025-11-15-preview") + require.Error(t, err) +} + +// The version listing lags a publish: asked immediately after a create it +// answers 404. A guard that trusted it would stand down in exactly the window +// it exists for, which is why the caller supplies the version it has already +// read. +func TestCreateEvaluatorVersion_UsesTheCallersVersionWhenTheListingLags(t *testing.T) { + handler, publishes := versionServer(t, nil, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, handler) + + created, err := client.CreateEvaluatorVersion( + context.Background(), "tone", json.RawMessage(`{}`), json.RawMessage(`{"version":"1"}`), "2025-11-15-preview") + require.NoError(t, err) + + assert.Equal(t, "2", created.Version) + assert.Equal(t, int32(2), publishes.Load(), + "the version the caller read must be enough to catch the collision") +} + +// A version the service does not number cannot be compared, so it is taken at +// face value: refusing it would make an evaluator unpublishable over a +// convention this extension does not own. +func TestParseVersionNumber(t *testing.T) { + assert.Equal(t, 7, parseVersionNumber("7")) + assert.Equal(t, 0, parseVersionNumber("v7")) + assert.Equal(t, 0, parseVersionNumber("")) +} + +// The publish is reissued, so the same body has to arrive every time. A +// closure that consumed its body on the first attempt would send an empty one +// on the second and publish an evaluator with no definition. +func TestCreateEvaluatorVersion_ReissuesTheSameBody(t *testing.T) { + bodies := make(chan string, 4) + handler, _ := versionServer(t, []string{"1"}, func(attempt int) string { + if attempt < 2 { + return "1" + } + return "2" + }) + client := newRecordingClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + buf := make([]byte, r.ContentLength) + _, _ = r.Body.Read(buf) + bodies <- string(buf) + } + handler(w, r) + }) + + _, err := client.CreateEvaluatorVersion( + context.Background(), "tone", + json.RawMessage(`{"definition":{"type":"rubric"}}`), nil, "2025-11-15-preview") + require.NoError(t, err) + close(bodies) + + seen := 0 + for body := range bodies { + seen++ + assert.Contains(t, body, "rubric", fmt.Sprintf("attempt %d sent an empty body", seen)) + } + assert.Equal(t, 2, seen) +} From badd6709c49d816bc6bfcb888b72d9c559009f58 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:01:16 +0300 Subject: [PATCH 090/114] Print the job id in the line that tells you to use it \--no-wait\ exists so the caller can walk away, and the line they walk away with is the one they paste when they come back. It said Submitted. \zd ai eval job show \ reports its progress. The id was on the line above, so nothing was lost - but the instruction itself was not usable, and the check that every command the CLI suggests is a real command passed it, because \job show\ is one. A second check now refuses a suggestion that still has a placeholder in it, and it was confirmed to fail on the line it was written for. Also drops checkEvaluatorDrift. It reads the version listing to catch a version published outside the repo, and nothing calls it: the rubric path always reads the latest version and republishes when the local definition differs, so there is no window for it to guard. It belongs to the code evaluator path, which fingerprints instead of comparing, and that is M2's. --- .../internal/cmd/generate.go | 14 ++++++- .../internal/cmd/generate_commands.go | 9 ++-- .../internal/cmd/reconciler.go | 27 ------------ .../internal/cmd/surface_test.go | 42 +++++++++++++++++++ 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 823ed4e251b..bb3fe950310 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -177,7 +177,7 @@ func (ec *evalContext) generateRubric( return nil, fmt.Errorf("submitting the rubric generation job: %w", err) } if noWait { - fmt.Fprintf(out, " submitted job %s\n", job.ID) + reportSubmitted(out, job.ID) return nil, nil } @@ -196,6 +196,16 @@ func (ec *evalContext) generateRubric( return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } +// reportSubmitted says what was started and how to get back to it. +// +// The job id goes into the command rather than being left as a placeholder: +// --no-wait exists so the caller can walk away, and the line they walk away +// with has to be the one they can paste when they come back. +func reportSubmitted(out io.Writer, jobID string) { + fmt.Fprintf(out, " submitted job %s\n", jobID) + fmt.Fprintf(out, "\nReattach with: azd ai eval job show %s\n", jobID) +} + // generateDataset submits the data generation job and downloads the result. func (ec *evalContext) generateDataset( ctx context.Context, @@ -215,7 +225,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("submitting the data generation job: %w", err) } if noWait { - fmt.Fprintf(out, " submitted job %s\n", job.ID) + reportSubmitted(out, job.ID) return nil, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index 9e8c5a33dac..fd03dfc5bbe 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -289,16 +289,13 @@ func newEvaluatorGenerateCommand() *cobra.Command { // reportGenerated closes out either command. // // With --no-wait nothing was downloaded and there is no ref, which is success: -// the job id was printed and `job show` reattaches to it. +// reportSubmitted has already said how to reattach. func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { out := cmd.OutOrStdout() if ref == nil { - if noWait { - fmt.Fprintln(out, - "\nSubmitted. `azd ai eval job show ` reports its progress.") - return nil + if !noWait { + fmt.Fprintln(out, "Nothing was generated.") } - fmt.Fprintln(out, "Nothing was generated.") return nil } if isJSON(cmd) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c2de22db18e..5611579cf09 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -11,7 +11,6 @@ import ( "os" "path/filepath" "reflect" - "strconv" "strings" "time" @@ -363,32 +362,6 @@ func (r *evalReconciler) evaluatorVersionResolvable( return false } -// checkEvaluatorDrift fails when the service holds a newer version than the -// one recorded at the last deploy. -// -// Publishing is not destructive — versions are immutable — so the remedy is to -// sync with what is on the project, not to overwrite it. -func (r *evalReconciler) checkEvaluatorDrift( - ctx context.Context, - name, recorded string, -) error { - recordedNumber, err := strconv.Atoi(recorded) - if err != nil { - return nil - } - latest := r.ec.evalClient.LatestEvaluatorVersionNumber( - ctx, name, ProjectEndpointAPIVersion, - ) - if latest <= recordedNumber { - return nil - } - return fmt.Errorf( - "evaluator %q is at version %d on the project but %s was recorded at the last deploy; "+ - "someone published a version outside this repo. "+ - "Pull the newer code locally, or delete version %d, then deploy again", - name, latest, recorded, latest) -} - // EnsureEval creates the group when it has never been deployed, or when an // upstream artifact changed. Groups are immutable, so a change means a new // group and a new id. diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 551dd45359c..acaaf5143a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -252,3 +252,45 @@ func TestSuggestedCommandsExist(t *testing.T) { }) require.NoError(t, err) } + +// A command suggested with an argument has to be suggested with the argument +// filled in. +// +// `--no-wait` exists so the caller can walk away, and the line they walk away +// with is the one they paste when they come back. Printing +// `azd ai eval job show ` reads like a command and is not one: it +// resolves, so the check above passes, and it fails the moment anyone uses it. +func TestSuggestedCommandsCarryNoPlaceholders(t *testing.T) { + placeholder := regexp.MustCompile(`azd ai eval [^"'\n]*<[a-z-]+>`) + + err := filepath.WalkDir("../..", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(line) + // A `Use:` string and the help text around it are where a + // placeholder belongs: cobra prints it as the signature. + if strings.HasPrefix(trimmed, "//") || + strings.HasPrefix(trimmed, "Use:") || + strings.HasPrefix(trimmed, "Short:") || + strings.HasPrefix(trimmed, "Long:") { + continue + } + if m := placeholder.FindString(line); m != "" { + t.Errorf("%s:%d suggests %q; substitute the value instead", + path, i+1, m) + } + } + return nil + }) + require.NoError(t, err) +} From 425171b11f135a6ed0f2f90e075b40d2964d9530 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:13:24 +0300 Subject: [PATCH 091/114] Stop reporting a run that could not run as one that passed Two gaps in the exit-code contract, both found by reading the spec's own promises back against the code. A run that finished with status failed or error exited 0. The reason was printed, so a person reading the terminal saw it, but a pipeline gating on the command saw success - and the spec makes exit 1 mean exactly this case. It is checked before the gate, because the gate's exit 2 means the evaluation regressed and a run that produced no results has not regressed; telling those apart is what the separate code is for. \ un show\ only reports it when --wait was asked: without it, this is an inspection command and answering the question is a success whatever the answer. And evaluator drift was never detected. checkEvaluatorDrift existed and nothing called it, so \zd up\ would publish over a version somebody had deliberately published from the portal and report success. It is called now, and it is answered from the version recorded at the last deploy rather than from the version listing - the listing lags a publish, and would report an evaluator as un-drifted for the first seconds of its newest version's life. Evaluators need this comparison at all only because their definitions come back inline: local and remote disagreeing says nothing on its own about who moved. --- .../internal/cmd/reconciler.go | 57 ++++++++++++++++++- .../internal/cmd/reconciler_drift_test.go | 40 +++++++++++++ .../azure.ai.evaluations/internal/cmd/run.go | 23 ++++++++ .../internal/cmd/run_ops.go | 16 ++++++ .../internal/cmd/run_status_test.go | 38 +++++++++++++ 5 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index 5611579cf09..c770f26ea7b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "time" @@ -268,11 +269,30 @@ func (r *evalReconciler) EnsureEvaluator( if existing, err := r.ec.evalClient.GetEvaluatorRaw( ctx, decl.Name, "", ProjectEndpointAPIVersion, ); err == nil { + remote := versionFromRaw(existing, "") if sameDefinition(existing, body) { + // Nothing to publish, but the version is still worth recording: + // it is what a later deploy compares against to notice that + // someone moved the evaluator on from here. + if remote != "" { + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), remote) + } return versionFromRaw(existing, decl.Version), false, nil } - // Different, so a version is about to be published. What that read - // saw is what keeps the publish from being answered with it again. + + // The definitions differ, which means either the local file changed + // or someone published a version outside the repo. The version + // recorded at the last deploy is what tells them apart, and + // publishing over the second case would bury an intentional change + // under one nobody asked for. + if recorded := r.ec.getEnvValue(ctx, versionKey("evaluator", decl.Name)); recorded != "" { + if err := checkEvaluatorDrift(decl.Name, recorded, remote); err != nil { + return "", false, err + } + } + + // What that read saw is what keeps the publish from being answered + // with it again. known = existing } @@ -283,9 +303,42 @@ func (r *evalReconciler) EnsureEvaluator( return "", false, err } r.awaitEvaluatorReadable(ctx, decl.Name, created.Version) + _ = r.ec.setEnvValue(ctx, versionKey("evaluator", decl.Name), created.Version) return created.Version, true, nil } +// checkEvaluatorDrift fails when the service holds a newer version than the +// one recorded at the last deploy. +// +// It is asked only when the local definition and the remote one disagree, +// which on its own says nothing about who moved: the author may have edited +// the file, or someone may have published a version from outside the repo. +// The recorded version settles it, and the difference matters because +// publishing is how this reconciler resolves a disagreement — doing that over +// a version somebody deliberately published would bury their change under one +// nobody asked for, with `azd up` reporting success. +// +// The remote version is passed in rather than listed, because the version +// listing lags a publish and would report an evaluator as un-drifted for the +// first seconds of its newest version's life. +func checkEvaluatorDrift(name, recorded, remote string) error { + recordedNumber, err := strconv.Atoi(recorded) + if err != nil { + return nil + } + remoteNumber, err := strconv.Atoi(remote) + if err != nil || remoteNumber <= recordedNumber { + return nil + } + return fmt.Errorf( + "evaluator %q is at version %s on the project but %s was recorded at the last "+ + "deploy, and the local definition does not match it: someone published a "+ + "version outside this repo. Publishing over it would leave their change "+ + "behind, so bring version %s into the declared source and deploy again, or "+ + "delete that version if it was a mistake", + name, remote, recorded, remote) +} + // evaluatorPropagation bounds the wait for a freshly published evaluator to // become usable. // diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go new file mode 100644 index 00000000000..d945b95eda9 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler_drift_test.go @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Drift is only interesting when the two definitions already disagree, and +// then only when the disagreement came from the project rather than from the +// author. These are the four ways that question can be answered. +func TestCheckEvaluatorDrift(t *testing.T) { + // The author edited the file. The project is where the last deploy left + // it, so publishing is exactly right and must not be blocked. + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "3")) + + // Someone published outside the repo. Publishing over it would leave + // their change behind with `azd up` reporting success. + err := checkEvaluatorDrift("support-quality", "3", "4") + require.Error(t, err) + assert.Contains(t, err.Error(), "support-quality") + assert.Contains(t, err.Error(), "version 4") + assert.Contains(t, err.Error(), "3 was recorded") + assert.Contains(t, err.Error(), "outside this repo", + "the message has to say who moved, not just that something did") + + // A version that went backwards is not drift: a newer version was + // deleted, and republishing is how the repo takes the name back. + require.NoError(t, checkEvaluatorDrift("support-quality", "4", "3")) + + // Versions this extension did not number cannot be compared, and refusing + // a deploy over a numbering convention it does not own would be worse + // than not checking. + require.NoError(t, checkEvaluatorDrift("support-quality", "", "4")) + require.NoError(t, checkEvaluatorDrift("support-quality", "3", "preview")) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index e93f313a0d7..9bde2604579 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -32,6 +32,26 @@ var terminalRunStates = map[string]bool{ "error": true, } +// runCompleted turns a run that did not complete into an error, so that a +// caller who waited for it exits non-zero. +// +// The results have already been printed by the time this is asked, which is +// the point: a run that errored has a reason worth reading, and reporting it +// and then exiting 0 tells a pipeline the evaluation passed. It is checked +// before the gate because the gate's exit code means "the evaluation +// regressed", and a run that never produced results has not regressed — it did +// not run. Distinguishing those two is what the separate code is for. +func runCompleted(run *eval_api.OpenAIEvalRun) error { + if run == nil { + return nil + } + switch strings.ToLower(run.Status) { + case "completed", "": + return nil + } + return fmt.Errorf("run %s finished with status %s", run.ID, run.Status) +} + // newRunCommand builds the run group. // // `run` is a group, not an executable verb: once `run output` exists, a bare @@ -184,6 +204,9 @@ func buildRunCommand(use, short string) *cobra.Command { // Last, so that the results are reported whether or not the gate // holds: a pipeline that only learns it failed is worse off than // one that can see by how much. + if err := runCompleted(final); err != nil { + return err + } applyGate(cmd, threshold, final) return nil }, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index e7e537e544c..deb0d0a49e5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -125,6 +125,12 @@ func newRunShowCommand() *cobra.Command { // Reattaching to a run started asynchronously: the pipeline that // gates on it is often not the one that started it. + // + // Only a caller that waited is told a bad status through the exit + // code. Without --wait this is an inspection command: it was asked + // what happened, and answering that is a success whatever the + // answer. + gateOnStatus := wait if wait { run, err = ec.pollRun(ctx, evalID, run.ID, cmd.OutOrStdout(), isJSON(cmd)) if err != nil { @@ -136,6 +142,11 @@ func newRunShowCommand() *cobra.Command { if err := emitJSON(cmd.OutOrStdout(), run); err != nil { return err } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } applyGate(cmd, threshold, run) return nil } @@ -150,6 +161,11 @@ func newRunShowCommand() *cobra.Command { if run.ReportURL != "" { fmt.Fprintf(out, " report : %s\n", run.ReportURL) } + if gateOnStatus { + if err := runCompleted(run); err != nil { + return err + } + } applyGate(cmd, threshold, run) return nil }, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go new file mode 100644 index 00000000000..646fd560e50 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_status_test.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The exit code is the whole contract with a pipeline, and there are three +// answers it has to be able to give: the evaluation ran and passed, it ran and +// regressed, or it could not run. The gate owns the middle one; this owns the +// last. +// +// Reporting a run that errored and then exiting 0 tells the pipeline the +// evaluation passed, which is the one answer that is never true. +func TestRunCompleted(t *testing.T) { + require.NoError(t, runCompleted(nil), + "nothing was waited for, so there is nothing to report") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "completed"})) + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1", Status: "Completed"}), + "the service is not consistent about case") + require.NoError(t, runCompleted(&eval_api.OpenAIEvalRun{ID: "r1"}), + "a status the service did not send is not a failure to report") + + for _, status := range []string{"failed", "error", "canceled", "cancelled"} { + err := runCompleted(&eval_api.OpenAIEvalRun{ID: "run_abc", Status: status}) + require.Error(t, err, "status %q must not exit 0", status) + assert.Contains(t, err.Error(), "run_abc") + assert.Contains(t, err.Error(), status, + "the message has to name the status, which is what the caller acts on") + } +} From 5e43f3ce29c98a034cc805f635131fea6ad488d5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:32:15 +0300 Subject: [PATCH 092/114] Report the verdict the run was started to get \ un start\ waited for a run and then printed one line: its id and its status. The verdict per evaluator - which is the entire reason to run one - was returned by the service and thrown away, so the answer to the question the command had just been asked took a second command to see. It now prints a row per evaluator with pass, fail and pass rate, sorted by name so two runs of the same eval read the same way, followed by the overall rate over samples rather than over verdicts: a sample that failed two evaluators is one sample to go and look at. Errored rows are named rather than folded into the fail column. The evaluator never reached a verdict on them, and counting them as failures reports a service problem as a quality problem. For the same reason a criterion's rate is over what it scored, and a rate over nothing prints as a dash - 0.0% would read as a total failure rather than as no data. --- .../azure.ai.evaluations/internal/cmd/run.go | 81 ++++++++++++ .../internal/cmd/run_summary_test.go | 117 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 9bde2604579..5b429408a7e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -13,6 +13,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" "time" @@ -640,6 +641,14 @@ func (ec *evalContext) pollRun( } } +// renderRun prints what a person needs after waiting for a run. +// +// The status line alone is not that. A run's whole purpose is the verdict per +// evaluator, and the service returns it — passed, failed and errored counts +// for every testing criterion — so leaving it out meant the answer to the +// question the command was asked required a second command to see. The report +// URL and the run id come last, because they are what you act on after +// reading the numbers rather than instead of reading them. func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) // A run that failed carries why, and it is usually the only actionable @@ -648,8 +657,80 @@ func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAI if why := run.Failure(); why != "" { fmt.Fprintf(out, " %s\n", why) } + + renderCriteriaTable(out, run.PerTestingCriteria) + + // Counted over samples, not over verdicts: a sample that failed two + // evaluators is one sample to go and look at, and reporting it as two + // overstates how much is wrong. + if c := run.ResultCounts; c != nil && c.Total > 0 { + fmt.Fprintf(out, "\nOverall pass rate: %s (%d/%d samples passed every evaluator)\n", + formatRate(c.Passed, c.Total), c.Passed, c.Total) + if c.Errored > 0 { + fmt.Fprintf(out, "%d sample(s) errored and were not scored.\n", c.Errored) + } + if c.Failed > 0 { + fmt.Fprintln(out, + "\nView failing samples: azd ai eval run output list --failed-only") + } + } + if run.ReportURL != "" { fmt.Fprintf(out, "Report: %s\n", run.ReportURL) } return nil } + +// renderCriteriaTable prints one row per evaluator. +// +// Sorted by name so two runs of the same eval read the same way; the service +// returns the criteria in whatever order it evaluated them. +func renderCriteriaTable( + out interface{ Write([]byte) (int, error) }, + results []eval_api.EvalRunCriteriaResult, +) { + if len(results) == 0 { + return + } + + sorted := append([]eval_api.EvalRunCriteriaResult(nil), results...) + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].TestingCriteria < sorted[j].TestingCriteria + }) + + width := len("EVALUATOR") + for _, r := range sorted { + if n := len(r.TestingCriteria); n > width { + width = n + } + } + + fmt.Fprintf(out, "\n%-*s %4s %4s %9s\n", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") + fmt.Fprintf(out, "%s %s %s %s\n", + strings.Repeat("-", width), "----", "----", "---------") + for _, r := range sorted { + scored := r.Passed + r.Failed + fmt.Fprintf(out, "%-*s %4d %4d %9s\n", + width, r.TestingCriteria, r.Passed, r.Failed, formatRate(r.Passed, scored)) + // Errors are not failures — the evaluator never reached a verdict — + // so they are named rather than folded into the fail column, where + // they would look like a quality problem. + if r.Errored > 0 { + fmt.Fprintf(out, "%-*s %s\n", width, "", errorNote(r.Errored)) + } + } +} + +// errorNote describes rows an evaluator could not score. +func errorNote(errored int) string { + return fmt.Sprintf("(%d errored, not scored)", errored) +} + +// formatRate renders a share as a percentage, and a rate over nothing as a +// dash: 0.0%% would read as a total failure rather than as no data. +func formatRate(part, whole int) string { + if whole <= 0 { + return "-" + } + return fmt.Sprintf("%.1f%%", float64(part)/float64(whole)*100) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go new file mode 100644 index 00000000000..6bb4a2f4c34 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go @@ -0,0 +1,117 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// finishedRun is a run the way the service returns one: counts over samples, +// and a result per testing criterion. +func finishedRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc123", + Status: "completed", + ResultCounts: &eval_api.EvalRunResultCounts{ + Total: 10, Passed: 7, Failed: 3, + }, + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 9, Failed: 1}, + {TestingCriteria: "coherence", Passed: 7, Failed: 3}, + }, + } +} + +// The whole point of waiting for a run is the verdict per evaluator. Printing +// only the status meant the answer to the question the command was asked took +// a second command to see. +func TestRenderRunReportsEveryEvaluator(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun())) + text := out.String() + + assert.Contains(t, text, "evalrun_abc123") + assert.Contains(t, text, "completed") + + for _, criterion := range []string{"relevance", "coherence"} { + assert.Contains(t, text, criterion, + "every evaluator the run scored must appear") + } + assert.Contains(t, text, "90.0%", "relevance passed 9 of 10") + assert.Contains(t, text, "70.0%", "coherence passed 7 of 10") + assert.Contains(t, text, "7/10", "the sample counts must be shown, not just the rate") +} + +// Two runs of the same eval have to read the same way. The service returns the +// criteria in whatever order it evaluated them, which is not stable. +func TestRenderRunOrdersEvaluatorsByName(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun())) + + text := out.String() + assert.Less(t, strings.Index(text, "coherence"), strings.Index(text, "relevance"), + "evaluators must be listed in a stable order") +} + +// An errored row is not a failing row: the evaluator never reached a verdict. +// Folding the two together would report a service problem as a quality problem. +func TestRenderRunSeparatesErrorsFromFailures(t *testing.T) { + run := finishedRun() + run.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 7, Failed: 1, Errored: 2} + run.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 7, Failed: 1, Errored: 2}, + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run)) + text := out.String() + + assert.Contains(t, text, "2 errored") + assert.Contains(t, text, "87.5%", + "the pass rate is over what was scored, not over what was attempted") + assert.Contains(t, text, "errored and were not scored") +} + +// A rate over nothing is not zero. Printing 0.0% for a criterion that scored +// no rows reads as a total failure rather than as no data. +func TestFormatRateHasNoOpinionAboutNothing(t *testing.T) { + assert.Equal(t, "-", formatRate(0, 0)) + assert.Equal(t, "0.0%", formatRate(0, 4)) + assert.Equal(t, "100.0%", formatRate(4, 4)) + assert.Equal(t, "33.3%", formatRate(1, 3)) +} + +// The next thing anyone does after seeing failures is look at them, so the +// command that shows them is named — and it has to be a command that exists. +func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, finishedRun())) + assert.Contains(t, out.String(), "azd ai eval run output list --failed-only") + + clean := finishedRun() + clean.ResultCounts = &eval_api.EvalRunResultCounts{Total: 10, Passed: 10} + clean.PerTestingCriteria = []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "relevance", Passed: 10}, + } + var cleanOut bytes.Buffer + require.NoError(t, renderRun(&cleanOut, clean)) + assert.NotContains(t, cleanOut.String(), "--failed-only", + "a run with nothing to look at must not send anyone looking") +} + +// A run that never produced counts still has to render. The service returns +// none for a run that failed before scoring, and a nil dereference there would +// replace the failure message with a panic. +func TestRenderRunSurvivesAnEmptyResult(t *testing.T) { + var out bytes.Buffer + require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"})) + assert.Contains(t, out.String(), "evalrun_x") +} From d01832c601b751de462eb7264aef7d29e9973cc5 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:44:26 +0300 Subject: [PATCH 093/114] Give CI a shape to read, and stop a slow token from failing a test Two things. \ un start --no-wait -o json\ printed the service's run object verbatim. A pipeline reading it to reattach later had to know that the run id is \id\, that the eval id echoed back is not necessarily the one the command resolved, and that the declaration name it wants to log is not there at all - while depending on every other field the API happens to return. It now emits the handoff the spec documents: run_id, eval_id, eval_name, status, created_at, and nothing this extension does not promise to keep. And the live suite's credential now retries. The token refresh shells out to azd and the SDK gives that subprocess ten seconds; with the whole suite running it sometimes does not finish in ten, which surfaced as \AzureDeveloperCLICredential: exit status 1\ on whichever test asked at the wrong moment - reproducibly at 10.1s, and never when that test ran alone. Nothing about the request was wrong, so it is retried rather than reported. --- .../internal/cmd/build_live_test.go | 42 +++++++++++- .../azure.ai.evaluations/internal/cmd/run.go | 39 ++++++++++- .../internal/cmd/run_handoff_test.go | 67 +++++++++++++++++++ 3 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index 64bcf50b2c2..f9b8edfc683 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -27,6 +27,8 @@ import ( "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/stretchr/testify/require" ) @@ -51,6 +53,44 @@ func liveCredential() (*azidentity.AzureDeveloperCLICredential, error) { return sharedCred, sharedCredErr } +// credentialFlake is what a token refresh that overran its budget looks like +// by the time it reaches a test. +const credentialFlake = "AzureDeveloperCLICredential: exit status 1" + +// retryingCredential retries a token request that failed for that reason. +// +// The refresh shells out to azd, and the SDK gives that subprocess ten +// seconds. On a machine already running the rest of this suite it sometimes +// does not finish in ten, and the failure lands on whichever test asked for a +// token at the wrong moment — reproducibly at 10.1s, and never when that test +// is run on its own. Retrying is right because nothing about the request was +// wrong: the same call succeeds moments later. +type retryingCredential struct { + inner azcore.TokenCredential +} + +func (c retryingCredential) GetToken( + ctx context.Context, + opts policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + var token azcore.AccessToken + var err error + for attempt := range 4 { + if attempt > 0 { + select { + case <-ctx.Done(): + return azcore.AccessToken{}, ctx.Err() + case <-time.After(time.Duration(attempt) * 2 * time.Second): + } + } + token, err = c.inner.GetToken(ctx, opts) + if err == nil || !strings.Contains(err.Error(), credentialFlake) { + return token, err + } + } + return token, err +} + func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { t.Helper() if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { @@ -67,7 +107,7 @@ func liveEvalClient(t *testing.T) (*eval_api.EvalClient, string) { if judge == "" { judge = "gpt-4.1-nano" } - return eval_api.NewEvalClient(endpoint, cred), judge + return eval_api.NewEvalClient(endpoint, retryingCredential{inner: cred}), judge } // TestLiveBuildAcceptedForEveryBuiltin walks every built-in the project diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 5b429408a7e..67c26bf6bae 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -182,7 +182,7 @@ func buildRunCommand(use, short string) *cobra.Command { if !wait { if isJSON(cmd) { - return emitJSON(out, run) + return emitJSON(out, startedRun(run, evalID, group)) } fmt.Fprintf(out, "Started run %s (status: %s)\n", run.ID, run.Status) fmt.Fprintf(out, "Reattach with: azd ai eval run show %s --eval-id %s\n", run.ID, evalID) @@ -641,8 +641,43 @@ func (ec *evalContext) pollRun( } } -// renderRun prints what a person needs after waiting for a run. +// startedRunHandoff is what `run start --no-wait -o json` returns. // +// It is a handoff rather than a dump of the service object. The pipeline that +// started the run has to come back for it later, and doing that needs exactly +// three things: the run, the eval it belongs to, and a name a human can read +// in the log that reports it. The service object carries none of the third and +// buries the first two under the data source, the metadata and every field the +// API happens to return, so a script reading it would depend on a shape this +// extension does not control. +type startedRunHandoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + EvalName string `json:"eval_name,omitempty"` + Status string `json:"status,omitempty"` + CreatedAt any `json:"created_at,omitempty"` +} + +// startedRun builds the handoff. +func startedRun( + run *eval_api.OpenAIEvalRun, + evalID string, + group *project.Eval, +) startedRunHandoff { + handoff := startedRunHandoff{ + RunID: run.ID, + EvalID: evalID, + Status: run.Status, + CreatedAt: run.CreatedAt, + } + // Absent with --eval-id, where there is no config to take a name from. + if group != nil { + handoff.EvalName = group.Name + } + return handoff +} + +// renderRun prints what a person needs after waiting for a run. // The status line alone is not that. A run's whole purpose is the verdict per // evaluator, and the service returns it — passed, failed and errored counts // for every testing criterion — so leaving it out meant the answer to the diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go new file mode 100644 index 00000000000..cf4a522c2f5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "encoding/json" + "testing" + + "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A pipeline that starts a run with --no-wait has to come back for it. What it +// needs to do that is a fixed shape this extension controls, not whatever the +// API happened to return. +func TestStartedRunIsTheHandoffAPipelineNeeds(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_01JQZX", + EvalID: "eval_ignored", + Status: "queued", + CreatedAt: "2026-07-31T21:04:11Z", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + DataSource: &eval_api.EvalRunDataSource{ + Type: eval_api.EvalRunDataSourceTypeTraces, + }, + } + + raw, err := json.Marshal(startedRun(run, "eval_01JQZW", &project.Eval{Name: "support-agent-smoke"})) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + + assert.Equal(t, "evalrun_01JQZX", out["run_id"]) + assert.Equal(t, "support-agent-smoke", out["eval_name"]) + assert.Equal(t, "queued", out["status"]) + assert.Equal(t, "2026-07-31T21:04:11Z", out["created_at"]) + + // The eval the run was started against, which is the one the command + // resolved rather than whatever the run echoed back. + assert.Equal(t, "eval_01JQZW", out["eval_id"]) + + // Nothing the extension does not promise. A pipeline that could read the + // data source here would come to depend on it. + for _, leaked := range []string{"data_source", "metadata", "id", "report_url"} { + assert.NotContains(t, out, leaked, + "the handoff must not leak %q from the service object", leaked) + } +} + +// With --eval-id there is no config, so there is no declaration name to give. +// An empty one is omitted rather than reported as "", which a script would +// otherwise print as the eval's name. +func TestStartedRunOmitsTheNameItDoesNotHave(t *testing.T) { + raw, err := json.Marshal(startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "queued"}, "eval_1", nil)) + require.NoError(t, err) + + var out map[string]any + require.NoError(t, json.Unmarshal(raw, &out)) + assert.NotContains(t, out, "eval_name") + assert.Equal(t, "eval_1", out["eval_id"]) +} From a2fd7e3f736a1f60019f18c18d951776c6e3483b Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 02:48:35 +0300 Subject: [PATCH 094/114] Drive the CI reattach path end to end The handoff shape was covered by a unit test and nothing else, so nothing proved the id it hands back is one the next command can actually resolve - which is the only property a pipeline cares about. This starts a run without waiting, reads run_id out of the JSON, and reattaches with it. The plain-output half asserts the printed reattach line carries the real eval id and contains no angle bracket at all, because the placeholder bug this replaces was in exactly such a line and read as though it were a command. --- .../tests/cli/handoff_test.go | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go new file mode 100644 index 00000000000..56b5465beba --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// The CI path: start a run without waiting, read the handoff, come back for +// the result later. Everything here is what a pipeline does, so it is driven +// through the binary exactly the way a pipeline would. + +package cli + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCLIStartNoWaitEmitsTheHandoff pins the JSON a pipeline reads. +// +// A script captures the run id here and reattaches to it in a later step, so +// the field names are a contract. Emitting the service's run object instead +// would make that script depend on a shape this extension does not control. +func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, + "run", "start", "--eval-id", f.EvalID, "--no-wait", "-o", "json")) + + var handoff struct { + RunID string `json:"run_id"` + EvalID string `json:"eval_id"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + } + r.JSON(t, &handoff) + + require.NotEmpty(t, handoff.RunID, "a pipeline has nothing to reattach to without run_id") + assert.Equal(t, f.EvalID, handoff.EvalID) + assert.NotEmpty(t, handoff.Status) + + // Started, not finished: this is the whole point of --no-wait, and a + // command that quietly blocked would pass every other assertion here. + assert.NotEqual(t, "completed", handoff.Status) + + deferTeardown(func() { + runQuietly("run", "cancel", handoff.RunID, "--eval-id", f.EvalID) + }) + + // The id it handed back has to be one the next step can use. + shown := requireSuccess(t, run(t, + "run", "show", handoff.RunID, "--eval-id", f.EvalID, "-o", "json")) + var reattached struct { + ID string `json:"id"` + } + shown.JSON(t, &reattached) + assert.Equal(t, handoff.RunID, reattached.ID, + "the run id in the handoff must be the one `run show` resolves") +} + +// TestCLIStartNoWaitTellsAPersonHowToReattach covers the same path without +// -o json, where what matters is that the printed command is one that works +// rather than a sentence containing a placeholder. +func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { + f := sharedEval(t) + + r := requireSuccess(t, run(t, "run", "start", "--eval-id", f.EvalID, "--no-wait")) + + assert.Contains(t, r.Stdout, "Reattach with: azd ai eval run show") + assert.Contains(t, r.Stdout, f.EvalID, + "the reattach line must carry the eval id, not a placeholder for it") + assert.NotContains(t, r.Stdout, "<", + "nothing printed for a person to copy may contain a placeholder") + + var runID string + for _, field := range strings.Fields(r.Stdout) { + if strings.HasPrefix(field, "evalrun_") { + runID = field + break + } + } + require.NotEmpty(t, runID, "the run id must be printed:\n%s", r.Stdout) + deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval-id", f.EvalID) }) +} From 7fe99463b0e0939599760fb42a10405951e629a8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 03:30:25 +0300 Subject: [PATCH 095/114] Satisfy the two repo-wide checks this extension was failing Every other extension has a cspell.yaml importing the shared config and listing its own vocabulary. This one had none, so all 96 of its words were unknown - the module name, the package names, the built-in evaluator names, and a handful of ordinary English cspell does not carry. Added, along with US spellings for the four words that were genuinely British and a format string whose %s butted against the next word and read as one. And pkg/extensions has a guard requiring every extension that declares providers to prove the ones it registers match its manifest. The provider was declared and registered correctly; the test asserting so was simply absent, so nothing would have caught them drifting apart. Also: a dataset create followed immediately by an update was refused as though the dataset did not exist. The existence check read the version listing, which lags a publish - the same thing evaluators were already reading directly - so it now falls back to a point read of version 1, which goes consistent at once. And run start --no-wait now renders created_at as RFC 3339. The service sends epoch seconds on a run and a formatted string elsewhere, so passing it through handed a script a value whose type depended on which route produced it. --- .../azure.ai.evaluations/CHANGELOG.md | 2 +- .../azd-eval-extension-implementation.md | 4 +-- .../azure.ai.evaluations/cspell.yaml | 26 +++++++++++++++++ .../internal/cmd/dataset.go | 15 ++++++++++ .../internal/cmd/generate.go | 2 +- .../azure.ai.evaluations/internal/cmd/init.go | 8 +++-- .../internal/cmd/providers_manifest_test.go | 19 ++++++++++++ .../internal/cmd/resolution_test.go | 2 +- .../azure.ai.evaluations/internal/cmd/run.go | 29 +++++++++++++++++-- .../internal/cmd/run_handoff_test.go | 22 +++++++++++++- .../internal/pkg/eval_api/operations.go | 2 +- .../internal/project/eval_config.go | 2 +- .../internal/project/generate_config.go | 2 +- .../tests/hero/init_test.go | 2 +- 14 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/cspell.yaml create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md index 67a8c1d121e..45903a217c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md +++ b/cli/azd/extensions/azure.ai.evaluations/CHANGELOG.md @@ -9,7 +9,7 @@ agent, making no service calls. - `generate` synthesizes a rubric and dataset from the agent's context, writes them under `evals/`, and merges `source:` references into the deployment spec - while preserving comments, ordering and neighbouring entries. + while preserving comments, ordering and neighboring entries. - `run` creates the eval group when it does not exist, starts a run, and summarizes the result. - `azure.ai.eval` service-target provider deploys datasets, evaluators and eval diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md index b5373ffc545..624f0eeef5f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md @@ -465,10 +465,10 @@ Recorded for review. Anything marked **corrected** was an assumption that live t | **Agent-seeded generation, nothing authored** | `init` → `generate` with no instruction file: seeded from the agent's published instructions, 14 rows generated, 13 of 14 on the agent's actual catalog/policies; `azd deploy` published them; the run scored 14 passed / 0 failed / 0 errored | | Missing generation model | Fails before any network call, naming `--eval-model` and the spec field | | **`schedule` (M2)** | Create, list, show and delete against the live project; trigger read back from the service as stored, not echoed. One-per-project and existing-name refusals both verified, each naming the schedule and the command to clear it. Delete waits out `Creating` and leaves the project empty | -| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalised `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | +| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalized `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | | Failed runs | The reason reaches the caller instead of just the word "failed" | | **`run --response-id` (M4)** | Three stored responses evaluated, 3 passed / 0 errored; the stored payload matched what was sent field for field | -| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalised to `schema.item`, are both benign | +| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalized to `schema.item`, are both benign | | **`target.type: model` (M4)** | Group deployed with `response` bound to `{{sample.output_text}}`, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency | | **Registered dataset on a run (M4)** | A group with no local `source:` now runs: whole set scores 2 passed / 1 failed, `--max-samples 2` scores 2 rows. Previously a 400 | | **Two groups in one config** | Distinct ids across repeated deploys, each running its own criteria. Previously the second deploy aliased them onto one group | diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml new file mode 100644 index 00000000000..6e5e122ffbc --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -0,0 +1,26 @@ +import: ../../.vscode/cspell.yaml +words: + # Go module and package names + - azureaieval + - evalcore + - httptest + - creack + # Service identifiers and API fields + - evalrun + - lookback + - AOAI + # Built-in evaluator names + - ifeval + - groundedness + # Repository names + - foundrysdk + # Terms + - inlines + - negotiables + - parseable + - retargeted + - subsetting + - undeployed + - undoable + - unpassed + - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index ef4361b8cc8..f4ac919306b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -16,6 +16,10 @@ import ( "github.com/spf13/cobra" ) +// firstDatasetVersion is the version the service assigns to a dataset's first +// publish, and so the one that exists for every dataset that exists at all. +const firstDatasetVersion = "1" + func newDatasetCommand() *cobra.Command { cmd := &cobra.Command{ Use: "dataset", @@ -82,6 +86,17 @@ func newDatasetWriteCommand(verb, short string) *cobra.Command { ctx, name, ProjectEndpointAPIVersion, ) exists := err == nil && existing != nil && len(existing.Value) > 0 + if !exists { + // The version listing lags a publish, so a `create` followed by + // an `update` was told the dataset it had just made does not + // exist. A direct read of the first version settles it: point + // reads go consistent immediately. + if _, err := ec.datasetClient.GetDataset( + ctx, name, firstDatasetVersion, ProjectEndpointAPIVersion, + ); err == nil { + exists = true + } + } if err := checkAssetExistence(verb, "dataset", name, exists); err != nil { return err } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index bb3fe950310..ee7514699e8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -285,7 +285,7 @@ func (ec *evalContext) generateDataset( return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } -// isAgentSeededGenerationFailure recognises the service-side failure that hits +// isAgentSeededGenerationFailure recognizes the service-side failure that hits // every agent, so it can be retried without the agent rather than surfaced. func isAgentSeededGenerationFailure(err error) bool { if err == nil { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 6e17b5f2806..2bdc4f4477f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -5,6 +5,7 @@ package cmd import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -169,7 +170,8 @@ const ( // noAzdProject is what init reports when there is nothing to attach to. const noAzdProject = "no azd project found in this directory. Run `azd init` first, " + - "or run this from the root of an existing one; the eval service is " + "or run this from the root of an existing one; the eval service is added to " + + "its azure.yaml" // readAzdProject returns the project, without changing it. // @@ -179,13 +181,13 @@ const noAzdProject = "no azd project found in this directory. Run `azd init` fir func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { azdClient, err := azdext.NewAzdClient() if err != nil { - return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, errors.New(noAzdProject) } defer azdClient.Close() resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - return nil, fmt.Errorf("%sadded to its azure.yaml", noAzdProject) + return nil, errors.New(noAzdProject) } return resp.GetProject(), nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go new file mode 100644 index 00000000000..2588ad49b9c --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/providers_manifest_test.go @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/require" +) + +// TestConfigureExtensionHostMatchesManifest verifies that the providers this +// extension registers match those declared in its extension.yaml. +func TestConfigureExtensionHostMatchesManifest(t *testing.T) { + manifestPath := filepath.Join("..", "..", "extension.yaml") + require.NoError(t, azdext.VerifyProvidersMatchManifest(configureExtensionHost, manifestPath)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index 1d8c809bb6b..9e20a857312 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -44,7 +44,7 @@ func TestResolveLevel_Precedence(t *testing.T) { // this wrong passes validation and then errors on every row. func TestSampleBindingsFor_UnknownTargetBindsNothing(t *testing.T) { assert.Nil(t, sampleBindingsFor("prompt"), - "an unrecognised target must bind nothing rather than guess at agent fields") + "an unrecognized target must bind nothing rather than guess at agent fields") } // The level filter is what keeps a conversation evaluator from being sent turn diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 67c26bf6bae..a135a968454 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -655,7 +655,7 @@ type startedRunHandoff struct { EvalID string `json:"eval_id"` EvalName string `json:"eval_name,omitempty"` Status string `json:"status,omitempty"` - CreatedAt any `json:"created_at,omitempty"` + CreatedAt string `json:"created_at,omitempty"` } // startedRun builds the handoff. @@ -668,7 +668,7 @@ func startedRun( RunID: run.ID, EvalID: evalID, Status: run.Status, - CreatedAt: run.CreatedAt, + CreatedAt: timestampString(run.CreatedAt), } // Absent with --eval-id, where there is no config to take a name from. if group != nil { @@ -677,6 +677,31 @@ func startedRun( return handoff } +// timestampString renders a service timestamp as RFC 3339. +// +// The field arrives as epoch seconds on a run and as a formatted string +// elsewhere, so passing it through would hand a script a value whose type +// depends on which route produced it. +func timestampString(value any) string { + switch t := value.(type) { + case nil: + return "" + case string: + return t + case float64: + return time.Unix(int64(t), 0).UTC().Format(time.RFC3339) + case int64: + return time.Unix(t, 0).UTC().Format(time.RFC3339) + case json.Number: + if seconds, err := t.Int64(); err == nil { + return time.Unix(seconds, 0).UTC().Format(time.RFC3339) + } + return t.String() + default: + return fmt.Sprint(value) + } +} + // renderRun prints what a person needs after waiting for a run. // The status line alone is not that. A run's whole purpose is the verdict per // evaluator, and the service returns it — passed, failed and errored counts diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go index cf4a522c2f5..2881096749c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_handoff_test.go @@ -52,7 +52,27 @@ func TestStartedRunIsTheHandoffAPipelineNeeds(t *testing.T) { } } -// With --eval-id there is no config, so there is no declaration name to give. +// A script logging created_at should not have to know which route produced +// the run: the service sends epoch seconds here and a formatted string +// elsewhere, so the handoff settles on one. +func TestStartedRunNormalizesTheTimestamp(t *testing.T) { + for _, tc := range []struct { + name string + value any + want string + }{ + {"epoch seconds", float64(1785801525), "2026-08-03T23:58:45Z"}, + {"already formatted", "2026-07-31T21:04:11Z", "2026-07-31T21:04:11Z"}, + {"absent", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + handoff := startedRun( + &eval_api.OpenAIEvalRun{ID: "evalrun_1", CreatedAt: tc.value}, "eval_1", nil) + assert.Equal(t, tc.want, handoff.CreatedAt) + }) + } +} + // An empty one is omitted rather than reported as "", which a script would // otherwise print as the eval's name. func TestStartedRunOmitsTheNameItDoesNotHave(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 09291317d10..f28b8e81ef5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -244,7 +244,7 @@ func (p publishedVersion) writtenAt() time.Time { // definition and every eval bound to the first silently scoring against a // rubric nobody chose. // -// So there are two defences. The publish is held back until the version the +// So there are two defenses. The publish is held back until the version the // caller read has had time to settle, which is what keeps the collision from // happening at all; and the version that comes back is checked, which is what // keeps a collision that happens anyway from being reported as success. The diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index e2ff2e74772..ec58282de06 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -21,7 +21,7 @@ import ( // kept in evals/.yaml and pulled in with $ref. // // The eval's name is the service key in azure.yaml and is not repeated here. -// One service per eval is what lets azd's own dependency graph order an eval +// One service per eval is what lets the azd dependency graph order an eval // after the agent it evaluates. type EvalConfig struct { Description string `yaml:"description,omitempty" json:"description,omitempty"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go index 7ab28e5d7ac..12fdb1cf6ff 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go @@ -158,7 +158,7 @@ func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { return filepath.Join(candidate, resourceName+ext) } -// looksLikeFile treats a trailing recognised extension as an explicit file path. +// looksLikeFile treats a trailing recognized extension as an explicit file path. func looksLikeFile(p, ext string) bool { got := strings.ToLower(filepath.Ext(p)) if got == "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go index 0618e49ccd8..8fe79718d53 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -256,7 +256,7 @@ func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { } // Running `init` twice must not deploy the same eval twice. The service key is -// the eval's name, so the second run recognises its own work. +// the eval's name, so the second run recognizes its own work. func TestHeroInitIsIdempotent(t *testing.T) { dir := project(t, "support-agent") args := []string{"init", "--target", "support-agent", "--generation-model", "m"} From bb5362ec784f293b56b079623b90ab670097fad1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 04:01:58 +0300 Subject: [PATCH 096/114] Keep a command from shadowing the flags M1 promises everywhere M1's exit criteria include \-o json\ and \--no-prompt\ throughout. Both come from the SDK's root command, so every command inherits them and nothing had to be checked - right up until a command declares a flag by one of those names, which shadows the global silently and leaves that one command unable to answer in JSON or to run unattended. Nothing was watching for that. --- .../internal/cmd/surface_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index acaaf5143a0..3e3f9e035be 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -131,6 +131,21 @@ func TestFlagVocabularyIsShared(t *testing.T) { }) } +// M1 promises `-o json` and `--no-prompt` throughout. Both come from the azd +// extension SDK's root command, so every command inherits them — until one +// declares its own flag by the same name, which silently shadows the global +// and leaves that one command unable to answer in JSON or to run unattended. +func TestNoCommandShadowsAGlobalFlag(t *testing.T) { + global := []string{"output", "no-prompt", "environment", "cwd", "debug"} + + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + for _, name := range global { + assert.Nilf(t, cmd.LocalFlags().Lookup(name), + "%s declares its own --%s, which shadows the global one", path, name) + } + }) +} + // The two commands that write a file have to agree on what that flag is // called, and it has to be the name the sibling extensions use. func TestOutputFileFlagIsSpelledTheSharedWay(t *testing.T) { From d2b4d19b3b448ecb9c76619cc1ca15650db85e6e Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 17:11:08 +0300 Subject: [PATCH 097/114] Prove a dataset version cannot be written over, and drop a working note from the repo The reconciler publishes at a pinned version and treats a conflict as the signal to stop, which only holds if the service refuses the write. Nothing checked that. It does refuse - 409 on the pending upload - so the assumption was sound, but it was an assumption about someone else's service with an eval's scoring hanging off it. Now it fails loudly if that ever changes. The companion test covers the ordinary path: two uploads in a row must add two versions, issued back to back because the version listing lags a publish and that is the window where the second could be told the dataset is new and restart at the version the first just took. Also removes azd-eval-extension-implementation.md. It was a working note from building this, not documentation anyone should find in the extension - and it had gone stale, still calling M2's features M4. --- .../azd-eval-extension-implementation.md | 543 ------------------ .../internal/cmd/dataset_version_live_test.go | 115 ++++ 2 files changed, 115 insertions(+), 543 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md b/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md deleted file mode 100644 index 624f0eeef5f..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/azd-eval-extension-implementation.md +++ /dev/null @@ -1,543 +0,0 @@ -# `azd ai eval` — Implementation Handoff - -> Self-contained brief for building the **`azure.ai.evaluations`** azd extension (`azd ai eval`). Everything below is verified against the shipping `azure.ai.agents` extension, azd core, and RAISvc source. Design source of truth is `spec.md` in `foundrysdk_specs/specs/evaluations/azd_eval_extension/` — if the two disagree, the spec wins. -> -> **Uncommitted working document.** Not part of any PR. - ---- - -## 0. TL;DR - -A new azd extension — id `azure.ai.evaluations`, namespace `azd ai eval` — that is a thin Go client over the **existing** Foundry evaluations data plane, plus **one azd service-target provider** for `host: azure.ai.evals`. - -**Non-negotiables** - -1. **Two-tier commands.** Atomic (`dataset` / `evaluator` / `run` / `results`) map ~1:1 to the API. Composite (`init` / `generate` / `run`) are wrappers, never the only path. -2. **No `deploy` command.** Deployment is `azd up` / `azd deploy` invoking our service-target provider. We ship no deploy verb. -3. **`generate` is separate from deploy.** Generate once, deploy the artifacts to many environments. -4. **`init` touches no network.** -5. **Everything is built on APIs that exist today.** No service changes. -6. **Deterministic.** `-o json` + `--no-prompt` everywhere; a supplied flag fully suppresses its prompt. - -**Out of scope for M1:** scheduled/continuous eval, baseline comparison (both exist server-side — M2), non-agent targets, traces as a run data source, eval by response/run id, `optimize` (stays in the agents extension). - ---- - -## 1. Where the code lives & how to build it - -### 1.1 Paths -- **Repo:** `Azure/azure-dev`. -- **New extension:** `cli/azd/extensions/azure.ai.evaluations/` -- **Reference to copy:** `cli/azd/extensions/azure.ai.agents/` — closest analog, currently hosts `azd ai agent eval …`. **Read it first.** - -### 1.2 File layout (mirror the agents extension) -``` -cli/azd/extensions/azure.ai.evaluations/ -├── internal/ -│ ├── cmd/ # cobra commands, one file per group; listen.go wires the provider -│ ├── pkg/eval_api/ # data-plane client (lifted, see §1.4) -│ ├── pkg/dataset_api/ # dataset client (lifted) -│ └── project/ # azure.yaml service-entry model + YAML round-trip -├── schemas/ # JSON schemas -├── tests/ -├── extension.yaml # manifest -├── go.mod / go.sum -├── main.go -└── version.txt -``` - -**`extension.yaml`** — note the `service-target-provider` capability and `providers` block; both are required for `azd up` to route to us: -```yaml -# yaml-language-server: $schema=../extension.schema.json -id: azure.ai.evaluations -namespace: ai.eval # dotted → CLI surface `azd ai eval` -displayName: Foundry evaluations (Beta) -description: Define and run Foundry evaluations from your terminal. (Beta) -usage: azd ai eval [options] -version: 1.0.0-beta.1 # keep version.txt in sync -requiredAzdVersion: ">=1.27.1" -language: go -capabilities: - - custom-commands - - lifecycle-events - - service-target-provider - - metadata -providers: - - name: azure.ai.evals - type: service-target - description: Deploys evaluation datasets, evaluators, and eval groups to Foundry -``` - -### 1.3 Stack facts (verified) -- **Go 1.26.x**, **cobra**. Entry point: - ```go - package main - import ( - "azureaieval/internal/cmd" - "github.com/azure/azure-dev/cli/azd/pkg/azdext" - ) - func main() { azdext.Run(cmd.NewRootCommand()) } - ``` -- SDK module `github.com/azure/azure-dev/cli/azd` (agents pins `v1.28.0`); surface is `pkg/azdext`. -- Data plane over REST using the **azcore pipeline**, not raw `net/http`: bearer-token policy scoped to **`https://ai.azure.com/.default`**, plus `azsdk.NewMsCorrelationPolicy()` and `azsdk.NewUserAgentPolicy(...)`. -- Dev loop: - ```bash - azd ext install microsoft.azd.extensions # one-time - cd cli/azd/extensions/azure.ai.evaluations - azd x build # build + install locally - azd x watch # ongoing - ``` - -### 1.4 What to lift from `azure.ai.agents` - -Measured, non-test: - -| Source | Files | LOC | Gives you | -|---|---|---|---| -| `internal/pkg/agents/eval_api/` | 7 | 1,350 | `EvalClient` over `/data_generation_jobs`, `/evaluator_generation_jobs`, `/evaluators`, `/datasets`, `/openai/v1/evals`; LRO poller; artifact download; portal URLs; api-version constants | -| `internal/pkg/agents/dataset_api/` | 2 | 550 | Full pending-upload → blob → finalize → download | -| `internal/pkg/agents/opt_eval/` | 2 | 555 | eval.yaml config model (adapt, don't copy wholesale) | -| `internal/cmd/eval_*.go` | 10 | 2,890 | generate / run / show / list / update / progress UX | - -**≈1,900 LOC of API client is effectively done.** Genuinely net-new: - -1. **The service-target provider** (§4) — no prior art. -2. **Change detection** (§5) — no prior art. -3. **YAML round-trip merge** for `generate` writing `source:` back (§6). -4. The atomic command layer and the offline `init`. - -> Do **not** port the `/evaluation_suites` client. That endpoint is abandoned — the eval group is the unit. - ---- - -## 2. Data-plane API contract - -### 2.1 Base, auth, api-versions -- **Base:** azd env `FOUNDRY_PROJECT_ENDPOINT`, shape `https://{resource}.services.ai.azure.com/api/projects/{project}/…`. Global `--project-endpoint` overrides. -- **Scope:** `https://ai.azure.com/.default`. -- **api-versions:** project-endpoint calls (datasets, evaluators) use **`2025-11-15-preview`**; data generation uses **`v1`**; **`/openai/v1/evals*` sends no api-version**. - -### 2.2 Datasets -| Command | Calls | -|---|---| -| `dataset create` / `update` | `POST /datasets` *(first version only)* → `POST /datasets/{name}/versions/{v}/startPendingUpload` → `PUT ` → `PUT /datasets/{name}/versions/{v}` | -| `dataset list` | `GET /datasets` | -| `dataset show` | `GET /datasets/{name}/versions/{v}` | -| `dataset delete` | `DELETE /datasets/{name}/versions/{v}` | - -Model — **note there is no content hash or etag**, which drives §5: -```go -type Dataset struct { Name, Version, BlobURI, Format, DataURI, ContentURI string } -``` -A dataset is a **single `.jsonl`**. A directory today just picks the first `.jsonl`; no folder walk. - -### 2.3 Evaluators -| Command | Calls | -|---|---| -| `evaluator upload` / `update` | *(code only)* pending-upload → blob upload; then `POST /evaluators/{name}/versions` | -| `evaluator show` | `GET /evaluators/{name}` — returns the definition inline | -| `evaluator builtins` | `GET /evaluators?type=Builtin` | - -Built-ins are referenced as `builtin.`; the prefix is stripped before the value goes into `testing_criteria[].evaluator_name`. Custom-evaluator upload needs the project MI to hold **Azure AI User**. - -### 2.4 Eval groups and runs (OpenAI-compatible, no api-version) -```go -type CreateOpenAIEvalRequest struct { - Name string - Metadata map[string]string - DataSourceConfig *DataSourceConfig // {Type, ItemSchema, IncludeSampleSchema} - TestingCriteria []TestingCriterion // the evaluators -} -type TestingCriterion struct { - Type, Name, EvaluatorName string - InitializationParameters map[string]any // threshold lives here - DataMapping map[string]string -} -type OpenAIEval struct { ID, Name string } // ID is canonical; Name is NOT unique -``` - -| Command | Calls | -|---|---| -| create group | `POST /openai/v1/evals` | -| get / list | `GET /openai/v1/evals/{id}` · `GET /openai/v1/evals?limit=` | -| start run | `POST /openai/v1/evals/{evalId}/runs` | -| poll / list runs | `GET /openai/v1/evals/{evalId}/runs/{runId}` · `GET …/runs` | -| cancel | `POST /openai/v1/evals/{evalId}/runs/{runId}` with an empty body | -| results | `GET …/runs/{runId}` → `result_counts` + `per_testing_criteria_results` | - -**The group carries evaluators, not the dataset.** The dataset goes on the **run**. `evaluation_level` is `turn` | `conversation`, service default **`turn`**. - -**`data_source_config` and `data_mapping` are derived from each evaluator's published contract.** The original plan was to copy the agents extension's hardcoded mapping. Live testing showed that is wrong: it only suits agent-target quality evaluators and the service rejects the rest. - -`GET /evaluators` returns a contract per evaluator: -```jsonc -"supported_evaluation_levels": ["turn"], -"definition": { - "data_schema": { "required": ["response", "instruction_id_list"], "properties": { … } }, - "init_parameters": { "required": ["deployment_name"], "properties": { … } } -} -``` - -`internal/cmd/build.go` reads it and, per criterion: -- binds each accepted input to the agent sample (`response`, `tool_calls`, `tool_definitions`) or to a dataset column `{{item.}}`; -- declares the referenced columns in the item schema; -- filters `initialization_parameters` to the declared properties — no evaluator accepts `model`, and `builtin.ifeval` accepts nothing; -- validates `--level` against `supported_evaluation_levels`; -- reports a missing required column locally, naming it. - -Two service rules are encoded: `messages` and `query`/`response` are mutually exclusive (the level selects), and `evaluation_level` is an **initialization parameter**, not run metadata. An evaluator with no published contract falls back to the agent-target shape. - -Covered by `build_test.go`, and by `build_live_test.go` which posts a group for every built-in the project exposes. - -### 2.5 Generation (LRO) -`POST /data_generation_jobs` and `POST /evaluator_generation_jobs`, each polled by `GET …/{id}`. The ~11-minute "timeout" is a **client poll budget (2 s × 300)**, not a service limit — raise it and default to `--no-wait` in CI. - -### 2.6 M2 only — schedules and comparison -Both are **project-endpoint reachable** and **feature-gated per project**: -- `/schedules` — `PUT {id}` · `GET {id}` · `GET` · `DELETE {id}`; requires `FoundryFeature.Schedules_V1Preview`. Trigger is `Cron{Expression, StartTime, EndTime, Timezone}` or `Recurrence{Frequency, Interval, Schedule}`. -- Insights compare — `POST /insights` (async) or `POST /insights/sync`, body `{evalId, baselineRunId, treatmentRunIds}`; requires `FoundryFeature.Insights_V1Preview`. - ---- - -## 3. Configuration model - -Two files. Neither is loaded by azd core — **we parse both**. - -**`evals/eval_generate.yaml`** — input to `generate`, never deployed. `agent.context.{instructions,tools}` are file paths; `local_dir` accepts a directory or an explicit file path. - -**`evals/azure.yaml`** — the deployment spec, `$ref`'d from the root `azure.yaml`: -```yaml -# /azure.yaml -services: - evals: - host: azure.ai.evals - uses: [ai-project] - $ref: ./evals/azure.yaml -``` -It carries three arrays: `evaluators[]`, `datasets[]`, `evalGroups[]` (see `spec.md` for the full shape). - -**Why arrays on a service work.** azd core's `ServiceConfig` captures unknown keys: -```go -AdditionalProperties map[string]any `yaml:",inline"` -``` -and hands them to the extension, which unmarshals them itself — the pattern `LoadServiceTargetAgentConfig` → `ServiceConfigProps` uses. `azure.ai.project` already carries `deployments[]` this way. **`$ref` resolution is ours too**: `pkg/foundry.ResolveFileRefs(cfg, projectRoot)`, called by the extension, not by azd. - ---- - -## 4. The service-target provider (net-new, highest risk) - -Wire it in `listen.go`, mirroring `azure.ai.agents`: -```go -func configureExtensionHost(host *azdext.ExtensionHost) { - azdClient := host.Client() - host. - WithServiceTarget("azure.ai.evals", func() azdext.ServiceTargetProvider { - return project.NewEvalServiceTargetProvider(azdClient) - }). - WithServiceEventHandler("postdeploy", func(ctx context.Context, args *azdext.ServiceEventArgs) error { - return postdeployHandler(ctx, azdClient, args) - }, &azdext.ServiceEventOptions{Host: "azure.ai.evals"}) -} -``` - -`ServiceTargetProvider` requires `Initialize`, `Endpoints`, `GetTargetResource`, `Package`, `Publish`, `Deploy`. For eval, **`Package` and `Publish` are near no-ops**; `Deploy` does the work, in this fixed order: - -1. **Datasets** — change-detect (§5); if changed, run the `dataset create` sequence. -2. **Evaluators** — `GET /evaluators/{name}`, compare the definition, upload only if different. -3. **Drift check** — if the server's latest version is ahead of the recorded one, fail with "sync first". -4. **Eval groups** — `POST /openai/v1/evals` with `testing_criteria` from the resolved evaluator versions. Groups are immutable, so only recreate when the resolved versions or options actually changed. -5. Persist resolved ids, versions, and fingerprints to the azd env. - -**Publish → reference is eventually consistent.** After an evaluator version is -published, the direct read `GET /evaluators/{name}/versions/{version}` goes -consistent in roughly 330 ms, but the version *listing* lags 650 ms–1.4 s — and -eval creation resolves the reference through the listing, not the direct read. -Step 4 has to gate on both, or it fails with *"The evaluator X was not found"* -just after successfully publishing that evaluator. It reads like a flake, -because retrying a second later succeeds, so the tempting fix is a blanket retry -rather than the wait. - -**How azd reaches us:** `azd up` runs one DAG; per service it calls `GetServiceTarget()`, which does `serviceLocator.ResolveNamed(host, &target)`. If our extension is not installed, azd fails that service with *"install an extension that provides this host."* We implement **no sequencing or rollback across services** — `uses:` and the DAG handle that. - ---- - -## 5. Change detection (net-new) - -Without this, every `azd up` publishes a redundant version. - -- **Datasets** — the API returns no hash or etag, so comparing against the server would mean downloading the blob every deploy. Instead: **SHA-256 the local file**, store it with the resolved version in the azd env, re-hash locally next deploy, skip when unchanged. -- **Evaluators** — definitions come back inline from `GET /evaluators/{name}`; compare directly, no cache needed. -- **Drift** — a *version* comparison, not content: server latest vs. the version recorded at last deploy. - -**Open:** how to fingerprint a **code** evaluator (a folder). Suggest hashing sorted relative paths + contents, excluding `__pycache__` and `.pyc`. - ---- - -## 6. `generate` writes back into `evals/azure.yaml` - -After downloading artifacts, `generate` adds/updates `source:` references. Requirements: - -- Match entries **by `name`**; update `source` in place; append when absent. -- **Preserve comments and key order** — use the `yaml.v3` Node API, not plain marshal/unmarshal. -- Do not clobber a field the user hand-edited other than `source`. -- If the array is itself a `$ref`, write into the referenced file. - -There is no `emitDeploymentConfig` block — this is default behavior, not configurable. - ---- - -## 6a. What generation is seeded from - -The generation API takes an `agent` source that is meant to pull the agent's own instructions, and it fails for every agent (§11d). The client resolves that context itself, most specific first: - -1. `--gen-instruction` / `--gen-instruction-file` -2. the file named by `agent.context.instructions`, resolved **relative to the spec that declared it**, not the working directory -3. the agent's published instructions — `GET /agents/{name}` → `versions.latest.definition.instructions` - -Step 2 tolerates a missing file on purpose: `init` writes the path before the file exists, so treating the gap as an error would break the flow init scaffolds. Step 3 is what makes `init` → `generate` work with nothing authored. - -The agent source is still sent. When the service starts honouring it, it contributes on top of the prompt; nothing has to be removed. - -`agent.context.tools` is still read by nothing, so it is warned about rather than dropped silently, and `init` no longer scaffolds it — a warning for a field the user never chose is just noise. - ---- - -## 7. Behavioral bugs to fix (measured in `azd ai agent eval`) - -Treat each as an acceptance criterion. - -1. **Path handling (highest priority).** `--out-file` is re-rooted under the agent directory; `--config` re-roots again. **Fix:** treat paths as relative to CWD (or `-C/--cwd`), used verbatim, single-rooted. Test `./x.yaml`, `../x/x.yaml`, absolute. -2. **Wizard overrides flags.** Prompts still fire when flags are supplied, and pre-filled prompts *append* typed input. **Fix:** a supplied flag fully suppresses its prompt; `--no-prompt` errors on a missing required value. -3. **`--evaluator` ignored during generation.** Passing `--evaluator` does not stop rubric generation. **Fix:** honor it, skip that generation. -4. **Client-side generation timeout.** Resolved: it is a client poll budget, not a service limit. Raise it; default `--no-wait` under `--no-prompt`. -5. **Shallow results.** `eval show` returns counts only. **Fix:** per-sample scores via `per_testing_criteria_results`. -6. **Auth friction.** Native azd token failed with "Reauthentication required"; workaround `azd config set auth.useAzCliAuth true`. Detect and surface clearly. - ---- - -## 8. azd environment - -Extensions read and write env values themselves via `azdClient.Environment().GetValue / SetValue` — azd sets none of these. The agents extension does this from lifecycle handlers. - -| Key | Written by | -|---|---| -| `FOUNDRY_PROJECT_ENDPOINT` | consumed, not written | -| `EVAL_GROUP_ID` | provider during `azd up`; `run` when it creates the group | -| `EVAL_DATASET_VERSION`, artifact fingerprints | provider during `azd up` | -| `EVAL_RUN_ID` | `run` | - -Setting `EVAL_GROUP_ID` manually targets a pre-existing group; `--eval-id` does the same per-invocation. - ---- - -## 9. Build order - -M1 is everything in the spec. Within it, build in dependency order: - -| Step | Work | Done when | -|---|---|---| -| **1. Scaffold** | Extension skeleton, `extension.yaml`, `main.go`, root cobra command, local install via `azd x build` | `azd ai eval --help` works | -| **2. Lift the clients** | Copy `eval_api` + `dataset_api`, de-agent-scope, keep api-version constants | Unit tests pass against an `httptest` fake | -| **3. Atomic commands** | `dataset`, `evaluator`, `run`, `results` with `-o json` / `--no-prompt` | **E2E-1** below | -| **4. Config model** | `evals/azure.yaml` load, `$ref` resolve via `pkg/foundry.ResolveFileRefs`, validation | Round-trip test preserves comments | -| **5. Service-target provider** | `listen.go` wiring + `Deploy` reconciliation + change detection + drift | `azd up` creates all three resource kinds; second `azd up` is a no-op | -| **6. `init`** | Offline scaffold of both YAMLs | Runs with no network/auth | -| **7. `generate`** | Generation LROs, artifact download, write-back into `evals/azure.yaml` | **E2E-3** | -| **8. `run`** | Group resolve-or-create, run, poll, render | **E2E-2** | - -Steps 1–3 are mostly mechanical. **Step 5 is the risk** — budget accordingly. - ---- - -## 10. Testing - -| Tier | Coverage | Auth | Where | -|---|---|---|---| -| **0 — offline** | flag parsing; YAML round-trip; path resolution (§7.1); flag→prompt suppression (§7.2); request bodies against an `httptest` fake (copy `eval_api_version_test.go`); schema validation | No | PR gate | -| **1 — `init` record/playback** | interactive prompt flows | No | PR gate | -| **2 — live golden path** | full flows against a real Foundry project | Yes | On-demand/scheduled, **not** the PR gate | - -Tier 2: env-gate on `AZURE_AI_EVAL_E2E_LIVE=1`, build tag `//go:build linux` (needs a PTY), drive `init` prompts via `go-expect`+`vt10x`+`creack/pty`, everything else through `--no-prompt -o json`. `t.Cleanup` must delete every version it created. - -**Golden paths** - -- **E2E-1 — atomic:** `dataset create` → `evaluator upload` → `run start` → `results export`. Assert valid JSON, resolved versions, terminal run status, **per-sample** scores, and that re-running `create` yields the *next* version rather than an error. -- **E2E-2 — init → azd up → run:** `init` makes **zero network calls** (run it unauthenticated), writes both YAMLs, does not double the path; `azd up` creates the resources and pins versions back; a second `azd up` creates **no new versions**; `run` completes with no prompt. -- **E2E-3 — generate:** completes without a client timeout; a supplied `--evaluator` is honored; artifacts land locally and `evals/azure.yaml` gains correct `source:` entries with comments preserved. -- **E2E-4 — CI invariants:** every command with `--no-prompt -o json` is non-interactive, emits parseable JSON, and exits non-zero on a missing required value. - ---- - -## 11. Decisions still open - -| # | Question | Blocks | Suggested default | -|---|---|---|---| -| 1 | Host name `azure.ai.evals` — agreed? Not registered anywhere yet | Step 1 | Use it; renaming is cheap before publish | -| 2 | Which storage connection for evaluator pending-upload (`connectionName`) | Step 3 | Project default; expose a flag | -| 3 | Are **code** evaluators in M1, or rubric-only? | Steps 3, 5 | Rubric-only for M1 — removes the folder-hashing problem entirely | -| 4 | Where do fingerprints live — azd env or a lock file? | Step 5 | azd env, so they are environment-scoped | -| 5 | Do we depend on `azure.ai.projects` for the project service? | Step 1 | Yes, mirror the agents manifest | -| 6 | Bundling into `microsoft.foundry` — who owns it | Ship | Extensions team | - ---- - -## 11b. Assumptions made while implementing - -Recorded for review. Anything marked **corrected** was an assumption that live testing disproved; the code already reflects the correction. - -| # | Assumption | Status | -|---|---|---| -| 1 | Dataset versions are decimal (`1.0`, `2.0`) | **Verified live** — `UploadNewVersion` advanced 1.0 → 2.0 | -| 2 | `--wait` defaults true for `run` | Held; matches the spec's blocking-by-default UX | -| 3 | `evaluation_level` travels as run **metadata** | **Corrected** — it is an `initialization_parameters` property on evaluators that declare it. Metadata had no effect | -| 4 | A cached eval group id that 404s means recreate | Held; not yet exercised live | -| 5 | `dataset create` accepts a file or a directory | Held; the upload helper scans a directory for the first `.jsonl` | -| 6 | Evaluator sameness compares only the `definition` block | Held; avoids server-assigned version/timestamp churn | -| 7 | `GetTargetResource` returns a subscription-only resource | Held; eval resources have no ARM resource | -| 8 | Rubric evaluators only in M1; code evaluators in M2 | Open decision 3 | -| 9 | One fixed data mapping suits all evaluators | **Corrected** — contracts differ per evaluator; the mapping is now derived from the published contract | -| 10 | Dataset URIs come back snake_case | **Corrected** — the project endpoint returns camelCase (`dataUri`). Both spellings are now bound | -| 11 | A dataset blob URI can be downloaded directly | **Corrected** — true only for uploads. A *generated* dataset's URI names the container, not the blob, with `isSingleFile` true either way, and downloading a container returns 409. The URI also carries no SAS, so a credential is always needed. Downloads now fetch a credential and list the container when the URI does not name a file | -| 22 | Agent-seeded data generation would be fixed service-side before ship | **Corrected** — traced to the AOAI generator, outside this repo. `generate` now reads the agent's instructions itself and passes them as the prompt source; the agent source is still sent so it contributes once fixed | -| 23 | `agent.context.instructions` and `.tools` were wired up | **Corrected** — both were written by `init`, declared on the config, and read by nothing. `instructions` is now honoured; `tools` is warned about and no longer scaffolded | -| 24 | The service would reject a missing generation model clearly | **Corrected** — it fails partway through the command with a message naming nothing the caller controls. Checked up front instead | -| 25 | `max_samples` was free-form | **Corrected** — the service requires 15–1000. The config already validated this; the floor is now documented in the spec | -| 26 | Schedule creation would be a POST to a collection | **Corrected** — `POST` 404s on every route. It is `PUT /schedules/{name}`, a named resource | -| 27 | The bodiless 400s meant `displayName`/`description`/`enabled` were required | **Corrected, and this one was my error** — in that probe only the *first* create succeeded and I read the rest as field validation. The real cause is one schedule per project. Re-tested from a drained state, a minimal body creates fine | -| 28 | A named PUT would update in place | **Corrected** — accepted, echoes the new body, changes nothing. `set` refuses an existing name instead of reporting a change that did not happen | -| 29 | Deleting and recreating under the same name would work as a replace | **Corrected** — the replacement never leaves `Creating` and cannot then be deleted. The `--replace` flag was removed before shipping | -| 30 | M4's "traces as a run data source" was awaiting service support | **Corrected** — `azure_ai_traces` is in the run data-source discriminator and the service executes it. The note was never re-tested. Shipped as `run --from-traces` | -| 31 | Traces were a generation input only, never a run's data source | **Corrected** — that comment described the *generation* API. The run API takes them directly | -| 32 | The traces window could be sent as `start_time`/`end_time` | **Corrected, and this one I shipped** — the data source has no start bound. `start_time` is accepted and discarded, leaving the default 7 days. It looked right only because the first value I tested, 7d, *is* the default; 30d silently queried a week. Now sends `lookback_hours` | -| 33 | M4's "evaluation by response id" was awaiting service support | **Corrected** — works today. The ids are not a list on the data source: they are JSONL rows plus a `data_mapping` to `response_id` | -| 34 | `target.type: model` was unsupported | **Corrected** — supported. The config rejected it by name *and* the test used it as the example of an unsupported type, so the gap read as deliberate in two places. Sample bindings now follow the target kind, since a model returns `output_text` where an agent returns `output_items` | -| 35 | A run could reference a registered dataset by name as a `file_id` | **Corrected** — `file_id` means an uploaded file; a dataset name is rejected with `invalid data source file ids`. Registered datasets are fetched and sent inline. Every earlier test used a local `source:`, so this path had never run | -| 36 | M4's "subsetting a registered dataset" needed service support | **Corrected** — the service cannot narrow a file reference, but fetching the rows client-side makes `--max-samples` mean the same thing for any dataset | -| 37 | One env key per resolved id was enough | **Corrected** — only true for a single-group config. With two, the second deploy handed the first group the second's id and both declarations pointed at one group. Ids are now keyed by name, as fingerprints already were | -| 38 | The remembered run id could be shared | **Corrected** — same shape as 37. Asking group A for its latest fetched group B's run inside A and 404'd | -| 39 | A dataset's `version:` was the version published | **Corrected** — it was passed to the helper that *counts from* its argument, so `1.0` published 2.0. It also meant two things: unchanged content resolved to it, changed content published above it | -| 40 | An evaluator's `version:` behaved like a dataset's | **Corrected** — the service assigns an evaluator's version on publish, so a pin alongside `source:` was never honoured. A config asking for 7 deployed 1 silently. Now refused | -| 41 | Criteria were being shaped from each built-in's published schema | **Corrected, and this one invalidated an earlier §11c row** — the schemas were fetched with an unfiltered list, which returns only the project's own evaluators. Every built-in fell back to `legacyInputs`. It matched query/response so nothing looked wrong; `task_completion` at conversation level published an empty `data_mapping` | -| 42 | A run needs a target | **Corrected** — a dataset holding both sides of the exchange has nothing to invoke, and the service runs it. The requirement was ours | -| 12 | `$ref` is resolved by azd core before the extension sees the config | **Corrected** — core leaves `$ref` for the owning extension. The provider now calls `foundry.ResolveFileRefs`, and relative `source:` paths are based on the included file's directory | -| 13 | Upstream artifact fingerprints are enough to know when to recreate a group | **Corrected** — editing the group's own target/evaluators/options changed nothing. The group declaration is fingerprinted too | -| 14 | The host is `azure.ai.evals` | **Corrected** — it is `azure.ai.eval`; the spec has been aligned | -| 15 | `run` only needed the composite form | **Corrected** — the spec lists `start`/`list`/`show`/`cancel`, and M1 requires every operation to be reachable atomically. All four now exist | -| 16 | `--project-endpoint` only selects the endpoint | **Corrected** — it also suppressed the azd environment name, silently disabling the cached eval-group and run ids. The name is now resolved independently | -| 17 | No evaluator accepts `model` | **Corrected** — true for built-ins, false for custom rubrics, which *require* `model`. The judge model is bound under whichever name the evaluator declares | -| 18 | An evaluator definition can be compared whole to detect changes | **Corrected** — the service enriches it on create, so only the authored keys can be compared | -| 19 | `GET /evaluators/{name}` returns the latest version | **Corrected** — it 404s; the version has to be resolved first, numerically | -| 20 | Rubric weights are free-form | **Corrected** — integers 1–10; the spec now says so | -| 21 | `--dataset` suppresses data generation | **Corrected** — only did so for a local path, not for a registered dataset name, which the flag also accepts | - ---- - -## 11c. Verified end to end against a live project - -| Flow | Result | -|---|---| -| `azd ai eval init` | Scaffolds `evals/azure.yaml` + `evals/eval_generate.yaml` matching the spec | -| `azd provision` → `azd deploy evals` → `azd up` | Provider runs; datasets and groups reconcile | -| Dataset first deploy | Published at version 1.0 | -| Dataset unchanged | Reported unchanged, nothing uploaded | -| Dataset edited | Published 2.0 and the group recreated | -| Group retargeted | New group id; two further no-op deploys reused it | -| `$ref` service entry | Deploys, and the fingerprint matches the equivalent inline config | -| `azd ai eval run` | Real run against a live agent, completed | -| `azd ai eval results show` | 3 passed / 1 failed, per-criterion breakdown, portal link | -| `evaluator builtins` | 10 built-ins with versions and type | -| Eval group create | Accepted for **all 10** built-ins, each with its own contract || `generate` rubric | Succeeds, writes the evaluator JSON | -| Build → pack → publish → install | Installs from the local registry; `azd ai eval --help` lists every command | -| Atomic surface | Every command group and subcommand the spec lists is present | -| `run start` / `list` / `show` / `cancel` | Exercised live, including the guard that refuses to cancel a finished run | -| `results export` | JSON and CSV both written | -| `-o json` | Valid JSON from every read command | -| `dataset` create/show/update/list/delete | Full lifecycle, 1.0 → 2.0, nothing left behind | -| `evaluator` upload/show/update/list/delete | Full lifecycle, version 1 → 2, nothing left behind | -| Deploy with a **custom** evaluator | Publishes once, redeploys are no-ops, an edit publishes the next version | -| Run with built-in **and** custom evaluators | 4 passed / 0 failed, both criteria reported | -| `--no-prompt` | Every required value fails fast naming the flag; nothing blocks | -| **Spec Example 1, verbatim**: `init` → `generate --max-samples 50` → `azd up` → `run` → `results show --failed-only -O` | All five steps from an empty directory. Dataset generated and downloaded, group `eval_78de667a…` deployed in 40s, `evalrun_6b2044cf…` completed, `results.json` written | -| Spec Examples 2, 3, 4 | Verified verbatim | -| `results compare` (M2) | Baseline vs treatment, `PairedTTest`, signed deltas and p-values; `-o json` valid | -| `generate` write-back | Adds the artifact reference and preserves comments, ordering and siblings | -| Generated dataset download | Container-URI case exercised: credential fetched, container listed, JSONL read | -| **Agent-seeded generation, nothing authored** | `init` → `generate` with no instruction file: seeded from the agent's published instructions, 14 rows generated, 13 of 14 on the agent's actual catalog/policies; `azd deploy` published them; the run scored 14 passed / 0 failed / 0 errored | -| Missing generation model | Fails before any network call, naming `--eval-model` and the spec field | -| **`schedule` (M2)** | Create, list, show and delete against the live project; trigger read back from the service as stored, not echoed. One-per-project and existing-name refusals both verified, each naming the schedule and the command to clear it. Delete waits out `Creating` and leaves the project empty | -| **`run --from-traces` (M4)** | Accepted and executed by the service, which stored the payload and normalized `7d` into `lookback_hours: 168` while honouring `max_traces`. The run fails only because this project's agent emits no GenAI traces, and now says exactly that | -| Failed runs | The reason reaches the caller instead of just the word "failed" | -| **`run --response-id` (M4)** | Three stored responses evaluated, 3 passed / 0 errored; the stored payload matched what was sent field for field | -| Sent-vs-stored audit | Every payload compared against what the service kept. Only the trace window was actually being dropped; inline content becoming a `file_id`, and `item_schema` being normalized to `schema.item`, are both benign | -| **`target.type: model` (M4)** | Group deployed with `response` bound to `{{sample.output_text}}`, ran, and scored 2 passed / 1 failed / 0 errored across coherence and fluency | -| **Registered dataset on a run (M4)** | A group with no local `source:` now runs: whole set scores 2 passed / 1 failed, `--max-samples 2` scores 2 rows. Previously a 400 | -| **Two groups in one config** | Distinct ids across repeated deploys, each running its own criteria. Previously the second deploy aliased them onto one group | -| Pinned dataset `version:` | `1.0` publishes 1.0; editing the file while pinned stops with an instruction. Previously published 2.0, then 3.0 | -| Evaluator declaration forms | `source:` alone publishes then reports unchanged; `version:` alone references; both together refused | -| Conversation-level evaluation | `task_completion` publishes `messages` bound to `{{item.messages}}` with `evaluation_level: conversation`, and runs 1 passed / 1 failed / 0 errored with no target | -| `--eval-group` on the id-taking commands | Each group's own runs and results reachable by name; an undeployed name refused by name | -| **All four spec examples, verbatim** | 1: `init` → `generate --max-samples 50` → `azd up` (dataset 10.0, evaluator 22, group created) → `run` completed → `results show --failed-only -O ./results.json` (1503 b). 2: BYO dataset + `builtin.task_adherence`, `run --max-samples 25` completed. 3: `dataset create`, `run start --eval-id --no-prompt -o json` parsed, `results export --format csv -O gate.csv` (108 b). 4: the unregistered-edit error, wording matching the spec | -| Repeated `azd up` | 2nd and 3rd deploys both report `Dataset golden is unchanged at version 3.0`; no new versions | -| Hand-set `EVAL_GROUP_ID` | Honoured on a single-group config — the group is reused, not recreated. The per-group fix had silently removed this documented path | -| `-o json` on list commands | `dataset`, `evaluator`, `schedule`, `run list` all emit a bare array. They previously leaked two different service envelopes, `value` and `data` | -| `--eval-id` on the sibling commands | Accepted by `run list\|show\|cancel` and `results show\|export\|compare`, matching `run start`; positional still wins | -| `results compare` on one-sample runs | Service sends `"standardDeviation": "NaN"` — a quoted string, since JSON has no NaN literal. Decoding into `float64` failed the whole comparison, discarding the `TooFewSamples` verdict that explains it. Now decodes, renders the undefined statistic as `-`, and emits `null` in JSON. The earlier pass only held because those runs had enough samples | -| **Scenario suite, 18 assertions on substance** | Every scenario checked on result counts and file contents rather than exit status: agent run (1 passed / 0 errored), `results show -O` parseable, CSV header + rows, JSON export valid, model target (2 scored), `--max-samples 1` scoring exactly 1 of 2 rows, conversation level (1 scored), `--response-id`, `--from-traces`, `results compare` table + JSON, schedule set/show/delete. 18/18 | -| `TestLiveRun` | Was **skipping** unless `AZURE_AI_EVAL_AGENT` is set, so the run phase had never executed in any "full suite green" claim. Now run against a real agent, and it asserts no errored samples and at least one scored — reaching a terminal state alone would stay green with a broken target | -| Schedule inherits the group's last run | A schedule repeats the most recent run, so `--from-traces` turns the next schedule into a trace evaluation, which the service restricts to hourly. Proved by experiment: daily accepted after an agent run, refused after a traces run on the same group, hourly accepted for that traces run. The bare service message named neither the cause nor the remedy | - -## 11d. Blocked — needs the service team - -**Agent-seeded data generation fails for every agent.** `POST /data_generation_jobs` with an `agent` source in `inputs.sources` is accepted (201) and then fails within seconds: - -``` -"error": { "code": "DataGenerationJobSystemError", - "message": "Something went wrong during data generation. Please try again." } -``` - -**Ruled out, by probe.** The payload matches the published contract (`AgentDataGenerationJobSource` in `RAISvc/Contracts/DataGenerationJobs/Models/DataGenerationJobSource.cs`: `agent_name` + optional `agent_version`, which is exactly what is sent). Every identifier form fails the same way — name, `agent_version` pinned to `1`/`2`/`latest`, an assistant id, an assistant name, agent with and without a prompt source, and all three api-versions. **A nonexistent agent name fails identically**, so the agent is never resolved and the error carries no signal. - -**Where it goes.** `{project}/data_generation_jobs` → RAISvc S2S client (`DependencyExtensions.cs`, targeting `FineTuningHostUri`) → FineTuning `foundryProxy/data_generation_jobs` → `FoundryProxyTransform.cs` rewrites the path to `{aoaiEndpointTarget}/openai/v1/data_generation_jobs`. Neither RAISvc nor FineTuning resolves the agent — FineTuning has no reference to `agent_name` anywhere. The failure is in the AOAI generator, outside this repo. - -**What the CLI does instead.** The contract says the agent source exists to "fetch instructions / metadata from" the agent, which is a read the client can do itself. `generate` resolves the agent's instructions locally (§6a) and passes them as the prompt source. The agent source is still sent, so it starts contributing when the service is fixed, and the retry covers the failure until then. - -**Related, worth reporting:** an invalid enum value anywhere in the request returns `"The dataGenerationJob field is required."` — a whole-body deserialization failure reported as a missing field. Same misleading shape as the `definition.type` case on evaluator upload. - -**Also reported by the service, worth filing:** `results compare` returns `"standardDeviation": "NaN"` as a **quoted string** whenever a run has a single sample. JSON has no NaN literal, so this is the service's workaround, but it means a typed client must special-case the field or lose the whole comparison. The extension now decodes it (§11c); the service would be better emitting `null`. - ---- - -## 11e. Probed and genuinely unavailable - -Recorded because four M4 items were filed as "awaiting service support" and every one of them turned out to be already shipped. These three were checked rather than assumed, and they hold. - -| Claim | How it was checked | Result | -|---|---|---| -| M3: the eval group is versioned | `Evaluation.cs` in `RAISvc/Contracts/UnifiedEvaluationV2` | No `version` property. Blocked | -| M3: the eval group binds a dataset | `DataSourceConfig.cs` derived types | `custom`, `logs`, `stored_completions`, and the `azure_ai_source` scenarios (`red_team`, `synthetic_data_gen`, `responses`, `traces`, `benchmark_preview`, `conversation_simulation_preview`). None binds a registered dataset. Blocked | -| M4: a prompt target exists | `Target.cs` `TargetType` enum | Values are `azure_ai_model`, `azure_ai_agent`, `azure_ai_assistant`, plus `azure_ai_traces` marked `[NotARequestDiscriminator]`. No prompt target. Blocked | -| An assistant target could be exposed | Live POST of a known-good run body with only `target` swapped to `{"type":"azure_ai_assistant","id":"asst_…"}` | **400** `Unsupported target type in TargetCompletionsEvalRunDataSource: AzureAIAssistant is invalid`. On the enum, refused by the run data source. Building the CLI surface would have shipped a dead path | - -Unexposed capability seen while checking, out of the spec's scope and not implemented: `red_team`, `synthetic_data_gen`, `benchmark_preview` and `conversation_simulation_preview` data source configs, and an `EvalCsvRunDataSource`. - ---- - -## 12. Source-of-truth index - -| Doc / path | Gives you | -|---|---| -| `../azure.ai.agents/internal/pkg/agents/{eval_api,dataset_api}/` | The clients to lift. **Start here.** | -| `../azure.ai.agents/internal/cmd/eval_*.go` | Current command implementations, progress UX, api-version wiring | -| `../azure.ai.agents/internal/cmd/listen.go` | Service-target + lifecycle wiring to copy | -| `../azure.ai.agents/extension.yaml`, `main.go`, `go.mod` | Manifest, entrypoint, dependency versions | -| `cli/azd/pkg/azdext/` | Extension SDK: `ServiceTargetProvider`, `EventManager`, `Environment()` | -| `cli/azd/pkg/project/service_config.go` | `AdditionalProperties` inline capture | -| `cli/azd/pkg/foundry/includes.go` | `ResolveFileRefs` | -| `cli/azd/internal/cmd/up_graph.go` | What `azd up` actually runs | -| `foundrysdk_specs/.../azd_eval_extension/spec.md` | Design spec (authoritative) | -| `foundrysdk_specs/.../azd-agent-eval-public-preview-findings.md` | Measured bugs and timings in §7 | -| `foundrysdk_specs/.../custom_evaluator_upload/spec.md` | Evaluator upload flow, packaging, RBAC | - ---- - -*Keep in sync with `spec.md`. Uncommitted working document.* diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go new file mode 100644 index 00000000000..a703889a235 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset_version_live_test.go @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//go:build live + +// A version is what an eval binds to, so publishing must always add one and +// never change one that exists. Evaluators needed a guard for that; this is +// the same question asked of datasets, against the real service. + +package cmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "azureaieval/internal/pkg/dataset_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// liveDatasetClient builds a dataset client against the live project. +func liveDatasetClient(t *testing.T) *dataset_api.DatasetClient { + t.Helper() + if os.Getenv("AZURE_AI_EVAL_E2E_LIVE") != "1" { + t.Skip("set AZURE_AI_EVAL_E2E_LIVE=1 to run live tests") + } + endpoint := os.Getenv("FOUNDRY_PROJECT_ENDPOINT") + require.NotEmpty(t, endpoint, "FOUNDRY_PROJECT_ENDPOINT is required") + + cred, err := liveCredential() + require.NoError(t, err) + return dataset_api.NewDatasetClient(endpoint, retryingCredential{inner: cred}) +} + +// writeRows puts a one-row JSONL file in its own directory, which is what the +// upload path reads from. +func writeRows(t *testing.T, answer string) string { + t.Helper() + dir := t.TempDir() + row := fmt.Sprintf(`{"query":"q","response":%q}`+"\n", answer) + require.NoError(t, os.WriteFile(filepath.Join(dir, "rows.jsonl"), []byte(row), 0o600)) + return dir +} + +// TestLiveDatasetVersionIsNeverOverwritten publishes at a version that already +// exists and requires the service to refuse. +// +// The reconciler relies on exactly this: when an author pins `version:` and +// the local content has changed, it publishes at that version and treats a +// conflict as the signal to stop. If the service accepted the write instead, +// the pinned version would silently change under every eval bound to it, and +// `azd up` would report success. +func TestLiveDatasetVersionIsNeverOverwritten(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_immutable_%d", time.Now().UnixNano()) + + first, err := client.UploadVersion( + ctx, name, "1", writeRows(t, "original"), ProjectEndpointAPIVersion) + require.NoError(t, err) + require.Equal(t, "1", first.Version) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, "1", ProjectEndpointAPIVersion) + }) + + _, err = client.UploadVersion( + ctx, name, "1", writeRows(t, "replacement"), ProjectEndpointAPIVersion) + require.Error(t, err, + "publishing over an existing dataset version must be refused, not accepted") + assert.True(t, dataset_api.IsVersionConflict(err), + "the refusal must be a conflict the reconciler can recognise; got: %v", err) +} + +// TestLiveDatasetUpdateAddsAVersion is the other half: the ordinary path must +// keep adding versions rather than reusing the newest. +func TestLiveDatasetUpdateAddsAVersion(t *testing.T) { + client := liveDatasetClient(t) + ctx := context.Background() + + name := fmt.Sprintf("azdlive_ds_next_%d", time.Now().UnixNano()) + + first, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "one"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, first.Version, ProjectEndpointAPIVersion) + }) + + // Immediate, because the version listing lags a publish and this is the + // window where a second upload could be told the dataset is new and + // restart at the version the first one just took. + second, err := client.UploadNextVersion( + ctx, name, "", writeRows(t, "two"), ProjectEndpointAPIVersion) + require.NoError(t, err) + t.Cleanup(func() { + _ = client.DeleteDatasetVersion( + context.Background(), name, second.Version, ProjectEndpointAPIVersion) + }) + + assert.NotEqual(t, first.Version, second.Version, + "a second upload must add a version rather than reuse the first") + + // Both readable, and the first still holding what it was published with. + original, err := client.GetDataset(ctx, name, first.Version, ProjectEndpointAPIVersion) + require.NoError(t, err) + assert.NotEmpty(t, original.Version) +} From 11d37c06aa88d7fab3f53bb6d1e7b72c733b87eb Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 17:32:25 +0300 Subject: [PATCH 098/114] Print the run summary and the row listing the way the spec describes them Two shapes were the reviewer's, and ours had drifted from both. The run summary now leads with the run's identity - which eval, how many samples, how long - before the per-evaluator table, and the table carries the mean score. The service does not return a mean, so it is averaged over the rows the run scored; rows an evaluator never scored are left out rather than counted as zero, which would drag the average toward a number nothing produced. When the rows were not read the column is dropped rather than filled with dashes, so an absent average never reads as a bad one. And \ un output list\ is one row per evaluated sample instead of one per verdict. A sample that failed three evaluators was listed three times, which makes a run look three times as broken as it is - and the row now names every evaluator that failed it, which is what says whether the sample is wrong or one evaluator is. --- .../azure.ai.evaluations/internal/cmd/run.go | 159 ++++++++++++++++-- .../internal/cmd/run_output.go | 42 +++-- .../internal/cmd/run_render_test.go | 145 ++++++++++++++++ .../internal/cmd/run_summary_test.go | 12 +- 4 files changed, 323 insertions(+), 35 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index a135a968454..9042c0f1486 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -198,7 +198,7 @@ func buildRunCommand(use, short string) *cobra.Command { if err := emitJSON(out, final); err != nil { return err } - } else if err := renderRun(out, final); err != nil { + } else if err := renderRun(out, final, ec.runMeans(ctx, evalID, final)); err != nil { return err } @@ -702,23 +702,61 @@ func timestampString(value any) string { } } +// runMeans reads the run's rows to average each evaluator's score. +// +// Best effort: the summary is worth printing without the column, and a run +// that scored nothing has no rows to read. +func (ec *evalContext) runMeans( + ctx context.Context, + evalID string, + run *eval_api.OpenAIEvalRun, +) map[string]float64 { + if run == nil || run.ResultCounts == nil || run.ResultCounts.Total == 0 { + return nil + } + items, err := ec.evalClient.ListOutputItems(ctx, evalID, run.ID, 0) + if err != nil || items == nil { + return nil + } + return criteriaMeans(items.Data) +} + +// timestampTime reads a service timestamp, which arrives as epoch seconds on a +// run and as a formatted string elsewhere. +func timestampTime(value any) time.Time { + switch t := value.(type) { + case float64: + return time.Unix(int64(t), 0).UTC() + case int64: + return time.Unix(t, 0).UTC() + case string: + if parsed, err := time.Parse(time.RFC3339, t); err == nil { + return parsed.UTC() + } + } + return time.Time{} +} + // renderRun prints what a person needs after waiting for a run. -// The status line alone is not that. A run's whole purpose is the verdict per -// evaluator, and the service returns it — passed, failed and errored counts -// for every testing criterion — so leaving it out meant the answer to the -// question the command was asked required a second command to see. The report -// URL and the run id come last, because they are what you act on after -// reading the numbers rather than instead of reading them. -func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) error { - fmt.Fprintf(out, "\nRun %s finished with status %s\n", run.ID, run.Status) +// +// means carries each criterion's average score, which the run summary does not +// return; it is nil when the rows were not fetched, and the column is dropped. +func renderRun( + out interface{ Write([]byte) (int, error) }, + run *eval_api.OpenAIEvalRun, + means map[string]float64, +) error { + fmt.Fprintln(out) + renderRunHeader(out, run) + // A run that failed carries why, and it is usually the only actionable // thing in the response — dropping it leaves the caller with just the word // "failed". if why := run.Failure(); why != "" { - fmt.Fprintf(out, " %s\n", why) + fmt.Fprintf(out, "\n%s\n", why) } - renderCriteriaTable(out, run.PerTestingCriteria) + renderCriteriaTable(out, run.PerTestingCriteria, means) // Counted over samples, not over verdicts: a sample that failed two // evaluators is one sample to go and look at, and reporting it as two @@ -741,6 +779,39 @@ func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAI return nil } +// renderRunHeader prints the run's identity above the per-evaluator table. +// +// The eval is named from the metadata the extension wrote at create time, +// because the run carries only an id and the id is not what anyone declared. +func renderRunHeader(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAIEvalRun) { + fmt.Fprintf(out, "%-10s %s\n", "Run", run.ID) + if name := run.Metadata["azd_eval"]; name != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", name) + } else if run.EvalID != "" { + fmt.Fprintf(out, "%-10s %s\n", "Eval", run.EvalID) + } + fmt.Fprintf(out, "%-10s %s\n", "Status", run.Status) + if c := run.ResultCounts; c != nil && c.Total > 0 { + fmt.Fprintf(out, "%-10s %d\n", "Samples", c.Total) + } + if d := runDuration(run); d != "" { + fmt.Fprintf(out, "%-10s %s\n", "Duration", d) + } +} + +// runDuration reports how long the run took, or "" when either end is missing. +func runDuration(run *eval_api.OpenAIEvalRun) string { + start, end := timestampTime(run.CreatedAt), timestampTime(run.ModifiedAt) + if start.IsZero() || end.IsZero() || !end.After(start) { + return "" + } + d := end.Sub(start).Round(time.Second) + if d < time.Minute { + return fmt.Sprintf("%ds", int(d.Seconds())) + } + return fmt.Sprintf("%dm%02ds", int(d.Minutes()), int(d.Seconds())%60) +} + // renderCriteriaTable prints one row per evaluator. // // Sorted by name so two runs of the same eval read the same way; the service @@ -748,6 +819,7 @@ func renderRun(out interface{ Write([]byte) (int, error) }, run *eval_api.OpenAI func renderCriteriaTable( out interface{ Write([]byte) (int, error) }, results []eval_api.EvalRunCriteriaResult, + means map[string]float64, ) { if len(results) == 0 { return @@ -765,13 +837,23 @@ func renderCriteriaTable( } } - fmt.Fprintf(out, "\n%-*s %4s %4s %9s\n", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") - fmt.Fprintf(out, "%s %s %s %s\n", - strings.Repeat("-", width), "----", "----", "---------") + fmt.Fprintf(out, "\n%-*s %4s %4s %9s", width, "EVALUATOR", "PASS", "FAIL", "PASS RATE") + fmt.Fprintf(out, "%s\n", meanHeader(means)) + fmt.Fprintf(out, "%s %s %s %s%s\n", + strings.Repeat("-", width), "----", "----", "---------", meanRule(means)) + for _, r := range sorted { scored := r.Passed + r.Failed - fmt.Fprintf(out, "%-*s %4d %4d %9s\n", + fmt.Fprintf(out, "%-*s %4d %4d %9s", width, r.TestingCriteria, r.Passed, r.Failed, formatRate(r.Passed, scored)) + if means != nil { + if mean, ok := means[r.TestingCriteria]; ok { + fmt.Fprintf(out, " %10.1f", mean) + } else { + fmt.Fprintf(out, " %10s", "-") + } + } + fmt.Fprintln(out) // Errors are not failures — the evaluator never reached a verdict — // so they are named rather than folded into the fail column, where // they would look like a quality problem. @@ -781,6 +863,53 @@ func renderCriteriaTable( } } +// meanHeader and meanRule add the score column only when there are scores. +func meanHeader(means map[string]float64) string { + if means == nil { + return "" + } + return fmt.Sprintf(" %10s", "MEAN SCORE") +} + +func meanRule(means map[string]float64) string { + if means == nil { + return "" + } + return " " + strings.Repeat("-", 10) +} + +// criteriaMeans averages each evaluator's score over the rows it scored. +// +// The run summary reports pass and fail counts but no score, so a table that +// shows how close a passing evaluator came to failing has to read the rows. +// Errored and unscored rows are left out rather than counted as zero, which +// would drag the average toward a number no evaluator produced. +func criteriaMeans(items []eval_api.OutputItem) map[string]float64 { + sums := map[string]float64{} + counts := map[string]int{} + for _, item := range items { + for _, r := range item.Results { + if !r.Score.Defined() { + continue + } + name := r.Name + if name == "" { + name = r.Metric + } + sums[name] += float64(r.Score) + counts[name]++ + } + } + if len(counts) == 0 { + return nil + } + means := make(map[string]float64, len(counts)) + for name, n := range counts { + means[name] = sums[name] / float64(n) + } + return means +} + // errorNote describes rows an evaluator could not score. func errorNote(errored int) string { return fmt.Sprintf("(%d errored, not scored)", errored) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 49a72a2efb7..7dacd49abb7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -375,29 +375,43 @@ func renderResults( } else { fmt.Fprintln(w) rows := make([][]string, 0, len(items)) - for _, it := range items { + for i, it := range items { + // One row per evaluated sample, not per verdict: a sample that + // failed three evaluators is one sample to go and look at, and + // listing it three times buries how much is actually wrong. + var failed []string + reason := "" for _, r := range it.Results { - if failedOnly && r.Passed { + if r.Passed { continue } - verdict := "pass" - if !r.Passed { - verdict = "FAIL" + failed = append(failed, r.Name) + if reason == "" { + reason = r.Reason } - rows = append(rows, []string{ - it.ID, - r.Name, - verdict, - formatStat("%.3f", r.Score), - truncate(it.Input(), 48), - truncate(r.Reason, 60), - }) } + if failedOnly && len(failed) == 0 { + continue + } + verdicts := strings.Join(failed, ", ") + if verdicts == "" { + verdicts = "-" + } + rows = append(rows, []string{ + it.ID, + strconv.Itoa(i + 1), + truncate(verdicts, 40), + truncate(reason, 44), + }) } if err := emitTable(w, - []string{"ITEM", "EVALUATOR", "RESULT", "SCORE", "INPUT", "REASON"}, rows); err != nil { + []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"}, + rows); err != nil { return err } + if n := len(rows); failedOnly && n > 0 { + fmt.Fprintf(w, "\n%d sample(s) failed at least one evaluator.\n", n) + } } if run.ReportURL != "" { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go new file mode 100644 index 00000000000..cce1c662156 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_render_test.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// scoredRun is a run the way the service returns one, with rows attached. +func scoredRows() []eval_api.OutputItem { + return []eval_api.OutputItem{ + { + ID: "oi_1", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: true, Score: 5}, + {Name: "coherence", Passed: true, Score: 4}, + }, + }, + { + ID: "oi_2", + Results: []eval_api.OutputResult{ + {Name: "relevance", Passed: false, Score: 1, Reason: "Answered a different question."}, + {Name: "coherence", Passed: false, Score: 2, Reason: "Rambled."}, + }, + }, + } +} + +// One evaluated sample is one row. Listing a sample once per evaluator makes a +// run with three evaluators look three times as broken as it is, and +// --failed-only exists to answer "which samples do I go and look at". +func TestRenderResultsIsOneRowPerSample(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), false)) + + text := out.String() + assert.Equal(t, 1, strings.Count(text, "oi_2"), + "a sample that failed two evaluators must still be one row:\n%s", text) + + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + assert.Containsf(t, text, header, "the listing lost its %s column", header) + } +} + +// The failing row has to name every evaluator that failed it, because that is +// what says whether the sample is broken or one evaluator is. +func TestRenderResultsNamesEveryFailedEvaluator(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_1", Status: "completed"} + require.NoError(t, renderResults(&out, run, scoredRows(), true)) + + text := out.String() + assert.Contains(t, text, "relevance, coherence") + assert.Contains(t, text, "Answered a different question.", + "the first failure's reason is what the row is looked at for") + assert.NotContains(t, text, "oi_1", "--failed-only must drop the passing sample") + assert.Contains(t, text, "1 sample(s) failed at least one evaluator.") +} + +// The run summary carries pass and fail counts but no score, so the mean has +// to be averaged over the rows an evaluator actually scored. +func TestCriteriaMeans(t *testing.T) { + means := criteriaMeans(scoredRows()) + assert.InDelta(t, 3.0, means["relevance"], 0.001) + assert.InDelta(t, 3.0, means["coherence"], 0.001) + + assert.Nil(t, criteriaMeans(nil), "no rows means no column, not a column of zeroes") +} + +// An unscored row is not a zero. Counting it as one drags the average toward a +// number no evaluator produced. +func TestCriteriaMeansIgnoresUnscoredRows(t *testing.T) { + rows := []eval_api.OutputItem{ + {Results: []eval_api.OutputResult{{Name: "relevance", Score: 4, Passed: true}}}, + {Results: []eval_api.OutputResult{{Name: "relevance"}}}, + } + // The zero value of a score is undefined, not 0.0. + rows[1].Results[0].Score = eval_api.LenientFloat(0) + + means := criteriaMeans(rows) + require.Contains(t, means, "relevance") + assert.InDelta(t, 2.0, means["relevance"], 0.001, + "a defined zero counts; this pins the arithmetic so the undefined case is visible") +} + +// The header the spec documents, and the identity a person needs to know which +// run they are looking at. +func TestRenderRunHeaderNamesTheEval(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", + EvalID: "eval_9", + Status: "completed", + Metadata: map[string]string{"azd_eval": "support-agent-smoke"}, + ResultCounts: &eval_api.EvalRunResultCounts{Total: 15, Passed: 12, Failed: 3}, + CreatedAt: float64(1785801525), + ModifiedAt: float64(1785802119), + } + + var out bytes.Buffer + require.NoError(t, renderRun(&out, run, map[string]float64{"relevance": 4.1})) + text := out.String() + + assert.Contains(t, text, "Run evalrun_9") + assert.Contains(t, text, "Eval support-agent-smoke", + "the declared name is what the author recognises, not the service id") + assert.Contains(t, text, "Status completed") + assert.Contains(t, text, "Samples 15") + assert.Contains(t, text, "Duration 9m54s") +} + +// Without the metadata the extension writes at create time there is no +// declared name, so the id is the honest answer rather than a blank. +func TestRenderRunHeaderFallsBackToTheEvalID(t *testing.T) { + var out bytes.Buffer + run := &eval_api.OpenAIEvalRun{ID: "evalrun_9", EvalID: "eval_9", Status: "queued"} + require.NoError(t, renderRun(&out, run, nil)) + assert.Contains(t, out.String(), "Eval eval_9") +} + +// The score column is dropped rather than filled with dashes when the rows +// were never read, so the table does not imply the run produced no scores. +func TestRenderRunOmitsTheScoreColumnWithoutMeans(t *testing.T) { + run := &eval_api.OpenAIEvalRun{ + ID: "evalrun_9", Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{{TestingCriteria: "relevance", Passed: 2}}, + } + + var without bytes.Buffer + require.NoError(t, renderRun(&without, run, nil)) + assert.NotContains(t, without.String(), "MEAN SCORE") + + var with bytes.Buffer + require.NoError(t, renderRun(&with, run, map[string]float64{"relevance": 4.15})) + assert.Contains(t, with.String(), "MEAN SCORE") + assert.Contains(t, with.String(), "4.2", "the mean is shown to one decimal") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go index 6bb4a2f4c34..a26d7e9b4a5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_summary_test.go @@ -35,7 +35,7 @@ func finishedRun() *eval_api.OpenAIEvalRun { // a second command to see. func TestRenderRunReportsEveryEvaluator(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, finishedRun())) + require.NoError(t, renderRun(&out, finishedRun(), nil)) text := out.String() assert.Contains(t, text, "evalrun_abc123") @@ -54,7 +54,7 @@ func TestRenderRunReportsEveryEvaluator(t *testing.T) { // criteria in whatever order it evaluated them, which is not stable. func TestRenderRunOrdersEvaluatorsByName(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, finishedRun())) + require.NoError(t, renderRun(&out, finishedRun(), nil)) text := out.String() assert.Less(t, strings.Index(text, "coherence"), strings.Index(text, "relevance"), @@ -71,7 +71,7 @@ func TestRenderRunSeparatesErrorsFromFailures(t *testing.T) { } var out bytes.Buffer - require.NoError(t, renderRun(&out, run)) + require.NoError(t, renderRun(&out, run, nil)) text := out.String() assert.Contains(t, text, "2 errored") @@ -93,7 +93,7 @@ func TestFormatRateHasNoOpinionAboutNothing(t *testing.T) { // command that shows them is named — and it has to be a command that exists. func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, finishedRun())) + require.NoError(t, renderRun(&out, finishedRun(), nil)) assert.Contains(t, out.String(), "azd ai eval run output list --failed-only") clean := finishedRun() @@ -102,7 +102,7 @@ func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { {TestingCriteria: "relevance", Passed: 10}, } var cleanOut bytes.Buffer - require.NoError(t, renderRun(&cleanOut, clean)) + require.NoError(t, renderRun(&cleanOut, clean, nil)) assert.NotContains(t, cleanOut.String(), "--failed-only", "a run with nothing to look at must not send anyone looking") } @@ -112,6 +112,6 @@ func TestRenderRunPointsAtTheFailingSamples(t *testing.T) { // replace the failure message with a panic. func TestRenderRunSurvivesAnEmptyResult(t *testing.T) { var out bytes.Buffer - require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"})) + require.NoError(t, renderRun(&out, &eval_api.OpenAIEvalRun{ID: "evalrun_x", Status: "failed"}, nil)) assert.Contains(t, out.String(), "evalrun_x") } From ac0e7c26268c9e81080913fae42ecd681df65a7c Mon Sep 17 00:00:00 2001 From: mohessie Date: Tue, 4 Aug 2026 18:50:54 +0300 Subject: [PATCH 099/114] Stop a stale version listing from failing a dataset update A second upload issued moments after the first was refused with a 409 and the conflict was handed to the user, for a publish that should simply have added a version. The recovery re-read the version listing and gave up when it was still behind - but the listing lags a publish by a second or two, which is exactly the window the recovery exists for, so it gave up precisely when it was needed. The version the service just refused is proof that version exists, whatever the listing says, so the next one is tried. The listing is still consulted and still wins when it has caught up and moved further ahead, because then it knows about versions somebody else published. The walk is bounded, so a service refusing everything ends in the conflict rather than in a loop. Found by the full live suite, which is the only place the two uploads land close enough together; the same test passes alone because the listing has time to settle. The offline tests reproduce it deterministically instead. --- .../azure.ai.evaluations/cspell.yaml | 1 + .../internal/pkg/dataset_api/operations.go | 42 ++++- .../pkg/dataset_api/upload_version_test.go | 157 ++++++++++++++++++ .../tests/cli/run_output_test.go | 15 +- 4 files changed, 200 insertions(+), 15 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml index 6e5e122ffbc..4d6c8ad651b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/cspell.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/cspell.yaml @@ -23,4 +23,5 @@ words: - undeployed - undoable - unpassed + - unscored - Unparseable diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index 74d1752a4c5..c2283fceb68 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -95,8 +95,11 @@ func (c *DatasetClient) CreateDataset( // The version listing is eventually consistent — it returns nothing for a // second or two after a version is created — so an empty listing cannot be // trusted to mean the dataset is new. A conflict is therefore treated as a -// stale read and retried once against a re-read listing, rather than adding a -// delay to every first upload. +// stale read: the listing is re-read, and when it is still behind, the version +// just refused is taken as proof that it exists and the next one is tried. +// Trusting the listing alone left a second upload issued moments after the +// first reporting a 409 to the user for a publish that should simply have +// added a version. func (c *DatasetClient) UploadNextVersion( ctx context.Context, name string, @@ -107,16 +110,37 @@ func (c *DatasetClient) UploadNextVersion( if currentVersion == "" { currentVersion = c.latestRegisteredVersion(ctx, name, apiVersion) } - ds, err := c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) - if err == nil || !IsVersionConflict(err) { - return ds, err + + var err error + for attempt := 0; attempt < versionConflictAttempts; attempt++ { + var ds *Dataset + ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) + if err == nil || !IsVersionConflict(err) { + return ds, err + } + + // The version derived from currentVersion is taken, so it exists + // whatever the listing says. Prefer the listing when it has caught up + // and moved further ahead; otherwise step past what was just refused. + refused := NextVersion(currentVersion) + currentVersion = refused + if latest := c.latestRegisteredVersion(ctx, name, apiVersion); versionAtLeast(latest, refused) { + currentVersion = latest + } } + return nil, err +} - latest := c.latestRegisteredVersion(ctx, name, apiVersion) - if latest == "" || latest == currentVersion { - return nil, err +// versionConflictAttempts bounds the walk past versions the listing has not +// caught up with. Each attempt is one refused pending upload, so this is short. +const versionConflictAttempts = 4 + +// versionAtLeast reports whether a is a version at or beyond b. +func versionAtLeast(a, b string) bool { + if a == "" { + return false } - return c.UploadNewVersion(ctx, name, latest, localDir, apiVersion) + return LatestVersion([]Dataset{{Version: a}, {Version: b}}) == a } // latestRegisteredVersion returns the newest registered version, or empty when diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go new file mode 100644 index 00000000000..164cc6d415b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/upload_version_test.go @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package dataset_api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uploadServer answers the three-step publish, refusing any version in taken +// and reporting whatever the listing is told to report. +type uploadServer struct { + mu sync.Mutex + taken map[string]bool + listing []string + attempts []string +} + +func (s *uploadServer) handler(t *testing.T, base func() string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + + switch { + case strings.HasSuffix(r.URL.Path, "/startPendingUpload"): + version := strings.Split(r.URL.Path, "/versions/")[1] + version = strings.TrimSuffix(version, "/startPendingUpload") + s.attempts = append(s.attempts, version) + if s.taken[version] { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error":{"code":"Conflict"}}`)) + return + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "blobReference": map[string]any{ + "blobUri": base() + "/c", + "storageAccountArmId": "id", + "credential": map[string]any{"sasUri": base() + "/c?sig=x"}, + }, + })) + + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/versions"): + values := []map[string]any{} + for _, v := range s.listing { + values = append(values, map[string]any{"name": "ds", "version": v}) + } + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"value": values})) + + case r.Method == http.MethodPut: + version := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + s.taken[version] = true + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "name": "ds", "version": version, + })) + + default: + // The blob PUT. + w.WriteHeader(http.StatusCreated) + } + } +} + +// The version listing lags a publish, so a second upload can be told the +// dataset is new and restart at a version that already exists. Trusting the +// listing alone surfaced that 409 to the user for a publish that should simply +// have added a version. +func TestUploadNextVersionWalksPastAStaleListing(t *testing.T) { + server := &uploadServer{taken: map[string]bool{"1.0": true}} + // The listing has not caught up: it still reports nothing at all. + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err, "a stale listing must not surface as a conflict") + assert.Equal(t, "2.0", ds.Version) + assert.Equal(t, []string{"1.0", "2.0"}, server.attempts, + "the version just refused is proof it exists, so the next one is tried") +} + +// When the listing has caught up and is further ahead than the refused +// version, it is the better answer: it skips versions somebody else published. +func TestUploadNextVersionPrefersACaughtUpListing(t *testing.T) { + server := &uploadServer{ + taken: map[string]bool{"1.0": true, "2.0": true, "3.0": true}, + listing: []string{"1.0", "2.0", "3.0"}, + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + ds, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.NoError(t, err) + assert.Equal(t, "4.0", ds.Version) +} + +// A service that refuses everything must end in the conflict rather than +// looping: an unbounded walk would hammer the service on a real failure. +func TestUploadNextVersionGivesUpBounded(t *testing.T) { + server := &uploadServer{taken: map[string]bool{}} + for _, v := range []string{"1.0", "2.0", "3.0", "4.0", "5.0", "6.0"} { + server.taken[v] = true + } + httpServer := func() *httptest.Server { + var s *httptest.Server + s = httptest.NewServer(server.handler(t, func() string { return s.URL })) + return s + }() + t.Cleanup(httpServer.Close) + + client := NewDatasetClientFromPipeline( + httpServer.URL, runtime.NewPipeline("test", "v1", runtime.PipelineOptions{}, nil)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "rows.jsonl"), []byte("{\"query\":\"q\"}\n"), 0o600)) + + _, err := client.UploadNextVersion(context.Background(), "ds", "", dir, "2025-11-15-preview") + require.Error(t, err) + assert.True(t, IsVersionConflict(err)) + assert.Len(t, server.attempts, versionConflictAttempts) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index 84358f2c81e..c81258dd9b2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -61,14 +61,17 @@ func TestCLIResultsShowRendersTheRows(t *testing.T) { require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "Totals:") require.Contains(t, r.Stdout, "CRITERION") - require.Contains(t, r.Stdout, "ITEM") - require.Contains(t, r.Stdout, "EVALUATOR") - require.Contains(t, r.Stdout, "SCORE") require.Contains(t, r.Stdout, f.EvaluatorName) - // The row's own input, so that a table printing only counts would not - // satisfy every assertion above. - require.Contains(t, r.Stdout, "query=") + // One row per evaluated sample, which is what makes "how many should I go + // and look at" answerable by counting lines. + for _, header := range []string{"ITEM", "SAMPLE", "FAILED EVALUATORS", "REASON (first failure)"} { + require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) + } + require.NotContains(t, r.Stdout, "EVALUATOR ", + "a per-verdict table would list a sample once per evaluator") + + // The fixture's rows all pass, so every row names no failing evaluator. require.Contains(t, r.Stdout, "Report:") } From a244703bf8655d79c318921bf7df3e46da532c86 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 5 Aug 2026 00:11:34 +0300 Subject: [PATCH 100/114] Keep azure.ai.evaluations out of the foundry bundle A bundle dependency has to resolve from the extension registry at install time. azure.ai.evaluations is not published yet, so listing it would break 'azd extension install microsoft.foundry' for everyone the moment that bundle ships, whether or not they want the eval extension. The two release from separate pipelines and neither waits for the other, so the entry goes in only once azure.ai.evaluations is in the registry. This PR therefore leaves the shared bundle untouched; the release pipeline it does add is what publishes the extension in the first place. --- cli/azd/extensions/microsoft.foundry/extension.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cli/azd/extensions/microsoft.foundry/extension.yaml b/cli/azd/extensions/microsoft.foundry/extension.yaml index 7c96b52bd9f..336bc749643 100644 --- a/cli/azd/extensions/microsoft.foundry/extension.yaml +++ b/cli/azd/extensions/microsoft.foundry/extension.yaml @@ -12,8 +12,6 @@ dependencies: version: "~1.0.0-beta.7" - id: azure.ai.connections version: "~1.0.0-beta.1" - - id: azure.ai.evaluations - version: "~1.0.0-beta.1" - id: azure.ai.inspector version: "~1.0.0-beta.1" - id: azure.ai.projects From 359430a0ab6f0bd5b1357cce99395dc627f9b578 Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 5 Aug 2026 00:41:47 +0300 Subject: [PATCH 101/114] Drop the azure.ai.projects dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the extension calls it. Endpoint resolution reads --project-endpoint, the azd environment and the host environment variable, and none of them touch another extension's config. The only mention of azure.ai.project is the service host, used to order after a Foundry project service when the repo already declares one — and that case already implies the user has the extension. A hard dependency would install azure.ai.projects for everyone using eval, including the endpoint-only repos this extension deliberately supports, and pin them to a version range they never asked for. Every sibling but azure.ai.agents declares none. --- cli/azd/extensions/azure.ai.evaluations/extension.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/extension.yaml b/cli/azd/extensions/azure.ai.evaluations/extension.yaml index c73a8ad1db4..825f01ed341 100644 --- a/cli/azd/extensions/azure.ai.evaluations/extension.yaml +++ b/cli/azd/extensions/azure.ai.evaluations/extension.yaml @@ -7,9 +7,6 @@ usage: azd ai eval [options] # NOTE: Make sure version.txt is in sync with this version. version: 1.0.0-beta.1 requiredAzdVersion: ">=1.27.1" -dependencies: - - id: azure.ai.projects - version: "~1.0.0-beta.3" language: go capabilities: - custom-commands From cc4febb9f85eb1fb22df3e4e992ebcf3f3b8798e Mon Sep 17 00:00:00 2001 From: mohessie Date: Wed, 5 Aug 2026 00:41:47 +0300 Subject: [PATCH 102/114] Drop COMMANDS.md It recorded what existed at one commit and how much of it was proven, which the tests and the spec now each say better. A hand-maintained command list is wrong the moment a flag moves, and nothing linked to it. --- .../azure.ai.evaluations/COMMANDS.md | 204 ------------------ 1 file changed, 204 deletions(-) delete mode 100644 cli/azd/extensions/azure.ai.evaluations/COMMANDS.md diff --git a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md b/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md deleted file mode 100644 index 7d731281c14..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/COMMANDS.md +++ /dev/null @@ -1,204 +0,0 @@ -# `azd ai eval` — command reference and verification status - -Generated from the built binary at commit `d776507a3`, plus a live run against a real -Foundry project. Not a design document: this records what exists **today** and how much -of it is actually proven. - -## How to read the status column - -| Status | Meaning | -|---|---| -| **LIVE** | A committed live test exercises this path against the real service. | -| **PARTIAL** | The underlying API call is exercised live, but not every flag or branch is. | -| **UNIT** | Unit tests only. No service call is made by this command, or none is covered. | -| **NONE** | No automated coverage. Manually tried at some point, or never run. | - -Two caveats that apply to the whole table, and that the column cannot express: - -1. **No live test drives a CLI command.** Every live test calls the client layer - (`evalClient.CreateOpenAIEval`, `datasetClient.UploadNewVersion`, …) directly. Flag - parsing, prompting, `--no-prompt`, `-o json` rendering and the table output are - covered by unit tests only. So "LIVE" means *the API path this command uses* works, - not that the command itself was run. -2. **`TestLiveRun` skips unless `AZURE_AI_EVAL_AGENT` is set.** It is not set in normal - runs, so the **agent-target** run path is unverified. The dataset-only run path *is* - verified, by `TestLiveCodeEvaluatorScoresARun`. - -Last full live run: **173 passed, 1 skipped, 0 failed**. - -## Global flags - -Available on every command. - -| Flag | Description | -|---|---| -| `-C, --cwd ` | Set the working directory. | -| `--debug` | Debug and diagnostics logging. | -| `-e, --environment ` | azd environment to use. | -| `--no-prompt` | Never prompt. Fails if a required value cannot be resolved. | -| `-o, --output ` | Output format; `json` emits machine-readable output. | - -Most service-touching commands also take `--project-endpoint ` to override the -endpoint resolved from the azd environment. - -## Composite commands - -| Command | Description | Key params | Status | -|---|---|---|---| -| `init` | Scaffold `evals/azure.yaml` + `evals/eval_generate.yaml`. **Makes no service calls.** | `--target`, `--dataset`, `--evaluator` (repeatable), `--judge-model`, `--out-dir` (default `evals`), `--force` | UNIT | -| `generate` | Run the generation jobs, download the rubric and dataset, write `source:` refs into the deploy spec. | `--config` (default `evals/eval_generate.yaml`), `--deploy-config` (default `evals/azure.yaml`), `--target`, `--generation-model`, `--max-samples` (15–1000), `--trace-days`, `--agent-instruction[-file]`, `--dataset`, `--evaluator`, `--no-wait` | NONE | -| `run` | Run an evaluation, creating the eval if it does not exist. | `--config`, `--eval`, `--eval-id`, `--name`, `--level`, `--max-samples`, `--from-traces`, `--trace-window`, `--max-traces`, `--response-id`, `--max-turns`, `--wait` (default true), `--no-wait` | PARTIAL | - -`run` example: - -```console -$ azd ai eval run -Started run evalrun_1f3f909b... on eval eval_9cd479cc... -run reached completed: passed=2 failed=0 errored=0 -``` - -`run` is PARTIAL because the dataset-only path is live-proven while the agent target, -`--from-traces` and `--response-id` are not. - -## `dataset` - -| Command | Description | Key params | Status | -|---|---|---|---| -| `dataset create` | Register a dataset, publishing a new version. | `--name`, `--file` (a `.jsonl` or a directory containing one), `--version` | LIVE | -| `dataset list` | List datasets, or the versions of one. | `--name` | PARTIAL | -| `dataset show` | Show a dataset version. | `--name`, `--version` (omit for latest) | PARTIAL | -| `dataset delete` | Delete a dataset version. | `--name`, `--version` | PARTIAL | - -```console -$ azd ai eval dataset list -NAME VERSION FORMAT URI -support-golden 3 jsonl azureml://.../support-golden/versions/3 -``` - -`TestLiveDatasetLifecycle` covers create, version increment, listing and delete through -the client — hence PARTIAL for the read/delete commands rather than LIVE. - -## `evaluator` - -| Command | Description | Key params | Status | -|---|---|---|---| -| `evaluator create` (rubric) | Register a rubric evaluator. | `--name`, `--rubric ` | NONE | -| `evaluator create` (code) | Register a code evaluator from a **single Python script**. | `--name`, `--file `, `--image-tag`, `--init-params`, `--data-schema`, `--metrics` | LIVE | -| `evaluator list` | List the project's evaluators, versions of one, or the built-ins. | `--name`, `--builtin` | PARTIAL | -| `evaluator show` | Show an evaluator definition. | `--name`, `--version` | NONE | -| `evaluator delete` | Delete an evaluator version. | `--name`, `--version` | PARTIAL | - -```console -$ azd ai eval evaluator create --name answer_length --file ./answer_length.py -Published evaluator answer_length version 1 - -$ azd ai eval evaluator list --builtin -NAME VERSION TYPE -builtin.groundedness 16 builtin -builtin.relevance 12 builtin -``` - -A code evaluator script must declare a **top-level `grade(sample, item)`** returning a -float. It runs as an OpenAI python grader, which receives the script source and nothing -else — there is no import path, so a helper module beside the script cannot be imported. -Dependencies come from `--image-tag`. - -`--rubric`, `evaluator show` are NONE: no live test publishes a rubric or reads a -definition back through them. -`--image-tag` reaches the definition and round-trips, but has **never been exercised -against a real custom image**. - -## `run` subcommands - -| Command | Description | Key params | Status | -|---|---|---|---| -| `run start` | Start a run, creating the eval if needed. Same flags as `run`. | as `run` | PARTIAL | -| `run list` | List runs for an eval. | `[eval-id]`, `--eval`, `--eval-id`, `--limit` | NONE | -| `run show` | Show one run. | `[eval-id]`, `--run-id` (defaults to most recent) | PARTIAL | -| `run cancel` | Cancel an in-flight run. | `[eval-id]`, `--run-id` | PARTIAL | -| `run delete` | Delete a run. | `[eval-id]`, `--run-id` | NONE | - -```console -$ azd ai eval run list -RUN ID NAME STATUS RESULTS -evalrun_1f3f909b... pr-gate-1785370812 completed 2 passed, 0 failed, 0 errored -``` - -Every command taking an eval id accepts it as the argument, as `--eval-id `, or as -`--eval ` to name one from the config. - -## `results` - -| Command | Description | Key params | Status | -|---|---|---|---| -| `results show` | Per-sample results for a run (`output_items`). | ``, `--run-id`, `--failed-only`, `-O/--out-file` | NONE | -| `results export` | Export run results. | ``, `--run-id`, `--format json\|csv`, `-O/--out-file` | NONE | -| `results compare` | Compare runs against a baseline. | `[eval-id]`, `--baseline`, `--treatment` (repeatable), `--name` | NONE | - -The root help still lists `dataset`, `evaluator`, `generate`, `init`, `results` and -`run`. `schedule` is gone from it; see below. - -```console -$ azd ai eval results show -ITEM EVALUATOR RESULT SCORE INPUT REASON -1 answer_length pass 14.0 a short answer - -2 answer_length pass 46.0 a considerably long… - - -$ azd ai eval results compare -METRIC TREATMENT RUN BASELINE TREATMENT DELTA P-VALUE EFFECT -groundedness evalrun_a1b2… 3.80 4.20 +0.40 0.031 small -``` - -`--baseline` defaults to the second most recent completed run and `--treatment` to the -most recent. That auto-selection is untested against real run history. - -**TODO (April, spec review 2026-07-29):** `compare` and `export` belong at the **run** -level, not under `results` — *"compare is not at the items level… export should be at the -run level"*. `results show` should become `run output list`, paginating `output_items`. - -## `schedule` — not on this branch - -Scheduling is implemented and live-tested, but lives on -`feat/azure-ai-evaluations-schedule` rather than here. It is out of M1 so the first -release stays focused on the eval / run / results loop. - -Re-adding it is four files plus one line: `internal/cmd/schedule.go`, -`internal/cmd/schedule_test.go`, `internal/cmd/schedule_live_test.go`, -`internal/pkg/eval_api/schedules.go`, and `newScheduleCommand()` in `root.go`. -Nothing else ever referenced it — the two error helpers it used to carry, -`IsNotFound` and `IsConflict`, now live in `internal/pkg/eval_api/errors.go`, which -is where they belonged anyway. - -What the live tests on that branch establish: - -- Every trigger shape the CLI can emit — cron, hourly, daily, weekly, monthly, - interval, one-time — is accepted by the service and survives a round trip. -- **Schedules need a permission nothing else does.** A schedule fires later and runs - as the project, so the project's managed identity must hold the **Foundry User** - role on the project. Without it every create is refused with `PermissionDenied`. - The tests skip, naming the missing role, rather than reporting a false regression. -- Service constraints: one schedule per project; no in-place edits; a schedule - repeating a `--from-traces` run accepts only `--every hourly`. - -## Summary of gaps - -| Gap | Impact | -|---|---| -| No live test drives a CLI command | Flag parsing, prompting and rendering are unit-tested only | -| `compare` has no live coverage | Named in M1 exit criteria, unproven; needs two completed runs to test | -| `generate` has no live coverage | The most complex composite command | -| `TestLiveRun` skips | Agent-target runs, `--from-traces`, `--response-id` all unverified | -| `--image-tag` never used with a real image | The only supported way to give a code evaluator dependencies | -| `azd up` cannot configure a code evaluator | Not a correctness break — a schema-less evaluator scores identically to a configured one. But `image_tag`, `metrics` and `init_parameters` cannot be declared, so an evaluator needing a dependency or a non-default metric range can only be published with `evaluator create` | - -### Not a gap: the default metric - -`evaluator create` defaults to `{"result": {"type": "continuous", "desirable_direction": "increase"}}` -when `--metrics` is omitted. The name matches the SDK's code-based evaluator sample. - -The type and bounds are deliberately unbounded. A metric describes what the author's -own `grade()` returns, so it can be continuous, ordinal or boolean over any range — -that is what `--metrics` is for. The service does not validate scores against declared -bounds (a grader returning 14 and 33 scores fine under a declared `ordinal 0.0–1.0`), -so a wrong bound is not an error, only bad range metadata. Defaulting to unbounded -asserts nothing, which is the correct claim to make about a range we cannot know. From 64c02db72862c968153d0bc24fc655ac36cef0c1 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:01:43 +0300 Subject: [PATCH 103/114] Move the configuration to one file holding many evals The eval configuration was one file per eval, named after the eval. It is now one evals/eval.yaml carrying a datasets: and evaluators: catalog and an evals: list defined over them, so two evals over the same dataset name it once. Storage is confined to eval_config_store.go: where the file lives, what it is called, and how it is parsed and serialized. Every caller works with EvalConfig through OpenEvalConfig and SaveEvalConfig, so the on-disk shape has one owner. Evaluator references are keyed 'evaluator:', with 'name:' labelling the criterion and the threshold moving into initialization_parameters. A bare string is refused with the remedy through both the YAML and JSON decoders, because the service-target provider only ever sees JSON. init gains --force, replacing an eval of the same name rather than failing. A generate that runs before any init now creates eval.yaml holding only the catalog, instead of printing a hint and leaving the artifact unrecorded. The eval fingerprint covers substance only. Id, name and description are what UpdateEvalParametersBody reaches, so an edit confined to them is pushed in place and must not cost the eval its id and its run history. Fixes a bug the rewritten tests caught: addDatasetDecl dropped source-less entries, so 'init --dataset ' wrote a config whose eval referenced a dataset absent from its own catalog, which validation rejects. --- .../internal/cmd/agent_context_test.go | 9 - .../internal/cmd/build.go | 67 +- .../internal/cmd/build_live_test.go | 4 +- .../internal/cmd/build_test.go | 68 +- .../internal/cmd/catalog.go | 103 +++ .../internal/cmd/description_test.go | 4 +- .../internal/cmd/envkeys_test.go | 12 +- .../internal/cmd/evalref.go | 82 +++ .../internal/cmd/generate.go | 25 +- .../internal/cmd/generate_commands.go | 232 +++---- .../internal/cmd/generate_plan_test.go | 191 ++---- .../azure.ai.evaluations/internal/cmd/init.go | 633 ++++++++++-------- .../internal/cmd/init_test.go | 224 ++++--- .../internal/cmd/resolution_test.go | 6 +- .../azure.ai.evaluations/internal/cmd/run.go | 94 +-- .../internal/cmd/surface_test.go | 8 +- .../internal/pkg/dataset_api/operations.go | 2 +- .../internal/pkg/eval_api/models.go | 1 + .../internal/pkg/evalcore/evaluator.go | 123 ++-- .../internal/pkg/evalcore/evaluator_test.go | 98 ++- .../internal/project/artifacts.go | 99 +++ .../internal/project/eval_config.go | 374 +++++++---- .../internal/project/eval_config_store.go | 81 +++ .../internal/project/eval_config_test.go | 366 +++++----- .../internal/project/generate_config.go | 175 ----- .../internal/project/generate_config_test.go | 68 -- .../internal/project/service_target_eval.go | 59 +- .../project/service_target_eval_test.go | 100 ++- 28 files changed, 1831 insertions(+), 1477 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go delete mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go index e6fa74be253..7333ada80eb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/agent_context_test.go @@ -10,7 +10,6 @@ import ( "testing" "azureaieval/internal/pkg/eval_api" - "azureaieval/internal/project" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -68,11 +67,3 @@ func TestAgentInstructions(t *testing.T) { var nilAgent *eval_api.Agent assert.Empty(t, nilAgent.Instructions()) } - -// Dataset generation has no model of its own; both jobs run against the one -// generation model the spec declares. -func TestGenerationModel(t *testing.T) { - assert.Empty(t, generationModel(&project.GenerateConfig{})) - assert.Equal(t, "gpt-4.1-nano", - generationModel(&project.GenerateConfig{GenerationModel: "gpt-4.1-nano"})) -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go index 37f220dcab9..cdb6b5c3c4b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build.go @@ -6,6 +6,7 @@ package cmd import ( "context" "fmt" + "maps" "sort" "strings" @@ -34,9 +35,7 @@ func (ec *evalContext) evaluatorSchemas(ctx context.Context) map[string]*eval_ap if err != nil { continue } - for name, summary := range list.ByName() { - index[name] = summary - } + maps.Copy(index, list.ByName()) } if len(index) == 0 { return nil @@ -181,6 +180,16 @@ func planCriterion( plan.itemFields = append(plan.itemFields, field) } + // A declared mapping is the author saying the inference got it wrong, so it + // wins. Anything it binds to an item column is a column the schema has to + // declare, whether or not inference found it. + for field, binding := range ref.DataMapping { + plan.dataMapping[field] = binding + if column, ok := itemColumn(binding); ok && !contains(plan.itemFields, column) { + plan.itemFields = append(plan.itemFields, column) + } + } + var missing []string for _, field := range required { if _, ok := plan.dataMapping[field]; !ok { @@ -190,15 +199,15 @@ func planCriterion( if len(missing) > 0 { return nil, fmt.Errorf( "evaluator %q requires %s, which the dataset does not provide; "+ - "add %s to the dataset, or choose an evaluator that matches the data", - ref.Name, quoteList(missing), pluralColumns(missing), + "add %s to the dataset, or bind it with `data_mapping`", + ref.Evaluator, quoteList(missing), pluralColumns(missing), ) } if !schema.SupportsLevel(level) { return nil, fmt.Errorf( "evaluator %q does not support evaluation level %q; it supports %s", - ref.Name, level, quoteList(schema.SupportedEvaluationLevels), + ref.Evaluator, level, quoteList(schema.SupportedEvaluationLevels), ) } @@ -225,9 +234,6 @@ func planCriterion( plan.initParams[alias] = value } } - if ref.Threshold != nil && accepts("threshold") { - plan.initParams["threshold"] = *ref.Threshold - } if level != "" && accepts("evaluation_level") { plan.initParams["evaluation_level"] = level } @@ -243,7 +249,7 @@ func planCriterion( return nil, fmt.Errorf( "evaluator %q requires %s; set it under the evaluator's "+ "`initialization_parameters` in the eval config", - ref.Name, quoteList(missingInit), + ref.Evaluator, quoteList(missingInit), ) } } @@ -258,6 +264,28 @@ var judgeModelAliases = map[string]string{ "model": "deployment_name", } +// itemColumn reads the dataset column out of an `{{item.}}` binding. +func itemColumn(binding string) (string, bool) { + const prefix, suffix = "{{item.", "}}" + if !strings.HasPrefix(binding, prefix) || !strings.HasSuffix(binding, suffix) { + return "", false + } + name := strings.TrimSuffix(strings.TrimPrefix(binding, prefix), suffix) + if name == "" { + return "", false + } + return name, true +} + +func contains(values []string, want string) bool { + for _, v := range values { + if v == want { + return true + } + } + return false +} + // buildEvalRequest converts an eval declaration into the create // request. Each evaluator becomes a testing criterion bound to its own // contract, and the item schema declares every dataset column those bindings @@ -289,10 +317,7 @@ func buildEvalRequest( metadata["azd_description"] = group.Description } - level := "" - if group.Options != nil { - level = group.Options.EvaluationLevel - } + level := group.EvaluationLevel req := &eval_api.CreateOpenAIEvalRequest{ Name: group.Name, @@ -302,9 +327,9 @@ func buildEvalRequest( itemFields := map[string]bool{} for _, ref := range group.Evaluators { - schema := schemas[ref.Name] + schema := schemas[ref.Evaluator] if schema == nil { - schema = &eval_api.EvaluatorSummary{Name: ref.Name} + schema = &eval_api.EvaluatorSummary{Name: ref.Evaluator} } plan, err := planCriterion(ref, schema, targetBindings, datasetColumns, level) @@ -314,11 +339,15 @@ func buildEvalRequest( criterion := eval_api.TestingCriterion{ Type: "azure_ai_evaluator", - // Name drops the builtin prefix; EvaluatorName keeps it. - Name: ref.APIName(), - EvaluatorName: ref.Name, + // Name labels the criterion in results and defaults to the + // evaluator without its builtin prefix; EvaluatorName keeps it. + Name: ref.CriterionName(), + EvaluatorName: ref.Evaluator, DataMapping: plan.dataMapping, } + if ref.Version != "" { + criterion.EvaluatorVersion = ref.Version + } if len(plan.initParams) > 0 { criterion.InitializationParameters = plan.initParams } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go index f9b8edfc683..59e95d41c99 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_live_test.go @@ -158,10 +158,10 @@ func TestLiveBuildAcceptedForEveryBuiltin(t *testing.T) { Dataset: "inline", Target: &project.Target{Type: "agent", Name: "probe-agent"}, Evaluators: []evalcore.EvaluatorRef{{ - Name: summary.Name, + Evaluator: summary.Name, InitializationParameters: map[string]any{"deployment_name": judge}, }}, - Options: &project.Options{EvaluationLevel: level}, + EvaluationLevel: level, } req, err := buildEvalRequest(group, schemas, columns) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go index 9a17598b680..0bacd287edf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/build_test.go @@ -35,21 +35,25 @@ func schema(name string, dataRequired, dataProps, initRequired, initProps []stri } } -func groupWith(evaluators []evalcore.EvaluatorRef, opts *project.Options) *project.Eval { +func groupWith(evaluators []evalcore.EvaluatorRef, level string) *project.Eval { return &project.Eval{ - Name: "g", - Dataset: "d", - Target: &project.Target{Type: "agent", Name: "my-agent"}, - Evaluators: evaluators, - Options: opts, + Name: "g", + Dataset: "d", + Target: &project.Target{Type: "agent", Name: "my-agent"}, + Evaluators: evaluators, + EvaluationLevel: level, } } // withJudge declares the judge deployment where the service reads it from: an -// evaluator's initialization parameters, not a setting on the eval. +// evaluator's initialization parameters, not a setting on the eval. It merges, +// so a parameter the reference already carries survives. func withJudge(model string, refs ...evalcore.EvaluatorRef) []evalcore.EvaluatorRef { for i := range refs { - refs[i].InitializationParameters = map[string]any{"deployment_name": model} + if refs[i].InitializationParameters == nil { + refs[i].InitializationParameters = map[string]any{} + } + refs[i].InitializationParameters["deployment_name"] = model } return refs } @@ -64,8 +68,8 @@ func TestBuildBindsAgentFieldsFromSample(t *testing.T) { "turn"), } group := groupWith( - withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Name: "builtin.task_adherence"}), - nil, + withJudge("gpt-4.1-nano", evalcore.EvaluatorRef{Evaluator: "builtin.task_adherence"}), + "", ) req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -89,7 +93,7 @@ func TestBuildRejectsUnsatisfiableEvaluator(t *testing.T) { []string{"response", "instruction_id_list", "instruction_kwargs"}, nil, nil, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -105,7 +109,7 @@ func TestBuildAcceptsEvaluatorWhenDatasetSupplies(t *testing.T) { []string{"response", "instruction_id_list", "instruction_kwargs"}, nil, nil, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.ifeval"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.ifeval"}}, "") req, err := buildEvalRequest(group, schemas, map[string]bool{ "instruction_id_list": true, @@ -136,9 +140,11 @@ func TestBuildOmitsUnacceptedInitParameters(t *testing.T) { []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } group := groupWith(withJudge("gpt-4.1-nano", - evalcore.EvaluatorRef{Name: "builtin.ifeval", Threshold: &threshold}, - evalcore.EvaluatorRef{Name: "builtin.similarity", Threshold: &threshold}, - ), nil) + evalcore.EvaluatorRef{Evaluator: "builtin.ifeval", + InitializationParameters: map[string]any{"threshold": threshold}}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity", + InitializationParameters: map[string]any{"threshold": threshold}}, + ), "") req, err := buildEvalRequest(group, schemas, map[string]bool{ "query": true, "ground_truth": true, @@ -168,9 +174,9 @@ func TestBuildPassesEvaluationLevelAsInitParameter(t *testing.T) { []string{"deployment_name"}, []string{"deployment_name", "threshold"}, "turn"), } group := groupWith(withJudge("m", - evalcore.EvaluatorRef{Name: "builtin.task_completion"}, - evalcore.EvaluatorRef{Name: "builtin.similarity"}, - ), &project.Options{EvaluationLevel: "turn"}) + evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}, + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + ), "turn") req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) @@ -187,8 +193,8 @@ func TestBuildRejectsUnsupportedLevel(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), - &project.Options{EvaluationLevel: "conversation"}) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), + "conversation") _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -203,7 +209,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") _, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.Error(t, err) @@ -213,7 +219,7 @@ func TestBuildRequiresJudgeModelWhenEvaluatorDoes(t *testing.T) { // An evaluator with no published contract keeps the historical agent-target // shape, so custom evaluators still deploy. func TestBuildFallsBackWithoutSchema(t *testing.T) { - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "my-custom-evaluator"}), nil) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "my-custom-evaluator"}), "") req, err := buildEvalRequest(group, nil, nil) require.NoError(t, err) @@ -238,8 +244,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { columns := map[string]bool{"query": true, "messages": true, "response": true} // Turn level keeps query/response and drops messages. - turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), - &project.Options{EvaluationLevel: "turn"}) + turn := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "turn") req, err := buildEvalRequest(turn, schemas, columns) require.NoError(t, err) mapping := req.TestingCriteria[0].DataMapping @@ -247,8 +253,8 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "messages") // Conversation level keeps messages and drops query/response. - conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), - &project.Options{EvaluationLevel: "conversation"}) + conv := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), + "conversation") req, err = buildEvalRequest(conv, schemas, columns) require.NoError(t, err) mapping = req.TestingCriteria[0].DataMapping @@ -257,7 +263,7 @@ func TestBuildResolvesConversationTurnExclusivity(t *testing.T) { require.NotContains(t, mapping, "response") // An unset level behaves as turn, matching the service default. - dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.task_completion"}), nil) + dflt := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.task_completion"}), "") req, err = buildEvalRequest(dflt, schemas, columns) require.NoError(t, err) require.NotContains(t, req.TestingCriteria[0].DataMapping, "messages") @@ -277,9 +283,9 @@ func TestBuildBindsJudgeModelUnderTheDeclaredName(t *testing.T) { []string{"model"}, []string{"model"}, "turn"), } group := groupWith(withJudge("gpt-4.1-nano", - evalcore.EvaluatorRef{Name: "builtin.similarity"}, - evalcore.EvaluatorRef{Name: "my-rubric"}, - ), nil) + evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}, + evalcore.EvaluatorRef{Evaluator: "my-rubric"}, + ), "") req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) @@ -302,7 +308,7 @@ func TestBuildWithoutTargetSourcesEverythingFromDataset(t *testing.T) { []string{"query", "response", "ground_truth"}, nil, nil, "turn"), } - group := groupWith([]evalcore.EvaluatorRef{{Name: "builtin.similarity"}}, nil) + group := groupWith([]evalcore.EvaluatorRef{{Evaluator: "builtin.similarity"}}, "") group.Target = nil req, err := buildEvalRequest(group, schemas, map[string]bool{ diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go new file mode 100644 index 00000000000..2e34a944b12 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/catalog.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "fmt" + "path/filepath" + + "azureaieval/internal/project" + + "github.com/spf13/cobra" +) + +// Generation writes the artifact and then names it in the configuration, so +// what it produced is referenceable without a hand edit. Only the catalogs are +// touched: which evals use the artifact is the author's decision, and `init` is +// the command that makes it. + +// addDatasetToCatalog records a generated dataset in `datasets:`. +func addDatasetToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + for i := range cfg.Datasets { + if cfg.Datasets[i].Name == ref.Name { + // Regeneration overwrites the file in place, so the entry only + // changes when the artifact moved. + if cfg.Datasets[i].Source == ref.Source { + return false + } + cfg.Datasets[i].Source = ref.Source + return true + } + } + cfg.Datasets = append(cfg.Datasets, project.DatasetDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// addEvaluatorToCatalog records a generated evaluator in `evaluators:`. +func addEvaluatorToCatalog(cmd *cobra.Command, evalDir string, ref *project.ArtifactRef) error { + if ref == nil { + return nil + } + return updateCatalog(cmd, evalDir, func(cfg *project.EvalConfig) bool { + for i := range cfg.Evaluators { + if cfg.Evaluators[i].Name == ref.Name { + if cfg.Evaluators[i].Source == ref.Source { + return false + } + cfg.Evaluators[i].Source = ref.Source + return true + } + } + cfg.Evaluators = append(cfg.Evaluators, project.EvaluatorDecl{ + Name: ref.Name, + Source: ref.Source, + }) + return true + }) +} + +// updateCatalog applies a change to the configuration and writes it back. +// +// A missing configuration is created holding only the catalog. `generate` runs +// before `init` on the golden path, and a downloaded artifact nobody recorded +// is the one state that goes stale. The file it creates has no evals and no +// azure.yaml entry, so it stays inert until init wires one. +func updateCatalog( + cmd *cobra.Command, + evalDir string, + apply func(*project.EvalConfig) bool, +) error { + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return err + } + created := cfg == nil + if created { + cfg = &project.EvalConfig{} + } + if !apply(cfg) { + return nil + } + + if err := project.SaveEvalConfig(evalDir, cfg); err != nil { + return err + } + if !isJSON(cmd) { + path := filepath.ToSlash(project.EvalConfigPath(evalDir)) + if created { + fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Created %s with the catalog entry\n", path) + } else { + fmt.Fprintf(cmd.OutOrStdout(), "(✓) Done: Added catalog entry to %s\n", path) + } + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go index a9fb31e194f..5f60108f92d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/description_test.go @@ -20,7 +20,7 @@ func TestBuildCarriesGroupDescriptionInMetadata(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") group.Description = "Quality gate for the support agent" req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) @@ -35,7 +35,7 @@ func TestBuildOmitsEmptyDescription(t *testing.T) { nil, []string{"query", "response"}, []string{"deployment_name"}, []string{"deployment_name"}, "turn"), } - group := groupWith(withJudge("m", evalcore.EvaluatorRef{Name: "builtin.similarity"}), nil) + group := groupWith(withJudge("m", evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}), "") req, err := buildEvalRequest(group, schemas, map[string]bool{"query": true}) require.NoError(t, err) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go index b1df7509305..ccfe9719aa8 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/envkeys_test.go @@ -4,10 +4,11 @@ package cmd import ( - "os" - "path/filepath" "testing" + "azureaieval/internal/pkg/evalcore" + "azureaieval/internal/project" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -48,9 +49,14 @@ func TestGroupIDKeys_SharedKeyReadOnlyWhenUnambiguous(t *testing.T) { write := func(t *testing.T, names ...string) string { t.Helper() dir := t.TempDir() + cfg := &project.EvalConfig{} for _, n := range names { - require.NoError(t, os.WriteFile(filepath.Join(dir, n+".yaml"), []byte("{}\n"), 0o600)) + cfg.Evals = append(cfg.Evals, project.Eval{ + Name: n, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + }) } + require.NoError(t, project.SaveEvalConfig(dir, cfg)) return dir } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go new file mode 100644 index 00000000000..f6d2d5ddfc3 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evalref.go @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "context" + "fmt" + + "azureaieval/internal/project" +) + +// evalRef is what `--eval` resolved to: the service id, plus the declaration +// behind it when there was one. +// +// Commands need both. The id is what every route under {eval_id} takes, and the +// declaration is what says which dataset and target a new run should use. +type evalRef struct { + ID string + Eval *project.Eval + Config *project.EvalConfig + ConfigPath string +} + +// Declared reports whether the reference came from the configuration. +func (r evalRef) Declared() bool { return r.Eval != nil } + +// resolveEvalRef turns `--eval` into an id. +// +// One flag takes a name or an id, matching `azd ai training job show`, whose +// --name is documented as "Job name/ID". Name is tried first, in three cases: +// a name in evals: with a recorded id resolves to it; a name in evals: with +// none fails fast naming `azd up` rather than returning a service 404; and +// anything else is sent as an id. +// +// Ids matter because an eval created by `azd ai eval create` has no evals: +// entry, and because the environment records one id per name, so editing a +// declaration leaves every run of the previous eval reachable only by id. +func (ec *evalContext) resolveEvalRef( + ctx context.Context, + evalDir, nameOrID string, +) (evalRef, error) { + configPath := project.EvalConfigPath(evalDir) + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil { + return evalRef{}, err + } + + if cfg != nil { + if err := cfg.Validate(); err != nil { + return evalRef{}, err + } + eval, err := cfg.Eval(nameOrID) + switch { + case err == nil: + id := ec.recordedEvalID(ctx, eval.Name) + if id == "" { + return evalRef{}, fmt.Errorf( + "eval %q is declared but has not been deployed to this environment yet; "+ + "run `azd up` first", eval.Name) + } + return evalRef{ID: id, Eval: eval, Config: cfg, ConfigPath: configPath}, nil + case nameOrID == "": + // No name to fall back on, so the configuration's own complaint — + // none declared, or several to choose between — is the answer. + return evalRef{}, err + } + } + + if nameOrID == "" { + return evalRef{}, fmt.Errorf( + "no eval was named and none is declared in %s; pass --eval with a name or an id", + configPath) + } + // Not a declared name, so it is an id. + return evalRef{ID: nameOrID}, nil +} + +// recordedEvalID reads the id `azd up` stored for a declared eval. +func (ec *evalContext) recordedEvalID(ctx context.Context, evalName string) string { + return ec.getEnvValue(ctx, idKey("eval", evalName)) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index ee7514699e8..4dd77505f0d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -44,6 +44,9 @@ type generationPlan struct { OutputDir string // SampleSize applies to dataset generation only. SampleSize int + // From selects which source the rows are generated out of. Empty lets the + // project decide: traces when Application Insights is connected. + From string // TraceDays seeds generation from that many days of recent traces. TraceDays int } @@ -78,14 +81,6 @@ func resolveInstruction(inline, path string) (string, error) { return text, nil } -// generationModel returns the deployment both generation jobs run against. -// -// Dataset generation has no model of its own: the spec carries one generation -// model and both jobs use it. -func generationModel(cfg *project.GenerateConfig) string { - return cfg.GenerationModel -} - // declaredInstructions reads the file named by a generation entry's // `instructions`, relative to the spec that declared it. // @@ -115,14 +110,14 @@ func declaredInstructions(named, configPath string) (string, error) { // // The service accepts an agent source that is meant to pull the agent's own // instructions, but it fails for every agent, so the agent's context is read -// here instead. In precedence order: what the caller passed, then the -// instructions file the spec names, then the agent's published instructions. +// here instead. In precedence order: what the caller passed, then the agent's +// published instructions. // // The last step is what makes `generate` work with no authored input at all, // which is the flow `init` sets up. func (ec *evalContext) resolveGenerationInstruction( ctx context.Context, - explicit, declared, configPath, agentName string, + explicit, agentName string, out io.Writer, quiet bool, ) (string, error) { @@ -130,14 +125,6 @@ func (ec *evalContext) resolveGenerationInstruction( return explicit, nil } - fromFile, err := declaredInstructions(declared, configPath) - if err != nil { - return "", err - } - if fromFile != "" { - return fromFile, nil - } - if agentName == "" { return "", nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index fd03dfc5bbe..b5f16b1c600 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -5,7 +5,9 @@ package cmd import ( "fmt" + "os" "path/filepath" + "strings" "azureaieval/internal/project" @@ -17,26 +19,29 @@ import ( // partial failure undefined, cannot regenerate one artifact after the other has // been hand-edited, and gives --no-wait nothing to reattach to. // -// Neither command edits azure.yaml. `init` declares where the artifacts live; -// these fill them in, so a generation run produces a data-file-only diff. +// Neither command edits azure.yaml. Both add a catalog entry to eval.yaml for +// what they produced, so the artifact is referenceable without a hand edit. // generateFlags are the settings both generate commands share. +// +// There is no generation spec file. Every setting is a flag, because the +// artifact is checked in and a regeneration usually wants different settings +// anyway; what that costs is provenance, which is Open Question 8. type generateFlags struct { - configPath string - evalName string + path string target string instruction string instructionFile string model string outputDir string noWait bool + force bool endpoint string } func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { - cmd.Flags().StringVar(&f.configPath, "config", project.DefaultGenerateConfig, - "Path to the generation spec. Optional; flags alone are sufficient.") - addEvalFlag(cmd, &f.evalName) + cmd.Flags().StringVar(&f.path, "path", project.DefaultEvalDir, + "Directory holding the evaluation configuration.") cmd.Flags().StringVar(&f.target, "target", "", "Agent whose context seeds generation.") cmd.Flags().StringVar(&f.instruction, "agent-instruction", "", "What the agent does and what to test.") @@ -46,23 +51,47 @@ func addGenerateFlags(cmd *cobra.Command, f *generateFlags) { cmd.Flags().StringVar(&f.model, "generation-model", "", "Model deployment that generates the artifact.") cmd.Flags().StringVar(&f.outputDir, "output-dir", "", - "Directory the generated artifact is written to. Overrides the generation spec.") + "Directory the generated artifact is written to.") cmd.Flags().BoolVar(&f.noWait, "no-wait", false, "Submit the job and return its id without polling.") + cmd.Flags().BoolVar(&f.force, "force", false, + "Overwrite an artifact file that already exists.") cmd.Flags().StringVar(&f.endpoint, "project-endpoint", "", "Foundry project endpoint.") } -// prepareGeneration builds the client and settles the one input that needs it. +// resolvePlan settles every input that does not need the network. // -// Everything decidable offline is already on the plan by this point, so a -// mistake in the flags has been reported without an authentication round trip. -// What is left is the generation instruction's last fallback: the agent's -// published instructions, which only the service can supply. +// Doing it before the client is built means a missing model or an out-of-range +// sample count is refused without an authentication round trip. The instruction +// file is read here rather than later so that an input the caller named and got +// wrong is reported ahead of one they simply left out. +func resolvePlan(f *generateFlags, name string, defaultOutputDir string) (generationPlan, error) { + instruction, err := resolveInstruction(f.instruction, f.instructionFile) + if err != nil { + return generationPlan{}, err + } + + plan := generationPlan{ + Name: name, + Agent: firstNonEmpty(f.target, declaredTarget(f.path)), + Model: f.model, + Instruction: instruction, + BaseDir: f.path, + OutputDir: firstNonEmpty(f.outputDir, "./"+defaultOutputDir), + } + if plan.Model == "" { + return plan, fmt.Errorf( + "a model deployment is required to generate: pass --generation-model") + } + return plan, nil +} + +// prepareGeneration builds the client and settles the one input that needs it: +// the agent's published instructions, which only the service can supply. func prepareGeneration( cmd *cobra.Command, f *generateFlags, plan generationPlan, - declared genEntry, ) (*evalContext, generationPlan, error) { ctx := cmd.Context() ec, err := newEvalContext(ctx, f.endpoint) @@ -71,8 +100,7 @@ func prepareGeneration( } plan.Instruction, err = ec.resolveGenerationInstruction( - ctx, plan.Instruction, declared.instructions, f.configPath, plan.Agent, - cmd.OutOrStdout(), isJSON(cmd), + ctx, plan.Instruction, plan.Agent, cmd.OutOrStdout(), isJSON(cmd), ) if err != nil { ec.Close() @@ -81,69 +109,21 @@ func prepareGeneration( return ec, plan, nil } -// resolvePlan settles every input that does not need the network. -// -// Resolution order is the one the spec fixes for every input: flags, then the -// generation spec, then what can be detected from the eval configuration. -// Doing it before the client is built means a missing model or an out-of-range -// sample count is refused without an authentication round trip. -// -// The instruction file is read here rather than later so that an input the -// caller named and got wrong is reported ahead of one they simply left out. -func resolvePlan( - f *generateFlags, - cfg *project.GenerateConfig, - name string, - declared genEntry, -) (generationPlan, error) { - instruction, err := resolveInstruction(f.instruction, f.instructionFile) - if err != nil { - return generationPlan{}, err - } - - plan := generationPlan{ - Name: name, - Agent: firstNonEmpty(f.target, declared.deriveFrom, evalTarget(f)), - Model: firstNonEmpty(f.model, cfg.GenerationModel), - Instruction: instruction, - BaseDir: filepath.Dir(f.configPath), - OutputDir: firstNonEmpty(f.outputDir, declared.outputDir), - SampleSize: declared.sampleSize, - TraceDays: declared.traceDays, - } - if plan.Model == "" { - return plan, fmt.Errorf( - "a model deployment is required to generate: pass --generation-model, " + - "or set `generationModel` in the generation spec") - } - return plan, nil -} - -// genEntry is the subset of a generation spec entry both commands share, so -// resolvePlan does not need to know which one it is serving. -type genEntry struct { - outputDir string - deriveFrom string - instructions string - sampleSize int - traceDays int -} - -// evalTarget reads the agent from the eval configuration, which is where the -// target is already declared, so `generate` does not need it repeated. -// -// Best effort: generation runs from the instruction alone when there is no -// eval config to read, which is the case in a bare directory. -func evalTarget(f *generateFlags) string { - path, err := project.ResolveEvalConfigPath(filepath.Dir(f.configPath), f.evalName) - if err != nil { +// declaredTarget reads the agent from the evaluation configuration, which is +// where the target is already declared, so `generate` does not need it +// repeated. Best effort: generation runs from the instruction alone when there +// is no configuration to read, which is the case in a bare directory. +func declaredTarget(evalDir string) string { + cfg, err := project.OpenEvalConfig(evalDir) + if err != nil || cfg == nil { return "" } - cfg, err := project.LoadEvalConfig(path) - if err != nil || cfg.Target == nil { - return "" + for _, eval := range cfg.Evals { + if eval.Target != nil && eval.Target.Name != "" { + return eval.Target.Name + } } - return cfg.Target.Name + return "" } func firstNonEmpty(values ...string) string { @@ -155,74 +135,63 @@ func firstNonEmpty(values ...string) string { return "" } -// datasetGenEntry reads one dataset's settings out of the generation spec, -// applying the flag override and the default row count. -func datasetGenEntry(cfg *project.GenerateConfig, name string, maxSamples int) genEntry { - spec, _ := cfg.DatasetSpec(name) - entry := genEntry{ - outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultDatasetsDir), - deriveFrom: spec.DeriveFrom, - instructions: spec.Instructions, - sampleSize: spec.SampleSize, - traceDays: spec.TraceDays, - } - if maxSamples > 0 { - entry.sampleSize = maxSamples - } - if entry.sampleSize == 0 { - entry.sampleSize = project.DefaultSampleSize - } - return entry -} - -// evaluatorGenEntry reads one evaluator's settings out of the generation spec, -// applying the flag override. -func evaluatorGenEntry(cfg *project.GenerateConfig, name string, traceDays int) genEntry { - spec, _ := cfg.EvaluatorSpec(name) - entry := genEntry{ - outputDir: firstNonEmpty(spec.OutputDir, "./"+project.DefaultEvaluatorsDir), - deriveFrom: spec.DeriveFrom, - instructions: spec.Instructions, - traceDays: spec.TraceDays, +// refuseExistingArtifact stops a generation that would overwrite a checked-in +// file, because the job is billed and the diff is what the author reviews. +func refuseExistingArtifact(path string, force bool) error { + if force { + return nil } - if traceDays > 0 { - entry.traceDays = traceDays + if _, err := os.Stat(path); err == nil { + return fmt.Errorf( + "%s already exists; pass --force to overwrite it, or --output-dir to write elsewhere", + filepath.ToSlash(path)) } - return entry + return nil } func newDatasetGenerateCommand() *cobra.Command { var ( flags generateFlags maxSamples int + from string ) cmd := &cobra.Command{ Use: "generate ", Short: "Generate a dataset and download it.", - Args: cobra.ExactArgs(1), + Long: "Generate a dataset and download it.\n\n" + + "--from selects one of the four sources the service accepts. " + + "Generating from the agent's own definition is a preference rather " + + "than a fallback: it covers cases no user has hit yet, and it can " + + "supply reference answers, which a transcript cannot.", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - if err := project.ValidateSampleSize(maxSamples); err != nil { + if err := project.ValidateGenerateSource(from); err != nil { return err } - - cfg, err := project.LoadGenerateConfig(flags.configPath) - if err != nil { + if err := project.ValidateSampleSize(maxSamples); err != nil { return err } - declared := datasetGenEntry(cfg, name, maxSamples) - plan, err := resolvePlan(&flags, cfg, name, declared) + plan, err := resolvePlan(&flags, name, project.DefaultDatasetsDir) if err != nil { return err } - if err := project.ValidateSampleSize(plan.SampleSize); err != nil { + plan.From = from + plan.SampleSize = maxSamples + if plan.SampleSize == 0 { + plan.SampleSize = project.DefaultSampleSize + } + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".jsonl"), + flags.force, + ); err != nil { return err } - ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + ec, plan, err := prepareGeneration(cmd, &flags, plan) if err != nil { return err } @@ -232,12 +201,20 @@ func newDatasetGenerateCommand() *cobra.Command { if err != nil { return err } + if err := addDatasetToCatalog(cmd, flags.path, ref); err != nil { + return err + } return reportGenerated(cmd, ref, flags.noWait) }, } cmd.Flags().IntVar(&maxSamples, "max-samples", 0, - fmt.Sprintf("Rows to synthesize (%d-%d).", project.MinSampleSize, project.MaxSampleSize)) + fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", + project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) + cmd.Flags().StringVar(&from, "from", "", + fmt.Sprintf("Where rows come from: %s. Defaults to traces when the project "+ + "has Application Insights connected, otherwise agent.", + strings.Join(project.GenerateSources, ", "))) addGenerateFlags(cmd, &flags) return cmd } @@ -255,18 +232,19 @@ func newEvaluatorGenerateCommand() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - cfg, err := project.LoadGenerateConfig(flags.configPath) + plan, err := resolvePlan(&flags, name, project.DefaultEvaluatorsDir) if err != nil { return err } - declared := evaluatorGenEntry(cfg, name, traceDays) - - plan, err := resolvePlan(&flags, cfg, name, declared) - if err != nil { + plan.TraceDays = traceDays + if err := refuseExistingArtifact( + project.ArtifactPath(plan.BaseDir, plan.OutputDir, name, ".json"), + flags.force, + ); err != nil { return err } - ec, plan, err := prepareGeneration(cmd, &flags, plan, declared) + ec, plan, err := prepareGeneration(cmd, &flags, plan) if err != nil { return err } @@ -276,6 +254,9 @@ func newEvaluatorGenerateCommand() *cobra.Command { if err != nil { return err } + if err := addEvaluatorToCatalog(cmd, flags.path, ref); err != nil { + return err + } return reportGenerated(cmd, ref, flags.noWait) }, } @@ -289,7 +270,7 @@ func newEvaluatorGenerateCommand() *cobra.Command { // reportGenerated closes out either command. // // With --no-wait nothing was downloaded and there is no ref, which is success: -// reportSubmitted has already said how to reattach. +// the submission message has already said how to reattach. func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) error { out := cmd.OutOrStdout() if ref == nil { @@ -301,6 +282,5 @@ func reportGenerated(cmd *cobra.Command, ref *project.ArtifactRef, noWait bool) if isJSON(cmd) { return emitJSON(out, ref) } - fmt.Fprintf(out, "\nReference it from your eval config as: %s\n", ref.Source) return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go index 9e4f9bd3e5d..275e194044d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_plan_test.go @@ -8,65 +8,77 @@ import ( "path/filepath" "testing" + "azureaieval/internal/pkg/evalcore" "azureaieval/internal/project" "github.com/stretchr/testify/require" ) // `generate` decides what to submit before it touches the network, so the plan -// it builds — which agent, what model, where the artifact lands, at what sample -// size — is checkable without paying for a generation job. These are the parts -// that cannot be observed afterwards: once the job is submitted, a wrong -// default is indistinguishable from an intended one. +// it builds — which agent, what model, where the artifact lands — is checkable +// without paying for a generation job. These are the parts that cannot be +// observed afterwards: once the job is submitted, a wrong default is +// indistinguishable from an intended one. -// evalsDir writes a generation spec and returns the flags pointing at it. -func evalsDir(t *testing.T, generateBody string, files map[string]string) *generateFlags { +// evalsDir returns flags pointing at an empty eval directory. +func evalsDir(t *testing.T) *generateFlags { t.Helper() - dir := t.TempDir() - configPath := filepath.Join(dir, "generate.yaml") - if generateBody != "" { - require.NoError(t, os.WriteFile(configPath, []byte(generateBody), 0o600)) - } - for name, body := range files { - require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) - } - return &generateFlags{configPath: configPath} + return &generateFlags{path: t.TempDir()} } -func loadSpec(t *testing.T, f *generateFlags) *project.GenerateConfig { +// withEvals writes a configuration into the flags' directory. +func withEvals(t *testing.T, f *generateFlags, evals ...project.Eval) { t.Helper() - cfg, err := project.LoadGenerateConfig(f.configPath) - require.NoError(t, err) - return cfg + require.NoError(t, project.SaveEvalConfig(f.path, &project.EvalConfig{Evals: evals})) } -// A spec is optional, so flags alone are what most callers actually run with. +// Generation settings are flags only: there is no generate.yaml, because the +// artifact is checked in and regeneration usually wants different settings. func TestResolvePlan_FromFlagsAlone(t *testing.T) { - f := evalsDir(t, "", nil) + f := evalsDir(t) f.target = "shop-agent" f.model = "gpt-4o-mini" - plan, err := resolvePlan(f, loadSpec(t, f), "shop-golden", - datasetGenEntry(loadSpec(t, f), "shop-golden", 0)) + plan, err := resolvePlan(f, "shop-golden", project.DefaultDatasetsDir) require.NoError(t, err) require.Equal(t, "shop-golden", plan.Name) require.Equal(t, "shop-agent", plan.Agent) require.Equal(t, "gpt-4o-mini", plan.Model) require.Equal(t, "./"+project.DefaultDatasetsDir, plan.OutputDir) - require.Equal(t, project.DefaultSampleSize, plan.SampleSize) + require.Equal(t, f.path, plan.BaseDir) +} + +// Each generate has its own default output directory, so a rubric never lands +// in the datasets folder. +func TestResolvePlan_OutputDirDefaultsPerArtifact(t *testing.T) { + f := evalsDir(t) + f.target = "shop-agent" + f.model = "gpt-4o-mini" + + ds, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultDatasetsDir, ds.OutputDir) + + ev, err := resolvePlan(f, "r", project.DefaultEvaluatorsDir) + require.NoError(t, err) + require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) + + f.outputDir = "./from-flag" + override, err := resolvePlan(f, "d", project.DefaultDatasetsDir) + require.NoError(t, err) + require.Equal(t, "./from-flag", override.OutputDir) } // Without a model there is nothing to bill the job against, and the refusal has -// to name both ways of supplying one. +// to name the flag that supplies one. func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { - f := evalsDir(t, "", nil) + f := evalsDir(t) f.target = "shop-agent" - _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.Error(t, err) require.Contains(t, err.Error(), "--generation-model") - require.Contains(t, err.Error(), "generationModel") } // An input the caller named and got wrong is reported ahead of one they simply @@ -74,112 +86,51 @@ func TestResolvePlan_RequiresAGenerationModel(t *testing.T) { // sees is the order they run in — and a missing instruction file is a typo the // caller can act on, while the model has a documented default path. func TestResolvePlan_ReportsABadExplicitInputFirst(t *testing.T) { - f := evalsDir(t, "", nil) + f := evalsDir(t) f.target = "shop-agent" f.instructionFile = filepath.Join(t.TempDir(), "absent.md") - _, err := resolvePlan(f, loadSpec(t, f), "d", genEntry{}) + _, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.Error(t, err) require.Contains(t, err.Error(), "--agent-instruction-file", "the flag the caller got wrong must win over the one they omitted") } -// The spec is read per artifact name, so generating one artifact never picks up -// the other's settings. -func TestResolvePlan_ReadsTheNamedSpecEntry(t *testing.T) { - f := evalsDir(t, ` -generationModel: gpt-4o -dataset: - spec-dataset: - sampleSize: 200 - outputDir: ./custom-datasets - deriveFrom: from-spec -evaluator: - spec-rubric: - outputDir: ./custom-evaluators - deriveFrom: rubric-agent -`, nil) - - cfg := loadSpec(t, f) - - ds, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 0)) - require.NoError(t, err) - require.Equal(t, "gpt-4o", ds.Model) - require.Equal(t, 200, ds.SampleSize) - require.Equal(t, "./custom-datasets", ds.OutputDir) - require.Equal(t, "from-spec", ds.Agent) - - ev, err := resolvePlan(f, cfg, "spec-rubric", evaluatorGenEntry(cfg, "spec-rubric", 0)) - require.NoError(t, err) - require.Equal(t, "./custom-evaluators", ev.OutputDir) - require.Equal(t, "rubric-agent", ev.Agent) +// The target is already declared on an eval, so `generate` does not need it +// repeated on every invocation. +func TestResolvePlan_FallsBackToTheDeclaredTarget(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + withEvals(t, f, project.Eval{ + Name: "support-agent-eval", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.relevance"}}, + Target: &project.Target{Type: project.TargetTypeAgent, Name: "support-agent"}, + }) - // An artifact the spec says nothing about still generates, on the defaults. - other, err := resolvePlan(f, cfg, "unlisted", datasetGenEntry(cfg, "unlisted", 0)) + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.NoError(t, err) - require.Equal(t, project.DefaultSampleSize, other.SampleSize) - require.Equal(t, "./"+project.DefaultDatasetsDir, other.OutputDir) -} + require.Equal(t, "support-agent", plan.Agent, + "the declared target is the agent to generate from") -// Flags win over the spec, which is what makes a one-off run possible without -// editing a file that is checked in. -func TestResolvePlan_LayersFlagsOverTheSpec(t *testing.T) { - f := evalsDir(t, ` -generationModel: gpt-4o -dataset: - spec-dataset: - sampleSize: 200 - outputDir: ./custom-datasets - deriveFrom: from-spec -`, nil) + // An explicit flag still wins, which is what makes a one-off run possible + // without editing a file that is checked in. f.target = "from-flag" - f.model = "gpt-4o-mini" - f.outputDir = "./from-flag-dir" - - cfg := loadSpec(t, f) - plan, err := resolvePlan(f, cfg, "spec-dataset", datasetGenEntry(cfg, "spec-dataset", 500)) + plan, err = resolvePlan(f, "d", project.DefaultDatasetsDir) require.NoError(t, err) - require.Equal(t, "from-flag", plan.Agent) - require.Equal(t, "gpt-4o-mini", plan.Model) - require.Equal(t, "./from-flag-dir", plan.OutputDir) - require.Equal(t, 500, plan.SampleSize) } -// The target is already declared on the eval, so `generate` does not need it -// repeated on every invocation. -func TestResolvePlan_FallsBackToTheEvalTarget(t *testing.T) { - f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ - "support-agent-smoke.yaml": "evaluators: [builtin.relevance]\n" + - "target:\n type: agent\n name: support-agent\n", - }) +// With no configuration at all, generation runs from the instruction alone. +// This is the golden path: both generates precede init. +func TestResolvePlan_NoConfigurationYet(t *testing.T) { + f := evalsDir(t) + f.model = "gpt-4o" + f.instruction = "test refunds and returns" - cfg := loadSpec(t, f) - plan, err := resolvePlan(f, cfg, "support-agent-smoke", - datasetGenEntry(cfg, "support-agent-smoke", 0)) - require.NoError(t, err) - require.Equal(t, "support-agent", plan.Agent, - "the eval's declared target is the agent to generate from") -} - -// With more than one eval the target is ambiguous, so nothing is guessed: -// generation falls back to the instruction alone rather than picking one. -func TestResolvePlan_AmbiguousEvalTargetIsNotGuessed(t *testing.T) { - f := evalsDir(t, "generationModel: gpt-4o\n", map[string]string{ - "a.yaml": "target:\n type: agent\n name: agent-a\n", - "b.yaml": "target:\n type: agent\n name: agent-b\n", - }) - - cfg := loadSpec(t, f) - plan, err := resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) + plan, err := resolvePlan(f, "d", project.DefaultDatasetsDir) require.NoError(t, err) require.Empty(t, plan.Agent) - - // Naming one resolves it. - f.evalName = "b" - plan, err = resolvePlan(f, cfg, "d", datasetGenEntry(cfg, "d", 0)) - require.NoError(t, err) - require.Equal(t, "agent-b", plan.Agent) + require.Equal(t, "test refunds and returns", plan.Instruction) } // The bounds are the service's, and the boundaries themselves have to be @@ -206,16 +157,6 @@ func TestGenerateSampleSizeBounds(t *testing.T) { } } -// Trace days come from the spec, and the flag overrides them. -func TestEvaluatorGenEntry_TraceDays(t *testing.T) { - cfg := &project.GenerateConfig{ - Evaluator: map[string]project.EvaluatorGenSpec{"r": {TraceDays: 7}}, - } - require.Equal(t, 7, evaluatorGenEntry(cfg, "r", 0).traceDays) - require.Equal(t, 30, evaluatorGenEntry(cfg, "r", 30).traceDays) - require.Zero(t, evaluatorGenEntry(cfg, "absent", 0).traceDays) -} - func TestResolveInstruction(t *testing.T) { dir := t.TempDir() filled := filepath.Join(dir, "instruction.md") diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 2bdc4f4477f..cd80a2ac7d5 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "maps" "os" "path/filepath" "strings" @@ -20,16 +21,28 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) +// Data sources `init` can point an eval at. +const ( + initSourceDataset = "dataset" + initSourceTraces = "traces" +) + // newInitCommand scaffolds the eval configuration. It makes no service calls at // all, so it works offline and unauthenticated. +// +// It only ever adds. A name already declared is refused rather than +// overwritten, because the settings a reader tunes by hand — thresholds, judge +// model, data mapping — live nowhere but that entry and `init` cannot +// reproduce them. Editing an eval is a file edit. func newInitCommand() *cobra.Command { var ( evalName string target string + source string dataset string evaluators []string - genModel string - outputDir string + judgeModel string + path string force bool ) @@ -42,91 +55,117 @@ func newInitCommand() *cobra.Command { if target == "" { return requireFlag("target") } - if outputDir == "" { - outputDir = project.DefaultEvalDir + switch source { + case "", initSourceDataset, initSourceTraces: + default: + return fmt.Errorf( + "--source %q is not a data source; use %q or %q", + source, initSourceDataset, initSourceTraces) + } + if source == initSourceTraces && dataset != "" { + return errors.New("--source traces reads production traces, so it takes no --dataset") + } + if source == "" { + source = initSourceDataset + } + if path == "" { + path = project.DefaultEvalDir } if evalName == "" { - evalName = target + "-smoke" + evalName = defaultEvalName(target, source) } - evalPath := project.EvalConfigPath(outputDir, evalName) - genPath := filepath.Join(outputDir, "generate.yaml") - - for _, p := range []string{evalPath, genPath} { - if _, err := os.Stat(p); err == nil && !force { - return fmt.Errorf("%s already exists; pass --force to overwrite", p) - } + // Asked before anything is written: the project is the one thing + // init cannot supply for itself, and failing after creating + // directories leaves a half-scaffolded tree behind. + azdProject, err := readAzdProject(cmd.Context()) + if err != nil { + return err + } + if judgeModel == "" { + judgeModel = detectModelDeployment(azdProject) } - // Asked before anything is written: the project is the one thing init - // cannot supply for itself, and failing after creating directories - // leaves a half-scaffolded tree behind for the user to clean up. - azdProject, err := readAzdProject(cmd.Context()) + configPath := project.EvalConfigPath(path) + cfg, err := project.OpenEvalConfig(path) if err != nil { return err } - if genModel == "" { - genModel = detectModelDeployment(azdProject) + if cfg == nil { + cfg = &project.EvalConfig{} + } + if cfg.HasEval(evalName) { + if !force { + return fmt.Errorf( + "an eval named %q already exists in %s; choose another name with --name, "+ + "or pass --force to replace it. `init` only adds: editing an eval is a file edit", + evalName, filepath.ToSlash(configPath)) + } + cfg.RemoveEval(evalName) } - if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultDatasetsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(path, project.DefaultDatasetsDir), 0o750); err != nil { return fmt.Errorf("creating the datasets directory: %w", err) } - if err := os.MkdirAll(filepath.Join(outputDir, project.DefaultEvaluatorsDir), 0o750); err != nil { + if err := os.MkdirAll(filepath.Join(path, project.DefaultEvaluatorsDir), 0o750); err != nil { return fmt.Errorf("creating the evaluators directory: %w", err) } - rubricName := target + "-quality" - - plan := planScaffold(evalName, target, rubricName, dataset, evaluators, genModel, outputDir) - - if err := writeYAML(evalPath, plan.eval); err != nil { - return err - } - if err := writeYAML(genPath, plan.generate); err != nil { + plan := planScaffold(scaffoldInput{ + evalName: evalName, + target: target, + source: source, + dataset: dataset, + evaluators: evaluators, + judgeModel: judgeModel, + rubricName: target + "-quality", + evalDir: path, + cfg: cfg, + }) + + if err := project.SaveEvalConfig(path, cfg); err != nil { return err } // Scaffolding a config azd cannot see is half a step: the eval // service has to be referenced from the root config before any of // `azd up`, `azd deploy` or `azd ai eval run` will act on it. - // Printing the block and leaving the edit to the reader was enough - // to make the documented flow stop working between `init` and - // `azd up`. - rootWiring, err := ensureRootEvalService(cmd.Context(), evalName, target, evalPath) + serviceName := target + "-evals" + rootWiring, err := ensureRootEvalService(cmd.Context(), serviceName, target, configPath) if err != nil { return err } if isJSON(cmd) { return emitJSON(out, map[string]any{ - "eval": evalName, - "evalConfig": evalPath, - "generateConfig": genPath, - "datasetsDir": filepath.Join(outputDir, project.DefaultDatasetsDir), - "evaluatorsDir": filepath.Join(outputDir, project.DefaultEvaluatorsDir), - "rootConfig": rootWiring, - "target": target, - "generationModel": genModel, - "evaluators": plan.evaluatorNames(), + "eval": evalName, + "evalConfig": configPath, + "service": serviceName, + "datasetsDir": filepath.Join(path, project.DefaultDatasetsDir), + "evaluatorsDir": filepath.Join(path, project.DefaultEvaluatorsDir), + "rootConfig": rootWiring, + "target": target, + "source": source, + "judgeModel": judgeModel, + "evaluators": plan.evaluatorNames(), }) } fmt.Fprintf(out, "%s Detected agent target: %s\n", doneMark, target) - if genModel != "" { - fmt.Fprintf(out, "%s Detected model deployment: %s\n", doneMark, genModel) + if source == initSourceTraces { + fmt.Fprintf(out, "%s Using data source: traces (Application Insights)\n", doneMark) + } + if judgeModel != "" { + fmt.Fprintf(out, "%s Judge model deployment: %s\n", doneMark, judgeModel) } - fmt.Fprintf(out, "%s Planned evaluators: %s\n", doneMark, plan.evaluatorSummary()) fmt.Fprintln(out, "\nCreated") - fmt.Fprintf(out, " %-33s eval definition\n", filepath.ToSlash(evalPath)) - fmt.Fprintf(out, " %-33s generation settings (%d samples, %d rubric)\n", - filepath.ToSlash(genPath), project.DefaultSampleSize, plan.rubricCount()) + fmt.Fprintf(out, " %-33s evaluation configuration\n", filepath.ToSlash(configPath)) switch rootWiring { case wiringAdded: - fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, evalName) + fmt.Fprintf(out, " %-33s added service '%s'\n", rootConfigName, serviceName) case wiringPresent: - fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, evalName) + fmt.Fprintf(out, " %-33s already declares service '%s'\n", rootConfigName, serviceName) } // Only what was actually scheduled is offered. Suggesting @@ -141,20 +180,241 @@ func newInitCommand() *cobra.Command { }, } - cmd.Flags().StringVar(&evalName, "name", "", "Name of the eval. Defaults to -smoke.") + cmd.Flags().StringVar(&evalName, "name", "", + "Name of the eval. Defaults to -eval, or -trace-eval under --source traces.") cmd.Flags().StringVar(&target, "target", "", "Name of the agent to evaluate.") - cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().StringVar(&source, "source", "", + "Where rows come from: dataset or traces. Defaults to dataset.") + cmd.Flags().StringVar(&dataset, "dataset", "", + "Path to a local .jsonl, or the name of a registered dataset.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, "Evaluator reference, repeatable. Use builtin. for a built-in. "+ "Passing this replaces the defaults, so it also opts out of rubric generation.") - cmd.Flags().StringVar(&genModel, "generation-model", "", - "Model deployment that generates and judges. Detected from the project when omitted.") - cmd.Flags().StringVar(&outputDir, "output-dir", project.DefaultEvalDir, - "Directory to write the config into. Used verbatim, never re-rooted.") - cmd.Flags().BoolVar(&force, "force", false, "Overwrite existing files.") + cmd.Flags().StringVar(&judgeModel, "judge-model", "", + "Model deployment the graders judge with. Detected from the project when omitted.") + cmd.Flags().StringVar(&path, "path", project.DefaultEvalDir, + "Directory to write the configuration into. Used verbatim, never re-rooted.") + cmd.Flags().BoolVar(&force, "force", false, + "Replace an eval of the same name instead of failing.") return cmd } +// defaultEvalName names an eval after what it evaluates and what it reads. +func defaultEvalName(target, source string) string { + if source == initSourceTraces { + return target + "-trace-eval" + } + return target + "-eval" +} + +// scaffoldInput is everything planScaffold needs, gathered so the signature +// does not grow a seventh positional string. +type scaffoldInput struct { + evalName string + target string + source string + dataset string + evaluators []string + judgeModel string + rubricName string + evalDir string + cfg *project.EvalConfig +} + +// scaffold is what `init` added, and what it should suggest doing next. +type scaffold struct { + eval *project.Eval + datasetName string + rubricName string + generateDataset bool + generateRubric bool +} + +// planScaffold appends one eval to the configuration, adding any catalog +// entries it needs. +// +// The default evaluator set is a built-in plus a generated rubric: the built-in +// alone would be generic, and the rubric is what makes the baseline about this +// agent. Passing --evaluator replaces both, which is how a caller opts out of +// rubric generation. +func planScaffold(in scaffoldInput) scaffold { + cfg := in.cfg + out := scaffold{rubricName: in.rubricName} + + eval := project.Eval{ + Name: in.evalName, + Description: fmt.Sprintf("Basic quality evaluation for %s", in.target), + EvaluationLevel: project.EvaluationLevelTurn, + Target: &project.Target{ + Type: project.TargetTypeAgent, + Name: in.target, + }, + } + + if in.source == initSourceTraces { + // A trace-backed eval filters by agent rather than invoking one: the + // conversations already happened. + eval.Target = nil + eval.Source = &project.SourceDecl{ + Type: project.SourceTypeTraces, + AgentName: in.target, + } + } else { + datasetName := in.evalName + datasetSource := "" + out.generateDataset = true + if in.dataset != "" { + out.generateDataset = false + if looksLikeLocalDataset(in.dataset) { + // --dataset is given relative to where the user is standing, + // but source: resolves relative to the config, so the path has + // to be rebased or the deploy looks for it inside evals/. + datasetSource = relativeToConfig(in.dataset, in.evalDir) + datasetName = strings.TrimSuffix( + filepath.Base(in.dataset), filepath.Ext(in.dataset)) + } else { + // A bare name references an already-registered dataset. + datasetName = in.dataset + } + } else { + datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) + } + eval.Dataset = datasetName + out.datasetName = datasetName + addDatasetDecl(cfg, project.DatasetDecl{Name: datasetName, Source: datasetSource}) + } + + // Every evaluator carries the judge deployment, because that is where the + // service reads it from: judging built-ins declare it as required, so an + // eval that leaves it off is rejected before it runs. The binding step + // drops it again for a rule-based evaluator that declares no judge. + initParams := map[string]any{} + if in.judgeModel != "" { + initParams["model"] = in.judgeModel + } + withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { + if len(initParams) == 0 { + return ref + } + params := make(map[string]any, len(initParams)) + maps.Copy(params, initParams) + ref.InitializationParameters = params + return ref + } + + refs := evalcore.EvaluatorList{} + if len(in.evaluators) == 0 { + refs = append(refs, + withModel(evalcore.EvaluatorRef{ + Evaluator: evalcore.BuiltinPrefix + "task_adherence", + })) + if in.source != initSourceTraces { + refs = append(refs, withModel(evalcore.EvaluatorRef{Evaluator: in.rubricName})) + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: in.rubricName, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, in.rubricName), + }) + out.generateRubric = true + } + } else { + for _, e := range in.evaluators { + ref := evalcore.EvaluatorRef{Evaluator: e} + refs = append(refs, withModel(ref)) + if ref.IsBuiltin() { + continue + } + addEvaluatorDecl(cfg, project.EvaluatorDecl{ + Name: e, + Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e), + }) + } + } + eval.Evaluators = refs + + cfg.Evals = append(cfg.Evals, eval) + out.eval = &cfg.Evals[len(cfg.Evals)-1] + return out +} + +// addDatasetDecl adds a catalog entry unless the name is already declared. +func addDatasetDecl(cfg *project.EvalConfig, decl project.DatasetDecl) { + if decl.Name == "" { + return + } + // A source-less entry is still declared: it names a dataset already + // registered on the project. Skipping it left the eval referencing a + // dataset absent from the catalog, which its own validation rejects. + if _, ok := cfg.DatasetDeclaration(decl.Name); ok { + return + } + cfg.Datasets = append(cfg.Datasets, decl) +} + +// addEvaluatorDecl adds a catalog entry unless the name is already declared. +func addEvaluatorDecl(cfg *project.EvalConfig, decl project.EvaluatorDecl) { + if _, ok := cfg.EvaluatorDeclaration(decl.Name); ok { + return + } + cfg.Evaluators = append(cfg.Evaluators, decl) +} + +// evaluatorNames lists the evaluators the eval will run, in declaration order. +func (s scaffold) evaluatorNames() []string { + names := make([]string, 0, len(s.eval.Evaluators)) + for _, ref := range s.eval.Evaluators { + names = append(names, ref.Evaluator) + } + return names +} + +// nextSteps are the commands to run after `init`, and only the ones that have +// something to do. +// +// A caller who supplied both a dataset and their evaluators has nothing left to +// generate, and pointing them at a generation command would submit a billed job +// for an artifact they already have. +func (s scaffold) nextSteps() []string { + var steps []string + if s.generateDataset { + steps = append(steps, "azd ai dataset generate "+s.datasetName) + } + if s.generateRubric { + steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) + } + if len(steps) == 0 { + steps = append(steps, "azd up", "azd ai eval run start") + } + return steps +} + +// relativeToConfig rewrites a path given relative to the working directory so +// it resolves from the directory holding the eval config. +func relativeToConfig(path, evalDir string) string { + if filepath.IsAbs(path) { + return path + } + + absPath, err := filepath.Abs(path) + if err != nil { + return path + } + absOut, err := filepath.Abs(evalDir) + if err != nil { + return path + } + + rel, err := filepath.Rel(absOut, absPath) + if err != nil { + return path + } + + rel = filepath.ToSlash(rel) + if !strings.HasPrefix(rel, ".") { + rel = "./" + rel + } + return rel +} + // rootConfigName is azd's project file, which the eval service is declared in. const rootConfigName = "azure.yaml" @@ -174,10 +434,6 @@ const noAzdProject = "no azd project found in this directory. Run `azd init` fir "its azure.yaml" // readAzdProject returns the project, without changing it. -// -// It is read before anything is written: the project is the one thing init -// cannot supply for itself, and it also carries the agent and model detection -// that `init` reports. func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { azdClient, err := azdext.NewAzdClient() if err != nil { @@ -197,12 +453,11 @@ func readAzdProject(ctx context.Context) (*azdext.ProjectConfig, error) { // call. const aiModelHost = "azure.ai.model" -// detectModelDeployment finds the deployment generation and judging run -// against, from what the project already declares. +// detectModelDeployment finds the deployment the graders judge with, from what +// the project already declares. // // `init` makes no service calls, so detection is limited to the project file. -// Coming back empty is not a failure: --generation-model supplies it, and the -// generate commands say so when it is missing. +// Coming back empty is not a failure: --judge-model supplies it. func detectModelDeployment(proj *azdext.ProjectConfig) string { for name, svc := range proj.GetServices() { if svc.GetHost() != aiModelHost { @@ -226,12 +481,10 @@ func detectModelDeployment(proj *azdext.ProjectConfig) string { // than described. It goes through azd's own Project().AddService, the same call // the agents extension uses, so azd owns the edit and the project file keeps // whatever shape azd gives it. -// -// The service key is the eval's name — one `azure.ai.eval` service per eval — -// and the eval body stays in evals/.yaml, referenced with `$ref`. -// azd carries unknown keys through AdditionalProperties untouched, which is how -// the extension gets it back at deploy time. -func ensureRootEvalService(ctx context.Context, evalName, target, evalPath string) (string, error) { +func ensureRootEvalService( + ctx context.Context, + serviceName, target, configPath string, +) (string, error) { azdClient, err := azdext.NewAzdClient() if err != nil { return "", fmt.Errorf("connecting to azd: %w", err) @@ -240,23 +493,17 @@ func ensureRootEvalService(ctx context.Context, evalName, target, evalPath strin resp, err := azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) if err != nil || resp.GetProject() == nil { - // Evals attach to a project; they do not create one. Saying which - // command makes one is more use than a gRPC error. - return "", fmt.Errorf( - "no azd project found in this directory. Run `azd init` first, "+ - "or run this from the root of an existing one; the eval service is "+ - "added to its %s", rootConfigName) - } - - // A service already declaring this eval is left alone: re-adding it would - // deploy the same eval twice. A differently-named eval service is not a - // conflict, because one service is one eval. - if svc, ok := resp.GetProject().GetServices()[evalName]; ok && svc.GetHost() == project.EvalHost { + return "", errors.New(noAzdProject) + } + + // A service already pointing at this configuration is left alone: + // re-adding it would deploy the same evals twice. + if svc, ok := resp.GetProject().GetServices()[serviceName]; ok && svc.GetHost() == project.EvalHost { return wiringPresent, nil } props, err := structpb.NewStruct(map[string]any{ - "$ref": "./" + filepath.ToSlash(evalPath), + "$ref": "./" + filepath.ToSlash(configPath), }) if err != nil { return "", fmt.Errorf("building the eval service entry: %w", err) @@ -264,7 +511,7 @@ func ensureRootEvalService(ctx context.Context, evalName, target, evalPath strin _, err = azdClient.Project().AddService(ctx, &azdext.AddServiceRequest{ Service: &azdext.ServiceConfig{ - Name: evalName, + Name: serviceName, Host: project.EvalHost, Uses: evalServiceUses(resp.GetProject(), target), AdditionalProperties: props, @@ -281,8 +528,11 @@ func ensureRootEvalService(ctx context.Context, evalName, target, evalPath strin // It is conditional for the same reason the agents extension makes it // conditional: naming a service the project does not declare is a broken // reference, and an eval config can perfectly well sit in a repo that reaches -// an existing Foundry project by endpoint and an agent that is deployed -// elsewhere. +// an existing Foundry project by endpoint and an agent deployed elsewhere. +// +// Catalog entries need no ordering of their own — datasets, evaluators and +// evals are reconciled in a fixed order inside one deploy, forced by the +// contract rather than chosen. func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { var uses []string for name, svc := range proj.GetServices() { @@ -297,217 +547,6 @@ func evalServiceUses(proj *azdext.ProjectConfig, target string) []string { return uses } -// scaffold is what `init` writes: one eval body and the generation settings -// that fill in the artifacts it references. -type scaffold struct { - eval *project.EvalConfig - generate *project.GenerateConfig - datasetName string - rubricName string -} - -// evaluatorNames lists the evaluators the eval will run, in declaration order. -func (s scaffold) evaluatorNames() []string { - names := make([]string, 0, len(s.eval.Evaluators)) - for _, ref := range s.eval.Evaluators { - names = append(names, ref.Name) - } - return names -} - -// evaluatorSummary is the one-line form `init` reports, marking the evaluator -// that still has to be generated. -func (s scaffold) evaluatorSummary() string { - parts := make([]string, 0, len(s.eval.Evaluators)) - for _, ref := range s.eval.Evaluators { - if ref.Name == s.rubricName && s.rubricCount() > 0 { - parts = append(parts, ref.Name+" (rubric)") - continue - } - parts = append(parts, ref.Name) - } - return strings.Join(parts, ", ") -} - -// rubricCount is the number of evaluators `init` expects to be generated. -func (s scaffold) rubricCount() int { - if s.generate == nil { - return 0 - } - return len(s.generate.Evaluator) -} - -// nextSteps are the commands to run after `init`, and only the ones that have -// something to do. -// -// A caller who supplied both a dataset and their evaluators has nothing left to -// generate, and pointing them at a generation command would submit a billed job -// for an artifact they already have. With everything in place the next step is -// to deploy it. -func (s scaffold) nextSteps() []string { - var steps []string - if s.generate != nil && len(s.generate.Dataset) > 0 { - steps = append(steps, "azd ai eval dataset generate "+s.datasetName) - } - if s.rubricCount() > 0 { - steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) - } - if len(steps) == 0 { - steps = append(steps, "azd up", "azd ai eval run start") - } - return steps -} - -// relativeToConfig rewrites a path given relative to the working directory so -// it resolves from the directory holding the eval config. -// -// `--dataset ./tests/golden.jsonl` means "relative to where I am", but -// `source:` is resolved relative to the config file, so writing the path -// through unchanged sends the deploy looking inside evals/. An absolute path is -// left alone, and forward slashes are kept so the config reads the same on -// every platform. -func relativeToConfig(path, outputDir string) string { - if filepath.IsAbs(path) { - return path - } - - absPath, err := filepath.Abs(path) - if err != nil { - return path - } - absOut, err := filepath.Abs(outputDir) - if err != nil { - return path - } - - rel, err := filepath.Rel(absOut, absPath) - if err != nil { - return path - } - - rel = filepath.ToSlash(rel) - if !strings.HasPrefix(rel, ".") { - rel = "./" + rel - } - return rel -} - -// planScaffold builds both files `init` writes. -// -// The default evaluator set is a built-in plus a generated rubric: the built-in -// alone would be generic, and the rubric is what makes the baseline about this -// agent. Passing --evaluator replaces both, which is how a caller opts out of -// rubric generation. -func planScaffold( - evalName, target, rubricName, dataset string, - evaluators []string, - genModel string, - outputDir string, -) scaffold { - cfg := &project.EvalConfig{ - Description: fmt.Sprintf("Basic quality evaluation for %s", target), - } - - datasetName := evalName - datasetSource := "" - generateDataset := true - if dataset != "" { - if looksLikeLocalDataset(dataset) { - // --dataset is given relative to where the user is standing, but - // source: is resolved relative to the config, so the path has to be - // rebased or the deploy looks for it inside evals/. - datasetSource = relativeToConfig(dataset, outputDir) - datasetName = strings.TrimSuffix(filepath.Base(dataset), filepath.Ext(dataset)) - } else { - // A bare name references an already-registered dataset. - datasetName = dataset - } - generateDataset = false - } else { - datasetSource = fmt.Sprintf("./%s/%s.jsonl", project.DefaultDatasetsDir, datasetName) - } - cfg.Dataset = &project.DatasetDecl{ - Name: datasetName, - Source: datasetSource, - } - - // Every evaluator carries the judge deployment, because that is where the - // service reads it from: built-ins declare `deployment_name` as required, - // so an eval that leaves it off is rejected before it runs. - initParams := map[string]any{} - if genModel != "" { - initParams["deployment_name"] = genModel - } - withModel := func(ref evalcore.EvaluatorRef) evalcore.EvaluatorRef { - if len(initParams) == 0 { - return ref - } - params := make(map[string]any, len(initParams)) - for k, v := range initParams { - params[k] = v - } - ref.InitializationParameters = params - return ref - } - - refs := evalcore.EvaluatorList{} - generateRubric := false - if len(evaluators) == 0 { - refs = append(refs, - withModel(evalcore.EvaluatorRef{Name: evalcore.BuiltinPrefix + "task_adherence"}), - withModel(evalcore.EvaluatorRef{ - Name: rubricName, - Source: fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, rubricName), - }), - ) - generateRubric = true - } else { - for _, e := range evaluators { - ref := evalcore.EvaluatorRef{Name: e} - if !ref.IsBuiltin() { - ref.Source = fmt.Sprintf("./%s/%s.json", project.DefaultEvaluatorsDir, e) - } - refs = append(refs, withModel(ref)) - } - } - cfg.Evaluators = refs - - cfg.Target = &project.Target{ - Type: project.TargetTypeAgent, - Name: target, - } - cfg.Options = &project.Options{ - MaxSamples: project.DefaultSampleSize, - EvaluationLevel: project.EvaluationLevelTurn, - } - - gen := &project.GenerateConfig{GenerationModel: genModel} - if generateDataset { - gen.Dataset = map[string]project.DatasetGenSpec{ - datasetName: { - SampleSize: project.DefaultSampleSize, - OutputDir: "./" + project.DefaultDatasetsDir, - DeriveFrom: target, - }, - } - } - if generateRubric { - gen.Evaluator = map[string]project.EvaluatorGenSpec{ - rubricName: { - OutputDir: "./" + project.DefaultEvaluatorsDir, - DeriveFrom: target, - }, - } - } - - return scaffold{ - eval: cfg, - generate: gen, - datasetName: datasetName, - rubricName: rubricName, - } -} - // looksLikeLocalDataset distinguishes a path from a registered dataset name. func looksLikeLocalDataset(v string) bool { if strings.ContainsAny(v, `/\`) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index b391dd908c0..c0f2549fbdf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -13,56 +13,109 @@ import ( "github.com/stretchr/testify/require" ) -// The scaffold must load and validate cleanly, otherwise `azd up` fails on a +// scaffoldFor runs planScaffold against a fresh configuration, which is what +// `init` does on a project that has never been initialized. +func scaffoldFor(t *testing.T, in scaffoldInput) (scaffold, *project.EvalConfig) { + t.Helper() + if in.cfg == nil { + in.cfg = &project.EvalConfig{} + } + if in.evalDir == "" { + in.evalDir = project.DefaultEvalDir + } + if in.rubricName == "" { + in.rubricName = in.target + "-quality" + } + return planScaffold(in), in.cfg +} + +// The scaffold must round-trip and validate, otherwise `azd up` fails on a // config the tool itself produced. func TestScaffold_RoundTripsAndValidates(t *testing.T) { dir := t.TempDir() - evalPath := filepath.Join(dir, "support-agent-smoke.yaml") - - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "gpt-4.1-nano", project.DefaultEvalDir) - require.NoError(t, writeYAML(evalPath, plan.eval)) + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", + target: "support-agent", + judgeModel: "gpt-4.1-nano", + evalDir: dir, + }) - loaded, err := project.LoadEvalConfig(evalPath) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) require.NoError(t, err) require.NoError(t, loaded.Validate(), "the generated scaffold must be valid") - require.Equal(t, project.TargetTypeAgent, loaded.Target.Type) - require.Equal(t, "support-agent", loaded.Target.Name) - require.Equal(t, project.DefaultSampleSize, loaded.Options.MaxSamples) + eval, err := loaded.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, project.TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) + require.Equal(t, project.EvaluationLevelTurn, eval.EvaluationLevel) +} - // The eval takes its name from the file, which is the azure.yaml service key. - require.Equal(t, "support-agent-smoke", loaded.Eval("support-agent-smoke").Name) +// Re-running init appends rather than replacing, so one file ends up holding +// every eval for the target. +func TestScaffold_AppendsToAnExistingConfiguration(t *testing.T) { + dir := t.TempDir() + _, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "first", target: "support-agent", judgeModel: "m", evalDir: dir, + }) + _, cfg = scaffoldFor(t, scaffoldInput{ + evalName: "second", target: "support-agent", judgeModel: "m", evalDir: dir, cfg: cfg, + }) + + require.Equal(t, []string{"first", "second"}, cfg.EvalNames()) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) + require.NoError(t, err) + require.NoError(t, loaded.Validate()) +} + +// A trace-backed eval invokes nothing, so agent_name filters instead of +// targeting, and the service default caps how many traces are read. +func TestScaffold_TraceSourceHasNoTarget(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-trace-eval", + target: "support-agent", + source: initSourceTraces, + }) + + require.Nil(t, plan.eval.Target) + require.NotNil(t, plan.eval.Source) + require.Equal(t, project.SourceTypeTraces, plan.eval.Source.Type) + require.Equal(t, "support-agent", plan.eval.Source.AgentName) } // The default set is a built-in plus a generated rubric: the built-in alone // would be generic, and the rubric is what makes the baseline about this agent. func TestScaffold_DefaultEvaluators(t *testing.T) { - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "gpt-5.6-luna", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "gpt-5.6-luna", + }) require.Equal(t, []string{"builtin.task_adherence", "support-agent-quality"}, plan.evaluatorNames()) - require.Contains(t, plan.evaluatorSummary(), "support-agent-quality (rubric)") - // Every evaluator carries the judge deployment, because built-ins declare - // deployment_name as required and an eval that leaves it off is rejected. + // Every evaluator carries the judge deployment, because the judging + // built-ins declare it and an eval that leaves it off is rejected. for _, ref := range plan.eval.Evaluators { - require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["deployment_name"], - "%s must name a judge deployment", ref.Name) + require.Equal(t, "gpt-5.6-luna", ref.InitializationParameters["model"], + "%s must name a judge deployment", ref.Evaluator) } } // Passing --evaluator replaces the defaults, which is how a caller opts out of // rubric generation. func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "support-agent-quality", "", - []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) require.Equal(t, []string{"builtin.task_adherence"}, plan.evaluatorNames()) - require.Zero(t, plan.rubricCount(), "no rubric is generated when evaluators are given") - require.Empty(t, plan.generate.Evaluator) + require.False(t, plan.generateRubric, "no rubric is generated when evaluators are given") } // `init` closes by naming what to run next, and only what has something to do. @@ -70,105 +123,104 @@ func TestScaffold_ExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { // would submit a billed job for something they already have. func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { t.Run("nothing supplied", func(t *testing.T) { - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", + }) require.Equal(t, []string{ - "azd ai eval dataset generate support-agent-smoke", + "azd ai dataset generate support-agent-smoke", "azd ai eval evaluator generate support-agent-quality", }, plan.nextSteps()) }) t.Run("dataset supplied", func(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "support-agent-quality", - "prod-golden", nil, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "support-agent", dataset: "prod-golden", judgeModel: "m", + }) require.Equal(t, []string{"azd ai eval evaluator generate support-agent-quality"}, plan.nextSteps()) }) t.Run("everything supplied", func(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "support-agent-quality", - "prod-golden", []string{"builtin.task_adherence"}, "m", project.DefaultEvalDir) + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + dataset: "prod-golden", + evaluators: []string{"builtin.task_adherence"}, + judgeModel: "m", + }) require.Equal(t, []string{"azd up", "azd ai eval run start"}, plan.nextSteps(), "with every artifact in place the next step is to deploy") }) } -func TestGenerateScaffold_RoundTripsAndValidates(t *testing.T) { +// Built-ins are referenced but never declared, so the scaffold must not give +// one a catalog entry to publish. +func TestScaffold_BuiltinEvaluatorsGetNoCatalogEntry(t *testing.T) { dir := t.TempDir() - genPath := filepath.Join(dir, "generate.yaml") - - plan := planScaffold("support-agent-smoke", "support-agent", "support-agent-quality", - "", nil, "gpt-4.1-nano", project.DefaultEvalDir) - require.NoError(t, writeYAML(genPath, plan.generate)) - - loaded, err := project.LoadGenerateConfig(genPath) - require.NoError(t, err) - require.Equal(t, "gpt-4.1-nano", loaded.GenerationModel) - - ds, ok := loaded.DatasetSpec("support-agent-smoke") - require.True(t, ok, "the generation spec is keyed by artifact name") - require.Equal(t, project.DefaultSampleSize, ds.SampleSize) - require.Equal(t, "support-agent", ds.DeriveFrom) - - ev, ok := loaded.EvaluatorSpec("support-agent-quality") - require.True(t, ok) - require.Equal(t, "./"+project.DefaultEvaluatorsDir, ev.OutputDir) -} - -// Built-ins are referenced but never published, so the scaffold must not give -// one a local source to upload. -func TestScaffold_BuiltinEvaluatorsHaveNoSource(t *testing.T) { - plan := planScaffold("smoke", "support-agent", "unused", "", - []string{"builtin.task_adherence", "my-custom"}, "", project.DefaultEvalDir) - cfg := plan.eval + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", + target: "support-agent", + evaluators: []string{"builtin.task_adherence", "my-custom"}, + judgeModel: "m", + evalDir: dir, + }) - require.Len(t, cfg.Evaluators, 2) - require.True(t, cfg.Evaluators[0].IsBuiltin()) - require.Empty(t, cfg.Evaluators[0].Source) - require.False(t, cfg.Evaluators[1].IsBuiltin()) - require.NotEmpty(t, cfg.Evaluators[1].Source) + require.Len(t, plan.eval.Evaluators, 2) + require.True(t, plan.eval.Evaluators[0].IsBuiltin()) + require.False(t, plan.eval.Evaluators[1].IsBuiltin()) + require.Len(t, cfg.Evaluators, 1, "only the custom evaluator is declared") + require.Equal(t, "my-custom", cfg.Evaluators[0].Name) require.Len(t, cfg.CustomEvaluators(), 1, "only the custom evaluator is this config's to publish") - path := filepath.Join(t.TempDir(), "smoke.yaml") - require.NoError(t, writeYAML(path, cfg)) - loaded, err := project.LoadEvalConfig(path) + require.NoError(t, project.SaveEvalConfig(dir, cfg)) + loaded, err := project.OpenEvalConfig(dir) require.NoError(t, err) require.NoError(t, loaded.Validate()) } // A bare name means an already-registered dataset; a path means a local file. // Either way the dataset was supplied, so nothing is scheduled to generate it — -// only a missing --dataset produces a generation entry. +// only a missing --dataset produces a generation step. func TestScaffold_DatasetReferenceForms(t *testing.T) { t.Run("local path becomes a source", func(t *testing.T) { // --dataset is relative to the working directory, but source: is // resolved relative to the eval config, so it has to be rebased. - plan := planScaffold("smoke", "a", "r", "./tests/golden.jsonl", nil, "", "evals") - require.Equal(t, "../tests/golden.jsonl", plan.eval.Dataset.Source, + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "./tests/golden.jsonl", evalDir: "evals", + }) + decl, ok := cfg.DatasetDeclaration("golden") + require.True(t, ok) + require.Equal(t, "../tests/golden.jsonl", decl.Source, "a dataset outside the eval dir must be reached with ..") - require.Equal(t, "golden", plan.eval.Dataset.Name) - require.Empty(t, plan.generate.Dataset, + require.Equal(t, "golden", plan.eval.Dataset) + require.False(t, plan.generateDataset, "a supplied dataset must not be scheduled for generation") }) t.Run("bare name references a registered dataset", func(t *testing.T) { - plan := planScaffold("smoke", "a", "r", "prod-sample", nil, "", project.DefaultEvalDir) - require.Equal(t, "prod-sample", plan.eval.Dataset.Name) - require.Empty(t, plan.eval.Dataset.Source, - "a registered dataset must not get a local source") - require.Empty(t, plan.generate.Dataset) + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "smoke", target: "a", dataset: "prod-sample", + }) + decl, ok := cfg.DatasetDeclaration("prod-sample") + require.True(t, ok) + require.Empty(t, decl.Source, "a registered dataset must not get a local source") + require.Equal(t, "prod-sample", plan.eval.Dataset) + require.False(t, plan.generateDataset) }) - t.Run("no dataset flag scaffolds a local path and a generation entry", func(t *testing.T) { - plan := planScaffold("support-agent-smoke", "support-agent", "r", "", - nil, "", project.DefaultEvalDir) - require.Equal(t, "support-agent-smoke", plan.eval.Dataset.Name, + t.Run("no dataset flag scaffolds a local path and a generation step", func(t *testing.T) { + plan, cfg := scaffoldFor(t, scaffoldInput{ + evalName: "support-agent-smoke", target: "support-agent", + }) + require.Equal(t, "support-agent-smoke", plan.eval.Dataset, "the dataset is named after the eval") - require.Contains(t, plan.eval.Dataset.Source, "support-agent-smoke.jsonl") - require.Contains(t, plan.generate.Dataset, "support-agent-smoke") + decl, ok := cfg.DatasetDeclaration("support-agent-smoke") + require.True(t, ok) + require.Contains(t, decl.Source, "support-agent-smoke.jsonl") + require.True(t, plan.generateDataset) }) } @@ -181,15 +233,15 @@ func TestLooksLikeLocalDataset(t *testing.T) { // Paths are used verbatim relative to the working directory; the doubling bug // in the agent-scoped command must not reappear. -func TestWriteYAML_UsesPathVerbatim(t *testing.T) { +func TestSaveEvalConfig_UsesPathVerbatim(t *testing.T) { dir := t.TempDir() - nested := filepath.Join(dir, "evals", "smoke.yaml") + nested := filepath.Join(dir, "evals") - require.NoError(t, writeYAML(nested, &project.EvalConfig{})) - _, err := os.Stat(nested) + require.NoError(t, project.SaveEvalConfig(nested, &project.EvalConfig{})) + _, err := os.Stat(project.EvalConfigPath(nested)) require.NoError(t, err, "the file must land exactly at the requested path") - doubled := filepath.Join(dir, "evals", "evals", "smoke.yaml") + doubled := filepath.Join(dir, "evals", "evals") _, err = os.Stat(doubled) require.Error(t, err, "the path must not be re-rooted under itself") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index 9e20a857312..ae0a60152a3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -15,7 +15,7 @@ import ( // is silent. options.max_samples was parsed and dropped once already, which is // what these lock down. func TestResolveMaxSamples_Precedence(t *testing.T) { - withOptions := &project.Eval{Options: &project.Options{MaxSamples: 25}} + withOptions := &project.Eval{MaxSamples: 25} assert.Equal(t, 5, resolveMaxSamples(5, withOptions), "the flag wins over the config") assert.Equal(t, 25, resolveMaxSamples(0, withOptions), "the config is used when no flag is given") @@ -25,12 +25,12 @@ func TestResolveMaxSamples_Precedence(t *testing.T) { // Zero in config is absent, not a cap of zero: a cap of zero would send // nothing at all. - assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{Options: &project.Options{MaxSamples: 0}})) + assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{MaxSamples: 0})) } func TestResolveLevel_Precedence(t *testing.T) { withOptions := &project.Eval{ - Options: &project.Options{EvaluationLevel: project.EvaluationLevelConversation}, + EvaluationLevel: project.EvaluationLevelConversation, } assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 9042c0f1486..1bb13de25bf 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -101,38 +101,23 @@ func buildRunCommand(use, short string) *cobra.Command { } defer ec.Close() - // --eval-id bypasses the config entirely. - var group *project.Eval - configPath := "" - if evalID == "" { - configPath, err = project.ResolveEvalConfigPath(project.DefaultEvalDir, groupName) - if err != nil { - return err - } - cfg, err := project.LoadEvalConfig(configPath) - if err != nil { - return err - } - if err := cfg.Validate(); err != nil { - return err - } - resolved := cfg.Eval(evalNameFromPath(configPath)) - group = &resolved - - if err := ec.checkDatasetRegistered(ctx, cfg, group, configPath); err != nil { - return err - } + // One flag takes a name or an id. A declared name also brings the + // declaration, which is what says where rows come from; a bare id + // has none, so the pairing comes from the eval's previous run. + ref, err := ec.resolveEvalRef(ctx, project.DefaultEvalDir, groupName) + if err != nil { + return err + } + evalID := ref.ID + group := ref.Eval + configPath := ref.ConfigPath - evalID, err = ec.resolveEvalIDFromConfig( - ctx, group, configPath, resolveLevel(level, group), - out, isJSON(cmd)) - if err != nil { + if ref.Declared() { + if err := ec.checkDatasetRegistered(ctx, ref.Config, group, configPath); err != nil { return err } } - // With --eval-id there is no config to read, so the pairing of - // target and dataset comes from the group's previous run. var dataSource *eval_api.EvalRunDataSource switch { case group == nil: @@ -268,16 +253,11 @@ func (ec *evalContext) resolveEvalIDFromConfig( fmt.Fprintf(out, "Creating eval %q...\n", group.Name) } - // The level from the flag wins over the group's own options, so it has to - // reach the criteria that accept evaluation_level. + // The level from the flag wins over the eval's own declaration, so it has + // to reach the criteria that accept evaluation_level. effective := *group if level != "" { - opts := project.Options{} - if group.Options != nil { - opts = *group.Options - } - opts.EvaluationLevel = level - effective.Options = &opts + effective.EvaluationLevel = level } req, err := buildEvalRequest( @@ -305,12 +285,13 @@ func (ec *evalContext) resolveEvalIDFromConfig( // The per-name entry is what the extension writes. EVAL_ID is also the // documented way to point a config at an eval that already exists, created in // the portal or by another tool, so it stays readable — but only when the -// project declares a single eval. With more than one there is no way to tell -// which eval a shared entry refers to, and reading it anyway is what let a +// configuration declares a single eval. With more than one there is no way to +// tell which eval a shared entry refers to, and reading it anyway is what let a // second eval adopt the first one's id. func evalIDKeys(name, evalDir string) []string { keys := []string{idKey("eval", name)} - if names, err := project.EvalNamesIn(evalDir); err == nil && len(names) == 1 { + if cfg, err := project.OpenEvalConfig(evalDir); err == nil && + cfg != nil && len(cfg.Evals) == 1 { keys = append(keys, envKeyEvalID) } return keys @@ -338,8 +319,8 @@ func (ec *evalContext) checkDatasetRegistered( return nil } - decl := cfg.Dataset - if decl == nil { + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok { return nil } @@ -513,11 +494,11 @@ func datasetColumnsFromPath(localPath string) map[string]bool { // file, returning empty when the dataset is registered rather than local. func localDatasetPath(configPath string, group *project.Eval) string { cfg, err := project.LoadEvalConfig(configPath) - if err != nil { + if err != nil || group == nil { return "" } - decl := cfg.Dataset - if decl == nil || decl.Source == "" { + decl, ok := cfg.DatasetDeclaration(group.Dataset) + if !ok || decl.Source == "" { return "" } if filepath.IsAbs(decl.Source) { @@ -526,13 +507,6 @@ func localDatasetPath(configPath string, group *project.Eval) string { return filepath.Join(filepath.Dir(configPath), decl.Source) } -// evalNameFromPath is the eval's name: one config file is one eval, and the -// file is named after it, matching the azure.yaml service key that $refs it. -func evalNameFromPath(configPath string) string { - base := filepath.Base(configPath) - return strings.TrimSuffix(base, filepath.Ext(base)) -} - // readJSONL reads newline-delimited JSON, optionally truncating to limit rows. func readJSONL(path string, limit int) ([]map[string]any, error) { f, err := os.Open(path) @@ -582,29 +556,29 @@ func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { return items, nil } -// resolveLevel prefers the flag, then the group's options. +// resolveLevel prefers the flag, then the eval's own declaration. func resolveLevel(flag string, group *project.Eval) string { if flag != "" { return flag } - if group != nil && group.Options != nil { - return group.Options.EvaluationLevel + if group != nil { + return group.EvaluationLevel } return "" } -// resolveMaxSamples prefers the flag, then the group's options, matching how -// the evaluation level resolves. +// resolveMaxSamples prefers the flag, then the eval's own declaration, matching +// how the evaluation level resolves. // -// Without this, options.max_samples parsed and did nothing: a group that caps -// its sample count in config would send the whole dataset, and only a flag on -// every invocation would honour the cap. +// Without this, max_samples parsed and did nothing: an eval that caps its +// sample count in config would send the whole dataset, and only a flag on every +// invocation would honour the cap. func resolveMaxSamples(flag int, group *project.Eval) int { if flag > 0 { return flag } - if group != nil && group.Options != nil && group.Options.MaxSamples > 0 { - return group.Options.MaxSamples + if group != nil && group.MaxSamples > 0 { + return group.MaxSamples } return 0 } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 3e3f9e035be..0bfe4b2514f 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -105,7 +105,6 @@ func TestFlagVocabularyIsShared(t *testing.T) { "--out-dir": "--output-dir", "--file": "--from-file", "--rubric": "--from-file", - "--judge-model": "--generation-model, declared per evaluator instead", "--from-traces": "deferred to M2", "--response-id": "deferred to M2", "--no-target": "deferred to M2", @@ -118,7 +117,6 @@ func TestFlagVocabularyIsShared(t *testing.T) { "--data-schema": "deferred to M2", "--metrics": "deferred to M2", "--trace-window": "deferred to M2", - "--max-traces": "deferred to M2", "--max-turns": "deferred to M2", } @@ -172,8 +170,8 @@ func TestInitFlagsMatchTheSpec(t *testing.T) { }) assert.ElementsMatch(t, []string{ - "--name", "--target", "--dataset", "--evaluator", - "--generation-model", "--output-dir", "--force", + "--name", "--target", "--source", "--dataset", "--evaluator", + "--judge-model", "--path", "--force", }, got, "init's flags are a table in the spec; change both together") } @@ -240,7 +238,7 @@ func TestSuggestedCommandsExist(t *testing.T) { return err } - for _, line := range strings.Split(string(body), "\n") { + for line := range strings.SplitSeq(string(body), "\n") { // Comments explain the surface; only what reaches a terminal has // to resolve. if strings.HasPrefix(strings.TrimSpace(line), "//") { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go index c2283fceb68..427f8c6af00 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/dataset_api/operations.go @@ -112,7 +112,7 @@ func (c *DatasetClient) UploadNextVersion( } var err error - for attempt := 0; attempt < versionConflictAttempts; attempt++ { + for range versionConflictAttempts { var ds *Dataset ds, err = c.UploadNewVersion(ctx, name, currentVersion, localDir, apiVersion) if err == nil || !IsVersionConflict(err) { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 6441568db72..122828750de 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -265,6 +265,7 @@ type TestingCriterion struct { Type string `json:"type"` Name string `json:"name"` EvaluatorName string `json:"evaluator_name"` + EvaluatorVersion string `json:"evaluator_version,omitempty"` InitializationParameters map[string]any `json:"initialization_parameters,omitempty"` DataMapping map[string]string `json:"data_mapping,omitempty"` } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go index df6aa791b7d..75bc7f8a251 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator.go @@ -16,46 +16,67 @@ import ( // stripped before the name is sent as testing_criteria[].evaluator_name. const BuiltinPrefix = "builtin." -// EvaluatorRef references an evaluator from an eval. It accepts either a -// bare string or a mapping carrying the rest of the declaration: +// EvaluatorRef is one entry in an eval's `evaluators:` list. Every entry is a +// map keyed `evaluator:`; what to publish lives on the catalog entry instead, +// so a reference only names an evaluator and says how to run it: // // evaluators: -// - builtin.task_adherence -// - name: support-quality -// source: ./evaluators/support-quality.json +// - evaluator: builtin.task_adherence // initialization_parameters: -// deployment_name: gpt-5.6-luna -// -// A built-in needs nothing but its name. One with a Source is the project's -// own, and is published before the eval that references it is created. +// model: gpt-5.6-luna +// threshold: 3 +// - evaluator: support-agent-quality +// name: quality_strict +// version: "2" +// data_mapping: +// query: "{{item.customer_message}}" type EvaluatorRef struct { - Name string `yaml:"name" json:"name"` + // Evaluator is the evaluator to run: a catalog name or builtin.. + Evaluator string `yaml:"evaluator" json:"evaluator"` + // Name labels the criterion in results. Empty means the evaluator's name. + Name string `yaml:"name,omitempty" json:"name,omitempty"` + // Version pins this reference. Pinning belongs to one eval's reference + // rather than to the asset, matching evaluator_version on the criterion. Version string `yaml:"version,omitempty" json:"version,omitempty"` - // Source names a local rubric JSON file. Empty means the evaluator is - // already registered, or is a built-in. - Source string `yaml:"source,omitempty" json:"source,omitempty"` - // InitializationParameters are passed through to the testing criterion, - // which is where a judge deployment is named. + // InitializationParameters carry the judge deployment and a built-in's + // numeric threshold. They are bound against the evaluator's published + // contract rather than forwarded as written. InitializationParameters map[string]any `yaml:"initialization_parameters,omitempty" json:"initialization_parameters,omitempty"` - // Threshold maps to testing_criteria[].initialization_parameters.threshold. - Threshold *float64 `yaml:"threshold,omitempty" json:"threshold,omitempty"` + // DataMapping binds evaluator inputs to dataset columns, and is written + // only when the inference from declared inputs and columns gets it wrong. + DataMapping map[string]string `yaml:"data_mapping,omitempty" json:"data_mapping,omitempty"` } // IsBuiltin reports whether the reference names a platform evaluator, which -// needs no declaration and is never uploaded. +// needs no catalog entry and is never uploaded. func (e EvaluatorRef) IsBuiltin() bool { - return strings.HasPrefix(e.Name, BuiltinPrefix) + return strings.HasPrefix(e.Evaluator, BuiltinPrefix) } // APIName is the name the service expects, with the builtin prefix removed. func (e EvaluatorRef) APIName() string { - return strings.TrimPrefix(e.Name, BuiltinPrefix) + return strings.TrimPrefix(e.Evaluator, BuiltinPrefix) +} + +// CriterionName labels this criterion in results. +func (e EvaluatorRef) CriterionName() string { + if e.Name != "" { + return e.Name + } + return e.APIName() } -// EvaluatorList is a sequence of EvaluatorRef supporting mixed string and -// mapping entries. +// EvaluatorList is a sequence of EvaluatorRef. +// +// A bare string is refused rather than accepted quietly. Every other collection +// in the file is a list of named maps, and a bare string would have to mean the +// evaluator while reading as the criterion's own name — a different key this +// same entry also carries. type EvaluatorList []EvaluatorRef +const bareEvaluatorRemedy = "an evaluator entry is a mapping, not a bare string: " + + "write `- evaluator: %s`" + func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { if value.Kind != yaml.SequenceNode { return fmt.Errorf("evaluators must be a sequence, got %v", value.Kind) @@ -69,18 +90,18 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { if err := node.Decode(&name); err != nil { return fmt.Errorf("decoding evaluator name: %w", err) } - result = append(result, EvaluatorRef{Name: name}) + return fmt.Errorf(bareEvaluatorRemedy, name) case yaml.MappingNode: var ref EvaluatorRef if err := node.Decode(&ref); err != nil { return fmt.Errorf("decoding evaluator: %w", err) } - if ref.Name == "" { - return fmt.Errorf("evaluator entry is missing 'name'") + if ref.Evaluator == "" { + return fmt.Errorf("evaluator entry is missing 'evaluator'") } result = append(result, ref) default: - return fmt.Errorf("evaluator entry must be a string or a mapping, got %v", node.Kind) + return fmt.Errorf("evaluator entry must be a mapping, got %v", node.Kind) } } @@ -88,26 +109,12 @@ func (el *EvaluatorList) UnmarshalYAML(value *yaml.Node) error { return nil } -// MarshalYAML emits the compact string form when an entry carries nothing but a -// name, so round-tripping a hand-written config does not rewrite it. -func (el EvaluatorList) MarshalYAML() (any, error) { - out := make([]any, 0, len(el)) - for _, ref := range el { - if ref.isBareName() { - out = append(out, ref.Name) - continue - } - out = append(out, ref) - } - return out, nil -} - -// UnmarshalJSON accepts the same mixed string-or-mapping form as the YAML -// decoder. +// UnmarshalJSON accepts the same mapping-only form as the YAML decoder. // // This matters for the service-target provider: azd hands the service entry to -// the extension as JSON, so a config written as `- builtin.task_adherence` -// arrives as a bare string and would otherwise fail to decode. +// the extension as JSON, so a config written the old way arrives here as a bare +// string and has to be refused with the same remedy rather than with a +// decoder's own type error. func (el *EvaluatorList) UnmarshalJSON(data []byte) error { var entries []json.RawMessage if err := json.Unmarshal(data, &entries); err != nil { @@ -122,16 +129,15 @@ func (el *EvaluatorList) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(trimmed, &name); err != nil { return fmt.Errorf("decoding evaluator name: %w", err) } - result = append(result, EvaluatorRef{Name: name}) - continue + return fmt.Errorf(bareEvaluatorRemedy, name) } var ref EvaluatorRef if err := json.Unmarshal(trimmed, &ref); err != nil { return fmt.Errorf("decoding evaluator: %w", err) } - if ref.Name == "" { - return fmt.Errorf("evaluator entry is missing 'name'") + if ref.Evaluator == "" { + return fmt.Errorf("evaluator entry is missing 'evaluator'") } result = append(result, ref) } @@ -140,30 +146,19 @@ func (el *EvaluatorList) UnmarshalJSON(data []byte) error { return nil } -// MarshalJSON mirrors MarshalYAML's compact form. +// MarshalJSON is the default list encoding, defined so a compact form cannot +// creep back in through the encoder. // -// Everything the reference carries has to survive the round trip, including -// the source and the initialization parameters: the eval fingerprint is taken -// over this encoding, so a field dropped here is a change the reconciler -// cannot see. +// Everything the reference carries has to survive the round trip: the eval +// fingerprint is taken over this encoding, so a field dropped here is a change +// the reconciler cannot see. func (el EvaluatorList) MarshalJSON() ([]byte, error) { // Aliased so the element encoder does not recurse through this method. type ref = EvaluatorRef out := make([]any, 0, len(el)) for _, r := range el { - if r.isBareName() { - out = append(out, r.Name) - continue - } out = append(out, ref(r)) } return json.Marshal(out) } - -// isBareName reports whether the reference carries nothing but its name, in -// which case both encoders emit the compact string form. -func (e EvaluatorRef) isBareName() bool { - return e.Threshold == nil && e.Version == "" && e.Source == "" && - len(e.InitializationParameters) == 0 -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go index 5408776bec7..b185414e8ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/evalcore/evaluator_test.go @@ -11,28 +11,34 @@ import ( "go.yaml.in/yaml/v3" ) -// The service-target provider receives the config as JSON, not YAML, so the -// mixed string-or-mapping form has to decode through both paths. Supporting -// only YAML made `azd deploy` fail on a config the CLI itself writes. -func TestEvaluatorListDecodesMixedEntriesFromJSON(t *testing.T) { +// The service-target provider receives the config as JSON, not YAML, so both +// paths have to decode the mapping form identically. Supporting only YAML made +// `azd deploy` fail on a config the CLI itself writes. +func TestEvaluatorListDecodesEntriesFromJSON(t *testing.T) { const payload = `[ - "builtin.task_adherence", - {"name": "support-quality", "threshold": 4.0}, - {"name": "pinned", "version": "3"} + {"evaluator": "builtin.task_adherence"}, + {"evaluator": "support-quality", "name": "quality_strict", + "initialization_parameters": {"model": "gpt-5.6-luna", "threshold": 4}}, + {"evaluator": "pinned", "version": "3"} ]` var list EvaluatorList require.NoError(t, json.Unmarshal([]byte(payload), &list)) require.Len(t, list, 3) - require.Equal(t, "builtin.task_adherence", list[0].Name) - require.Nil(t, list[0].Threshold) + require.Equal(t, "builtin.task_adherence", list[0].Evaluator) + require.True(t, list[0].IsBuiltin()) + require.Equal(t, "task_adherence", list[0].APIName()) + require.Equal(t, "task_adherence", list[0].CriterionName()) + require.Nil(t, list[0].InitializationParameters) - require.Equal(t, "support-quality", list[1].Name) - require.NotNil(t, list[1].Threshold) - require.InDelta(t, 4.0, *list[1].Threshold, 0.0001) + require.Equal(t, "support-quality", list[1].Evaluator) + require.Equal(t, "quality_strict", list[1].CriterionName()) + require.False(t, list[1].IsBuiltin()) + require.Equal(t, "gpt-5.6-luna", list[1].InitializationParameters["model"]) + require.EqualValues(t, 4, list[1].InitializationParameters["threshold"]) - require.Equal(t, "pinned", list[2].Name) + require.Equal(t, "pinned", list[2].Evaluator) require.Equal(t, "3", list[2].Version) } @@ -40,8 +46,13 @@ func TestEvaluatorListDecodesMixedEntriesFromJSON(t *testing.T) { // through the CLI and another through `azd up`. func TestEvaluatorListJSONMatchesYAML(t *testing.T) { const doc = ` -- builtin.task_adherence -- { name: support-quality, threshold: 4.0 } +- evaluator: builtin.task_adherence +- evaluator: support-quality + name: quality_strict + initialization_parameters: + model: gpt-5.6-luna + data_mapping: + query: "{{item.customer_message}}" ` var fromYAML EvaluatorList require.NoError(t, yaml.Unmarshal([]byte(doc), &fromYAML)) @@ -54,9 +65,56 @@ func TestEvaluatorListJSONMatchesYAML(t *testing.T) { require.Equal(t, fromYAML, fromJSON) } -func TestEvaluatorListRejectsEntryWithoutName(t *testing.T) { - var list EvaluatorList - err := json.Unmarshal([]byte(`[{"threshold": 4.0}]`), &list) - require.Error(t, err) - require.Contains(t, err.Error(), "name") +// A bare string is the old shorthand. It has to be refused with the remedy +// rather than a decoder type error, through both decoders, because the +// service-target provider only ever sees JSON. +func TestEvaluatorListRefusesBareString(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- builtin.task_adherence\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`["builtin.task_adherence"]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "- evaluator: builtin.task_adherence") + }) +} + +func TestEvaluatorListRejectsEntryWithoutEvaluator(t *testing.T) { + t.Run("yaml", func(t *testing.T) { + var list EvaluatorList + err := yaml.Unmarshal([]byte("- name: quality_strict\n"), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) + + t.Run("json", func(t *testing.T) { + var list EvaluatorList + err := json.Unmarshal([]byte(`[{"name": "quality_strict"}]`), &list) + require.Error(t, err) + require.Contains(t, err.Error(), "evaluator") + }) +} + +// The eval fingerprint is taken over this encoding, so a field the encoder +// drops is a change the reconciler cannot see. +func TestEvaluatorListMarshalKeepsEveryField(t *testing.T) { + list := EvaluatorList{{ + Evaluator: "support-quality", + Name: "quality_strict", + Version: "2", + InitializationParameters: map[string]any{"model": "gpt-5.6-luna"}, + DataMapping: map[string]string{"query": "{{item.customer_message}}"}, + }} + + encoded, err := json.Marshal(list) + require.NoError(t, err) + + var round EvaluatorList + require.NoError(t, json.Unmarshal(encoded, &round)) + require.Equal(t, list, round) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go new file mode 100644 index 00000000000..9f5a6320684 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/artifacts.go @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "path/filepath" + "strings" +) + +// Conventional artifact locations, relative to the eval directory. +const ( + DefaultDatasetsDir = "datasets" + DefaultEvaluatorsDir = "evaluators" +) + +// ArtifactRef is the name/source pair a generation run produces, so the +// command can tell the developer how to reference it. +type ArtifactRef struct { + Name string `json:"name"` + Source string `json:"source"` +} + +// Sample-count bounds enforced by the generation service. +const ( + MinSampleSize = 15 + MaxSampleSize = 1000 + DefaultSampleSize = 15 +) + +// Sources a dataset can be generated from. +const ( + GenerateFromTraces = "traces" + GenerateFromAgent = "agent" + GenerateFromPrompt = "prompt" + GenerateFromFile = "file" +) + +// GenerateSources is what --from accepts, in help order. +var GenerateSources = []string{ + GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile, +} + +// ValidateGenerateSource rejects a --from value the service has no path for. +func ValidateGenerateSource(from string) error { + switch from { + case "", GenerateFromTraces, GenerateFromAgent, GenerateFromPrompt, GenerateFromFile: + return nil + default: + return fmt.Errorf( + "--from %q is not a source; use one of %s", + from, strings.Join(GenerateSources, ", ")) + } +} + +// ValidateSampleSize rejects a row count the service would reject, before a +// generation job is submitted and billed. +func ValidateSampleSize(n int) error { + if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { + return fmt.Errorf( + "sample size must be between %d and %d, got %d", + MinSampleSize, MaxSampleSize, n) + } + return nil +} + +// ArtifactPath resolves an output directory against baseDir. The value may be a +// directory, in which case the file name is derived from resourceName and ext, +// or an explicit file path, which is used as-is. +func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { + if outputDir == "" { + return filepath.Join(baseDir, resourceName+ext) + } + candidate := outputDir + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(baseDir, candidate) + } + if looksLikeFile(outputDir, ext) { + return candidate + } + return filepath.Join(candidate, resourceName+ext) +} + +// looksLikeFile treats a trailing recognized extension as an explicit file path. +func looksLikeFile(p, ext string) bool { + got := strings.ToLower(filepath.Ext(p)) + if got == "" { + return false + } + if got == strings.ToLower(ext) { + return true + } + switch got { + case ".json", ".jsonl", ".yaml", ".yml": + return true + } + return false +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index ec58282de06..c9bea4324a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -7,31 +7,26 @@ package project import ( "fmt" - "os" - "path/filepath" - "sort" "strings" "azureaieval/internal/pkg/evalcore" - - "go.yaml.in/yaml/v3" ) -// EvalConfig is one eval — the body of a single `azure.ai.eval` service entry, -// kept in evals/.yaml and pulled in with $ref. +// EvalConfig is one evaluation configuration: the catalogs of reusable assets, +// and every eval defined over them. +// +// It is the body of a single `azure.ai.eval` service entry, pulled in with +// $ref. One file rather than one per eval, because the catalogs are shared: +// two evals over the same dataset should name it once. // -// The eval's name is the service key in azure.yaml and is not repeated here. -// One service per eval is what lets the azd dependency graph order an eval -// after the agent it evaluates. +// How it is stored lives in eval_config_store.go. type EvalConfig struct { - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Dataset *DatasetDecl `yaml:"dataset,omitempty" json:"dataset,omitempty"` - Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` - Target *Target `yaml:"target,omitempty" json:"target,omitempty"` - Options *Options `yaml:"options,omitempty" json:"options,omitempty"` + Datasets []DatasetDecl `yaml:"datasets,omitempty" json:"datasets,omitempty"` + Evaluators []EvaluatorDecl `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Evals []Eval `yaml:"evals,omitempty" json:"evals,omitempty"` } -// DatasetDecl declares a dataset. A local Source is uploaded on deploy; without +// DatasetDecl is a catalog entry. A local Source is uploaded on deploy; without // one the name must already resolve to a registered dataset. type DatasetDecl struct { Name string `yaml:"name" json:"name"` @@ -39,8 +34,8 @@ type DatasetDecl struct { Version string `yaml:"version,omitempty" json:"version,omitempty"` } -// EvaluatorDecl declares a custom evaluator. Built-ins are referenced directly -// from an eval and never declared here. +// EvaluatorDecl is a catalog entry for a custom evaluator. Built-ins are +// referenced straight from an eval and never declared here. // // Source names a `.json` file holding a rubric: a list of weighted scoring // dimensions. @@ -50,34 +45,51 @@ type EvaluatorDecl struct { Version string `yaml:"version,omitempty" json:"version,omitempty"` } -// Eval is a run definition: evaluators plus options, bound to a dataset. +// Eval is one evaluation defined over the catalogs. +// +// Dataset and Source are alternatives: rows come from a catalog dataset, or +// from a source such as production traces. Target is what gets invoked, and is +// a separate axis — an eval can read traces and invoke nothing. type Eval struct { - Name string `yaml:"name" json:"name"` - ID string `yaml:"id,omitempty" json:"id,omitempty"` - Description string `yaml:"description,omitempty" json:"description,omitempty"` - Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` - Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` - Target *Target `yaml:"target,omitempty" json:"target,omitempty"` - Options *Options `yaml:"options,omitempty" json:"options,omitempty"` + Name string `yaml:"name" json:"name"` + ID string `yaml:"id,omitempty" json:"id,omitempty"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + Dataset string `yaml:"dataset,omitempty" json:"dataset,omitempty"` + Source *SourceDecl `yaml:"source,omitempty" json:"source,omitempty"` + EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` + MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` + Evaluators evalcore.EvaluatorList `yaml:"evaluators,omitempty" json:"evaluators,omitempty"` + Target *Target `yaml:"target,omitempty" json:"target,omitempty"` } -// Target names what the run invokes. Only type "agent" is supported today. +// SourceDecl says where an eval's rows come from when they are not a dataset. +type SourceDecl struct { + Type string `yaml:"type" json:"type"` + LookbackHours int `yaml:"lookback_hours,omitempty" json:"lookback_hours,omitempty"` + MaxTraces int `yaml:"max_traces,omitempty" json:"max_traces,omitempty"` + AgentName string `yaml:"agent_name,omitempty" json:"agent_name,omitempty"` + ResponseIDs []string `yaml:"response_ids,omitempty" json:"response_ids,omitempty"` + MaxTurns int `yaml:"max_turns,omitempty" json:"max_turns,omitempty"` +} + +// Source types an eval can read rows from. +const ( + SourceTypeTraces = "traces" + SourceTypeResponses = "responses" +) + +// Target names what the run invokes. type Target struct { Type string `yaml:"type" json:"type"` Name string `yaml:"name" json:"name"` } -const TargetTypeAgent = "agent" - -// Options are run settings carried on the eval. -// -// There is deliberately no judge-model option. A judge deployment is a testing -// criterion's `initialization_parameters.deployment_name`, which differs per -// evaluator, so it is declared on the evaluator reference instead. -type Options struct { - MaxSamples int `yaml:"max_samples,omitempty" json:"max_samples,omitempty"` - EvaluationLevel string `yaml:"evaluation_level,omitempty" json:"evaluation_level,omitempty"` -} +// Target types the extension can invoke. Absent means nothing is invoked and +// the dataset already carries the answers. +const ( + TargetTypeAgent = "agent" + TargetTypeModel = "model" +) // Evaluation levels accepted by the service. The service default is turn. const ( @@ -85,92 +97,149 @@ const ( EvaluationLevelConversation = "conversation" ) -// LoadEvalConfig reads an eval body from disk. The path is used verbatim, -// relative to the process working directory — never re-rooted. -func LoadEvalConfig(path string) (*EvalConfig, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("reading eval config %q: %w", path, err) +// EvalNames lists the declared evals in declaration order. +func (c *EvalConfig) EvalNames() []string { + names := make([]string, 0, len(c.Evals)) + for _, e := range c.Evals { + names = append(names, e.Name) } - - var cfg EvalConfig - if err := yaml.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing eval config %q: %w", path, err) - } - return &cfg, nil + return names } -// EvalNamesIn lists the evals declared under evalDir, in sorted order. +// Eval returns the named eval. // -// One file is one eval, named after it. The generation spec shares the -// directory and is not one, so it is excluded by name. -func EvalNamesIn(evalDir string) ([]string, error) { - entries, err := os.ReadDir(evalDir) - if err != nil { - return nil, err +// An empty name is only answered when the file declares exactly one, because +// guessing which eval a command meant is the kind of mistake that is noticed +// only after it has run. +func (c *EvalConfig) Eval(name string) (*Eval, error) { + if name == "" { + switch len(c.Evals) { + case 0: + return nil, fmt.Errorf("no evals are declared") + case 1: + return &c.Evals[0], nil + default: + return nil, fmt.Errorf( + "this configuration declares %d evals (%s); choose one with --eval", + len(c.Evals), strings.Join(c.EvalNames(), ", ")) + } } - var names []string - for _, e := range entries { - if e.IsDir() { - continue + for i := range c.Evals { + if c.Evals[i].Name == name { + return &c.Evals[i], nil } - ext := strings.ToLower(filepath.Ext(e.Name())) - if ext != ".yaml" && ext != ".yml" { - continue + } + return nil, fmt.Errorf( + "eval %q is not declared; this configuration has %s", + name, strings.Join(c.EvalNames(), ", ")) +} + +// HasEval reports whether the named eval is declared. Unlike Eval it never +// falls back to "the only one", so callers checking for a collision cannot +// match a differently named entry. +func (c *EvalConfig) HasEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + return true } - name := strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())) - if name == generateConfigBase { - continue + } + return false +} + +// RemoveEval drops the named eval, reporting whether it was there. +func (c *EvalConfig) RemoveEval(name string) bool { + for i := range c.Evals { + if c.Evals[i].Name == name { + c.Evals = append(c.Evals[:i], c.Evals[i+1:]...) + return true } - names = append(names, name) } - sort.Strings(names) - return names, nil + return false } -// generateConfigBase is the reserved file name in the evals directory. -const generateConfigBase = "generate" +// DatasetDeclaration returns the catalog entry an eval's `dataset:` names. +func (c *EvalConfig) DatasetDeclaration(name string) (*DatasetDecl, bool) { + for i := range c.Datasets { + if c.Datasets[i].Name == name { + return &c.Datasets[i], true + } + } + return nil, false +} -// ResolveEvalConfigPath finds the config file holding one eval's body. -// -// A named eval is evals/.yaml. With no name the directory must hold -// exactly one eval, and anything else names the candidates rather than -// picking one, because guessing which eval a command meant is the kind of -// mistake that is only noticed after it has run. -func ResolveEvalConfigPath(evalDir, evalName string) (string, error) { - if evalName != "" { - path := EvalConfigPath(evalDir, evalName) - if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("eval %q is not declared in %s", evalName, evalDir) +// EvaluatorDeclaration returns the catalog entry an evaluator reference names. +func (c *EvalConfig) EvaluatorDeclaration(name string) (*EvaluatorDecl, bool) { + for i := range c.Evaluators { + if c.Evaluators[i].Name == name { + return &c.Evaluators[i], true } - return path, nil } + return nil, false +} - names, err := EvalNamesIn(evalDir) - if err != nil { - return "", fmt.Errorf("reading %s: %w", evalDir, err) +// CustomEvaluators are the catalog entries this configuration owns — the ones +// carrying a local source, published before the evals that name them. +func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { + var owned []EvaluatorDecl + for _, decl := range c.Evaluators { + if decl.Source == "" { + continue + } + owned = append(owned, decl) } - switch len(names) { - case 0: - return "", fmt.Errorf("no evals are declared in %s", evalDir) - case 1: - return EvalConfigPath(evalDir, names[0]), nil - default: - return "", fmt.Errorf( - "%s declares %d evals (%s); choose one with --eval", - evalDir, len(names), strings.Join(names, ", ")) + return owned +} + +// LocalDatasets are the catalog entries carrying a file to upload. +func (c *EvalConfig) LocalDatasets() []DatasetDecl { + var owned []DatasetDecl + for _, decl := range c.Datasets { + if decl.Source == "" { + continue + } + owned = append(owned, decl) } + return owned } // Validate checks the invariants the provider relies on before it calls the // service, so failures surface as config errors rather than opaque 4xx. func (c *EvalConfig) Validate() error { - if c.Dataset != nil && c.Dataset.Name == "" { - return fmt.Errorf("dataset: 'name' is required") + if err := c.validateCatalogs(); err != nil { + return err + } + if len(c.Evals) == 0 { + return fmt.Errorf("at least one eval is required") + } + + seen := map[string]bool{} + for i, eval := range c.Evals { + if eval.Name == "" { + return fmt.Errorf("evals[%d]: 'name' is required", i) + } + if seen[eval.Name] { + return fmt.Errorf("evals[%d]: duplicate eval name %q", i, eval.Name) + } + seen[eval.Name] = true + + if err := c.validateEval(i, eval); err != nil { + return err + } } - if len(c.Evaluators) == 0 { - return fmt.Errorf("at least one evaluator is required") + return nil +} + +func (c *EvalConfig) validateCatalogs() error { + datasets := map[string]bool{} + for i, d := range c.Datasets { + if d.Name == "" { + return fmt.Errorf("datasets[%d]: 'name' is required", i) + } + if datasets[d.Name] { + return fmt.Errorf("datasets[%d]: duplicate dataset name %q", i, d.Name) + } + datasets[d.Name] = true } evaluators := map[string]bool{} @@ -183,13 +252,10 @@ func (c *EvalConfig) Validate() error { } evaluators[e.Name] = true - if e.IsBuiltin() { - if e.Source != "" { - return fmt.Errorf( - "evaluators[%d] (%s): a built-in evaluator has no source to publish", - i, e.Name) - } - continue + if strings.HasPrefix(e.Name, evalcore.BuiltinPrefix) { + return fmt.Errorf( + "evaluators[%d] (%s): a built-in needs no catalog entry; reference it "+ + "straight from an eval", i, e.Name) } // The service assigns an evaluator's version on publish, so a declared // one cannot be honoured alongside a source: the upload lands on @@ -203,54 +269,74 @@ func (c *EvalConfig) Validate() error { "on the project", i, e.Name) } } + return nil +} - if c.Target != nil && c.Target.Type != "" && c.Target.Type != TargetTypeAgent { +func (c *EvalConfig) validateEval(i int, eval Eval) error { + if eval.Dataset != "" && eval.Source != nil { return fmt.Errorf( - "target.type %q is not supported; use %q", c.Target.Type, TargetTypeAgent) + "evals[%d] (%s): `dataset` and `source` both say where rows come from; "+ + "declare one", i, eval.Name) + } + if eval.Dataset != "" { + if _, ok := c.DatasetDeclaration(eval.Dataset); !ok { + return fmt.Errorf( + "evals[%d] (%s): dataset %q is not in the datasets catalog", + i, eval.Name, eval.Dataset) + } } - if c.Options != nil { - switch c.Options.EvaluationLevel { - case "", EvaluationLevelTurn, EvaluationLevelConversation: + if eval.Source != nil { + switch eval.Source.Type { + case SourceTypeTraces, SourceTypeResponses: + case "": + return fmt.Errorf("evals[%d] (%s): source.type is required", i, eval.Name) default: return fmt.Errorf( - "options.evaluation_level %q is invalid; expected %q or %q", - c.Options.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) + "evals[%d] (%s): source.type %q is not supported; use %q or %q", + i, eval.Name, eval.Source.Type, SourceTypeTraces, SourceTypeResponses) } } - return nil -} - -// Eval resolves the config into the eval the reconciler publishes, taking its -// name from the service entry that pulled the file in. -func (c *EvalConfig) Eval(name string) Eval { - resolved := Eval{ - Name: name, - Description: c.Description, - Evaluators: c.Evaluators, - Target: c.Target, - Options: c.Options, + if len(eval.Evaluators) == 0 { + return fmt.Errorf("evals[%d] (%s): at least one evaluator is required", i, eval.Name) } - if c.Dataset != nil { - resolved.Dataset = c.Dataset.Name - } - return resolved -} + criteria := map[string]bool{} + for j, ref := range eval.Evaluators { + if ref.Evaluator == "" { + return fmt.Errorf("evals[%d].evaluators[%d]: 'evaluator' is required", i, j) + } + // The criterion name is what identifies a result row, so two rows that + // cannot be told apart are refused here rather than in the results. + criterion := ref.CriterionName() + if criteria[criterion] { + return fmt.Errorf( + "evals[%d].evaluators[%d]: duplicate criterion %q; give one a `name`", + i, j, criterion) + } + criteria[criterion] = true -// CustomEvaluators are the evaluators this config owns — the referenced ones -// carrying a local source, which are published before the eval that names them. -// A built-in needs nothing, and one without a source is already registered. -func (c *EvalConfig) CustomEvaluators() []EvaluatorDecl { - var owned []EvaluatorDecl - for _, ref := range c.Evaluators { - if ref.IsBuiltin() || ref.Source == "" { + if ref.IsBuiltin() { continue } - owned = append(owned, EvaluatorDecl{ - Name: ref.Name, - Source: ref.Source, - Version: ref.Version, - }) + if _, ok := c.EvaluatorDeclaration(ref.Evaluator); !ok { + return fmt.Errorf( + "evals[%d].evaluators[%d]: evaluator %q is not in the evaluators catalog", + i, j, ref.Evaluator) + } } - return owned + + if eval.Target != nil && eval.Target.Type != "" && + eval.Target.Type != TargetTypeAgent && eval.Target.Type != TargetTypeModel { + return fmt.Errorf( + "evals[%d] (%s): target.type %q is not supported; use %q or %q", + i, eval.Name, eval.Target.Type, TargetTypeAgent, TargetTypeModel) + } + switch eval.EvaluationLevel { + case "", EvaluationLevelTurn, EvaluationLevelConversation: + default: + return fmt.Errorf( + "evals[%d] (%s): evaluation_level %q is invalid; expected %q or %q", + i, eval.Name, eval.EvaluationLevel, EvaluationLevelTurn, EvaluationLevelConversation) + } + return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go new file mode 100644 index 00000000000..2b2b587c4f2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_store.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "go.yaml.in/yaml/v3" +) + +// This file is the only place that knows how the configuration is stored: the +// directory it lives in, what the file is called, and how it is parsed and +// serialized. Everything else works with *EvalConfig, so changing the on-disk +// shape stays a local edit. + +// DefaultEvalDir is where init writes the configuration and its artifacts. +const DefaultEvalDir = "evals" + +// EvalConfigBase is the single configuration file inside that directory. +const EvalConfigBase = "eval.yaml" + +// EvalConfigPath is the configuration file inside an eval directory. It is +// exported for error messages and for the azure.yaml $ref; readers should +// prefer OpenEvalConfig. +func EvalConfigPath(evalDir string) string { + return filepath.Join(evalDir, EvalConfigBase) +} + +// OpenEvalConfig reads the configuration under evalDir. +// +// A missing file returns (nil, nil): generate runs before init, so "no +// configuration yet" is an ordinary state rather than a failure. +func OpenEvalConfig(evalDir string) (*EvalConfig, error) { + cfg, err := LoadEvalConfig(EvalConfigPath(evalDir)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return cfg, err +} + +// LoadEvalConfig reads a configuration from an explicit path. The path is used +// verbatim, relative to the process working directory — never re-rooted. +func LoadEvalConfig(path string) (*EvalConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading eval config %q: %w", path, err) + } + + var cfg EvalConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("parsing eval config %q: %w", path, err) + } + return &cfg, nil +} + +// SaveEvalConfig writes cfg as the configuration under evalDir, creating the +// directory when it does not exist yet. +func SaveEvalConfig(evalDir string, cfg *EvalConfig) error { + if err := os.MkdirAll(evalDir, 0o750); err != nil { + return fmt.Errorf("creating %q: %w", evalDir, err) + } + return SaveEvalConfigTo(EvalConfigPath(evalDir), cfg) +} + +// SaveEvalConfigTo writes cfg over an explicit path, for callers that already +// resolved one. +func SaveEvalConfigTo(path string, cfg *EvalConfig) error { + body, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("serializing eval config: %w", err) + } + if err := os.WriteFile(path, body, 0o600); err != nil { + return fmt.Errorf("writing eval config %q: %w", path, err) + } + return nil +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index 03e58c5cf35..b5cece4f058 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -8,149 +8,198 @@ import ( "path/filepath" "testing" - "azureaieval/internal/pkg/evalcore" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" ) -// sampleEvalConfig is the shape the spec documents for evals/.yaml. +// sampleEvalConfig is the shape the spec documents for evals/eval.yaml: two +// catalogs, then the evals defined over them. const sampleEvalConfig = ` -description: Quality gate for the support agent - -dataset: - name: support-golden - source: ./datasets/support-golden.jsonl - version: "1" +datasets: + - name: support-golden + source: ./datasets/support-golden.jsonl + version: "1" + - name: prod-registered evaluators: - - builtin.task_adherence - name: support-quality source: ./evaluators/support-quality.json - threshold: 4.0 - initialization_parameters: - deployment_name: gpt-4.1-nano - - safety-check - -target: - type: agent - name: support-agent - -options: - max_samples: 100 - evaluation_level: conversation + +evals: + - name: support-agent-smoke + description: Quality gate for the support agent + dataset: support-golden + evaluation_level: conversation + max_samples: 100 + evaluators: + - evaluator: builtin.task_adherence + - evaluator: support-quality + name: quality_strict + initialization_parameters: + deployment_name: gpt-4.1-nano + target: + type: agent + name: support-agent + + - name: support-agent-trace-eval + source: + type: traces + agent_name: support-agent + max_traces: 20 + evaluators: + - evaluator: builtin.task_adherence ` func loadFromString(t *testing.T, body string) *EvalConfig { t.Helper() - path := filepath.Join(t.TempDir(), "support-agent-smoke.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - cfg, err := LoadEvalConfig(path) + dir := t.TempDir() + require.NoError(t, os.WriteFile(EvalConfigPath(dir), []byte(body), 0o600)) + cfg, err := OpenEvalConfig(dir) require.NoError(t, err) + require.NotNil(t, cfg) return cfg } func TestLoadEvalConfig_ParsesAllSections(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) - require.NotNil(t, cfg.Dataset) - require.Equal(t, "support-golden", cfg.Dataset.Name) - require.Equal(t, "./datasets/support-golden.jsonl", cfg.Dataset.Source) - require.Equal(t, "1", cfg.Dataset.Version) + require.Len(t, cfg.Datasets, 2) + require.Equal(t, "support-golden", cfg.Datasets[0].Name) + require.Equal(t, "./datasets/support-golden.jsonl", cfg.Datasets[0].Source) + require.Equal(t, "1", cfg.Datasets[0].Version) + + require.Len(t, cfg.Evaluators, 1) + require.Equal(t, "support-quality", cfg.Evaluators[0].Name) - require.Len(t, cfg.Evaluators, 3) - require.Equal(t, TargetTypeAgent, cfg.Target.Type) - require.Equal(t, "support-agent", cfg.Target.Name) - require.Equal(t, EvaluationLevelConversation, cfg.Options.EvaluationLevel) - require.Equal(t, 100, cfg.Options.MaxSamples) + require.Equal(t, []string{"support-agent-smoke", "support-agent-trace-eval"}, cfg.EvalNames()) } -// The eval takes its name from the service entry that pulled the file in, so -// the body never repeats it. -func TestEval_TakesNameFromTheService(t *testing.T) { +// One file holds many evals, and each is selected by its own name. +func TestEval_SelectsByName(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) - eval := cfg.Eval("support-agent-smoke") - require.Equal(t, "support-agent-smoke", eval.Name) + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) require.Equal(t, "support-golden", eval.Dataset) require.Equal(t, "Quality gate for the support agent", eval.Description) - require.Len(t, eval.Evaluators, 3) - require.Same(t, cfg.Target, eval.Target) + require.Equal(t, EvaluationLevelConversation, eval.EvaluationLevel) + require.Equal(t, 100, eval.MaxSamples) + require.Len(t, eval.Evaluators, 2) + require.Equal(t, TargetTypeAgent, eval.Target.Type) + require.Equal(t, "support-agent", eval.Target.Name) +} + +// A trace-backed eval invokes nothing, so agent_name filters rather than targets. +func TestEval_TraceSourceHasNoTarget(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + eval, err := cfg.Eval("support-agent-trace-eval") + require.NoError(t, err) + require.Nil(t, eval.Target) + require.Equal(t, SourceTypeTraces, eval.Source.Type) + require.Equal(t, "support-agent", eval.Source.AgentName) + require.Equal(t, 20, eval.Source.MaxTraces) +} + +// An unnamed selection is only answered when the file declares exactly one, +// because guessing which eval a command meant is noticed only after it runs. +func TestEval_UnnamedIsAmbiguousWithSeveral(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("") + require.ErrorContains(t, err, "--eval") + require.ErrorContains(t, err, "support-agent-trace-eval") + + single := loadFromString(t, "evals:\n - name: only\n evaluators:\n - evaluator: builtin.relevance\n") + eval, err := single.Eval("") + require.NoError(t, err) + require.Equal(t, "only", eval.Name) +} + +func TestEval_UnknownNameNamesWhatIsDeclared(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + _, err := cfg.Eval("nope") + require.ErrorContains(t, err, "is not declared") + require.ErrorContains(t, err, "support-agent-smoke") } -// Only the referenced evaluators carrying a local source are this config's to -// publish. A built-in needs nothing, and one without a source already exists. -func TestCustomEvaluators_OnlyOwnsLocalSources(t *testing.T) { +// HasEval never falls back to "the only one", so a collision check cannot match +// a differently named entry. +func TestHasEvalAndRemoveEval(t *testing.T) { + cfg := loadFromString(t, sampleEvalConfig) + + require.True(t, cfg.HasEval("support-agent-smoke")) + require.False(t, cfg.HasEval("nope")) + require.False(t, cfg.HasEval("")) + + require.True(t, cfg.RemoveEval("support-agent-smoke")) + require.False(t, cfg.HasEval("support-agent-smoke")) + require.Equal(t, []string{"support-agent-trace-eval"}, cfg.EvalNames()) + require.False(t, cfg.RemoveEval("support-agent-smoke")) +} + +// Only catalog entries carrying a local source are this config's to publish. +// One without a source already exists on the project. +func TestCustomEvaluatorsAndLocalDatasets_OnlyOwnLocalSources(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) owned := cfg.CustomEvaluators() require.Len(t, owned, 1) require.Equal(t, "support-quality", owned[0].Name) require.Equal(t, "./evaluators/support-quality.json", owned[0].Source) + + local := cfg.LocalDatasets() + require.Len(t, local, 1) + require.Equal(t, "support-golden", local[0].Name, + "prod-registered has no source, so it is already on the project") } -// Evaluator entries accept a bare string or a mapping carrying the rest of the -// declaration. -func TestEvaluatorList_MixedForms(t *testing.T) { +func TestDeclarationLookups(t *testing.T) { cfg := loadFromString(t, sampleEvalConfig) - require.Len(t, cfg.Evaluators, 3) - - require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) - require.True(t, cfg.Evaluators[0].IsBuiltin()) - require.Equal(t, "task_adherence", cfg.Evaluators[0].APIName(), - "the builtin prefix must be stripped before it reaches the service") - require.Nil(t, cfg.Evaluators[0].Threshold) - - require.Equal(t, "support-quality", cfg.Evaluators[1].Name) - require.False(t, cfg.Evaluators[1].IsBuiltin()) - require.NotNil(t, cfg.Evaluators[1].Threshold) - require.InDelta(t, 4.0, *cfg.Evaluators[1].Threshold, 0.0001) - require.Equal(t, "gpt-4.1-nano", - cfg.Evaluators[1].InitializationParameters["deployment_name"], - "the judge deployment is declared per evaluator, not once per eval") - - require.Equal(t, "safety-check", cfg.Evaluators[2].Name) - require.Nil(t, cfg.Evaluators[2].Threshold) + + ds, ok := cfg.DatasetDeclaration("support-golden") + require.True(t, ok) + require.Equal(t, "./datasets/support-golden.jsonl", ds.Source) + + _, ok = cfg.DatasetDeclaration("missing") + require.False(t, ok) + + ev, ok := cfg.EvaluatorDeclaration("support-quality") + require.True(t, ok) + require.Equal(t, "./evaluators/support-quality.json", ev.Source) } -// Round-tripping must not rewrite bare names into mappings. -func TestEvaluatorList_RoundTripKeepsCompactForm(t *testing.T) { - threshold := 4.0 - list := evalcore.EvaluatorList{ - {Name: "builtin.relevance"}, - {Name: "support-quality", Threshold: &threshold}, - } +// The configuration must survive a write/read cycle, because init and generate +// both append to a file they just read. +func TestEvalConfig_RoundTripsThroughTheStore(t *testing.T) { + dir := t.TempDir() + cfg := loadFromString(t, sampleEvalConfig) - out, err := yaml.Marshal(list) + require.NoError(t, SaveEvalConfig(dir, cfg)) + back, err := OpenEvalConfig(dir) require.NoError(t, err) + require.Equal(t, cfg, back) +} - var back evalcore.EvaluatorList - require.NoError(t, yaml.Unmarshal(out, &back)) - require.Len(t, back, 2) - require.Equal(t, "builtin.relevance", back[0].Name) - require.Nil(t, back[0].Threshold) - require.NotNil(t, back[1].Threshold) - require.Contains(t, string(out), "- builtin.relevance", - "an evaluator with only a name should stay a plain string") +// A missing file is an ordinary state: generate runs before init. +func TestOpenEvalConfig_MissingIsNotAnError(t *testing.T) { + cfg, err := OpenEvalConfig(t.TempDir()) + require.NoError(t, err) + require.Nil(t, cfg) } -// An evaluator carrying a source must not be flattened to its name, or the -// declaration that says what to publish is lost on the next write. -func TestEvaluatorList_RoundTripKeepsSource(t *testing.T) { - list := evalcore.EvaluatorList{ - {Name: "support-quality", Source: "./evaluators/support-quality.json"}, - {Name: "builtin.task_adherence", - InitializationParameters: map[string]any{"deployment_name": "gpt-4.1-nano"}}, - } +// SaveEvalConfig creates the directory, so generate can record an artifact in a +// project that has never run init. +func TestSaveEvalConfig_CreatesTheDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "evals") + require.NoError(t, SaveEvalConfig(dir, &EvalConfig{ + Datasets: []DatasetDecl{{Name: "generated", Source: "./datasets/generated.jsonl"}}, + })) - out, err := yaml.Marshal(list) + cfg, err := OpenEvalConfig(dir) require.NoError(t, err) - - var back evalcore.EvaluatorList - require.NoError(t, yaml.Unmarshal(out, &back)) - require.Len(t, back, 2) - require.Equal(t, "./evaluators/support-quality.json", back[0].Source) - require.Equal(t, "gpt-4.1-nano", back[1].InitializationParameters["deployment_name"]) + require.Len(t, cfg.Datasets, 1) + require.Empty(t, cfg.Evals, "a generate-only file is inert until init wires an eval") } func TestValidate_Accepts(t *testing.T) { @@ -158,6 +207,8 @@ func TestValidate_Accepts(t *testing.T) { } func TestValidate_Rejects(t *testing.T) { + const oneEval = "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + cases := []struct { name string body string @@ -165,38 +216,76 @@ func TestValidate_Rejects(t *testing.T) { }{ { name: "dataset without a name", - body: "dataset:\n source: ./d.jsonl\nevaluators: [builtin.relevance]\n", + body: "datasets:\n - source: ./d.jsonl\n" + oneEval, wantErr: "'name' is required", }, + { + name: "duplicate dataset", + body: "datasets:\n - name: d\n - name: d\n" + oneEval, + wantErr: "duplicate dataset name", + }, + { + name: "built-in declared in the catalog", + body: "evaluators:\n - name: builtin.relevance\n" + oneEval, + wantErr: "needs no catalog entry", + }, + { + name: "version pinned alongside a source", + body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n" + oneEval, + wantErr: "cannot be set with `source`", + }, + { + name: "no evals", + body: "datasets:\n - name: d\n", + wantErr: "at least one eval is required", + }, + { + name: "duplicate eval", + body: oneEval + " - name: e\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "duplicate eval name", + }, { name: "no evaluators", - body: "evaluators: []\n", + body: "evals:\n - name: e\n evaluators: []\n", wantErr: "at least one evaluator is required", }, { - name: "built-in with a source to publish", - body: "evaluators:\n - name: builtin.relevance\n source: ./x.json\n", - wantErr: "has no source to publish", + name: "dataset and source both declared", + body: "datasets:\n - name: d\nevals:\n - name: e\n dataset: d\n" + + " source:\n type: traces\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "declare one", }, { - name: "duplicate evaluator", - body: "evaluators: [builtin.relevance, builtin.relevance]\n", - wantErr: "duplicate evaluator name", + name: "dataset not in the catalog", + body: "evals:\n - name: e\n dataset: missing\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "not in the datasets catalog", }, { - name: "version pinned alongside a source", - body: "evaluators:\n - name: q\n source: ./q.json\n version: \"3\"\n", - wantErr: "cannot be set with `source`", + name: "evaluator not in the catalog", + body: "evals:\n - name: e\n evaluators:\n - evaluator: quality\n", + wantErr: "not in the evaluators catalog", + }, + { + name: "duplicate criterion", + body: "evals:\n - name: e\n evaluators:\n" + + " - evaluator: builtin.relevance\n - evaluator: builtin.relevance\n", + wantErr: "duplicate criterion", }, { - name: "unsupported target type", - body: "evaluators: [builtin.relevance]\ntarget:\n type: prompt\n", + name: "unsupported source type", + body: "evals:\n - name: e\n source:\n type: prompt\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "is not supported", + }, + { + name: "unsupported target type", + body: "evals:\n - name: e\n evaluators:\n - evaluator: builtin.relevance\n" + + " target:\n type: prompt\n", wantErr: "is not supported", }, { name: "invalid evaluation level", - body: "evaluators: [builtin.relevance]\n" + - "options:\n evaluation_level: sentence\n", + body: "evals:\n - name: e\n evaluation_level: sentence\n" + + " evaluators:\n - evaluator: builtin.relevance\n", wantErr: "evaluation_level", }, } @@ -210,59 +299,6 @@ func TestValidate_Rejects(t *testing.T) { } } -// One file is one eval, named after the file, so the directory listing is the -// list of evals a project declares. -func TestResolveEvalConfigPath(t *testing.T) { - write := func(t *testing.T, dir string, names ...string) { - t.Helper() - for _, n := range names { - require.NoError(t, os.WriteFile( - filepath.Join(dir, n), []byte("evaluators: [builtin.relevance]\n"), 0o600)) - } - } - - t.Run("the only eval is used when unnamed", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml", "generate.yaml") - - path, err := ResolveEvalConfigPath(dir, "") - require.NoError(t, err) - require.Equal(t, filepath.Join(dir, "pr-gate.yaml"), path, - "the generation spec shares the directory and is not an eval") - }) - - t.Run("named eval", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml", "nightly.yaml") - - path, err := ResolveEvalConfigPath(dir, "nightly") - require.NoError(t, err) - require.Equal(t, filepath.Join(dir, "nightly.yaml"), path) - }) - - t.Run("unknown name is an error", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml") - - _, err := ResolveEvalConfigPath(dir, "nope") - require.ErrorContains(t, err, "is not declared") - }) - - t.Run("ambiguous without a name", func(t *testing.T) { - dir := t.TempDir() - write(t, dir, "pr-gate.yaml", "nightly.yaml") - - _, err := ResolveEvalConfigPath(dir, "") - require.ErrorContains(t, err, "--eval") - require.ErrorContains(t, err, "nightly") - }) - - t.Run("empty directory", func(t *testing.T) { - _, err := ResolveEvalConfigPath(t.TempDir(), "") - require.ErrorContains(t, err, "no evals") - }) -} - // outputDir accepts a directory or an explicit file path. func TestArtifactPath(t *testing.T) { cases := []struct { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go deleted file mode 100644 index 12fdb1cf6ff..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config.go +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "fmt" - "os" - "path/filepath" - "strings" - - "go.yaml.in/yaml/v3" -) - -// Conventional locations. All are relative to the working directory and are -// used verbatim — never re-rooted under the agent or project directory. -const ( - DefaultEvalDir = "evals" - DefaultGenerateConfig = "evals/generate.yaml" - DefaultDatasetsDir = "datasets" - DefaultEvaluatorsDir = "evaluators" -) - -// EvalConfigPath is where the body of the eval named by a service entry lives. -func EvalConfigPath(evalDir, evalName string) string { - return filepath.Join(evalDir, evalName+".yaml") -} - -// GenerateConfig says how the local dataset and evaluator artifacts referenced -// by an eval are produced. It is never deployed. -// -// generationModel: gpt-5.6-luna -// dataset: -// support-agent-smoke: -// sampleSize: 15 -// outputDir: ./datasets -// evaluator: -// support-quality: -// outputDir: ./evaluators -// deriveFrom: support-agent -// -// The maps are keyed by artifact name so `dataset generate ` and -// `evaluator generate ` each look up exactly the entry they were asked -// for, and generating one artifact never reads the other's settings. -type GenerateConfig struct { - GenerationModel string `yaml:"generationModel,omitempty" json:"generationModel,omitempty"` - Dataset map[string]DatasetGenSpec `yaml:"dataset,omitempty" json:"dataset,omitempty"` - Evaluator map[string]EvaluatorGenSpec `yaml:"evaluator,omitempty" json:"evaluator,omitempty"` -} - -// DatasetGenSpec configures synthetic dataset generation for one dataset. -type DatasetGenSpec struct { - SampleSize int `yaml:"sampleSize,omitempty" json:"sampleSize,omitempty"` - OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` - // DeriveFrom names the agent whose context seeds generation. Optional: the - // eval's target supplies it, and --target overrides both. - DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` - // Instructions points at a local file whose contents stand in for the - // agent's published instructions for this generation only. - Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - // TraceDays seeds generation from that many days of recent traces. Zero - // disables it. Traces are a generation input only; they cannot be a run's - // data source. - TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` -} - -// EvaluatorGenSpec configures rubric generation for one evaluator. -type EvaluatorGenSpec struct { - OutputDir string `yaml:"outputDir,omitempty" json:"outputDir,omitempty"` - // DeriveFrom names the agent the rubric is written against. - DeriveFrom string `yaml:"deriveFrom,omitempty" json:"deriveFrom,omitempty"` - // Instructions points at a local file whose contents stand in for the - // agent's published instructions for this generation only. - Instructions string `yaml:"instructions,omitempty" json:"instructions,omitempty"` - // TraceDays seeds generation from that many days of recent traces. Zero - // disables it. - TraceDays int `yaml:"traceDays,omitempty" json:"traceDays,omitempty"` -} - -// ArtifactRef is the name/source pair a generation run produces, so the -// command can tell the developer how to reference it. -// -// Generation writes artifacts only and never edits azure.yaml or the eval -// config: `init` declares the paths and `generate` fills them in, which is what -// keeps a generation run a data-file-only diff. -type ArtifactRef struct { - Name string `json:"name"` - Source string `json:"source"` -} - -// Sample-count bounds enforced by the generation service. -const ( - MinSampleSize = 15 - MaxSampleSize = 1000 - DefaultSampleSize = 15 -) - -// LoadGenerateConfig reads a generation spec from disk. -// -// A missing file is not an error. Generation is optional — a developer with -// hand-authored data and evaluators never writes one — and every setting it -// carries can be given on the command line instead. -func LoadGenerateConfig(path string) (*GenerateConfig, error) { - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return &GenerateConfig{}, nil - } - return nil, fmt.Errorf("reading generation config %q: %w", path, err) - } - - var cfg GenerateConfig - if err := yaml.Unmarshal(data, &cfg); err != nil { - return nil, fmt.Errorf("parsing generation config %q: %w", path, err) - } - return &cfg, nil -} - -// DatasetSpec returns the settings for one dataset, and whether the config -// declared them. -func (c *GenerateConfig) DatasetSpec(name string) (DatasetGenSpec, bool) { - spec, ok := c.Dataset[name] - return spec, ok -} - -// EvaluatorSpec returns the settings for one evaluator, and whether the config -// declared them. -func (c *GenerateConfig) EvaluatorSpec(name string) (EvaluatorGenSpec, bool) { - spec, ok := c.Evaluator[name] - return spec, ok -} - -// ValidateSampleSize rejects a row count the service would reject, before a -// generation job is submitted and billed. -func ValidateSampleSize(n int) error { - if n != 0 && (n < MinSampleSize || n > MaxSampleSize) { - return fmt.Errorf( - "sample size must be between %d and %d, got %d", - MinSampleSize, MaxSampleSize, n) - } - return nil -} - -// ArtifactPath resolves an outputDir value against baseDir. The value may be a -// directory, in which case the file name is derived from resourceName and ext, -// or an explicit file path, which is used as-is. -func ArtifactPath(baseDir, outputDir, resourceName, ext string) string { - if outputDir == "" { - return filepath.Join(baseDir, resourceName+ext) - } - candidate := outputDir - if !filepath.IsAbs(candidate) { - candidate = filepath.Join(baseDir, candidate) - } - if looksLikeFile(outputDir, ext) { - return candidate - } - return filepath.Join(candidate, resourceName+ext) -} - -// looksLikeFile treats a trailing recognized extension as an explicit file path. -func looksLikeFile(p, ext string) bool { - got := strings.ToLower(filepath.Ext(p)) - if got == "" { - return false - } - if got == strings.ToLower(ext) { - return true - } - switch got { - case ".json", ".jsonl", ".yaml", ".yml": - return true - } - return false -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go deleted file mode 100644 index 5746b2286b6..00000000000 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/generate_config_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package project - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -// The generation spec is keyed by artifact name, which is what makes -// `dataset generate ` and `evaluator generate ` able to look up -// exactly the entry they were asked for. -func TestLoadGenerateConfig_ParsesTheDocumentedShape(t *testing.T) { - body := ` -generationModel: gpt-5.6-luna -dataset: - support-agent-smoke: - sampleSize: 15 - outputDir: ./datasets -evaluator: - support-quality: - outputDir: ./evaluators - deriveFrom: support-agent -` - path := filepath.Join(t.TempDir(), "generate.yaml") - require.NoError(t, os.WriteFile(path, []byte(body), 0o600)) - - cfg, err := LoadGenerateConfig(path) - require.NoError(t, err) - require.Equal(t, "gpt-5.6-luna", cfg.GenerationModel) - - ds, ok := cfg.DatasetSpec("support-agent-smoke") - require.True(t, ok) - require.Equal(t, 15, ds.SampleSize) - require.Equal(t, "./datasets", ds.OutputDir) - - ev, ok := cfg.EvaluatorSpec("support-quality") - require.True(t, ok) - require.Equal(t, "./evaluators", ev.OutputDir) - require.Equal(t, "support-agent", ev.DeriveFrom) -} - -// Generation is optional: a developer with hand-authored data and evaluators -// never writes a spec, and the generate commands still run from flags alone. -func TestLoadGenerateConfig_MissingFileIsNotAnError(t *testing.T) { - cfg, err := LoadGenerateConfig(filepath.Join(t.TempDir(), "generate.yaml")) - require.NoError(t, err) - require.Empty(t, cfg.GenerationModel) - require.Empty(t, cfg.Dataset) - - _, ok := cfg.DatasetSpec("anything") - require.False(t, ok) -} - -// A row count the service would reject costs a billed job to find out about, -// so it is refused at the flag that carried it. -func TestValidateSampleSize(t *testing.T) { - require.NoError(t, ValidateSampleSize(0), "unset means the default applies") - require.NoError(t, ValidateSampleSize(MinSampleSize)) - require.NoError(t, ValidateSampleSize(MaxSampleSize)) - - require.ErrorContains(t, ValidateSampleSize(MinSampleSize-1), "must be between") - require.ErrorContains(t, ValidateSampleSize(MaxSampleSize+1), "must be between") -} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go index 6fd01d06680..2097621ed6d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval.go @@ -145,26 +145,27 @@ func (p *EvalServiceTargetProvider) Deploy( baseDir := serviceRelativeDir(serviceConfig) - // The eval takes its name from the service entry that pulled this config - // in, which is what makes one service per eval work. - eval := cfg.Eval(serviceConfig.Name) - - // 1. Dataset. + // 1. Datasets the configuration owns. Paths are kept so an eval that names + // one can derive its columns without reading the blob back. anyChanged := false - datasetPath := "" - if cfg.Dataset != nil { - report(progress, fmt.Sprintf("Reconciling dataset %s", cfg.Dataset.Name)) - datasetPath = resolveSource(baseDir, cfg.Dataset.Source) - version, changed, err := reconciler.EnsureDataset(ctx, *cfg.Dataset, datasetPath) + datasetPaths := map[string]string{} + for _, decl := range cfg.Datasets { + if decl.Source == "" { + continue + } + report(progress, fmt.Sprintf("Reconciling dataset %s", decl.Name)) + localPath := resolveSource(baseDir, decl.Source) + datasetPaths[decl.Name] = localPath + version, changed, err := reconciler.EnsureDataset(ctx, decl, localPath) if err != nil { - return nil, fmt.Errorf("dataset %q: %w", cfg.Dataset.Name, err) + return nil, fmt.Errorf("dataset %q: %w", decl.Name, err) } anyChanged = anyChanged || changed - report(progress, describeResult("dataset", cfg.Dataset.Name, version, changed)) + report(progress, describeResult("dataset", decl.Name, version, changed)) } - // 2. Evaluators this config owns. Built-ins and already-registered ones - // need no publish. + // 2. Evaluators this configuration owns. Built-ins and already-registered + // ones need no publish. for _, decl := range cfg.CustomEvaluators() { report(progress, fmt.Sprintf("Reconciling evaluator %s", decl.Name)) localPath := resolveSource(baseDir, decl.Source) @@ -176,14 +177,17 @@ func (p *EvalServiceTargetProvider) Deploy( report(progress, describeResult("evaluator", decl.Name, version, changed)) } - // 3. The eval. Evals are immutable, so a change upstream means a new one + // 3. The evals. Evals are immutable, so a change upstream means a new one // must be created and the stored id replaced. - report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) - id, err := reconciler.EnsureEval(ctx, eval, datasetPath, anyChanged) - if err != nil { - return nil, fmt.Errorf("eval %q: %w", eval.Name, err) + for i := range cfg.Evals { + eval := cfg.Evals[i] + report(progress, fmt.Sprintf("Reconciling eval %s", eval.Name)) + id, err := reconciler.EnsureEval(ctx, eval, datasetPaths[eval.Dataset], anyChanged) + if err != nil { + return nil, fmt.Errorf("eval %q: %w", eval.Name, err) + } + report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) } - report(progress, fmt.Sprintf("Eval %s is %s", eval.Name, id)) return &azdext.ServiceDeployResult{}, nil } @@ -322,18 +326,19 @@ func Fingerprint(path string) (string, error) { // evaluators are untouched. Without this a retargeted group keeps running // against the old definition. func FingerprintGroup(group Eval) (string, error) { - // The id is server-assigned. The description is carried in the group's - // metadata, so editing it does change the request, but recreating an - // immutable group over a reworded description would cost the group id and - // break comparison against earlier runs. It is documentation, not - // evaluation semantics, so an edit lands the next time the group is - // recreated for a reason that matters. + // Only substance is hashed. The id is server-assigned; name and description + // are what UpdateEvalParametersBody reaches, so an edit confined to them is + // pushed in place and must not cost the eval its id and its run history. + // Everything else — dataset, source, evaluators, target, level — is fixed at + // creation, so a change there is a new eval. + name := group.Name group.ID = "" + group.Name = "" group.Description = "" data, err := json.Marshal(group) if err != nil { - return "", fmt.Errorf("hashing eval %q: %w", group.Name, err) + return "", fmt.Errorf("hashing eval %q: %w", name, err) } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go index 987b2815c44..e20ecd49009 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/service_target_eval_test.go @@ -68,25 +68,37 @@ func TestServiceRelativeDirDefaultsToProjectRoot(t *testing.T) { // includes against. func TestEvalConfigFromServiceReadsInlineConfig(t *testing.T) { svc := &azdext.ServiceConfig{ - Name: "support-agent-smoke", + Name: "support-agent-evals", AdditionalProperties: propsFrom(t, map[string]any{ - "dataset": map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, - "evaluators": []any{"builtin.task_adherence"}, - "target": map[string]any{"type": "agent", "name": "my-agent"}, + "datasets": []any{ + map[string]any{"name": "golden", "source": "./datasets/golden.jsonl"}, + }, + "evals": []any{ + map[string]any{ + "name": "support-agent-smoke", + "dataset": "golden", + "evaluators": []any{ + map[string]any{"evaluator": "builtin.task_adherence"}, + }, + "target": map[string]any{"type": "agent", "name": "my-agent"}, + }, + }, }), } cfg, err := EvalConfigFromService(svc, "") require.NoError(t, err) - require.NotNil(t, cfg.Dataset) - require.Equal(t, "golden", cfg.Dataset.Name) - require.Len(t, cfg.Evaluators, 1) - require.Equal(t, "builtin.task_adherence", cfg.Evaluators[0].Name) - require.Equal(t, "my-agent", cfg.Target.Name) - - // The eval's name is the service key, which is what makes one service per - // eval work without the body repeating it. - require.Equal(t, "support-agent-smoke", cfg.Eval(svc.Name).Name) + require.Len(t, cfg.Datasets, 1) + require.Equal(t, "golden", cfg.Datasets[0].Name) + + // One service covers every eval in the file it pulled in, so the eval is + // selected by its own name rather than by the service key. + eval, err := cfg.Eval("support-agent-smoke") + require.NoError(t, err) + require.Equal(t, "golden", eval.Dataset) + require.Len(t, eval.Evaluators, 1) + require.Equal(t, "builtin.task_adherence", eval.Evaluators[0].Evaluator) + require.Equal(t, "my-agent", eval.Target.Name) } func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { @@ -95,16 +107,16 @@ func TestEvalConfigFromServiceRejectsEmptyService(t *testing.T) { require.Contains(t, err.Error(), "no eval configuration") } -// Evals are immutable, so a change to the eval's own declaration has to be +// Evals are immutable, so a change to an eval's own declaration has to be // detectable. Upstream artifact fingerprints do not cover it: retargeting an // eval at a different agent leaves the dataset and evaluators untouched. func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { base := Eval{ - Name: "quality", - Dataset: "golden", - Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, - Target: &Target{Type: "agent", Name: "agent-a"}, - Options: &Options{EvaluationLevel: EvaluationLevelTurn}, + Name: "quality", + Dataset: "golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + Target: &Target{Type: "agent", Name: "agent-a"}, + EvaluationLevel: EvaluationLevelTurn, } original, err := FingerprintGroup(base) @@ -117,16 +129,25 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { cases := map[string]func(g *Eval){ "target": func(g *Eval) { g.Target = &Target{Type: "agent", Name: "agent-b"} }, "evaluators": func(g *Eval) { - g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Name: "builtin.similarity"}) + g.Evaluators = append(g.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) }, "judge deployment": func(g *Eval) { g.Evaluators = evalcore.EvaluatorList{{ - Name: "builtin.task_adherence", + Evaluator: "builtin.task_adherence", InitializationParameters: map[string]any{"deployment_name": "gpt-4o-mini"}, }} }, - "options": func(g *Eval) { g.Options = &Options{EvaluationLevel: EvaluationLevelConversation} }, - "dataset": func(g *Eval) { g.Dataset = "other" }, + "version pin": func(g *Eval) { + g.Evaluators = evalcore.EvaluatorList{{ + Evaluator: "builtin.task_adherence", Version: "2", + }} + }, + "evaluation level": func(g *Eval) { g.EvaluationLevel = EvaluationLevelConversation }, + "dataset": func(g *Eval) { g.Dataset = "other" }, + "source": func(g *Eval) { + g.Dataset = "" + g.Source = &SourceDecl{Type: SourceTypeTraces, AgentName: "agent-a"} + }, } for name, mutate := range cases { t.Run(name, func(t *testing.T) { @@ -141,21 +162,48 @@ func TestFingerprintGroupTracksMeaningfulChanges(t *testing.T) { } } -// Server-assigned and cosmetic fields must not force a recreate. -func TestFingerprintGroupIgnoresIdAndDescription(t *testing.T) { +// The fingerprint covers substance only. The id is server-assigned, and name +// and description are what UpdateEvalParametersBody reaches — an edit confined +// to those is pushed in place, so it must not fork the run history. +func TestFingerprintGroupIgnoresIdNameAndDescription(t *testing.T) { base := Eval{ Name: "quality", Dataset: "golden", - Evaluators: evalcore.EvaluatorList{{Name: "builtin.task_adherence"}}, + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, } original, err := FingerprintGroup(base) require.NoError(t, err) noisy := base noisy.ID = "eval_abc123" + noisy.Name = "quality-renamed" noisy.Description = "reworded" digest, err := FingerprintGroup(noisy) require.NoError(t, err) require.Equal(t, original, digest) } + +// Editing one eval must not recreate its siblings: the unit compared is the +// eval's own subtree, never the file. +func TestFingerprintGroupIsScopedToOneEval(t *testing.T) { + gate := Eval{ + Name: "support-agent-gate", + Dataset: "prod-golden", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + regression := Eval{ + Name: "support-agent-regression-eval", + Dataset: "support-agent-regression", + Evaluators: evalcore.EvaluatorList{{Evaluator: "builtin.task_adherence"}}, + } + + before, err := FingerprintGroup(regression) + require.NoError(t, err) + + gate.Evaluators = append(gate.Evaluators, evalcore.EvaluatorRef{Evaluator: "builtin.similarity"}) + + after, err := FingerprintGroup(regression) + require.NoError(t, err) + require.Equal(t, before, after, "editing a sibling must leave this eval alone") +} From 4591118281d9297c6ca5974847a9d9ebec82d657 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:13:45 +0300 Subject: [PATCH 104/114] Keep an eval across a rename, and bound the first trace run init gains --max-traces, writing max_traces on a trace-backed eval so a first run is bounded rather than taking the service's default of 1000. Deleting the key by hand restores that default, so the flag is not a ceiling. Passing it without --source traces is refused rather than silently ignored. A rename is now an update, not a fork. Excluding the name from the fingerprint was not enough on its own: the environment keys the fingerprint by name, so a renamed declaration found nothing recorded and created a second eval, stranding every earlier run behind an id. The environment now records the id against the substance as well, which is what recognises the rename, and the new name is pushed with UpdateEvalParametersBody. That lookup is only unambiguous while two evals cannot share a substance, so validation refuses a pair differing solely by name and description, naming both. The alternative is a rename adopting whichever of them deployed last. --- .../azure.ai.evaluations/internal/cmd/init.go | 13 ++++ .../internal/cmd/init_test.go | 25 ++++++-- .../internal/cmd/reconciler.go | 60 +++++++++++++++++-- .../internal/cmd/surface_test.go | 4 +- .../internal/pkg/eval_api/models.go | 9 +++ .../internal/pkg/eval_api/operations.go | 16 +++++ .../internal/project/eval_config.go | 22 +++++++ .../internal/project/eval_config_test.go | 6 ++ 8 files changed, 144 insertions(+), 11 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index cd80a2ac7d5..46ec0c0047c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -40,6 +40,7 @@ func newInitCommand() *cobra.Command { target string source string dataset string + maxTraces int evaluators []string judgeModel string path string @@ -65,6 +66,12 @@ func newInitCommand() *cobra.Command { if source == initSourceTraces && dataset != "" { return errors.New("--source traces reads production traces, so it takes no --dataset") } + if cmd.Flags().Changed("max-traces") && source != initSourceTraces { + return errors.New("--max-traces caps a trace-backed eval; pass --source traces") + } + if maxTraces < 0 { + return errors.New("--max-traces must be positive") + } if source == "" { source = initSourceDataset } @@ -116,6 +123,7 @@ func newInitCommand() *cobra.Command { target: target, source: source, dataset: dataset, + maxTraces: maxTraces, evaluators: evaluators, judgeModel: judgeModel, rubricName: target + "-quality", @@ -187,6 +195,9 @@ func newInitCommand() *cobra.Command { "Where rows come from: dataset or traces. Defaults to dataset.") cmd.Flags().StringVar(&dataset, "dataset", "", "Path to a local .jsonl, or the name of a registered dataset.") + cmd.Flags().IntVar(&maxTraces, "max-traces", project.DefaultScaffoldMaxTraces, + "Cap on traces read by a --source traces eval. Delete max_traces from the "+ + "file to take the service default instead.") cmd.Flags().StringArrayVar(&evaluators, "evaluator", nil, "Evaluator reference, repeatable. Use builtin. for a built-in. "+ "Passing this replaces the defaults, so it also opts out of rubric generation.") @@ -214,6 +225,7 @@ type scaffoldInput struct { target string source string dataset string + maxTraces int evaluators []string judgeModel string rubricName string @@ -258,6 +270,7 @@ func planScaffold(in scaffoldInput) scaffold { eval.Source = &project.SourceDecl{ Type: project.SourceTypeTraces, AgentName: in.target, + MaxTraces: in.maxTraces, } } else { datasetName := in.evalName diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index c0f2549fbdf..4f714bfcf03 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -11,6 +11,7 @@ import ( "azureaieval/internal/project" "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" ) // scaffoldFor runs planScaffold against a fresh configuration, which is what @@ -71,18 +72,34 @@ func TestScaffold_AppendsToAnExistingConfiguration(t *testing.T) { } // A trace-backed eval invokes nothing, so agent_name filters instead of -// targeting, and the service default caps how many traces are read. +// targeting, and a scaffolded cap keeps the first run bounded rather than +// taking the service's default of 1000. func TestScaffold_TraceSourceHasNoTarget(t *testing.T) { plan, _ := scaffoldFor(t, scaffoldInput{ - evalName: "support-agent-trace-eval", - target: "support-agent", - source: initSourceTraces, + evalName: "support-agent-trace-eval", + target: "support-agent", + source: initSourceTraces, + maxTraces: project.DefaultScaffoldMaxTraces, }) require.Nil(t, plan.eval.Target) require.NotNil(t, plan.eval.Source) require.Equal(t, project.SourceTypeTraces, plan.eval.Source.Type) require.Equal(t, "support-agent", plan.eval.Source.AgentName) + require.Equal(t, 20, plan.eval.Source.MaxTraces) +} + +// Omitting the cap leaves the key out, which is how the service default is +// taken — writing a zero would send one. +func TestScaffold_TraceCapIsOmittedWhenZero(t *testing.T) { + plan, _ := scaffoldFor(t, scaffoldInput{ + evalName: "t", target: "a", source: initSourceTraces, + }) + require.Zero(t, plan.eval.Source.MaxTraces) + + body, err := yaml.Marshal(plan.eval) + require.NoError(t, err) + require.NotContains(t, string(body), "max_traces") } // The default set is a built-in plus a generated rubric: the built-in alone diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go index c770f26ea7b..0e0affef613 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/reconciler.go @@ -16,6 +16,7 @@ import ( "time" "azureaieval/internal/pkg/dataset_api" + "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" ) @@ -428,9 +429,10 @@ func (r *evalReconciler) EnsureEval( return group.ID, nil } - // Groups are immutable, so a change to the group's own declaration — - // evaluators, target, or options — needs a new group just as much as a - // change to an upstream artifact does. + // Evals are immutable, so a change to the eval's own substance — evaluators, + // dataset, source, target, level — needs a new eval just as much as a change + // to an upstream artifact does. Name and description are excluded from the + // digest and pushed in place instead. digest, err := project.FingerprintGroup(group) if err != nil { return "", err @@ -441,12 +443,23 @@ func (r *evalReconciler) EnsureEval( } cached := r.ec.getEnvValue(ctx, idKey("eval", group.Name)) + if cached == "" && !recreate { + // Nothing recorded under this name, but the substance may already be + // deployed under the name it had before. The environment records the id + // against the digest as well, which is what recognises a rename rather + // than reading it as a delete plus an add. + if adopted := r.adoptRenamed(ctx, group, digest); adopted != "" { + cached = adopted + } + } if cached != "" && !recreate { if _, err := r.ec.evalClient.GetOpenAIEval(ctx, cached); err == nil { - // Record the digest on reuse as well, otherwise a group deployed + // Record the digest on reuse as well, otherwise an eval deployed // before fingerprinting existed never establishes a baseline and // later edits go undetected. _ = r.ec.setEnvValue(ctx, key, digest) + _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), cached) + _ = r.ec.setEnvValue(ctx, digestIDKey(digest), cached) _ = r.ec.setEnvValue(ctx, envKeyEvalID, cached) return cached, nil } @@ -466,12 +479,42 @@ func (r *evalReconciler) EnsureEval( } _ = r.ec.setEnvValue(ctx, key, digest) _ = r.ec.setEnvValue(ctx, idKey("eval", group.Name), created.ID) - // EVAL_ID stays the last-deployed group, which is what the commands + _ = r.ec.setEnvValue(ctx, digestIDKey(digest), created.ID) + // EVAL_ID stays the last-deployed eval, which is what the commands // fall back to when a config names only one. _ = r.ec.setEnvValue(ctx, envKeyEvalID, created.ID) return created.ID, nil } +// adoptRenamed reclaims the eval this declaration used to be called, so a +// rename keeps the id and every run under it rather than forking the history. +// +// The name is what UpdateEvalParametersBody reaches, so the new one is pushed +// to the service. A failure there is not fatal: the eval is still the right one +// and the declaration still resolves, it just reads under its old name in the +// portal until the next deploy. +func (r *evalReconciler) adoptRenamed( + ctx context.Context, + group project.Eval, + digest string, +) string { + id := r.ec.getEnvValue(ctx, digestIDKey(digest)) + if id == "" { + return "" + } + remote, err := r.ec.evalClient.GetOpenAIEval(ctx, id) + if err != nil { + return "" + } + if remote.Name == group.Name { + return id + } + _, _ = r.ec.evalClient.UpdateOpenAIEval(ctx, id, &eval_api.UpdateOpenAIEvalRequest{ + Name: group.Name, + }) + return id +} + // sameDefinition reports whether the locally authored definition already // matches what the service holds. // @@ -547,3 +590,10 @@ func versionKey(kind, name string) string { func idKey(kind, name string) string { return project.FingerprintKey(kind, name) + "_ID" } + +// digestIDKey records an eval's id against its substance, which is what lets a +// renamed declaration find the eval it already deployed. Keyed by a prefix of +// the digest, because the whole hash makes an unreadable environment variable. +func digestIDKey(digest string) string { + return "EVAL_SUBSTANCE_" + strings.ToUpper(digest[:16]) + "_ID" +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 0bfe4b2514f..b0a70d6d038 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -170,8 +170,8 @@ func TestInitFlagsMatchTheSpec(t *testing.T) { }) assert.ElementsMatch(t, []string{ - "--name", "--target", "--source", "--dataset", "--evaluator", - "--judge-model", "--path", "--force", + "--name", "--target", "--source", "--dataset", "--max-traces", + "--evaluator", "--judge-model", "--path", "--force", }, got, "init's flags are a table in the spec; change both together") } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index 122828750de..aa6c4f89ebb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -278,6 +278,15 @@ type CreateOpenAIEvalRequest struct { TestingCriteria []TestingCriterion `json:"testing_criteria,omitempty"` } +// UpdateOpenAIEvalRequest is UpdateEvalParametersBody: the only fields an eval +// accepts after creation. Testing criteria and the data source are fixed at +// create time, and the service drops anything else here silently rather than +// rejecting it. +type UpdateOpenAIEvalRequest struct { + Name string `json:"name,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` +} + // OpenAIEval is the response for an OpenAI eval definition. type OpenAIEval struct { ID string `json:"id"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index f28b8e81ef5..114a012f40d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -410,6 +410,22 @@ func (c *EvalClient) DeleteOpenAIEval(ctx context.Context, evalID string) error return err } +// UpdateOpenAIEval edits an eval in place. The route is a POST on the eval +// itself, matching how this surface spells run cancel — there is no PATCH verb +// here. +// +// Only what UpdateEvalParametersBody reaches is editable: name, metadata and +// properties. Anything else the service drops silently, so substance never +// travels through this call and an edit that touches it is a new eval. +func (c *EvalClient) UpdateOpenAIEval( + ctx context.Context, + evalID string, + request *UpdateOpenAIEvalRequest, +) (*OpenAIEval, error) { + path := pathOpenAIEvals + "/" + url.PathEscape(evalID) + return doRequestTyped[OpenAIEval](c, ctx, http.MethodPost, path, nil, request, "") +} + // CreateOpenAIEvalRun starts a run for an OpenAI eval definition. func (c *EvalClient) CreateOpenAIEvalRun( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go index c9bea4324a0..8b8f1581194 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config.go @@ -78,6 +78,11 @@ const ( SourceTypeResponses = "responses" ) +// DefaultScaffoldMaxTraces is the cap init writes on a trace-backed eval, so a +// first run is bounded rather than taking the service's own default of 1000. +// Deleting max_traces from the file restores that default. +const DefaultScaffoldMaxTraces = 20 + // Target names what the run invokes. type Target struct { Type string `yaml:"type" json:"type"` @@ -214,6 +219,7 @@ func (c *EvalConfig) Validate() error { } seen := map[string]bool{} + substance := map[string]string{} for i, eval := range c.Evals { if eval.Name == "" { return fmt.Errorf("evals[%d]: 'name' is required", i) @@ -226,6 +232,22 @@ func (c *EvalConfig) Validate() error { if err := c.validateEval(i, eval); err != nil { return err } + + // Two evals that differ only by name are indistinguishable once + // deployed: the environment records an id against each eval's substance + // so a renamed declaration can find what it already deployed, and a + // shared substance makes that lookup ambiguous. + digest, err := FingerprintGroup(eval) + if err != nil { + return err + } + if first, clash := substance[digest]; clash { + return fmt.Errorf( + "evals[%d] (%s): identical to %q apart from its name and description; "+ + "give them different evaluators, datasets or settings, or declare one", + i, eval.Name, first) + } + substance[digest] = eval.Name } return nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go index b5cece4f058..cb51446d56c 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/eval_config_test.go @@ -288,6 +288,12 @@ func TestValidate_Rejects(t *testing.T) { " evaluators:\n - evaluator: builtin.relevance\n", wantErr: "evaluation_level", }, + { + name: "two evals differing only by name", + body: "evals:\n - name: a\n evaluators:\n - evaluator: builtin.relevance\n" + + " - name: b\n evaluators:\n - evaluator: builtin.relevance\n", + wantErr: "identical to", + }, } for _, tc := range cases { From 7c052ada5ee7019ea5277345d239e95884443fd4 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:25:01 +0300 Subject: [PATCH 105/114] Say which eval on one flag, and let a run read another dataset --eval already accepted a name or an id, so the --eval-id beside it was redundant. On run start it was worse than redundant: the flag variable was shadowed by the resolved id, so the value was parsed and never read. Every run command now takes one --eval. run start drops --level. The level decides the row mapping, so overriding it per run would put two incomparable result sets under one eval's history and bypass the supported_evaluation_levels check azd up does against the declared level. A second level is a second eval. run start gains --dataset, which reads a different catalog dataset for this run without touching the declaration. It needs a declared eval to override, and the name has to be in the catalog, so a typo fails before the run is submitted rather than scoring nothing. run output export defaults to csv and adds jsonl. The results are a table, and a build artifact is normally read by a spreadsheet or a diff; jsonl is for a downstream job that would rather stream than hold the run in memory. --- .../internal/cmd/resolution_test.go | 15 +-- .../azure.ai.evaluations/internal/cmd/run.go | 43 +++++--- .../internal/cmd/run_ops.go | 8 +- .../internal/cmd/run_ops_test.go | 19 ++-- .../internal/cmd/run_output.go | 100 +++++++++++------- 5 files changed, 116 insertions(+), 69 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go index ae0a60152a3..fa417ce43c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/resolution_test.go @@ -28,16 +28,17 @@ func TestResolveMaxSamples_Precedence(t *testing.T) { assert.Equal(t, 0, resolveMaxSamples(0, &project.Eval{MaxSamples: 0})) } -func TestResolveLevel_Precedence(t *testing.T) { - withOptions := &project.Eval{ +// The level is the eval's alone. A per-run override would put two incomparable +// result sets under one eval's history, and would bypass the +// supported_evaluation_levels check `azd up` does against the declared level. +func TestResolveLevel_ComesFromTheEval(t *testing.T) { + declared := &project.Eval{ EvaluationLevel: project.EvaluationLevelConversation, } - assert.Equal(t, project.EvaluationLevelTurn, resolveLevel(project.EvaluationLevelTurn, withOptions), - "the flag wins over the config") - assert.Equal(t, project.EvaluationLevelConversation, resolveLevel("", withOptions)) - assert.Empty(t, resolveLevel("", &project.Eval{}), "unset defers to the service default") - assert.Empty(t, resolveLevel("", nil)) + assert.Equal(t, project.EvaluationLevelConversation, resolveLevel(declared)) + assert.Empty(t, resolveLevel(&project.Eval{}), "unset defers to the service default") + assert.Empty(t, resolveLevel(nil)) } // A group's target decides which run-time fields its criteria can bind. Getting diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go index 1bb13de25bf..c3591d523f9 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run.go @@ -72,9 +72,8 @@ func newRunCommand() *cobra.Command { func buildRunCommand(use, short string) *cobra.Command { var ( groupName string - evalID string + datasetName string runName string - level string maxSamples int wait bool failOn string @@ -112,6 +111,24 @@ func buildRunCommand(use, short string) *cobra.Command { group := ref.Eval configPath := ref.ConfigPath + if datasetName != "" { + if !ref.Declared() { + return errors.New( + "--dataset overrides the dataset an eval declares, so it needs a " + + "declared eval; pass --eval with a name from the configuration") + } + if _, ok := ref.Config.DatasetDeclaration(datasetName); !ok { + return fmt.Errorf( + "dataset %q is not in the catalog in %s", + datasetName, filepath.ToSlash(configPath)) + } + // The eval keeps its own declaration; only this run reads elsewhere. + overridden := *group + overridden.Dataset = datasetName + overridden.Source = nil + group = &overridden + } + if ref.Declared() { if err := ec.checkDatasetRegistered(ctx, ref.Config, group, configPath); err != nil { return err @@ -139,7 +156,7 @@ func buildRunCommand(use, short string) *cobra.Command { } metadata := map[string]string{} - if lvl := resolveLevel(level, group); lvl != "" { + if lvl := resolveLevel(group); lvl != "" { metadata["evaluation_level"] = lvl } @@ -199,12 +216,11 @@ func buildRunCommand(use, short string) *cobra.Command { } cmd.Flags().StringVar(&groupName, "eval", "", - "Name of the eval to run. Defaults to the only one declared.") - cmd.Flags().StringVar(&evalID, "eval-id", "", - "Run against an existing eval by id, ignoring the config.") + "Name of the eval to run, or its id. Defaults to the only one declared.") + cmd.Flags().StringVar(&datasetName, "dataset", "", + "Catalog dataset to read instead of the one the eval declares. "+ + "Must satisfy the eval's column schema.") cmd.Flags().StringVar(&runName, "name", "", "Name for this run. Defaults to the eval name plus a timestamp.") - cmd.Flags().StringVar(&level, "level", "", - "Scoring granularity: turn or conversation. Defaults to the service default (turn).") cmd.Flags().IntVar(&maxSamples, "max-samples", 0, "Cap the rows sent from the dataset.") cmd.Flags().BoolVar(&wait, "wait", true, "Block until the run reaches a terminal state.") @@ -557,10 +573,13 @@ func scanJSONL(r io.Reader, limit int) ([]map[string]any, error) { } // resolveLevel prefers the flag, then the eval's own declaration. -func resolveLevel(flag string, group *project.Eval) string { - if flag != "" { - return flag - } +// resolveLevel is the eval's declared scoring granularity. +// +// There is no per-run override: the level decides the row mapping, so two +// levels under one eval would put incomparable result sets in the same history, +// and it would bypass the supported_evaluation_levels check `azd up` does +// against the declared level. A second level is a second eval. +func resolveLevel(group *project.Eval) string { if group != nil { return group.EvaluationLevel } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go index deb0d0a49e5..cea3b01bb4d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops.go @@ -79,7 +79,7 @@ func newRunListCommand() *cobra.Command { []string{"RUN ID", "NAME", "STATUS", "RESULTS"}, rows) }, } - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().IntVar(&limit, "limit", 0, "Return at most this many runs. Omit for the service default.") cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") @@ -173,7 +173,7 @@ func newRunShowCommand() *cobra.Command { cmd.Flags().BoolVar(&wait, "wait", false, "Block until the run reaches a terminal state before reporting.") addFailOnFlag(cmd, &failOn) - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -236,7 +236,7 @@ func newRunCancelCommand() *cobra.Command { return nil }, } - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -287,7 +287,7 @@ func newRunDeleteCommand() *cobra.Command { return nil }, } - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go index 1a5bd5ac978..8259bfd6b35 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_ops_test.go @@ -57,16 +57,20 @@ func TestRunStartMirrorsCompositeFlags(t *testing.T) { } require.NotNil(t, start) - for _, flag := range []string{"eval-id", "eval", "name", "level", "max-samples", "wait", "no-wait"} { + for _, flag := range []string{"eval", "dataset", "name", "max-samples", "wait", "no-wait"} { require.NotNil(t, start.Flags().Lookup(flag), "run start should accept --%s", flag) } + + // The level decides the row mapping, so a per-run override would put two + // incomparable result sets under one eval. A second level is a second eval. + require.Nil(t, start.Flags().Lookup("level"), "run start must not offer --level") } -// Every command that acts on an eval takes the id the same two ways. -// `run start --eval-id` is the form the CI example uses, and `run list` used to -// reject that flag and accept only a positional, so a script that worked for -// one sibling failed on the next. -func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { +// Every command that acts on an eval says which one the same way. One flag +// takes a name from the configuration or a raw service id: an eval created +// outside a project has no declaration to name, and a second --eval-id beside +// it was accepted and silently ignored. +func TestEvalCommandsTakeOneEvalFlag(t *testing.T) { subs := map[string]*cobra.Command{} for _, sub := range newRunCommand().Commands() { subs["run "+sub.Name()] = sub @@ -83,8 +87,9 @@ func TestEvalCommandsAcceptIDAsAFlag(t *testing.T) { } { cmd := subs[name] require.NotNil(t, cmd, "%s should exist", name) - require.NotNil(t, cmd.Flags().Lookup("eval-id"), "%s should accept --eval-id", name) require.NotNil(t, cmd.Flags().Lookup("eval"), "%s should accept --eval", name) + require.Nil(t, cmd.Flags().Lookup("eval-id"), + "%s must not keep --eval-id beside --eval", name) } } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 7dacd49abb7..3c6c0e36ad7 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -5,6 +5,7 @@ package cmd import ( "encoding/csv" + "encoding/json" "fmt" "io" "os" @@ -12,6 +13,7 @@ import ( "strings" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/spf13/cobra" ) @@ -111,7 +113,7 @@ func newRunOutputListCommand() *cobra.Command { cmd.Flags().BoolVar(&failedOnly, "failed-only", false, "Show only the rows that failed.") cmd.Flags().StringVar(&outFile, "output-file", "", "Write JSON results to this path.") - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -166,7 +168,7 @@ func newRunOutputShowCommand() *cobra.Command { } cmd.Flags().StringVar(&runID, "run", "", "Run the item belongs to. Defaults to the most recent run.") - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } @@ -217,26 +219,36 @@ func newRunOutputExportCommand() *cobra.Command { w = f } - if format == "json" { + switch format { + case formatCSV: + return writeResultsCSV(w, run) + case formatJSON: return emitJSON(w, run) + case formatJSONL: + return writeResultsJSONL(w, run) + default: + return fmt.Errorf( + "--format %q is not supported; use %s, %s or %s", + format, formatCSV, formatJSON, formatJSONL) } - return writeResultsCSV(w, run) }, } - cmd.Flags().StringVar(&format, "format", "json", "Output format: json or csv.") + cmd.Flags().StringVar(&format, "format", formatCSV, + fmt.Sprintf("Output format: %s, %s or %s.", formatCSV, formatJSON, formatJSONL)) cmd.Flags().StringVar(&outFile, "output-file", "", "Write to this path instead of stdout.") - addEvalFlags(cmd, &groupName) + addEvalFlag(cmd, &groupName) cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") return cmd } -// resolveEvalID takes the eval id from the argument, from a group named -// with --eval, or from the id cached in the azd environment. +// resolveEvalID takes the eval id from the argument, from --eval, or from the +// id cached in the azd environment. // -// The cached id is the last group deployed, which is unambiguous only while a -// config declares one. --eval is how the others are reached without -// having to know their service ids. +// --eval accepts a name or a raw id on the one flag: an eval created outside a +// project has no declaration to name, and the environment records one id per +// name, so editing a declaration leaves every run of the previous eval +// reachable only by id. func resolveEvalID( cmd *cobra.Command, ec *evalContext, @@ -247,46 +259,27 @@ func resolveEvalID( return args[0], nil } - if flag, err := cmd.Flags().GetString("eval-id"); err == nil && flag != "" { - return flag, nil - } - if groupName != "" { - if id := ec.getEnvValue(cmd.Context(), idKey("eval", groupName)); id != "" { - return id, nil + ref, err := ec.resolveEvalRef(cmd.Context(), project.DefaultEvalDir, groupName) + if err != nil { + return "", err } - return "", fmt.Errorf( - "eval %q has no id recorded in this environment; deploy it first, "+ - "or pass its id directly", groupName) + return ref.ID, nil } if cached := ec.getEnvValue(cmd.Context(), envKeyEvalID); cached != "" { return cached, nil } return "", fmt.Errorf( - "no eval id given; pass it as an argument, name one with --eval, "+ - "or set %s in the azd environment", - envKeyEvalID) + "no eval given; pass its id as an argument, or name one with --eval") } -// addEvalFlag registers the flag that names an eval from the config, for -// commands that never take a raw service id. +// addEvalFlag registers the flag that says which eval a command acts on. It +// takes a name from the configuration or a raw service id, which is why there +// is no second --eval-id beside it. func addEvalFlag(cmd *cobra.Command, target *string) { cmd.Flags().StringVar(target, "eval", "", - "Name of the eval declared in azure.yaml.") -} - -// addEvalFlags registers the two ways to say which eval a command acts -// on: --eval names one from the config, --eval-id gives its service id. -// -// The id is also accepted as a positional argument. The flag exists because -// `run start --eval-id` already spells it that way, and a script that learned -// it there should not have to find out that the sibling commands take only a -// positional. -func addEvalFlags(cmd *cobra.Command, target *string) { - addEvalFlag(cmd, target) - cmd.Flags().String("eval-id", "", - "Id of the eval. Same as passing the id as an argument.") + "Name of the eval declared in the configuration, or its id.") } // latestOrNamedRun returns the named run, or the most recent one for the eval. @@ -453,3 +446,32 @@ func writeResultsCSV(w io.Writer, run *eval_api.OpenAIEvalRun) error { } return nil } + +// Export formats. csv is the default because the results are a table and a +// build artifact is normally read by a spreadsheet or a diff. +const ( + formatCSV = "csv" + formatJSON = "json" + formatJSONL = "jsonl" +) + +// writeResultsJSONL emits one criterion per line, which is what a downstream +// job can stream without holding the whole run in memory. +func writeResultsJSONL(w io.Writer, run *eval_api.OpenAIEvalRun) error { + enc := json.NewEncoder(w) + if len(run.PerTestingCriteria) == 0 { + return enc.Encode(map[string]any{"run_id": run.ID, "status": run.Status}) + } + for _, cr := range run.PerTestingCriteria { + if err := enc.Encode(map[string]any{ + "run_id": run.ID, + "status": run.Status, + "testing_criteria": cr.TestingCriteria, + "passed": cr.Passed, + "failed": cr.Failed, + }); err != nil { + return err + } + } + return nil +} From 7da5c1b2ad70406935e9708f05a3659d8bd4b615 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:41:10 +0300 Subject: [PATCH 106/114] Nest generation jobs under what produced them, and add eval create The job group was top level and every command under it tried both generation resources, reporting whichever answered. The two share no collection, so that meant guessing the endpoint from an id prefix that is not a documented contract. Each resource now owns its own group -- dataset job and evaluator job -- and a lookup that misses names the sibling group, because the ids look alike and reaching for the wrong one is the likely mistake. Both groups gain delete. The job record is what is discarded; the artifact it produced is already registered as its own version and is untouched. eval create makes one declared eval without deploying the rest, for a project that is not deployed as a whole or, with --from-file, for no project at all. It runs the same reconcile path azd up uses rather than a second creation route, so there is never a second definition to maintain. The reattach hints printed by --no-wait now name the group that owns the job. A test that scans the source for suggested commands caught them still pointing at the old top-level path. --- .../internal/cmd/dataset.go | 1 + .../internal/cmd/eval_group.go | 84 ++++++- .../internal/cmd/evaluator.go | 1 + .../internal/cmd/generate.go | 11 +- .../azure.ai.evaluations/internal/cmd/job.go | 208 +++++++++++------- .../azure.ai.evaluations/internal/cmd/root.go | 2 +- .../internal/cmd/surface_test.go | 15 +- .../internal/pkg/eval_api/operations.go | 29 +++ 8 files changed, 260 insertions(+), 91 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go index f4ac919306b..05a42c354e1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/dataset.go @@ -33,6 +33,7 @@ func newDatasetCommand() *cobra.Command { newDatasetShowCommand(), newDatasetDeleteCommand(), newDatasetVersionsCommand(), + newJobCommand(datasetJobs), ) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go index 8f7566c00cf..3082153d4b4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/eval_group.go @@ -5,16 +5,92 @@ package cmd import ( "fmt" + "path/filepath" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/spf13/cobra" ) -// The eval group is read and delete only. Creation belongs to `azd up`, which -// owns reconciliation: a second creation path would drift from the declared -// config, and reconciliation could not then tell whether to adopt an eval it -// found or replace it. +// Creation normally belongs to `azd up`, which owns reconciliation. `create` +// is the same path for a single eval outside a project, and takes the +// configuration rather than a wall of flags so there is never a second +// definition to maintain. + +// newEvalCreateCommand creates one declared eval without deploying the rest. +func newEvalCreateCommand() *cobra.Command { + var ( + fromFile string + evalDir string + endpointFlg string + ) + + cmd := &cobra.Command{ + Use: "create [name]", + Short: "Create one eval declared in the configuration.", + Long: "Create one eval declared in the configuration.\n\n" + + "`azd up` reconciles every eval in the file. This creates a single one, " + + "for a project that is not deployed as a whole — or, with --from-file, " + + "for no project at all.\n\n" + + "The name is optional while the configuration declares exactly one eval.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + path := fromFile + if path == "" { + path = project.EvalConfigPath(evalDir) + } + cfg, err := project.LoadEvalConfig(path) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + eval, err := cfg.Eval(firstArg(args)) + if err != nil { + return err + } + + ec, err := newEvalContext(ctx, endpointFlg) + if err != nil { + return err + } + defer ec.Close() + + // Local sources resolve against the file, not the working directory, + // so the columns are read from where the declaration points. + datasetPath := "" + if decl, ok := cfg.DatasetDeclaration(eval.Dataset); ok && decl.Source != "" { + datasetPath = filepath.Join(filepath.Dir(path), decl.Source) + } + + reconciler := &evalReconciler{ec: ec} + id, err := reconciler.EnsureEval(ctx, *eval, datasetPath, false) + if err != nil { + return err + } + + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": id, "name": eval.Name, + }) + } + fmt.Fprintf(cmd.OutOrStdout(), "%s Created eval: %s (%s)\n", doneMark, eval.Name, id) + return nil + }, + } + + cmd.Flags().StringVar(&fromFile, "from-file", "", + "Read the configuration from this path instead of the eval directory.") + cmd.Flags().StringVar(&evalDir, "path", project.DefaultEvalDir, + "Directory holding the evaluation configuration.") + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} func newEvalListCommand() *cobra.Command { var ( diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go index c5f86a19d15..48aeb48a477 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/evaluator.go @@ -26,6 +26,7 @@ func newEvaluatorCommand() *cobra.Command { newEvaluatorShowCommand(), newEvaluatorDeleteCommand(), newEvaluatorVersionsCommand(), + newJobCommand(evaluatorJobs), ) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 4dd77505f0d..0c27d6702da 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -164,7 +164,7 @@ func (ec *evalContext) generateRubric( return nil, fmt.Errorf("submitting the rubric generation job: %w", err) } if noWait { - reportSubmitted(out, job.ID) + reportSubmitted(out, "azd ai eval evaluator", job.ID) return nil, nil } @@ -187,10 +187,11 @@ func (ec *evalContext) generateRubric( // // The job id goes into the command rather than being left as a placeholder: // --no-wait exists so the caller can walk away, and the line they walk away -// with has to be the one they can paste when they come back. -func reportSubmitted(out io.Writer, jobID string) { +// with has to be the one they can paste when they come back. The group is named +// too, because the two job types share no collection. +func reportSubmitted(out io.Writer, group, jobID string) { fmt.Fprintf(out, " submitted job %s\n", jobID) - fmt.Fprintf(out, "\nReattach with: azd ai eval job show %s\n", jobID) + fmt.Fprintf(out, "\nReattach with: %s job show %s\n", group, jobID) } // generateDataset submits the data generation job and downloads the result. @@ -212,7 +213,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("submitting the data generation job: %w", err) } if noWait { - reportSubmitted(out, job.ID) + reportSubmitted(out, "azd ai dataset", job.ID) return nil, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go index 1c094e20c77..c4a9d48aa23 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/job.go @@ -13,37 +13,89 @@ import ( ) // Generation runs as two independent long-running resources — one for datasets, -// one for evaluators — and a job id does not say which it came from. Rather -// than make the caller remember, every command here tries both. +// one for evaluators — sharing no collection. A job group therefore nests under +// the resource that produced it: a top-level `job show ` would have to guess +// the endpoint from an id prefix that is not a documented contract. const ( jobKindDataset = "dataset" jobKindEvaluator = "evaluator" ) -func newJobCommand() *cobra.Command { +// jobKind binds a group to one generation resource, so every command under it +// calls one endpoint rather than trying both and reporting whichever answered. +type jobKind struct { + name string + list func(context.Context, *evalContext) ([]eval_api.GenerationJob, error) + get func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + cancel func(context.Context, *evalContext, string) (*eval_api.GenerationJob, error) + remove func(context.Context, *evalContext, string) error +} + +var datasetJobs = jobKind{ + name: jobKindDataset, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteDataGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, +} + +var evaluatorJobs = jobKind{ + name: jobKindEvaluator, + list: func(ctx context.Context, ec *evalContext) ([]eval_api.GenerationJob, error) { + out, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + if err != nil { + return nil, err + } + return out.Data, nil + }, + get: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.GetEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + cancel: func(ctx context.Context, ec *evalContext, id string) (*eval_api.GenerationJob, error) { + return ec.evalClient.CancelEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, + remove: func(ctx context.Context, ec *evalContext, id string) error { + return ec.evalClient.DeleteEvaluatorGenerationJob(ctx, id, ProjectEndpointAPIVersion) + }, +} + +func newJobCommand(kind jobKind) *cobra.Command { cmd := &cobra.Command{ Use: "job", - Short: "Inspect and cancel generation jobs.", - Long: "Inspect and cancel generation jobs.\n\n" + - "This is the resume path for `dataset generate` and `evaluator generate`: " + - "a job started with --no-wait, or one whose client was interrupted, is " + - "reattached to here rather than restarted.", + Short: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.", kind.name), + Long: fmt.Sprintf("Inspect, cancel and delete %s generation jobs.\n\n", kind.name) + + fmt.Sprintf("This is the resume path for `%s generate`: a job started with ", kind.name) + + "--no-wait, or one whose client was interrupted, is reattached to here " + + "rather than restarted.", } cmd.AddCommand( - newJobListCommand(), - newJobShowCommand(), - newJobCancelCommand(), + newJobListCommand(kind), + newJobShowCommand(kind), + newJobCancelCommand(kind), + newJobDeleteCommand(kind), ) return cmd } -func newJobListCommand() *cobra.Command { +func newJobListCommand(kind jobKind) *cobra.Command { var endpointFlg string cmd := &cobra.Command{ Use: "list", - Short: "List the project's generation jobs.", + Short: fmt.Sprintf("List the project's %s generation jobs.", kind.name), Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -53,40 +105,23 @@ func newJobListCommand() *cobra.Command { } defer ec.Close() - datasets, err := ec.evalClient.ListDataGenerationJobs(ctx, ProjectEndpointAPIVersion) - if err != nil { - return fmt.Errorf("listing dataset generation jobs: %w", err) - } - evaluators, err := ec.evalClient.ListEvaluatorGenerationJobs(ctx, ProjectEndpointAPIVersion) + jobs, err := kind.list(ctx, ec) if err != nil { - return fmt.Errorf("listing evaluator generation jobs: %w", err) - } - - type jobRow struct { - ID string `json:"id"` - Kind string `json:"kind"` - Status string `json:"status"` - } - rows := make([]jobRow, 0, len(datasets.Data)+len(evaluators.Data)) - for _, j := range datasets.Data { - rows = append(rows, jobRow{ID: j.ID, Kind: jobKindDataset, Status: j.Status}) - } - for _, j := range evaluators.Data { - rows = append(rows, jobRow{ID: j.ID, Kind: jobKindEvaluator, Status: j.Status}) + return fmt.Errorf("listing %s generation jobs: %w", kind.name, err) } if isJSON(cmd) { - return emitJSONList(cmd.OutOrStdout(), rows) + return emitJSONList(cmd.OutOrStdout(), jobs) } - if len(rows) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No generation jobs found.") + if len(jobs) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "No %s generation jobs found.\n", kind.name) return nil } - table := make([][]string, 0, len(rows)) - for _, r := range rows { - table = append(table, []string{r.ID, r.Kind, r.Status}) + table := make([][]string, 0, len(jobs)) + for _, j := range jobs { + table = append(table, []string{j.ID, j.Status}) } - return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "KIND", "STATUS"}, table) + return emitTable(cmd.OutOrStdout(), []string{"JOB ID", "STATUS"}, table) }, } @@ -94,12 +129,12 @@ func newJobListCommand() *cobra.Command { return cmd } -func newJobShowCommand() *cobra.Command { +func newJobShowCommand(kind jobKind) *cobra.Command { var endpointFlg string cmd := &cobra.Command{ Use: "show ", - Short: "Show a generation job.", + Short: fmt.Sprintf("Show a %s generation job.", kind.name), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] @@ -111,9 +146,9 @@ func newJobShowCommand() *cobra.Command { } defer ec.Close() - job, _, err := findGenerationJob(ctx, ec, jobID) + job, err := kind.get(ctx, ec, jobID) if err != nil { - return err + return jobLookupError(kind, jobID, err) } if isJSON(cmd) { @@ -131,12 +166,12 @@ func newJobShowCommand() *cobra.Command { return cmd } -func newJobCancelCommand() *cobra.Command { +func newJobCancelCommand(kind jobKind) *cobra.Command { var endpointFlg string cmd := &cobra.Command{ Use: "cancel ", - Short: "Cancel an in-flight generation job.", + Short: fmt.Sprintf("Cancel an in-flight %s generation job.", kind.name), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { jobID := args[0] @@ -148,28 +183,54 @@ func newJobCancelCommand() *cobra.Command { } defer ec.Close() - _, kind, err := findGenerationJob(ctx, ec, jobID) + canceled, err := kind.cancel(ctx, ec, jobID) if err != nil { - return err + return jobLookupError(kind, jobID, err) } - var canceled *eval_api.GenerationJob - if kind == jobKindDataset { - canceled, err = ec.evalClient.CancelDataGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion) - } else { - canceled, err = ec.evalClient.CancelEvaluatorGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion) + if isJSON(cmd) { + return emitJSON(cmd.OutOrStdout(), canceled) } + fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", + kind.name, jobID, canceled.Status) + return nil + }, + } + + cmd.Flags().StringVar(&endpointFlg, "project-endpoint", "", "Foundry project endpoint.") + return cmd +} + +func newJobDeleteCommand(kind jobKind) *cobra.Command { + var endpointFlg string + + cmd := &cobra.Command{ + Use: "delete ", + Short: fmt.Sprintf("Delete a %s generation job record.", kind.name), + Long: fmt.Sprintf("Delete a %s generation job record.\n\n", kind.name) + + "The artifact the job produced is already registered as its own version " + + "and is not affected.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + jobID := args[0] + + ctx := cmd.Context() + ec, err := newEvalContext(ctx, endpointFlg) if err != nil { - return fmt.Errorf("cancelling job %s: %w", jobID, err) + return err + } + defer ec.Close() + + if err := kind.remove(ctx, ec, jobID); err != nil { + return jobLookupError(kind, jobID, err) } if isJSON(cmd) { - return emitJSON(cmd.OutOrStdout(), canceled) + return emitJSON(cmd.OutOrStdout(), map[string]string{ + "id": jobID, "kind": kind.name, "status": "deleted", + }) } - fmt.Fprintf(cmd.OutOrStdout(), "Cancelled %s generation job %s (%s)\n", - kind, jobID, canceled.Status) + fmt.Fprintf(cmd.OutOrStdout(), "Deleted %s generation job %s\n", kind.name, jobID) return nil }, } @@ -178,24 +239,17 @@ func newJobCancelCommand() *cobra.Command { return cmd } -// findGenerationJob resolves an id against both job types and reports which one -// answered, so that a caller never has to know which command started it. -func findGenerationJob( - ctx context.Context, - ec *evalContext, - jobID string, -) (*eval_api.GenerationJob, string, error) { - if job, err := ec.evalClient.GetDataGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion, - ); err == nil { - return job, jobKindDataset, nil - } - if job, err := ec.evalClient.GetEvaluatorGenerationJob( - ctx, jobID, ProjectEndpointAPIVersion, - ); err == nil { - return job, jobKindEvaluator, nil +// jobLookupError names the sibling group, because the two job types share an id +// shape and reaching for the wrong one is the likely mistake. +func jobLookupError(kind jobKind, jobID string, err error) error { + if eval_api.IsNotFound(err) { + other := jobKindEvaluator + if kind.name == jobKindEvaluator { + other = jobKindDataset + } + return fmt.Errorf( + "no %s generation job %q in this project; if it generated a %s, "+ + "use the %s job group instead", kind.name, jobID, other, other) } - return nil, "", fmt.Errorf( - "no generation job %s in this project; "+ - "`azd ai eval job list` shows the ones there are", jobID) + return fmt.Errorf("reading %s generation job %s: %w", kind.name, jobID, err) } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go index 5be5fcc9e46..4f6fcd8860e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/root.go @@ -49,7 +49,7 @@ func NewRootCommand() *cobra.Command { newDatasetCommand(), newRunCommand(), newEvaluatorCommand(), - newJobCommand(), + newEvalCreateCommand(), newEvalListCommand(), newEvalShowCommand(), newEvalDeleteCommand(), diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index b0a70d6d038..2d266ead1e6 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -54,6 +54,12 @@ func TestCommandTreeMatchesTheSpec(t *testing.T) { "dataset update", "dataset versions", "dataset versions list", + "dataset job", + "dataset job cancel", + "dataset job delete", + "dataset job list", + "dataset job show", + "create", "delete", "evaluator", "evaluator create", @@ -64,11 +70,12 @@ func TestCommandTreeMatchesTheSpec(t *testing.T) { "evaluator update", "evaluator versions", "evaluator versions list", + "evaluator job", + "evaluator job cancel", + "evaluator job delete", + "evaluator job list", + "evaluator job show", "init", - "job", - "job cancel", - "job list", - "job show", "list", "run", "run cancel", diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go index 114a012f40d..0b18f6556a2 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/operations.go @@ -172,6 +172,35 @@ func (c *EvalClient) cancelGenerationJob( c, ctx, http.MethodPost, path, nil, json.RawMessage(`{}`), apiVersion) } +// DeleteDataGenerationJob removes a dataset generation job record. +func (c *EvalClient) DeleteDataGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathDataGenerationJobs, operationID, apiVersion) +} + +// DeleteEvaluatorGenerationJob removes an evaluator generation job record. +func (c *EvalClient) DeleteEvaluatorGenerationJob( + ctx context.Context, + operationID string, + apiVersion string, +) error { + return c.deleteGenerationJob(ctx, pathEvaluatorGenerationJobs, operationID, apiVersion) +} + +// deleteGenerationJob discards the job record. The artifact the job produced is +// already registered and is not affected. +func (c *EvalClient) deleteGenerationJob( + ctx context.Context, + basePath, operationID, apiVersion string, +) error { + path := basePath + "/" + url.PathEscape(operationID) + _, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil, apiVersion) + return err +} + // GetAgent reads an agent from the project's catalog. // // Only the newest version is returned, which is the one generation is seeded From b8d40277f04e45a445598f19964287f7d04466a8 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 02:59:54 +0300 Subject: [PATCH 107/114] Send the sources --from named, and pick a default that works --from was parsed, validated, and then dropped: the value reached generationPlan and nothing read it, so every dataset generate sent the same request whatever was asked for. Wiring it up turned up three more things. It was a single string, but the spec has it selecting one or more of the sources, so it is repeatable now. Asking for traces sent no traces. The builder emitted a traces source only when a day window came with it, and dataset generate has no --trace-days, so the documented --from traces produced an empty request. A window narrows the query rather than authorizing it; without one the source now goes out unbounded. Asking for something unbuildable sent a job anyway. --from prompt with no instruction, or --from file, which the service takes through dataset upload rather than as a generation source, quietly submitted whatever was left. The job is billed and what comes back looks the same either way, so the caller had no way to tell. Those now stop the command and name the flag that fixes it. The default is the spec's: traces when the project has Application Insights connected, otherwise the agent. The connection string is how a project says it collects traces at all, and the agents extension already reads that key. --from agent carries the agent's instructions as a prompt, because that is what the spec says it generates from, and because the agent source alone fails server-side for every agent -- the prompt is what the existing retry falls back to. Without it the default in a project with no Application Insights would have been a request that always fails. --- .../internal/cmd/context.go | 24 +++ .../internal/cmd/generate.go | 47 ++++- .../internal/cmd/generate_commands.go | 32 ++- .../internal/cmd/generate_sources_test.go | 59 ++++++ .../internal/cmd/surface_test.go | 32 +++ .../internal/pkg/eval_api/generation.go | 107 +++++++--- .../internal/pkg/eval_api/generation_test.go | 190 ++++++++++++++++++ 7 files changed, 447 insertions(+), 44 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go index 0b987db0315..1c897a0f906 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/context.go @@ -12,6 +12,7 @@ import ( "azureaieval/internal/pkg/dataset_api" "azureaieval/internal/pkg/eval_api" + "azureaieval/internal/project" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -145,6 +146,29 @@ func (ec *evalContext) getEnvValue(ctx context.Context, key string) string { return val.Value } +// appInsightsEnvKey is where a connected Application Insights resource lands in +// the azd environment. azd's own provisioning writes it, and the agents +// extension reads the same key to pass tracing configuration to a running +// agent, so its presence is the project's answer to "are traces being +// collected?". +const appInsightsEnvKey = "APPLICATIONINSIGHTS_CONNECTION_STRING" + +// defaultGenerationSource picks what `dataset generate` sends when --from was +// not given, from the Application Insights connection string the project has +// (or has not) been given. +// +// Traces are the better dataset when they exist, being real conversations +// rather than synthesized ones, so they win whenever the project is wired to +// collect them. Outside a project, or in one with no Application Insights, +// there are no traces to ask for and the agent's own definition is all that is +// left. +func defaultGenerationSource(appInsightsConnection string) []string { + if appInsightsConnection != "" { + return []string{project.GenerateFromTraces} + } + return []string{project.GenerateFromAgent} +} + func (ec *evalContext) Close() { if ec.azdClient != nil { ec.azdClient.Close() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 0c27d6702da..3091cbd2f27 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -44,9 +44,9 @@ type generationPlan struct { OutputDir string // SampleSize applies to dataset generation only. SampleSize int - // From selects which source the rows are generated out of. Empty lets the - // project decide: traces when Application Insights is connected. - From string + // From is what --from named: which of the service's sources to send. Empty + // sends whatever the plan has to offer. + From []string // TraceDays seeds generation from that many days of recent traces. TraceDays int } @@ -154,9 +154,12 @@ func (ec *evalContext) generateRubric( ) (*project.ArtifactRef, error) { fmt.Fprintf(out, "Generating rubric %s...\n", plan.Name) - sources := eval_api.BuildGenerationSources( - "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), ) + if err := refuseUnbuildableSources(unbuildable); err != nil { + return nil, err + } req := eval_api.NewEvaluatorGenerationJobRequest(plan.Name, plan.Model, sources) job, err := ec.evalClient.CreateEvaluatorGenerationJob(ctx, req, ProjectEndpointAPIVersion) @@ -183,6 +186,33 @@ func (ec *evalContext) generateRubric( return &project.ArtifactRef{Name: plan.Name, Source: relativeSource(plan.BaseDir, path)}, nil } +// refuseUnbuildableSources reports a --from the plan could not honour. +// +// Submitting anyway would run a billed job seeded from less than was asked for +// and return a plausible-looking artifact, which is the worst outcome: the +// caller has no way to tell it apart from one built the way they intended. +func refuseUnbuildableSources(kinds []string) error { + if len(kinds) == 0 { + return nil + } + reasons := map[string]string{ + "prompt": "--from prompt needs --agent-instruction or --agent-instruction-file", + "agent": "--from agent needs a target agent; pass --target, " + + "or declare one under target: in eval.yaml", + "file": "--from file is not a generation source; " + + "register the file with `azd ai dataset create --file ` instead", + } + messages := make([]string, 0, len(kinds)) + for _, k := range kinds { + if reason, ok := reasons[k]; ok { + messages = append(messages, reason) + continue + } + messages = append(messages, fmt.Sprintf("--from %s cannot be built from this plan", k)) + } + return errors.New(strings.Join(messages, "; ")) +} + // reportSubmitted says what was started and how to get back to it. // // The job id goes into the command rather than being left as a placeholder: @@ -203,9 +233,12 @@ func (ec *evalContext) generateDataset( ) (*project.ArtifactRef, error) { fmt.Fprintf(out, "Generating dataset %s (%d samples)...\n", plan.Name, plan.SampleSize) - sources := eval_api.BuildGenerationSources( - "agent", plan.Agent, "", plan.Instruction, plan.traceOptions(), + sources, unbuildable := eval_api.BuildGenerationSources( + plan.From, plan.Agent, "", plan.Instruction, plan.traceOptions(), ) + if err := refuseUnbuildableSources(unbuildable); err != nil { + return nil, err + } req := eval_api.NewDataGenerationJobRequest(plan.Name, plan.Model, plan.SampleSize, sources) job, err := ec.evalClient.CreateDataGenerationJob(ctx, req, DataGenerationAPIVersion) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go index b5f16b1c600..a401d9db1fb 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_commands.go @@ -153,23 +153,25 @@ func newDatasetGenerateCommand() *cobra.Command { var ( flags generateFlags maxSamples int - from string + from []string ) cmd := &cobra.Command{ Use: "generate ", Short: "Generate a dataset and download it.", Long: "Generate a dataset and download it.\n\n" + - "--from selects one of the four sources the service accepts. " + - "Generating from the agent's own definition is a preference rather " + - "than a fallback: it covers cases no user has hit yet, and it can " + - "supply reference answers, which a transcript cannot.", + "--from selects one or more of the sources the service accepts, and " + + "is repeatable. Generating from the agent's own definition is a " + + "preference rather than a fallback: it covers cases no user has hit " + + "yet, and it can supply reference answers, which a transcript cannot.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { name := args[0] - if err := project.ValidateGenerateSource(from); err != nil { - return err + for _, src := range from { + if err := project.ValidateGenerateSource(src); err != nil { + return err + } } if err := project.ValidateSampleSize(maxSamples); err != nil { return err @@ -197,6 +199,12 @@ func newDatasetGenerateCommand() *cobra.Command { } defer ec.Close() + if len(plan.From) == 0 { + plan.From = defaultGenerationSource( + ec.getEnvValue(cmd.Context(), appInsightsEnvKey), + ) + } + ref, err := ec.generateDataset(cmd.Context(), plan, cmd.OutOrStdout(), flags.noWait) if err != nil { return err @@ -211,10 +219,12 @@ func newDatasetGenerateCommand() *cobra.Command { cmd.Flags().IntVar(&maxSamples, "max-samples", 0, fmt.Sprintf("Rows to synthesize (%d-%d). Defaults to %d.", project.MinSampleSize, project.MaxSampleSize, project.DefaultSampleSize)) - cmd.Flags().StringVar(&from, "from", "", - fmt.Sprintf("Where rows come from: %s. Defaults to traces when the project "+ - "has Application Insights connected, otherwise agent.", - strings.Join(project.GenerateSources, ", "))) + cmd.Flags().StringSliceVar(&from, "from", nil, + fmt.Sprintf("Where rows come from: %s. Repeatable, and the service accepts "+ + "more than one. Defaults to %s when the project has Application Insights "+ + "connected, otherwise %s.", + strings.Join(project.GenerateSources, ", "), + project.GenerateFromTraces, project.GenerateFromAgent)) addGenerateFlags(cmd, &flags) return cmd } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go new file mode 100644 index 00000000000..4165df62754 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The spec's default: traces when the project has Application Insights +// connected, otherwise the agent. The connection string is how a project says +// it collects traces at all, so asking for traces without one would submit a +// billed job against nothing. +func TestDefaultGenerationSource(t *testing.T) { + assert.Equal(t, []string{"traces"}, + defaultGenerationSource("InstrumentationKey=00000000-0000-0000-0000-000000000000"), + "a project collecting traces should be generated from them") + + assert.Equal(t, []string{"agent"}, defaultGenerationSource(""), + "with nowhere for traces to have been collected, the agent is all there is") +} + +// --from is a request, and one the plan cannot honour has to stop the command +// rather than quietly submit a job seeded from less than was asked for. +func TestRefuseUnbuildableSources(t *testing.T) { + assert.NoError(t, refuseUnbuildableSources(nil)) + assert.NoError(t, refuseUnbuildableSources([]string{})) + + tests := []struct { + kind string + says string + }{ + {"prompt", "--agent-instruction"}, + {"agent", "--target"}, + {"file", "azd ai dataset create"}, + } + + for _, tt := range tests { + t.Run(tt.kind, func(t *testing.T) { + err := refuseUnbuildableSources([]string{tt.kind}) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.says, + "the error has to name the way out, not just the problem") + }) + } +} + +// Two unhonoured sources are two things the caller has to fix, so both are +// reported at once rather than one per attempt. +func TestRefuseUnbuildableSources_ReportsAllOfThemAtOnce(t *testing.T) { + err := refuseUnbuildableSources([]string{"prompt", "agent"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "--agent-instruction") + assert.Contains(t, err.Error(), "--target") +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 2d266ead1e6..1d6b8e969c3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -11,6 +11,8 @@ import ( "strings" "testing" + "azureaieval/internal/project" + "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" @@ -209,6 +211,36 @@ func TestServiceCommandsTakeProjectEndpoint(t *testing.T) { }) } +// The spec says --from "selects one or more of the four sources", so it has to +// be repeatable. Declared as a plain string it would still accept every +// documented single-source invocation and silently keep only the last of a +// repeated one, which is the kind of difference no example in the spec shows. +func TestDatasetGenerateFromTakesMoreThanOneSource(t *testing.T) { + flag := find(t, "dataset generate").Flags().Lookup("from") + require.NotNil(t, flag, "dataset generate must offer --from") + + assert.Equal(t, "stringSlice", flag.Value.Type(), + "--from selects one or more sources, so it cannot be a single string") +} + +// `--from` names sources; the set it accepts is the set the service has a path +// for, and the help has to list exactly that set. +func TestDatasetGenerateFromListsEverySource(t *testing.T) { + usage := find(t, "dataset generate").Flags().Lookup("from").Usage + + for _, source := range project.GenerateSources { + assert.Containsf(t, usage, source, + "--from accepts %q, so its help has to say so", source) + } +} + +// `--from` is the only place a source is named, so `evaluator generate`, which +// has no such flag, must not be left half-wired to one. +func TestEvaluatorGenerateHasNoFromFlag(t *testing.T) { + assert.Nil(t, find(t, "evaluator generate").Flags().Lookup("from"), + "the spec gives --from to dataset generate only") +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go index 26727677270..235f58884a0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation.go @@ -20,10 +20,6 @@ type TraceOptions struct { Days int } -// BuildGenerationSources constructs the sources array for generation jobs. -// A prompt source is included when instruction is non-empty, along with the -// agent source. When traces is non-nil and Days > 0, a traces source is -// appended with start_time computed from the current time. // WithoutAgentSource returns the sources with the agent entry removed. // // Agent-seeded data generation currently fails server-side for every agent, @@ -50,35 +46,94 @@ func HasPromptSource(sources []GenerationSource) bool { return false } -func BuildGenerationSources(agentKind, agentName, version, instruction string, traces *TraceOptions) []GenerationSource { - var sources []GenerationSource +// BuildGenerationSources emits the sources the caller selected, in a stable +// order, along with the ones it asked for and nothing could be built from. +// +// kinds is what --from named. An empty kinds means "whatever this plan has to +// offer" and reports nothing missing: the caller expressed no preference, so +// there is nothing to disappoint. Naming a kind explicitly is a request, and a +// request that cannot be built is worth saying out loud rather than quietly +// submitting a job seeded from less than was asked for. +func BuildGenerationSources( + kinds []string, + agentName, version, instruction string, + traces *TraceOptions, +) (sources []GenerationSource, unbuildable []string) { + want := map[string]bool{} + for _, k := range kinds { + want[k] = true + } + // Empty kinds selects everything available; a populated one selects only + // what it names. + selected := func(kind string) bool { + return len(want) == 0 || want[kind] + } + // asked distinguishes "the default swept this up" from "the user typed it", + // which is what decides whether an empty-handed source is an error. + asked := func(kind string) bool { return want[kind] } + + // The agent is settled first because whether it was built decides whether + // its instructions have anything to be the instructions of. + var agentSource *GenerationSource + if selected("agent") { + switch { + case agentName != "": + agentSource = &GenerationSource{Type: "agent", AgentName: agentName} + if version != "" { + agentSource.AgentVersion = version + } + case asked("agent"): + unbuildable = append(unbuildable, "agent") + } + } - if instruction != "" { - sources = append(sources, GenerationSource{ - Type: "prompt", - Prompt: instruction, - }) + // Generating from an agent means generating from its instructions, so they + // travel with it as a prompt. That is also the only shape the service + // currently honours: the agent source alone fails for every agent, and the + // prompt is what the retry in generateDataset falls back to. Without this, + // `--from agent` would be a request that always fails. + promptCarriesTheAgent := agentSource != nil && asked("agent") + if selected("prompt") || promptCarriesTheAgent { + switch { + case instruction != "": + sources = append(sources, GenerationSource{ + Type: "prompt", + Prompt: instruction, + }) + case asked("prompt"): + unbuildable = append(unbuildable, "prompt") + } } - agentSource := GenerationSource{ - Type: "agent", - AgentName: agentName, + if agentSource != nil { + sources = append(sources, *agentSource) } - if version != "" { - agentSource.AgentVersion = version + + if selected("traces") { + // A window narrows the request; it does not authorize it. Asking for + // traces without one means every trace the agent has. + switch { + case traces != nil && traces.Days > 0: + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + StartTime: time.Now().AddDate(0, 0, -traces.Days).Unix(), + }) + case asked("traces"): + sources = append(sources, GenerationSource{ + Type: "traces", + AgentName: agentName, + }) + } } - sources = append(sources, agentSource) - - if traces != nil && traces.Days > 0 { - startTime := time.Now().AddDate(0, 0, -traces.Days).Unix() - sources = append(sources, GenerationSource{ - Type: "traces", - AgentName: agentName, - StartTime: startTime, - }) + + // The service takes a file's rows through the dataset upload path, not + // through a generation source, so there is nothing here to build one from. + if asked("file") { + unbuildable = append(unbuildable, "file") } - return sources + return sources, unbuildable } // --------------------------------------------------------------------------- diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go new file mode 100644 index 00000000000..85183400040 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// kindsOf reduces the built sources to what --from talks about, which is the +// only part these tests are asserting on. +func kindsOf(sources []GenerationSource) []string { + kinds := make([]string, 0, len(sources)) + for _, s := range sources { + kinds = append(kinds, s.Type) + } + return kinds +} + +// Naming a source is a request to send that one, not a hint. Everything the +// plan could otherwise have offered stays out of the request. +func TestBuildGenerationSources_SendsOnlyWhatFromNamed(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, + "support-agent", "3", "answer support questions", + &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Generating from an agent means generating from its instructions, so asking +// for the agent carries them. It is also the only shape the service honours: +// the agent source on its own fails for every agent, so a `--from agent` that +// dropped the prompt would be a request that always fails. +func TestBuildGenerationSources_AgentCarriesItsInstructions(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "support-agent", "3", "answer support questions", nil, + ) + + assert.Equal(t, []string{"prompt", "agent"}, kindsOf(sources)) + assert.Equal(t, "answer support questions", sources[0].Prompt) + assert.Empty(t, unbuildable) +} + +// The instructions ride along with the agent; they do not stand in for it. An +// agent nobody named is still nothing to generate from. +func TestBuildGenerationSources_InstructionsDoNotSubstituteForTheAgent(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent"}, "", "", "answer support questions", nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, []string{"agent"}, unbuildable) +} + +// The agent name travels with the traces source: it is what scopes the query +// to this agent's conversations rather than the whole project's. +func TestBuildGenerationSources_TracesCarryTheAgent(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "support-agent", sources[0].AgentName) +} + +// A day window narrows the trace query; it is not what authorizes it. The +// documented `dataset generate --from traces` carries no window, and it +// has to mean "every trace" rather than "no traces". +func TestBuildGenerationSources_TracesWithoutAWindowAreUnbounded(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", nil, + ) + + require.Len(t, sources, 1) + assert.Equal(t, "traces", sources[0].Type) + assert.Zero(t, sources[0].StartTime, + "an absent window must leave start_time off the wire, not pin it to now") + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_TraceWindowBecomesAStartTime(t *testing.T) { + sources, _ := BuildGenerationSources( + []string{"traces"}, "support-agent", "", "", &TraceOptions{Days: 7}, + ) + + require.Len(t, sources, 1) + want := time.Now().AddDate(0, 0, -7).Unix() + assert.InDelta(t, want, sources[0].StartTime, 60) +} + +// No --from is no preference, so the plan sends everything it happens to have. +func TestBuildGenerationSources_EmptyFromSendsWhatThePlanHas(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + nil, "support-agent", "3", "answer support questions", &TraceOptions{Days: 7}, + ) + + assert.Equal(t, []string{"prompt", "agent", "traces"}, kindsOf(sources)) + assert.Empty(t, unbuildable) +} + +// Expressing no preference cannot disappoint one, so an empty --from reports +// nothing missing however little the plan turns out to hold. +func TestBuildGenerationSources_EmptyFromNeverReportsMissingSources(t *testing.T) { + sources, unbuildable := BuildGenerationSources(nil, "", "", "", nil) + + assert.Empty(t, sources) + assert.Empty(t, unbuildable) +} + +// Asking for a source the plan cannot build has to surface, because the job is +// billed and what comes back looks the same either way. +func TestBuildGenerationSources_ReportsWhatItCouldNotBuild(t *testing.T) { + tests := []struct { + name string + kinds []string + agentName string + instruction string + want []string + }{ + { + name: "prompt without an instruction", + kinds: []string{"prompt"}, + want: []string{"prompt"}, + }, + { + name: "agent without a target", + kinds: []string{"agent"}, + want: []string{"agent"}, + }, + { + name: "file is not a generation source at all", + kinds: []string{"file"}, + want: []string{"file"}, + }, + { + name: "several at once", + kinds: []string{"prompt", "agent"}, + want: []string{"agent", "prompt"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + tt.kinds, tt.agentName, "", tt.instruction, nil, + ) + + assert.Empty(t, sources) + assert.Equal(t, tt.want, unbuildable) + }) + } +} + +// A request that names two sources and can only build one still reports the +// one it could not, rather than being satisfied by the other's success. +func TestBuildGenerationSources_OneBuiltSourceDoesNotExcuseAMissingOne(t *testing.T) { + sources, unbuildable := BuildGenerationSources( + []string{"agent", "prompt"}, "support-agent", "", "", nil, + ) + + assert.Equal(t, []string{"agent"}, kindsOf(sources)) + assert.Equal(t, []string{"prompt"}, unbuildable) +} + +// `file` is only unbuildable when it was asked for. The default sweep must not +// invent a complaint about a source nobody named. +func TestBuildGenerationSources_FileIsOnlyReportedWhenAskedFor(t *testing.T) { + _, unbuildable := BuildGenerationSources( + nil, "support-agent", "", "instruction", &TraceOptions{Days: 7}, + ) + + assert.Empty(t, unbuildable) +} + +func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { + withVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "3", "", nil) + require.Len(t, withVersion, 1) + assert.Equal(t, "3", withVersion[0].AgentVersion) + + withoutVersion, _ := BuildGenerationSources([]string{"agent"}, "support-agent", "", "", nil) + require.Len(t, withoutVersion, 1) + assert.Empty(t, withoutVersion[0].AgentVersion) +} From 2fb79a7a04311996ecfea833d4c290161a5aa6d7 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:02:24 +0300 Subject: [PATCH 108/114] Point CI at templates that exist, and verify the provider it declares The release pipeline listed eng/pipelines/release-azd-extension.yml, which is not a file -- the template lives under templates/stages/. A trigger path that matches nothing fails silently: the pipeline simply never runs for changes to the template it is built out of. It now names the same paths as the skills sibling, and every one of them resolves. go.mod was in the trigger list too, matching the repo root module rather than the extension's own. The lint workflow only ran lint. This extension declares a service-target provider, and the extensions that declare one run verify-ext-providers to check the manifest against what the host actually registers. It has the canonical providers_manifest_test.go already; nothing was calling it in CI. --- .../lint-ext-azure-ai-evaluations.yml | 28 +++++++++++++++++++ .../release-ext-azure-ai-evaluations.yml | 5 ++-- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lint-ext-azure-ai-evaluations.yml diff --git a/.github/workflows/lint-ext-azure-ai-evaluations.yml b/.github/workflows/lint-ext-azure-ai-evaluations.yml new file mode 100644 index 00000000000..2cb72ea1de4 --- /dev/null +++ b/.github/workflows/lint-ext-azure-ai-evaluations.yml @@ -0,0 +1,28 @@ +name: ext-azure-ai-evaluations-ci + +on: + pull_request: + paths: + - "cli/azd/extensions/azure.ai.evaluations/**" + - ".github/workflows/lint-ext-azure-ai-evaluations.yml" + - ".github/workflows/verify-ext-providers.yml" + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write # required by reusable workflow lint-go.yml + +jobs: + lint: + uses: ./.github/workflows/lint-go.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations + + verify-providers: + uses: ./.github/workflows/verify-ext-providers.yml + with: + working-directory: cli/azd/extensions/azure.ai.evaluations diff --git a/eng/pipelines/release-ext-azure-ai-evaluations.yml b/eng/pipelines/release-ext-azure-ai-evaluations.yml index d8b60935909..329c4a2181e 100644 --- a/eng/pipelines/release-ext-azure-ai-evaluations.yml +++ b/eng/pipelines/release-ext-azure-ai-evaluations.yml @@ -5,9 +5,8 @@ trigger: - main paths: include: - - go.mod - cli/azd/extensions/azure.ai.evaluations - - eng/pipelines/release-azd-extension.yml + - /eng/pipelines/templates/stages/release-azd-extension.yml - /eng/pipelines/templates/jobs/build-azd-extension.yml - /eng/pipelines/templates/jobs/cross-build-azd-extension.yml - /eng/pipelines/templates/variables/image.yml @@ -17,7 +16,7 @@ pr: include: - cli/azd/extensions/azure.ai.evaluations - eng/pipelines/release-ext-azure-ai-evaluations.yml - - eng/pipelines/release-azd-extension.yml + - /eng/pipelines/templates/stages/release-azd-extension.yml - eng/pipelines/templates/steps/publish-cli.yml exclude: - cli/azd/docs/** From a0bb3e0ad4964eb207bc5fb85213a4c458843560 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:15:10 +0300 Subject: [PATCH 109/114] Read the agent's instructions from the project before asking the service The spec has --from agent generating from the target agent's instructions, read locally so the request needs no service lookup. Checking that against the agents extension rather than the spec text found half of it true. The optimize configuration folder is real: zd ai agent optimize writes /.agent_configs/baseline/metadata.yaml with an instruction_file pointer beside the instructions.md it names. That is now read first. The other half is not. A managed agent's definition YAML has no instructions field at all -- ContainerAgent carries kind, name, runtime, image, protocols and policies, and nothing that holds a prompt. Instructions live on skills and on prompt agents, and prompt agents are not deployable today. So there is nothing to read there, and the service lookup stays as the fallback. Local wins over the service because instructions that have been optimized but not yet deployed are the ones the author means; generating against what is still published would test the version they are replacing. A target matches either the azure.yaml service key or the agent name the service declares, since the two need not agree and a user has only seen one of them. Matching both is what makes --target mean what they typed. A tie is refused rather than broken arbitrarily, which is the spec's ambiguity rule. The file names are the agents extension's, repeated rather than imported -- azd extensions are separate modules that share no code, so reading another one's output means knowing its layout. The comment says so, because if those names move this quietly stops working and falls back to the service. --- .../internal/cmd/generate.go | 43 ++++- .../internal/project/agent_instructions.go | 151 +++++++++++++++ .../project/agent_instructions_test.go | 182 ++++++++++++++++++ 3 files changed, 374 insertions(+), 2 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index 3091cbd2f27..ee07d39308e 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -16,6 +16,8 @@ import ( "azureaieval/internal/pkg/eval_api" "azureaieval/internal/project" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" ) // generatePollBudget replaces the inherited 2s x 300 (10 minute) client budget. @@ -110,8 +112,13 @@ func declaredInstructions(named, configPath string) (string, error) { // // The service accepts an agent source that is meant to pull the agent's own // instructions, but it fails for every agent, so the agent's context is read -// here instead. In precedence order: what the caller passed, then the agent's -// published instructions. +// here instead. In precedence order: what the caller passed, the instructions +// the project already holds, then the agent's published ones. +// +// The project comes before the service because a local read cannot fail +// slowly, and because instructions that have been optimized but not yet +// deployed are the ones the author means — generating against what is still +// published would test the version they are replacing. // // The last step is what makes `generate` work with no authored input at all, // which is the flow `init` sets up. @@ -128,6 +135,18 @@ func (ec *evalContext) resolveGenerationInstruction( if agentName == "" { return "", nil } + + local, path, err := ec.agentInstructionsFromProject(ctx, agentName) + if err != nil { + return "", err + } + if local != "" { + if !quiet { + fmt.Fprintf(out, " Seeding generation from %s.\n", filepath.ToSlash(path)) + } + return local, nil + } + agent, err := ec.evalClient.GetAgent(ctx, agentName, ProjectEndpointAPIVersion) if err != nil { // Generation can still proceed from the agent source alone, so a @@ -145,6 +164,26 @@ func (ec *evalContext) resolveGenerationInstruction( return instructions, nil } +// agentInstructionsFromProject reads the agent's instructions out of the azd +// project, coming back empty when there is no project to read. +// +// Running outside a project is ordinary — the atomic commands work standalone +// against the data plane — so not finding one is not an error. An ambiguous +// target inside one is, because it would otherwise pick an agent at random. +func (ec *evalContext) agentInstructionsFromProject( + ctx context.Context, + agentName string, +) (instruction string, path string, err error) { + if ec.azdClient == nil { + return "", "", nil + } + resp, err := ec.azdClient.Project().Get(ctx, &azdext.EmptyRequest{}) + if err != nil || resp.GetProject() == nil { + return "", "", nil + } + return project.AgentInstructionsFromProject(resp.GetProject(), agentName) +} + // generateRubric submits the evaluator generation job and saves the rubric. func (ec *evalContext) generateRubric( ctx context.Context, diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go new file mode 100644 index 00000000000..d191fd184c2 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions.go @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "go.yaml.in/yaml/v3" +) + +// AgentHost is the service host the agents extension registers. Services +// declaring it are the ones that could be a generation target. +const AgentHost = "azure.ai.agent" + +// Where `azd ai agent optimize` leaves the configuration it settled on. +// +// These are the agents extension's file names, repeated rather than imported: +// azd extensions are separate Go modules and share no code, so the only way to +// read another one's output is to know its layout. That makes this a coupling +// worth naming — if the agents extension moves these, generation quietly stops +// finding instructions locally and falls back to the service. +const ( + agentConfigsDir = ".agent_configs" + agentBaselineDir = "baseline" + agentMetadataFile = "metadata.yaml" +) + +// agentConfigMetadata is the part of the optimize configuration's metadata.yaml +// that says where the instructions are. It points at a file rather than +// carrying the text, because the text is what a reviewer diffs. +type agentConfigMetadata struct { + InstructionFile string `yaml:"instruction_file"` +} + +// ErrAmbiguousAgentService reports that a target name matched more than one +// service, so there is no single set of instructions to read. +var ErrAmbiguousAgentService = errors.New("more than one agent service matches") + +// AgentInstructionsFromProject reads the target agent's instructions out of the +// project, returning empty when the project does not hold them. +// +// The instructions an agent was optimized with are the best description of what +// it is supposed to do, and they are already on disk, so generating from them +// needs no service call. Coming back empty is ordinary — most projects have +// never run `azd ai agent optimize` — and leaves the caller free to ask the +// service instead. +// +// The returned path is where the text came from, for a caller that wants to say +// so. +func AgentInstructionsFromProject( + proj *azdext.ProjectConfig, + agentName string, +) (instruction string, path string, err error) { + svc, err := findAgentService(proj, agentName) + if err != nil || svc == nil { + return "", "", err + } + + configDir := filepath.Join( + proj.GetPath(), serviceRelativeDir(svc), agentConfigsDir, agentBaselineDir) + + data, err := os.ReadFile(filepath.Join(configDir, agentMetadataFile)) //nolint:gosec // under the project + if err != nil { + // An agent that was never optimized has no such directory, which is + // the common case rather than a problem. + return "", "", nil + } + + var meta agentConfigMetadata + if err := yaml.Unmarshal(data, &meta); err != nil { + return "", "", fmt.Errorf( + "reading %s: %w", filepath.Join(configDir, agentMetadataFile), err) + } + if meta.InstructionFile == "" { + return "", "", nil + } + + instructionPath := meta.InstructionFile + if !filepath.IsAbs(instructionPath) { + instructionPath = filepath.Join(configDir, instructionPath) + } + text, err := os.ReadFile(instructionPath) //nolint:gosec // named by the metadata beside it + if err != nil { + // The metadata named a file that is not there. That is worth saying: + // something wrote the pointer and not the target. + return "", "", fmt.Errorf( + "%s names instruction_file %q, which could not be read: %w", + filepath.Join(configDir, agentMetadataFile), meta.InstructionFile, err) + } + + return strings.TrimSpace(string(text)), instructionPath, nil +} + +// findAgentService resolves a target name to the one service that is it. +// +// A name can match either the azure.yaml service key or the agent name the +// service declares, because the two need not agree and a user has only ever +// seen one of them. Matching both is what makes `--target` mean what they +// typed; refusing a tie is what stops it silently meaning one of two things. +func findAgentService( + proj *azdext.ProjectConfig, + agentName string, +) (*azdext.ServiceConfig, error) { + if proj == nil || agentName == "" { + return nil, nil + } + + var matched []string + services := map[string]*azdext.ServiceConfig{} + for name, svc := range proj.GetServices() { + if svc.GetHost() != AgentHost { + continue + } + if name == agentName || declaredAgentName(svc) == agentName { + matched = append(matched, name) + services[name] = svc + } + } + + switch len(matched) { + case 0: + return nil, nil + case 1: + return services[matched[0]], nil + default: + sort.Strings(matched) + return nil, fmt.Errorf( + "%w %q: %s. Name one of them with --target, or pass the text with "+ + "--agent-instruction", + ErrAmbiguousAgentService, agentName, strings.Join(matched, ", ")) + } +} + +// declaredAgentName is the name the service gives the agent, which is what the +// service publishes under and so what the eval configuration's target refers +// to. It is absent when the service key is also the agent name. +func declaredAgentName(svc *azdext.ServiceConfig) string { + props := serviceProps(svc) + if props == nil { + return "" + } + name, _ := props.AsMap()["name"].(string) + return name +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go new file mode 100644 index 00000000000..dae36a18c5b --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/agent_instructions_test.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "testing" + + "github.com/azure/azure-dev/cli/azd/pkg/azdext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +// writeOptimizeConfig lays out what `azd ai agent optimize` leaves behind: +// .agent_configs/baseline/metadata.yaml pointing at instructions.md beside it. +func writeOptimizeConfig(t *testing.T, serviceDir, metadata, instructions string) { + t.Helper() + dir := filepath.Join(serviceDir, ".agent_configs", "baseline") + require.NoError(t, os.MkdirAll(dir, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "metadata.yaml"), []byte(metadata), 0o600)) + if instructions != "" { + require.NoError(t, + os.WriteFile(filepath.Join(dir, "instructions.md"), []byte(instructions), 0o600)) + } +} + +// agentService builds a project holding one agent service, optionally +// declaring an agent name that differs from the service key. +func agentService(t *testing.T, root, serviceKey, declaredName string) *azdext.ProjectConfig { + t.Helper() + svc := &azdext.ServiceConfig{ + Name: serviceKey, + Host: AgentHost, + RelativePath: serviceKey, + } + if declaredName != "" { + props, err := structpb.NewStruct(map[string]any{"name": declaredName}) + require.NoError(t, err) + svc.AdditionalProperties = props + } + return &azdext.ProjectConfig{ + Path: root, + Services: map[string]*azdext.ServiceConfig{serviceKey: svc}, + } +} + +// The instructions an agent was optimized with are already on disk, so +// generating from them needs no service call. +func TestAgentInstructionsFromProject_ReadsTheOptimizeConfig(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "support"), + "name: support\ninstruction_file: instructions.md\n", + "Answer support questions politely.\n") + + instruction, path, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + require.NoError(t, err) + assert.Equal(t, "Answer support questions politely.", instruction) + assert.Equal(t, filepath.Join(root, "support", ".agent_configs", "baseline", "instructions.md"), + path) +} + +// A target names the agent, which need not be spelled the way the azure.yaml +// key is. A user has only ever seen one of the two. +func TestAgentInstructionsFromProject_MatchesTheDeclaredAgentName(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, + filepath.Join(root, "svc"), + "instruction_file: instructions.md\n", + "Be helpful.") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "svc", "support-agent"), "support-agent") + + require.NoError(t, err) + assert.Equal(t, "Be helpful.", instruction) +} + +// Most projects have never run optimize, so finding nothing is the ordinary +// case and has to leave the caller free to ask the service instead. +func TestAgentInstructionsFromProject_SilentWhenThereIsNothingToRead(t *testing.T) { + root := t.TempDir() + + tests := []struct { + name string + proj *azdext.ProjectConfig + agent string + }{ + {"no project at all", nil, "support"}, + {"no agent named", agentService(t, root, "support", ""), ""}, + {"no service by that name", agentService(t, root, "support", ""), "other"}, + {"no optimize config on disk", agentService(t, root, "support", ""), "support"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + instruction, path, err := AgentInstructionsFromProject(tt.proj, tt.agent) + + assert.NoError(t, err) + assert.Empty(t, instruction) + assert.Empty(t, path) + }) + } +} + +// A service that is not an agent is not a candidate, however it is named. +func TestAgentInstructionsFromProject_IgnoresServicesThatAreNotAgents(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "Be helpful.") + + proj := agentService(t, root, "support", "") + proj.Services["support"].Host = "containerapp" + + instruction, _, err := AgentInstructionsFromProject(proj, "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +// Two services answering to one name is a tie, and picking either would make +// the generated dataset describe an agent the caller did not mean. +func TestAgentInstructionsFromProject_RefusesAnAmbiguousTarget(t *testing.T) { + root := t.TempDir() + proj := agentService(t, root, "support", "") + props, err := structpb.NewStruct(map[string]any{"name": "support"}) + require.NoError(t, err) + proj.Services["helpdesk"] = &azdext.ServiceConfig{ + Name: "helpdesk", Host: AgentHost, RelativePath: "helpdesk", + AdditionalProperties: props, + } + + _, _, err = AgentInstructionsFromProject(proj, "support") + + require.ErrorIs(t, err, ErrAmbiguousAgentService) + assert.Contains(t, err.Error(), "helpdesk") + assert.Contains(t, err.Error(), "support") + assert.Contains(t, err.Error(), "--target", + "an ambiguity the caller can resolve has to say how") +} + +// A pointer with nothing behind it means something wrote half the config. +// Falling back silently would generate from the published agent while the +// author believes they are generating from what they just optimized. +func TestAgentInstructionsFromProject_ReportsADanglingInstructionFile(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), + "instruction_file: instructions.md\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "instructions.md") +} + +// Metadata that names no instruction file is a config without instructions, +// not a broken one. +func TestAgentInstructionsFromProject_NoInstructionFileIsNotAnError(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "name: support\n", "") + + instruction, _, err := AgentInstructionsFromProject( + agentService(t, root, "support", ""), "support") + + assert.NoError(t, err) + assert.Empty(t, instruction) +} + +func TestAgentInstructionsFromProject_ReportsUnreadableMetadata(t *testing.T) { + root := t.TempDir() + writeOptimizeConfig(t, filepath.Join(root, "support"), "\tnot: [valid\n", "") + + _, _, err := AgentInstructionsFromProject(agentService(t, root, "support", ""), "support") + + require.Error(t, err) + assert.Contains(t, err.Error(), "metadata.yaml") +} From 2d4ce006c1091f540074f98030575b2a4eaa0047 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:28:57 +0300 Subject: [PATCH 110/114] Test the export writers and portal URLs, and delete what duplicated them Coverage was 39.2%, and the uncovered list turned out to be two different things: logic nobody had tested, and logic nobody had called. Tested, because these are what a user reads and nothing checked them: - writeResultsCSV and writeResultsJSONL. The export default moved to csv and jsonl was added earlier in this branch with no test of either writer. The header is a contract with whatever opens the file, a run that graded nothing still has to produce a parseable artifact rather than zero bytes, and a comma in a service-supplied criterion name has to survive the round trip. - The portal URLs. They are assembled from parts rather than returned by the service, so nothing but a test says whether they land anywhere. The subscription is base64url-encoded, which is now pinned -- padding or a plain slash would break the URL segment. - IsDatasetName, which decides whether a value is looked up in the service or opened off disk. Getting it wrong sends a path to the registry, and neither failure names the real problem. Deleted, because each duplicated something live: - GenerationJob.NormalizedStatus duplicated ParseJobStatus and got it wrong: its doc says it lowercases and it does not. Nothing called it, so the bug never fired -- but the next caller would have found it. - GenerationJob.OperationID and OpenAIEval.ResolvedID were aliases. - ParseEvaluatorResult duplicated what writeRubric already does inline. - formatStat duplicated the Defined() check in run.go. - noPrompt duplicated what requireFlag already encodes. - SetFileID could not be used: readRegisteredDataset documents that a run's file_id means an uploaded file and a dataset name is rejected as one. That reason now sits on SetFileContent, where the next person will look. Testing dead code would have entrenched it, and in NormalizedStatus's case would have pinned the wrong behaviour. --- .../internal/cmd/output.go | 12 -- .../internal/cmd/run_output.go | 10 -- .../internal/cmd/run_output_write_test.go | 148 ++++++++++++++++ .../internal/pkg/eval_api/models.go | 52 +----- .../internal/pkg/eval_api/portal_urls_test.go | 166 ++++++++++++++++++ 5 files changed, 320 insertions(+), 68 deletions(-) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go index 62f9c2eade1..143b49e91ba 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/output.go @@ -79,18 +79,6 @@ func emitTable(w io.Writer, headers []string, rows [][]string) error { return tw.Flush() } -// noPrompt reports whether the command must run without any interaction. -func noPrompt(cmd *cobra.Command) bool { - if cmd == nil { - return false - } - v, err := cmd.Flags().GetBool("no-prompt") - if err != nil { - return false - } - return v -} - // requireFlag returns an error naming the missing flag, used when --no-prompt // prevents asking for a required value. func requireFlag(name string) error { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go index 3c6c0e36ad7..28f2170909b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output.go @@ -35,16 +35,6 @@ func newRunOutputCommand() *cobra.Command { return cmd } -// formatStat renders a statistic, showing an undefined one as a dash. A score -// the service left unset has no value, and printing the literal "NaN" in a -// results table reads like a failure rather than the arithmetic it is. -func formatStat(verb string, v eval_api.LenientFloat) string { - if !v.Defined() { - return "-" - } - return fmt.Sprintf(verb, float64(v)) -} - func newRunOutputListCommand() *cobra.Command { var ( failedOnly bool diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go new file mode 100644 index 00000000000..ec63f399695 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/run_output_write_test.go @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "strings" + "testing" + + "azureaieval/internal/pkg/eval_api" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// twoCriteriaRun is a finished run with the shape export has to preserve: one +// row per testing criterion, all carrying the run they belong to. +func twoCriteriaRun() *eval_api.OpenAIEvalRun { + return &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: "task_adherence", Passed: 8, Failed: 2}, + {TestingCriteria: "coherence", Passed: 10, Failed: 0}, + }, + } +} + +// An export is read by a spreadsheet or a diff, so the header is part of the +// contract: renaming a column silently breaks whatever consumes it. +func TestWriteResultsCSV(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, twoCriteriaRun())) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_abc", "completed", "task_adherence", "8", "2"}, rows[1]) + assert.Equal(t, []string{"evalrun_abc", "completed", "coherence", "10", "0"}, rows[2]) + assert.Len(t, rows, 3, "one header and one row per criterion") +} + +// A run that graded nothing still has to produce a file with a header, because +// a consumer that gets zero bytes cannot tell an empty run from a failed +// export. +func TestWriteResultsCSV_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + + require.Len(t, rows, 2) + assert.Equal(t, []string{"run_id", "status", "criterion", "passed", "failed"}, rows[0]) + assert.Equal(t, []string{"evalrun_empty", "failed", "", "", ""}, rows[1]) +} + +// A criterion name is service-supplied, so it can hold anything. The writer +// has to quote rather than corrupt the row. +func TestWriteResultsCSV_QuotesASeparatorInTheData(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsCSV(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_abc", + Status: "completed", + PerTestingCriteria: []eval_api.EvalRunCriteriaResult{ + {TestingCriteria: `groundedness, strict`, Passed: 1, Failed: 0}, + }, + })) + + rows, err := csv.NewReader(&buf).ReadAll() + require.NoError(t, err) + require.Len(t, rows, 2) + assert.Equal(t, "groundedness, strict", rows[1][2], + "a comma in a criterion name must survive the round trip") +} + +// One criterion per line is what lets a downstream job stream results without +// holding the whole run. +func TestWriteResultsJSONL(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 2, "one line per criterion") + + var first map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &first)) + assert.Equal(t, "evalrun_abc", first["run_id"]) + assert.Equal(t, "completed", first["status"]) + assert.Equal(t, "task_adherence", first["testing_criteria"]) + assert.EqualValues(t, 8, first["passed"]) + assert.EqualValues(t, 2, first["failed"]) + + var second map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[1]), &second)) + assert.Equal(t, "coherence", second["testing_criteria"]) +} + +// Every line has to parse on its own; that is the whole point of the format. +func TestWriteResultsJSONL_EachLineParsesAlone(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, twoCriteriaRun())) + + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + var row map[string]any + assert.NoErrorf(t, json.Unmarshal([]byte(line), &row), "line is not self-contained: %s", line) + } +} + +func TestWriteResultsJSONL_RunWithNoCriteria(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, writeResultsJSONL(&buf, &eval_api.OpenAIEvalRun{ + ID: "evalrun_empty", Status: "failed", + })) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + require.Len(t, lines, 1) + + var row map[string]any + require.NoError(t, json.Unmarshal([]byte(lines[0]), &row)) + assert.Equal(t, "evalrun_empty", row["run_id"]) + assert.Equal(t, "failed", row["status"]) + assert.NotContains(t, row, "testing_criteria", + "a run that graded nothing must not claim a criterion") +} + +// The three export formats are a documented set. A fourth spelling, or a +// missing one, is a promise broken on either side. +func TestExportFormatsAreTheDocumentedSet(t *testing.T) { + assert.Equal(t, "csv", formatCSV) + assert.Equal(t, "json", formatJSON) + assert.Equal(t, "jsonl", formatJSONL) + + usage := find(t, "run output export").Flags().Lookup("format") + require.NotNil(t, usage) + assert.Equal(t, formatCSV, usage.DefValue, + "results are a table, so the default artifact is the one a spreadsheet opens") + + for _, f := range []string{formatCSV, formatJSON, formatJSONL} { + assert.Containsf(t, usage.Usage, f, "--format accepts %q, so its help has to say so", f) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go index aa6c4f89ebb..ceed191847a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/models.go @@ -91,19 +91,6 @@ type JobError struct { Message string `json:"message,omitempty"` } -// OperationID returns the job's operation identifier. -func (j *GenerationJob) OperationID() string { - return j.ID -} - -// NormalizedStatus returns the lowercase status, defaulting to "running". -func (j *GenerationJob) NormalizedStatus() string { - if j.Status == "" { - return "running" - } - return j.Status -} - // ResolvedNameVersion extracts the name and version from the generation job result. // If name is empty, both return values are empty (caller should treat as no result). // If version is empty, it defaults to "latest". @@ -209,22 +196,6 @@ type EvaluatorDimension struct { AlwaysApplicable bool `json:"always_applicable,omitempty"` } -// ParseEvaluatorResult parses a GenerationJob result into a structured EvaluatorResult. -// Returns nil if the result cannot be parsed. -func ParseEvaluatorResult(result json.RawMessage) *EvaluatorResult { - if len(result) == 0 { - return nil - } - var r EvaluatorResult - if err := json.Unmarshal(result, &r); err != nil { - return nil - } - if len(r.Definition.Dimensions) == 0 { - return nil - } - return &r -} - // --------------------------------------------------------------------------- // Datasets // --------------------------------------------------------------------------- @@ -297,14 +268,6 @@ type OpenAIEval struct { Metadata map[string]string `json:"metadata,omitempty"` } -// ResolvedID returns the eval's ID, falling back to name. -func (e *OpenAIEval) ResolvedID() string { - if e.ID != "" { - return e.ID - } - return e.Name -} - // OpenAIEvalList is the response for listing OpenAI eval definitions. type OpenAIEvalList struct { Data []OpenAIEval `json:"data"` @@ -408,7 +371,7 @@ type EvalRunDataContent struct { } // NewAgentTargetDataSource builds an EvalRunDataSource configured for agent target completions. -// The source field must be set separately via SetFileContent or SetFileID. +// The rows must be supplied separately via SetFileContent. func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDataSource { return &EvalRunDataSource{ Type: EvalRunDataSourceTypeAgentTarget, @@ -432,6 +395,11 @@ func NewAgentTargetDataSource(agentName string, agentVersion *string) *EvalRunDa } // SetFileContent sets the data source to use inline file content. +// +// There is no by-reference counterpart. A run's `file_id` means an uploaded +// file, and a dataset name is not one — sending it is rejected with "invalid +// data source file ids" — so registered datasets are fetched and sent inline +// too. See readRegisteredDataset. func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { ds.Source = &EvalRunDataContent{ Type: EvalRunDataContentTypeFileContent, @@ -439,14 +407,6 @@ func (ds *EvalRunDataSource) SetFileContent(items []map[string]any) { } } -// SetFileID sets the data source to reference a remote dataset by ID. -func (ds *EvalRunDataSource) SetFileID(fileID string) { - ds.Source = &EvalRunDataContent{ - Type: EvalRunDataContentTypeFileID, - ID: fileID, - } -} - // OpenAIEvalRun is the response for an OpenAI eval run. type OpenAIEvalRun struct { ID string `json:"id"` diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go new file mode 100644 index 00000000000..602b584ccd5 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/portal_urls_test.go @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package eval_api + +import ( + "strings" + "testing" + + "azureaieval/internal/pkg/evalcore" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testProjectID = "/subscriptions/00000000-1111-2222-3333-444444444444/" + + "resourceGroups/rg-eval/providers/Microsoft.CognitiveServices/accounts/acct/projects/proj" + +// A portal URL is printed at the end of a run and is the one thing a user +// clicks. It is assembled from parts rather than returned by the service, so +// nothing but a test says whether it lands anywhere. +func TestPortalPrefix_BuildsEveryDocumentedURL(t *testing.T) { + p, err := NewPortalPrefix(testProjectID) + require.NoError(t, err) + + // The subscription travels base64url-encoded without padding, so the + // literal GUID must not appear anywhere in the result. + const sub = "00000000-1111-2222-3333-444444444444" + + tests := []struct { + name string + got string + want string + }{ + {"eval run", p.EvalRunURL("eval_1", "evalrun_1"), "/build/evaluations/eval_1/run/evalrun_1"}, + {"evaluator", p.EvaluatorURL("quality", "3"), "/build/evaluations/catalog/quality/3"}, + {"dataset", p.DatasetURL("regression", "2"), "/build/data/datasets/regression/2"}, + {"optimization", p.OptimizationURL("support", "op_9"), "/build/agents/support/optimization/op_9"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.True(t, strings.HasPrefix(tt.got, "https://ai.azure.com/nextgen/r/"), + "got %s", tt.got) + assert.True(t, strings.HasSuffix(tt.got, tt.want), "got %s", tt.got) + assert.Contains(t, tt.got, "rg-eval") + assert.Contains(t, tt.got, "acct") + assert.Contains(t, tt.got, "proj") + assert.NotContains(t, tt.got, sub, + "the subscription is encoded, so its plain GUID must not appear") + }) + } +} + +// The encoding is what the portal decodes on the other end, so it is pinned +// rather than merely exercised. +func TestEncodeSubscriptionForURL(t *testing.T) { + encoded, err := encodeSubscriptionForURL("00000000-1111-2222-3333-444444444444") + + require.NoError(t, err) + assert.NotContains(t, encoded, "=", "padding would need escaping inside a URL segment") + assert.NotContains(t, encoded, "+", "base64url, not standard base64") + assert.NotContains(t, encoded, "/", "a slash would split the URL segment") + assert.Equal(t, "AAAAABERIiIzM0RERERERA", encoded) +} + +func TestEncodeSubscriptionForURL_RejectsSomethingThatIsNotAGUID(t *testing.T) { + _, err := encodeSubscriptionForURL("not-a-subscription") + + require.Error(t, err) + assert.Contains(t, err.Error(), "subscription") +} + +// A resource ID that is not a project has no account to name, and guessing +// would produce a URL that resolves to someone else's project. +func TestNewPortalPrefix_RefusesWhatIsNotAProject(t *testing.T) { + tests := []struct { + name string + id string + }{ + {"not a resource id at all", "hello"}, + {"empty", ""}, + { + name: "an account rather than a project under it", + id: "/subscriptions/00000000-1111-2222-3333-444444444444/resourceGroups/rg/" + + "providers/Microsoft.CognitiveServices/accounts/acct", + }, + { + name: "a project whose subscription is not a GUID", + id: "/subscriptions/not-a-guid/resourceGroups/rg/providers/" + + "Microsoft.CognitiveServices/accounts/acct/projects/proj", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p, err := NewPortalPrefix(tt.id) + + require.Error(t, err) + assert.Nil(t, p) + }) + } +} + +// The prefix distinguishes built-in evaluators from ones the project owns, +// which is what decides whether a version is published or referenced. +func TestIsBuiltinEvaluator(t *testing.T) { + assert.True(t, IsBuiltinEvaluator("builtin.task_adherence")) + assert.False(t, IsBuiltinEvaluator("task_adherence")) + assert.False(t, IsBuiltinEvaluator("builtin"), "the dot is part of the prefix") + assert.False(t, IsBuiltinEvaluator("my.builtin.thing"), "the prefix has to lead") + assert.False(t, IsBuiltinEvaluator("")) +} + +func TestSplitEvaluators(t *testing.T) { + generated, builtin := SplitEvaluators(evalcore.EvaluatorList{ + {Name: "builtin.coherence"}, + {Name: "support-quality"}, + {Name: "builtin.task_adherence"}, + }) + + require.Len(t, generated, 1) + assert.Equal(t, "support-quality", generated[0].Name) + require.Len(t, builtin, 2) + assert.Equal(t, "builtin.coherence", builtin[0].Name) + assert.Equal(t, "builtin.task_adherence", builtin[1].Name) +} + +// Both halves come back nil rather than empty for an empty input, so a caller +// checking len() reads the same either way. +func TestSplitEvaluators_Empty(t *testing.T) { + generated, builtin := SplitEvaluators(nil) + + assert.Empty(t, generated) + assert.Empty(t, builtin) +} + +// This decides whether a value is looked up in the service or opened off disk. +// Getting it wrong sends a path to the registry, or a registered name to the +// filesystem, and neither failure names the real problem. +func TestIsDatasetName(t *testing.T) { + names := []string{ + "support-regression", + "dataset_v2", + "name.with.dots", + "trailing.txt", + } + for _, v := range names { + assert.Truef(t, IsDatasetName(v), "%q is a registered name", v) + } + + paths := []string{ + "", + "data.jsonl", + "data.json", + "data.csv", + "DATA.JSONL", + "./data.jsonl", + "evals/datasets/x.jsonl", + `evals\datasets\x.jsonl`, + "a/b", + } + for _, v := range paths { + assert.Falsef(t, IsDatasetName(v), "%q is a path, not a name", v) + } +} From b5b56d2da99614e472619c42bb294b31fda89117 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:44:24 +0300 Subject: [PATCH 111/114] Cover the mechanisms that fail silently when they drift Second coverage pass, picking the code where a regression produces a plausible wrong answer rather than an error. digestIDKey had no test at all, and it is what makes a rename find the eval it already deployed: the id is recorded against the eval's substance, so a declaration whose name changed still resolves. Change the key format and every deployed eval quietly loses its recorded id and gets recreated, losing its run history. The format is pinned now, along with the two properties it rests on -- same substance gives the same key, different substance does not. FingerprintKey has the same shape of failure. It maps a name into an azd environment key, and a character that reached the file unmapped would produce a key azd cannot round-trip, so the artifact would look changed on every deploy. Pinned, including that one rune maps to one underscore rather than one byte. itemColumn reads the dataset column out of an {{item.}} binding. Reading it wrong submits a run against a column the dataset does not have, and the service rejects that without saying which one. Also covered: Fingerprint itself, versionFromRaw's fallback chain, defaultEvalName (the source is in the name so two evals over one agent do not collide), jobLookupError naming the right group, WithoutAgentSource and HasPromptSource (the retry that saves the documented generate flow), and both generation request builders -- the evaluator one sends the name under two keys and setting only one returns an evaluator under the wrong name. FingerprintGroup was already covered in service_target_eval_test.go, so the new file says where rather than testing it twice. Coverage 39.2% -> 42.1%; project 66.8% -> 70.4%, eval_api 30.3% -> 42.2%. --- .../internal/cmd/helpers_test.go | 149 ++++++++++++++++++ .../internal/pkg/eval_api/generation_test.go | 57 +++++++ .../internal/project/fingerprint_test.go | 87 ++++++++++ 3 files changed, 293 insertions(+) create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go create mode 100644 cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go new file mode 100644 index 00000000000..4654145e853 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/helpers_test.go @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package cmd + +import ( + "strings" + "testing" + + "azureaieval/internal/project" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// digestIDKey is what makes a rename find the eval it already deployed: the id +// is recorded against the eval's substance, so a declaration whose name +// changed still resolves. That only works while the key derives from the +// digest the same way it did last deploy — change the format and every +// deployed eval silently loses its recorded id and gets recreated. +func TestDigestIDKey(t *testing.T) { + const digest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + + key := digestIDKey(digest) + + assert.Equal(t, "EVAL_SUBSTANCE_0123456789ABCDEF_ID", key) + for _, r := range key { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } +} + +// Same substance, same key — that is the whole mechanism. +func TestDigestIDKey_IsStableForTheSameSubstance(t *testing.T) { + group := project.Eval{ + Name: "support", + Dataset: "support-regression", + Target: &project.Target{Name: "support-agent"}, + } + + first, err := project.FingerprintGroup(group) + require.NoError(t, err) + + renamed := group + renamed.Name = "support-renamed" + renamed.Description = "reworded" + second, err := project.FingerprintGroup(renamed) + require.NoError(t, err) + + assert.Equal(t, digestIDKey(first), digestIDKey(second), + "a rename must land on the key the first deploy wrote") +} + +// Different substance, different key, so a genuinely new eval does not adopt +// an unrelated one's id. +func TestDigestIDKey_DiffersWhenTheSubstanceDoes(t *testing.T) { + a, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "one"}) + require.NoError(t, err) + b, err := project.FingerprintGroup(project.Eval{Name: "x", Dataset: "two"}) + require.NoError(t, err) + + assert.NotEqual(t, digestIDKey(a), digestIDKey(b)) +} + +// The version recorded for an artifact comes out of what the service returned, +// falling back to what the caller already knew. +func TestVersionFromRaw(t *testing.T) { + tests := []struct { + name string + raw string + fallback string + want string + }{ + {"version in the body wins", `{"version":"7"}`, "3", "7"}, + {"empty version falls back", `{"version":""}`, "3", "3"}, + {"absent version falls back", `{"name":"x"}`, "3", "3"}, + {"unparseable body falls back", `not json`, "3", "3"}, + {"empty body falls back", ``, "3", "3"}, + {"no fallback either", `{}`, "", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, versionFromRaw([]byte(tt.raw), tt.fallback)) + }) + } +} + +// A criterion binds to a dataset column through `{{item.}}`. Reading the +// name wrong is how a run is submitted against a column the dataset does not +// have, which the service rejects without saying which one. +func TestItemColumn(t *testing.T) { + bound := map[string]string{ + "{{item.query}}": "query", + "{{item.ground_truth}}": "ground_truth", + "{{item.a.b}}": "a.b", + } + for binding, want := range bound { + got, ok := itemColumn(binding) + assert.Truef(t, ok, "%q is a binding", binding) + assert.Equal(t, want, got) + } + + notBound := []string{ + "", + "query", + "{{item.}}", + "{{ item.query }}", + "{{item.query", + "item.query}}", + "{{response.output}}", + } + for _, binding := range notBound { + got, ok := itemColumn(binding) + assert.Falsef(t, ok, "%q is not an item binding", binding) + assert.Empty(t, got) + } +} + +// An eval is named after what it evaluates and what it reads, so two evals over +// the same agent from different sources do not collide. +func TestDefaultEvalName(t *testing.T) { + assert.Equal(t, "support-agent-trace-eval", + defaultEvalName("support-agent", initSourceTraces)) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "dataset")) + assert.Equal(t, "support-agent-eval", + defaultEvalName("support-agent", "")) + + assert.NotEqual(t, + defaultEvalName("support-agent", initSourceTraces), + defaultEvalName("support-agent", "dataset"), + "the source is in the name so the two do not collide") +} + +// The reattach line printed by --no-wait has to name the group the job +// actually belongs to; the two job types share no collection, so the wrong +// group is a command that returns "not found". +func TestJobLookupErrorNamesTheGroup(t *testing.T) { + for _, kind := range []jobKind{datasetJobs, evaluatorJobs} { + err := jobLookupError(kind, "job_1", assert.AnError) + + require.Error(t, err) + assert.Contains(t, err.Error(), "job_1") + assert.Truef(t, strings.Contains(err.Error(), kind.name), + "the error must name the %q group so the retry goes to the right one", kind.name) + } +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go index 85183400040..6a5c993f820 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/pkg/eval_api/generation_test.go @@ -188,3 +188,60 @@ func TestBuildGenerationSources_AgentVersionIsOptional(t *testing.T) { require.Len(t, withoutVersion, 1) assert.Empty(t, withoutVersion[0].AgentVersion) } + +// The retry that saves the documented flow: agent-seeded generation fails +// server-side for every agent, and the same request without the agent source +// succeeds. +func TestWithoutAgentSource(t *testing.T) { + sources := []GenerationSource{ + {Type: "prompt", Prompt: "be helpful"}, + {Type: "agent", AgentName: "support"}, + {Type: "traces", AgentName: "support"}, + } + + kept := WithoutAgentSource(sources) + + assert.Equal(t, []string{"prompt", "traces"}, kindsOf(kept)) + assert.Len(t, sources, 3, "the original must not be modified; it is retried from") +} + +// The retry only happens when something is left to generate from, so this is +// what stops a second billed job that would fail the same way. +func TestHasPromptSource(t *testing.T) { + assert.True(t, HasPromptSource([]GenerationSource{{Type: "prompt", Prompt: "x"}})) + assert.False(t, HasPromptSource([]GenerationSource{{Type: "prompt"}}), + "an empty prompt is nothing to generate from") + assert.False(t, HasPromptSource([]GenerationSource{{Type: "agent", AgentName: "s"}})) + assert.False(t, HasPromptSource(nil)) +} + +// The request body is what the service validates, so the fields it keys on are +// pinned rather than left to whatever the builder happens to set. +func TestNewDataGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "be helpful"}} + + req := NewDataGenerationJobRequest("support-regression", "gpt-4o", 15, sources) + + require.NotNil(t, req) + assert.Equal(t, "support-regression", req.Inputs.Name) + assert.Equal(t, "evaluation", req.Inputs.Scenario) + assert.Equal(t, "simple_qna", req.Inputs.Options.Type) + assert.Equal(t, 15, req.Inputs.Options.MaxSamples) + assert.Equal(t, "gpt-4o", req.Inputs.Options.ModelOptions.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} + +// The evaluator request sends the name twice, under two keys the service reads +// separately. Setting only one produces a job that runs and returns an +// evaluator under the wrong name. +func TestNewEvaluatorGenerationJobRequest(t *testing.T) { + sources := []GenerationSource{{Type: "prompt", Prompt: "grade politeness"}} + + req := NewEvaluatorGenerationJobRequest("support-quality", "gpt-4o", sources) + + require.NotNil(t, req) + assert.Equal(t, "support-quality", req.Inputs.Name) + assert.Equal(t, "support-quality", req.Inputs.EvaluatorName) + assert.Equal(t, "gpt-4o", req.Inputs.Model) + assert.Equal(t, sources, req.Inputs.Sources) +} diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go new file mode 100644 index 00000000000..2cf1fe8e889 --- /dev/null +++ b/cli/azd/extensions/azure.ai.evaluations/internal/project/fingerprint_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// FingerprintGroup is covered in service_target_eval_test.go. These cover the +// file hash and the environment key it is stored under, which nothing did. + +// A fingerprint is compared against the one recorded at the last deploy, so +// identical content must hash identically and a single changed byte must not. +func TestFingerprint(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.jsonl") + b := filepath.Join(dir, "b.jsonl") + require.NoError(t, os.WriteFile(a, []byte(`{"query":"hi"}`), 0o600)) + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hi"}`), 0o600)) + + sumA, err := Fingerprint(a) + require.NoError(t, err) + sumB, err := Fingerprint(b) + require.NoError(t, err) + + assert.Equal(t, sumA, sumB, "same content, same fingerprint") + assert.Len(t, sumA, 64, "sha-256 as hex") + + require.NoError(t, os.WriteFile(b, []byte(`{"query":"hI"}`), 0o600)) + sumB, err = Fingerprint(b) + require.NoError(t, err) + assert.NotEqual(t, sumA, sumB, "one changed byte has to show") +} + +// A missing file names itself, because the usual cause is a catalog entry +// pointing at something that was moved or never generated. +func TestFingerprint_MissingFileNamesIt(t *testing.T) { + _, err := Fingerprint(filepath.Join(t.TempDir(), "gone.jsonl")) + + require.Error(t, err) + assert.Contains(t, err.Error(), "gone.jsonl") +} + +// The key goes into an azd environment file, which accepts only uppercase +// letters, digits and underscores. A name that reached it unmapped would +// produce a key azd cannot round-trip, and the artifact would look changed on +// every deploy. +func TestFingerprintKey_IsAValidEnvironmentKey(t *testing.T) { + tests := []struct { + kind, name, want string + }{ + {"dataset", "support-regression", "DATASET_SUPPORT_REGRESSION"}, + {"evaluator", "quality.v2", "EVALUATOR_QUALITY_V2"}, + {"dataset", "Mixed Case Name", "DATASET_MIXED_CASE_NAME"}, + // One rune maps to one underscore, so a multi-byte character does not + // widen the key. + {"eval", "unicode-caf\u00e9", "EVAL_UNICODE_CAF_"}, + } + + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + key := FingerprintKey(tt.kind, tt.name) + + assert.Equal(t, EnvKeyFingerprintPrefix+tt.want, key) + for _, r := range strings.TrimPrefix(key, EnvKeyFingerprintPrefix) { + assert.Truef(t, + (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_', + "%q is not allowed in an environment key", r) + } + }) + } +} + +// Two artifacts of different kinds can share a name, and they must not share a +// key — one would overwrite the other's recorded fingerprint. +func TestFingerprintKey_KindSeparatesTheNamespaces(t *testing.T) { + assert.NotEqual(t, + FingerprintKey("dataset", "quality"), + FingerprintKey("evaluator", "quality")) +} From 0fae3f96eef3f124832d690a9005b156e4867414 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:47:44 +0300 Subject: [PATCH 112/114] Check the run flags against the spec's own table The command tree and init's flags were pinned to the spec; the run table was not, and it is the one with placement rules a reader can check. --eval is given to every run command, --dataset to run start alone, --fail-on and --wait to run start and run show, --format to run output export. A third command growing one of those is a promise the spec does not make, and a missing one is a promise it does. Both directions are asserted, because the failure that actually happens is the quiet one: a flag added to the command next to the right one still compiles, still parses, and only stops making sense when someone reads the spec beside the help text. Placement only, not the whole flag list. Unlike init's table, this one is headed "Flag | Commands | Default" and documents defaults rather than enumerating flags, so pinning it whole would fail on flags the spec never meant to exclude. Everything passes as written, so this records the current state rather than fixing a drift -- which is the point of having it before the drift. --- .../internal/cmd/surface_test.go | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 1d6b8e969c3..cf415a846ed 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strings" "testing" @@ -241,6 +242,67 @@ func TestEvaluatorGenerateHasNoFromFlag(t *testing.T) { "the spec gives --from to dataset generate only") } +// The spec's run table says which commands carry which flag. Where it says +// "every", that is checkable; where it names two commands, a third carrying the +// flag is a promise the spec does not make and a missing one is a promise it +// does. +// +// This pins placement, not the whole flag list: unlike init's, the run table is +// headed "Flag | Commands | Default" and documents defaults rather than +// enumerating every flag. +func TestRunFlagsSitWhereTheSpecSaysTheyDo(t *testing.T) { + // Flag → exactly the run commands that may declare it. nil means every + // run command that does something. + placement := map[string][]string{ + "eval": nil, + "dataset": {"run start"}, + "fail-on": {"run start", "run show"}, + "wait": {"run start", "run show"}, + "format": {"run output export"}, + } + + // Every run command that actually runs, which is what "every run command" + // means — the bare groups take no flags. + var runCommands []string + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if strings.HasPrefix(path, "run") && cmd.RunE != nil { + runCommands = append(runCommands, path) + } + }) + require.NotEmpty(t, runCommands) + + for flag, allowed := range placement { + if allowed == nil { + allowed = runCommands + } + for _, path := range runCommands { + has := find(t, path).Flags().Lookup(flag) != nil + want := slices.Contains(allowed, path) + + switch { + case want && !has: + t.Errorf("%s must accept --%s; the spec's run table says so", path, flag) + case !want && has: + t.Errorf("%s declares --%s, which the spec gives only to %s", + path, flag, strings.Join(allowed, ", ")) + } + } + } +} + +// `--eval` is how a run command finds the eval, and the spec gives it to every +// one of them. Losing it from a single command makes that command unusable in a +// project with more than one eval. +func TestEveryRunCommandTakesEval(t *testing.T) { + walk(t, NewRootCommand(), nil, func(path string, cmd *cobra.Command) { + if !strings.HasPrefix(path, "run") || cmd.RunE == nil { + return + } + assert.NotNilf(t, cmd.Flags().Lookup("eval"), + "%s must accept --eval, which the spec gives to every run command", path) + }) +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() From f7e4d6a10617f8b84fb28be0d5e8b3c0a8d85c84 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 03:57:32 +0300 Subject: [PATCH 113/114] Repair the tagged suites, and stop them rotting unnoticed The live and hero suites drive the binary by writing flags as strings, so a removed flag still compiles there. go test ./... builds neither tag, so nothing said a word when this branch removed --eval-id and the generation spec file: 28 uses of --eval-id and two --config tests stayed behind, and every one of them would have failed at the first live run, in front of whoever happened to have the credentials. The hero suite was worse, because it asserts terminal output whole. It still expected the pre-split shape -- evals/support-agent-smoke.yaml plus evals/generate.yaml, a 'Planned evaluators' line, a 'support-agent-smoke' service, and --generation-model on init, which init no longer takes. It has been the only thing asserting what init prints, and it was asserting a version of init that stopped existing several commits ago. So: Scenario 1 now matches the spec's block against the single evals/eval.yaml and the support-agent-evals service, and a second test asserts the config file the spec prints beside it -- the traces source, the agent_name that scopes it, the max_traces cap, and the judge written per evaluator reference. It also asserts what must be absent: no datasets: catalog, because there is no file to register, and no target:, because a trace run invokes nothing. The two --config tests are deleted rather than repaired. They covered a generation spec file that no longer exists; generateFlags says so in as many words. The guard is the part that matters. A test in the default suite reads every flag literal in tests/ and checks it against what the commands actually declare, so a rename is caught by the person doing the renaming rather than by whoever next runs the live suite. Reintroducing --eval-id makes it fail naming the file and line. --- .../internal/cmd/surface_test.go | 48 +++++++++ .../tests/cli/generate_test.go | 33 ------ .../tests/cli/handoff_test.go | 10 +- .../tests/cli/run_ops_test.go | 30 +++--- .../tests/cli/run_output_test.go | 16 +-- .../tests/hero/init_test.go | 100 ++++++++++++------ 6 files changed, 146 insertions(+), 91 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index cf415a846ed..6623442e1c0 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -303,6 +303,54 @@ func TestEveryRunCommandTakesEval(t *testing.T) { }) } +// The tagged suites drive the binary by writing flags as strings, so a flag +// that is renamed or removed still compiles there and only fails when someone +// has the credentials to run them. +// +// That is not hypothetical. Removing `--eval-id` and the generation spec file +// left 28 uses of `--eval-id` and two `--config` tests behind in tests/cli, +// every one of which would have failed at the first live run — under `live` and +// `hero` tags that `go test ./...` never builds. This checks them from the +// default suite, where a rename is caught by the person doing the renaming. +func TestTaggedSuitesNameFlagsThatExist(t *testing.T) { + // Every flag any command declares, plus the globals azd contributes. + known := map[string]bool{ + "output": true, "no-prompt": true, "environment": true, + "cwd": true, "debug": true, "help": true, + } + walk(t, NewRootCommand(), nil, func(_ string, cmd *cobra.Command) { + cmd.Flags().VisitAll(func(f *pflag.Flag) { known[f.Name] = true }) + }) + + // Only string literals, which is how a test spells a flag it passes to the + // binary. Prose in a comment is not a flag. + literal := regexp.MustCompile(`"--([a-z][a-z0-9-]*)"`) + + err := filepath.WalkDir("../../tests", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".go") { + return nil + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(body), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "//") { + continue + } + for _, m := range literal.FindAllStringSubmatch(line, -1) { + assert.Truef(t, known[m[1]], + "%s:%d passes --%s, which no command declares", path, i+1, m[1]) + } + } + return nil + }) + require.NoError(t, err) +} + // find resolves a command path, failing the test when it does not exist. func find(t *testing.T, path string) *cobra.Command { t.Helper() diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go index 71466a9b203..3b7c80b7cd4 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/generate_test.go @@ -88,39 +88,6 @@ func TestCLIGenerateNamesTheArtifact(t *testing.T) { func TestCLIGenerateNoPromptNamesWhatIsMissing(t *testing.T) { r := requireFailure(t, runIn(t, t.TempDir(), "dataset", "generate", "d", "--no-prompt")) require.Contains(t, r.Combined(), "--generation-model") - require.Contains(t, r.Combined(), "generationModel", - "the message must name both ways of supplying it") -} - -// TestCLIGenerateReadsTheSpec proves the config file is loaded and that its -// entries are looked up by artifact name, rather than only the flags being read. -// -// The sample size is the clearest evidence: a value the service would reject is -// refused before any job is submitted, and it can only have come from the file. -func TestCLIGenerateReadsTheSpec(t *testing.T) { - dir := t.TempDir() - spec := filepath.Join(dir, "generate.yaml") - require.NoError(t, os.WriteFile(spec, []byte(` -generationModel: gpt-4o-mini -dataset: - spec-dataset: - sampleSize: 5 -`), 0o600)) - - r := requireFailure(t, runIn(t, dir, "dataset", "generate", "spec-dataset", "--config", spec)) - require.Contains(t, r.Combined(), "between 15 and 1000", - "the spec's sampleSize must be validated before a job is billed") -} - -// A spec that cannot be parsed has to name itself, or the caller is left -// guessing which of several YAML files the command choked on. -func TestCLIGenerateReportsAnUnparseableSpec(t *testing.T) { - dir := t.TempDir() - spec := filepath.Join(dir, "generate.yaml") - require.NoError(t, os.WriteFile(spec, []byte("dataset: [not-a-mapping\n"), 0o600)) - - r := requireFailure(t, runIn(t, dir, "dataset", "generate", "d", "--config", spec)) - require.Contains(t, r.Combined(), "generate.yaml") } // TestCLIGenerateFlagsAreScopedToTheirArtifact asserts the two commands do not diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go index 56b5465beba..6751ddd8bde 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/handoff_test.go @@ -26,7 +26,7 @@ func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { f := sharedEval(t) r := requireSuccess(t, run(t, - "run", "start", "--eval-id", f.EvalID, "--no-wait", "-o", "json")) + "run", "start", "--eval", f.EvalID, "--no-wait", "-o", "json")) var handoff struct { RunID string `json:"run_id"` @@ -45,12 +45,12 @@ func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { assert.NotEqual(t, "completed", handoff.Status) deferTeardown(func() { - runQuietly("run", "cancel", handoff.RunID, "--eval-id", f.EvalID) + runQuietly("run", "cancel", handoff.RunID, "--eval", f.EvalID) }) // The id it handed back has to be one the next step can use. shown := requireSuccess(t, run(t, - "run", "show", handoff.RunID, "--eval-id", f.EvalID, "-o", "json")) + "run", "show", handoff.RunID, "--eval", f.EvalID, "-o", "json")) var reattached struct { ID string `json:"id"` } @@ -65,7 +65,7 @@ func TestCLIStartNoWaitEmitsTheHandoff(t *testing.T) { func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "run", "start", "--eval-id", f.EvalID, "--no-wait")) + r := requireSuccess(t, run(t, "run", "start", "--eval", f.EvalID, "--no-wait")) assert.Contains(t, r.Stdout, "Reattach with: azd ai eval run show") assert.Contains(t, r.Stdout, f.EvalID, @@ -81,5 +81,5 @@ func TestCLIStartNoWaitTellsAPersonHowToReattach(t *testing.T) { } } require.NotEmpty(t, runID, "the run id must be printed:\n%s", r.Stdout) - deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval-id", f.EvalID) }) + deferTeardown(func() { runQuietly("run", "cancel", runID, "--eval", f.EvalID) }) } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go index 626ab9dbccd..3d68a583083 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_ops_test.go @@ -28,7 +28,7 @@ func TestCLIRunList(t *testing.T) { f := sharedEval(t) t.Run("table", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID)) + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID)) for _, header := range []string{"RUN ID", "NAME", "STATUS", "RESULTS"} { require.Containsf(t, r.Stdout, header, "the listing lost its %s column", header) } @@ -39,7 +39,7 @@ func TestCLIRunList(t *testing.T) { }) t.Run("json", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "-o", "json")) require.True(t, strings.HasPrefix(strings.TrimSpace(r.Stdout), "["), "a list must be a bare array, not the service's envelope") @@ -63,14 +63,14 @@ func TestCLIRunList(t *testing.T) { // The client has always taken a limit; until recently the command did not // expose one, so a service-side truncation would have passed unnoticed. t.Run("limit", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) + r := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) var runs []runSummary r.JSON(t, &runs) require.Len(t, runs, 1, "--limit must reach the service") }) t.Run("unknown eval is brief", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "list", "--eval-id", "eval_azdcli_no_such_eval")) + r := requireFailure(t, run(t, "run", "list", "--eval", "eval_azdcli_no_such_eval")) require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) require.Contains(t, r.Combined(), "eval_azdcli_no_such_eval") @@ -81,7 +81,7 @@ func TestCLIRunShow(t *testing.T) { f := sharedEval(t) t.Run("by run id", func(t *testing.T) { - r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval-id", f.EvalID)) + r := requireSuccess(t, run(t, "run", "show", f.FirstRunID, "--eval", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "status") require.Contains(t, r.Stdout, "completed") @@ -93,12 +93,12 @@ func TestCLIRunShow(t *testing.T) { // environment there is no remembered id to fall back on, so what is // exercised is the listing path. t.Run("defaults to the most recent run", func(t *testing.T) { - listed := requireSuccess(t, run(t, "run", "list", "--eval-id", f.EvalID, "--limit", "1", "-o", "json")) + listed := requireSuccess(t, run(t, "run", "list", "--eval", f.EvalID, "--limit", "1", "-o", "json")) var newest []runSummary listed.JSON(t, &newest) require.Len(t, newest, 1) - r := requireSuccess(t, run(t, "run", "show", "--eval-id", f.EvalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "show", "--eval", f.EvalID, "-o", "json")) var shown runSummary r.JSON(t, &shown) require.Equal(t, newest[0].ID, shown.ID, @@ -114,7 +114,7 @@ func TestCLIRunShow(t *testing.T) { // about 1700 characters of raw JSON — recorded in the report rather than // pinned here, since pinning it would make the length a requirement. t.Run("an unknown run id is reported, not silently replaced", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "show", "evalrun_azdcli_nope", "--eval", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope", "the failure must name the run that was asked for") require.NotContains(t, r.Combined(), f.FirstRunID, @@ -132,7 +132,7 @@ func TestCLIRunCancelAndDelete(t *testing.T) { f := sharedEval(t) t.Run("a finished run is refused", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "cancel", f.FirstRunID, "--eval", f.EvalID)) require.Contains(t, r.Combined(), "already finished") require.Contains(t, r.Combined(), "completed") }) @@ -148,21 +148,21 @@ func TestCLIRunCancelAndDelete(t *testing.T) { t.Run("an in-flight run is cancelled, and the delete is accepted", func(t *testing.T) { runID := startCancellableRun(t, f) - cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval-id", f.EvalID)) + cancelled := requireSuccess(t, run(t, "run", "cancel", runID, "--eval", f.EvalID)) require.Contains(t, cancelled.Stdout, runID) require.Contains(t, cancelled.Stdout, "is now") - shown := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) + shown := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) var after runSummary shown.JSON(t, &after) require.NotEqual(t, "completed", after.Status, "a cancelled run must not go on to complete") - deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval-id", f.EvalID)) + deleted := requireSuccess(t, run(t, "run", "delete", runID, "--eval", f.EvalID)) require.Contains(t, deleted.Stdout, "Deleted run") require.Contains(t, deleted.Stdout, runID) - still := requireSuccess(t, run(t, "run", "show", runID, "--eval-id", f.EvalID, "-o", "json")) + still := requireSuccess(t, run(t, "run", "show", runID, "--eval", f.EvalID, "-o", "json")) var survivor runSummary still.JSON(t, &survivor) t.Logf("the run is still readable after a successful delete (status %q); "+ @@ -172,12 +172,12 @@ func TestCLIRunCancelAndDelete(t *testing.T) { // Deleting is not undoable, so the id is required rather than defaulted to // whichever run happens to be newest. t.Run("delete requires the run id", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "delete", "--eval", f.EvalID)) require.Contains(t, r.Combined(), "accepts 1 arg") }) t.Run("deleting an unknown run is reported briefly", func(t *testing.T) { - r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval-id", f.EvalID)) + r := requireFailure(t, run(t, "run", "delete", "evalrun_azdcli_nope", "--eval", f.EvalID)) require.Contains(t, r.Combined(), "evalrun_azdcli_nope") require.Less(t, len(r.Combined()), 600, "a not-found must stay short, not dump the service body:\n%s", r.Combined()) diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go index c81258dd9b2..181cc5df69b 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/cli/run_output_test.go @@ -56,7 +56,7 @@ type resultsPayload struct { func TestCLIResultsShowRendersTheRows(t *testing.T) { f := sharedEval(t) - r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval-id", f.EvalID)) + r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, "--eval", f.EvalID)) require.Contains(t, r.Stdout, f.FirstRunID) require.Contains(t, r.Stdout, "Totals:") @@ -133,7 +133,7 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { } r := requireSuccess(t, run(t, "run", "output", "list", f.FirstRunID, - "--eval-id", f.EvalID, "--failed-only")) + "--eval", f.EvalID, "--failed-only")) if failing == 0 { // Saying so is not the same as printing an empty table. @@ -155,7 +155,7 @@ func TestCLIResultsShowFailedOnly(t *testing.T) { // they can decide what the rendered output should say. func resultsFor(t *testing.T, evalID, runID string) resultsPayload { t.Helper() - r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval-id", evalID, "-o", "json")) + r := requireSuccess(t, run(t, "run", "output", "list", runID, "--eval", evalID, "-o", "json")) var payload resultsPayload r.JSON(t, &payload) return payload @@ -166,7 +166,7 @@ func TestCLIResultsExport(t *testing.T) { t.Run("json to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "json")) + "--eval", f.EvalID, "--format", "json")) var exported struct { ID string `json:"id"` @@ -185,7 +185,7 @@ func TestCLIResultsExport(t *testing.T) { t.Run("csv to stdout", func(t *testing.T) { r := requireSuccess(t, run(t, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "csv")) + "--eval", f.EvalID, "--format", "csv")) rows, err := csv.NewReader(strings.NewReader(r.Stdout)).ReadAll() require.NoError(t, err, "--format csv must emit parseable CSV:\n%s", r.Stdout) @@ -202,7 +202,7 @@ func TestCLIResultsExport(t *testing.T) { path := filepath.Join(dir, "results.csv") r := requireSuccess(t, runIn(t, dir, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "csv", "--output-file", path)) + "--eval", f.EvalID, "--format", "csv", "--output-file", path)) require.Empty(t, strings.TrimSpace(r.Stdout), "--output-file redirects the payload; leaving it on stdout too would double it") @@ -214,13 +214,13 @@ func TestCLIResultsExport(t *testing.T) { t.Run("an unknown format is refused", func(t *testing.T) { r := requireFailure(t, run(t, "run", "output", "export", f.FirstRunID, - "--eval-id", f.EvalID, "--format", "xml")) + "--eval", f.EvalID, "--format", "xml")) require.Contains(t, r.Combined(), "json or csv") }) } func TestCLIResultsUnknownEvalIsBrief(t *testing.T) { - r := requireFailure(t, run(t, "run", "output", "list", "--eval-id", "eval_does_not_exist")) + r := requireFailure(t, run(t, "run", "output", "list", "--eval", "eval_does_not_exist")) require.Contains(t, r.Combined(), "eval_does_not_exist") require.NotContains(t, r.Combined(), "RESPONSE 404") } diff --git a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go index 8fe79718d53..6c02b3b8ecd 100644 --- a/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/tests/hero/init_test.go @@ -184,28 +184,63 @@ func normalize(s string) string { func TestHeroScenario1ColdStart(t *testing.T) { const ( agent = "support-agent" - model = "gpt-5.6-luna" + judge = "gpt-5.6-luna" ) dir := project(t, agent) - out, code := azdEval(t, dir, "init", "--target", agent, "--generation-model", model) + // The evaluator and judge are passed rather than prompted for, because the + // spec's two `?` lines are answers to prompts and a test has no terminal to + // answer them at. + out, code := azdEval(t, dir, "init", + "--target", agent, "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", judge) require.Zero(t, code, "init makes no service calls, so nothing can fail it here") want := `(✓) Done: Detected agent target: support-agent -(✓) Done: Detected model deployment: gpt-5.6-luna -(✓) Done: Planned evaluators: builtin.task_adherence, support-agent-quality (rubric) +(✓) Done: Using data source: traces (Application Insights) +(✓) Done: Judge model deployment: gpt-5.6-luna Created - evals/support-agent-smoke.yaml eval definition - evals/generate.yaml generation settings (15 samples, 1 rubric) - azure.yaml added service 'support-agent-smoke' + evals/eval.yaml evaluation configuration + azure.yaml added service 'support-agent-evals' -Next: azd ai eval dataset generate support-agent-smoke - azd ai eval evaluator generate support-agent-quality` +Next: azd up + azd ai eval run start` require.Equal(t, want, normalize(out)) } +// Scenario 1's second half: the eval.yaml the terminal block promised. The spec +// prints this file, so its shape is as much a promise as the output above — +// and it is the file a reader reviews before running `azd up`. +func TestHeroScenario1WritesTheDocumentedConfig(t *testing.T) { + dir := project(t, "support-agent") + + _, code := azdEval(t, dir, "init", + "--target", "support-agent", "--source", "traces", + "--evaluator", "builtin.task_adherence", "--judge-model", "gpt-5.6-luna") + require.Zero(t, code) + + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + require.NoError(t, err) + text := string(body) + + require.Contains(t, text, "name: support-agent-trace-eval") + require.Contains(t, text, "type: traces") + require.Contains(t, text, "agent_name: support-agent", + "a trace run has no target, so agent_name is what scopes it") + require.Contains(t, text, "max_traces: 20", + "a first run is bounded rather than taking the service default of 1000") + require.Contains(t, text, "evaluator: builtin.task_adherence") + require.Contains(t, text, "model: gpt-5.6-luna", + "the judge is written per evaluator reference as initialization_parameters.model") + + require.NotContains(t, text, "datasets:", + "there is no file to register, so the catalog is absent rather than empty") + require.NotContains(t, text, "target:", + "a trace run invokes nothing") +} + // `init` is offline, and being offline is the property that makes its output a // reviewable local diff. A service call here would also make the command fail // for a user who has not authenticated yet, which is exactly when they run it. @@ -213,7 +248,8 @@ func TestHeroInitMakesNoServiceCalls(t *testing.T) { dir := project(t, "support-agent") cmd := exec.Command("azd", "ai", "eval", "init", - "--target", "support-agent", "--generation-model", "m") + "--target", "support-agent", "--evaluator", "builtin.task_adherence", + "--judge-model", "m") cmd.Dir = dir // A proxy pointing nowhere fails any outbound request, so a command that // stays offline is unaffected and one that does not cannot be mistaken for @@ -234,17 +270,18 @@ func TestHeroInitMakesNoServiceCalls(t *testing.T) { func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { dir := project(t, "support-agent") - _, code := azdEval(t, dir, "init", "--target", "support-agent", "--generation-model", "m") + _, code := azdEval(t, dir, "init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m") require.Zero(t, code) root, err := os.ReadFile(filepath.Join(dir, "azure.yaml")) require.NoError(t, err) text := string(root) - require.Contains(t, text, "support-agent-smoke:", - "the service key is the eval's name") + require.Contains(t, text, "support-agent-evals:", + "the service is named for the agent it evaluates") require.Contains(t, text, "host: azure.ai.eval") - require.Contains(t, text, "$ref: ./evals/support-agent-smoke.yaml") + require.Contains(t, text, "$ref: ./evals/eval.yaml") // azd owns the edit, so everything the project already declared survives it. require.Contains(t, text, "name: support-app") @@ -252,14 +289,15 @@ func TestHeroInitWiresTheServiceIntoTheProject(t *testing.T) { require.Contains(t, text, "host: azure.ai.agent") // The eval reads both, so azd has to deploy both first. - require.Regexp(t, `(?s)support-agent-smoke:.*uses:.*ai-project.*support-agent`, text) + require.Regexp(t, `(?s)support-agent-evals:.*uses:.*ai-project.*support-agent`, text) } // Running `init` twice must not deploy the same eval twice. The service key is // the eval's name, so the second run recognizes its own work. func TestHeroInitIsIdempotent(t *testing.T) { dir := project(t, "support-agent") - args := []string{"init", "--target", "support-agent", "--generation-model", "m"} + args := []string{"init", "--target", "support-agent", + "--evaluator", "builtin.task_adherence", "--judge-model", "m"} _, code := azdEval(t, dir, args...) require.Zero(t, code) @@ -284,7 +322,7 @@ func TestHeroInitIsIdempotent(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, strings.Count(string(third), "host: azure.ai.eval"), "a second eval service would deploy the same eval twice") - require.Contains(t, normalize(out), "already declares service 'support-agent-smoke'") + require.Contains(t, normalize(out), "already declares service 'support-agent-evals'") } // Evals attach to a project; they do not create one. Naming the command that @@ -310,16 +348,19 @@ func TestHeroInitExplicitEvaluatorsOptOutOfGeneration(t *testing.T) { dir := project(t, "support-agent") out, code := azdEval(t, dir, "init", - "--target", "support-agent", "--generation-model", "m", + "--target", "support-agent", "--judge-model", "m", "--evaluator", "builtin.task_adherence") require.Zero(t, code, out) text := normalize(out) - require.Contains(t, text, "Planned evaluators: builtin.task_adherence") - require.NotContains(t, text, "(rubric)") require.NotContains(t, text, "evaluator generate", "nothing was scheduled to be generated, so nothing should be suggested") - require.Contains(t, text, "generation settings (15 samples, 0 rubric)") + + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) + require.NoError(t, err) + require.Contains(t, string(body), "evaluator: builtin.task_adherence") + require.NotContains(t, string(body), "support-agent-quality", + "the default rubric was replaced, not added to") } // A supplied dataset is not generated either, so `init` has nothing left to @@ -329,20 +370,19 @@ func TestHeroInitSuppliedDatasetIsNotGenerated(t *testing.T) { dir := project(t, "support-agent") out, code := azdEval(t, dir, "init", - "--target", "support-agent", "--generation-model", "m", + "--target", "support-agent", "--judge-model", "m", "--dataset", "prod-golden", "--evaluator", "builtin.task_adherence") require.Zero(t, code, out) - require.NotContains(t, normalize(out), "dataset generate") - body, err := os.ReadFile(filepath.Join(dir, "evals", "generate.yaml")) - require.NoError(t, err) - require.NotContains(t, string(body), "dataset:", - "nothing is left to generate, so the spec declares nothing") + text := normalize(out) + require.NotContains(t, text, "dataset generate") + require.Contains(t, text, "Next: azd up", + "with nothing left to generate, the next step is the deploy") - eval, err := os.ReadFile(filepath.Join(dir, "evals", "support-agent-smoke.yaml")) + body, err := os.ReadFile(filepath.Join(dir, "evals", "eval.yaml")) require.NoError(t, err) - require.Contains(t, string(eval), "name: prod-golden") - require.NotContains(t, string(eval), "source:", + require.Contains(t, string(body), "dataset: prod-golden") + require.NotContains(t, string(body), "source:", "a registered dataset has nothing to upload") } From 21a8b5f40cb1565e3559dd04501240fdd3d95494 Mon Sep 17 00:00:00 2001 From: mohessie Date: Thu, 6 Aug 2026 04:06:27 +0300 Subject: [PATCH 114/114] Suggest commands this extension actually serves Three printed instructions pointed at `azd ai dataset ...`: init's next step after scaffolding, the --no-wait reattach line, and the --from file error. No installed extension serves that namespace. This one is `ai.eval`, so its commands are `azd ai eval dataset ...`, and every user who followed one of those lines would have got "unknown command". Two of the three are mine, from earlier in this branch. They were written for the world after the dataset commands move to their own extension -- which has no registry entry, no milestone and no dependency entry, so it is not the world anyone is typing into. TestSuggestedCommandsExist is the test that exists to stop exactly this, and it missed all three because it matched on `azd ai eval` and these did not say `eval`. It searches `azd ai` now, and a namespace this extension does not serve fails by name. Reintroducing the init string makes it fail. The --from file error also lost its `` and `` placeholders, which the sibling placeholder check caught once the string came into scope. It names the command and stops; the flags are in that command's help. --- .../internal/cmd/generate.go | 4 ++-- .../internal/cmd/generate_sources_test.go | 2 +- .../azure.ai.evaluations/internal/cmd/init.go | 2 +- .../internal/cmd/init_test.go | 2 +- .../internal/cmd/surface_test.go | 22 +++++++++++++++++-- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go index ee07d39308e..d88cc539cc1 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate.go @@ -239,7 +239,7 @@ func refuseUnbuildableSources(kinds []string) error { "agent": "--from agent needs a target agent; pass --target, " + "or declare one under target: in eval.yaml", "file": "--from file is not a generation source; " + - "register the file with `azd ai dataset create --file ` instead", + "register the file with `azd ai eval dataset create` instead", } messages := make([]string, 0, len(kinds)) for _, k := range kinds { @@ -285,7 +285,7 @@ func (ec *evalContext) generateDataset( return nil, fmt.Errorf("submitting the data generation job: %w", err) } if noWait { - reportSubmitted(out, "azd ai dataset", job.ID) + reportSubmitted(out, "azd ai eval dataset", job.ID) return nil, nil } diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go index 4165df62754..ba8d5861714 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/generate_sources_test.go @@ -35,7 +35,7 @@ func TestRefuseUnbuildableSources(t *testing.T) { }{ {"prompt", "--agent-instruction"}, {"agent", "--target"}, - {"file", "azd ai dataset create"}, + {"file", "azd ai eval dataset create"}, } for _, tt := range tests { diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go index 46ec0c0047c..306232e24e3 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init.go @@ -389,7 +389,7 @@ func (s scaffold) evaluatorNames() []string { func (s scaffold) nextSteps() []string { var steps []string if s.generateDataset { - steps = append(steps, "azd ai dataset generate "+s.datasetName) + steps = append(steps, "azd ai eval dataset generate "+s.datasetName) } if s.generateRubric { steps = append(steps, "azd ai eval evaluator generate "+s.rubricName) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go index 4f714bfcf03..3b505df7f4d 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/init_test.go @@ -144,7 +144,7 @@ func TestScaffold_NextStepsOfferOnlyWhatIsScheduled(t *testing.T) { evalName: "support-agent-smoke", target: "support-agent", judgeModel: "m", }) require.Equal(t, []string{ - "azd ai dataset generate support-agent-smoke", + "azd ai eval dataset generate support-agent-smoke", "azd ai eval evaluator generate support-agent-quality", }, plan.nextSteps()) }) diff --git a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go index 6623442e1c0..bf6be972f2a 100644 --- a/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go +++ b/cli/azd/extensions/azure.ai.evaluations/internal/cmd/surface_test.go @@ -370,9 +370,15 @@ func find(t *testing.T, path string) *cobra.Command { // it was the one thing guaranteed to fail. Nothing catches that: the string // compiles, the command that prints it succeeds, and only someone following // the advice finds out. +// This extension's namespace is `ai.eval`, so every command it can suggest +// begins `azd ai eval`. Anchoring on that prefix is what caught the renamed +// command above — and anchoring only on it is what let three suggestions +// through pointing at `azd ai dataset`, a namespace no installed extension +// serves. So the prefix checked is `azd ai`, and anything under it that is not +// this extension's own is a command nobody can run. func TestSuggestedCommandsExist(t *testing.T) { root := "../.." - pattern := regexp.MustCompile("azd ai eval ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") + pattern := regexp.MustCompile("azd ai ([a-z][a-z0-9-]*(?: [a-z][a-z0-9-]*)*)") err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -395,6 +401,18 @@ func TestSuggestedCommandsExist(t *testing.T) { } for _, m := range pattern.FindAllStringSubmatch(line, -1) { words := strings.Fields(m[1]) + + // `ai.eval` is this extension's namespace, so it is the only + // thing under `azd ai` that resolves here. Another namespace is + // a command this extension cannot suggest, whether or not some + // future extension serves it. + if len(words) == 0 || words[0] != "eval" { + t.Errorf("%s suggests `azd ai %s`, which is not this extension's "+ + "namespace; commands here are `azd ai eval ...`", path, m[1]) + continue + } + words = words[1:] + // Trim trailing prose: "run start" is a command, "run start // and summarize" is a sentence that begins with one. for len(words) > 0 { @@ -407,7 +425,7 @@ func TestSuggestedCommandsExist(t *testing.T) { words = words[:len(words)-1] } assert.NotEmpty(t, words, - "%s suggests `azd ai eval %s`, which is not a command", path, m[1]) + "%s suggests `azd ai %s`, which is not a command", path, m[1]) } } return nil