ci: verify examples and Copilot integration tests

.github/workflows/live-examples.yml

@@ -0,0 +1,85 @@
+name: Live examples
+
+on:
+  schedule:
+    - cron: "17 9 * * 1-5"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  copilot-requests: write
+
+concurrency:
+  group: live-examples
+  cancel-in-progress: false
+
+jobs:
+  verify:
+    name: Verify examples
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 22
+
+      - name: Install Copilot CLI
+        shell: bash
+        run: npm install --global @github/copilot@1.0.75
+
+      - name: Verify examples
+        id: examples
+        continue-on-error: true
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          set +e
+          go run ./cmd/verifyexamples \
+            --parallel 4 \
+            --log live-examples.log \
+            --md live-examples.md
+          status=$?
+          set -e
+
+          if [[ -f live-examples.md ]]; then
+            cat live-examples.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "The example verifier exited before it could write a report." >> "$GITHUB_STEP_SUMMARY"
+          fi
+          exit "$status"
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: live-example-results
+          path: |
+            live-examples.log
+            live-examples.md
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Check results
+        if: always()
+        shell: bash
+        env:
+          EXAMPLES_OUTCOME: ${{ steps.examples.outcome }}
+        run: |
+          if [[ "$EXAMPLES_OUTCOME" != "success" ]]; then
+            echo "Example verification failed."
+            exit 1
+          fi

.github/workflows/test.yml

@@ -40,8 +40,41 @@ jobs:
     - name: Run tests
       run: go test -v ${{ matrix.target.name != 'windows-arm64' && '-race' || '' }} -shuffle=on ./...
 
+  copilot-integration:
+    name: GitHub Copilot integration
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      copilot-requests: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        with:
+          persist-credentials: false
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 22
+
+      - name: Install Copilot CLI
+        shell: bash
+        run: npm install --global @github/copilot@1.0.75
+
+      - name: Run Copilot integration tests
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: go test -count=1 -v ./provider/copilotprovider -run '^TestE2E_'
+
   conclusion:
-    needs: [test]
+    needs: [test, copilot-integration]
     runs-on: ubuntu-latest
     if: ${{ always() }}
     steps:

cmd/verifyexamples/examples.go

@@ -708,14 +708,13 @@ var agentsExamples = []ExampleDefinition{
 		SkipReason:                   "Spawns a local stdio MCP server subprocess (go run) that requires live Foundry credentials.",
 	},
 	{
-		Name:        "02_agents_providers_github_copilot",
-		ProjectPath: "examples/02-agents/providers/github-copilot",
-		Inputs:      inputLines("Y", "Y", "Y"),
-		InputDelay:  3 * time.Second,
-		ExpectedOutputDescription: []string{
-			"The output should contain a response listing files in the current directory.",
-			"The output should not contain error messages or stack traces.",
-		},
+		Name:            "02_agents_providers_github_copilot",
+		ProjectPath:     "examples/02-agents/providers/github-copilot",
+		IsDeterministic: true,
+		Inputs:          inputLines("Y", "Y", "Y"),
+		InputDelay:      3 * time.Second,
+		MustContain:     []string{"main.go"},
+		MustNotContain:  []string{"panic:", "Permission denied"},
 	},
 	{
 		Name:                         "02_agents_skills_step01_file_based_skills",
@@ -927,19 +926,16 @@ var workflowExamples = []ExampleDefinition{
 		},
 	},
 	{
-		Name:        "03_workflows_checkpoint_checkpoint_with_human_in_the_loop",
-		ProjectPath: "examples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop",
-		Inputs:      inputLines("50", "25", "40", "45", "42", "50", "25", "40", "45", "42"),
-		InputDelay:  time.Second,
+		Name:            "03_workflows_checkpoint_checkpoint_with_human_in_the_loop",
+		ProjectPath:     "examples/03-workflows/checkpoint/checkpoint_with_human_in_the_loop",
+		IsDeterministic: true,
+		Inputs:          inputLines("50", "25", "40", "45", "42", "50", "25", "40", "45", "42"),
+		InputDelay:      time.Second,
 		MustContain: []string{
 			"Workflow completed with result:",
 			"Number of checkpoints created:",
 			"Restored run completed with result:",
 		},
-		ExpectedOutputDescription: []string{
-			"The output should show a number guessing workflow with higher/lower hints that eventually reaches the correct number.",
-			"The output should demonstrate checkpoint save and restore behavior.",
-		},
 	},
 	{
 		Name:        "03_workflows_concurrent_map_reduce",
@@ -992,14 +988,12 @@ var workflowExamples = []ExampleDefinition{
 		MustContain:     []string{"[intermediate] normalized hello workflow", "[final] Summary of"},
 	},
 	{
-		Name:        "03_workflows_human_in_the_loop_human_in_the_loop_basic",
-		ProjectPath: "examples/03-workflows/human-in-the-loop/human_in_the_loop_basic",
-		Inputs:      inputLines("50", "25", "40", "45", "42"),
-		InputDelay:  time.Second,
-		MustContain: []string{"found in"},
-		ExpectedOutputDescription: []string{
-			"The output should show a number guessing game with higher/lower hints that eventually reaches the correct number 42.",
-		},
+		Name:            "03_workflows_human_in_the_loop_human_in_the_loop_basic",
+		ProjectPath:     "examples/03-workflows/human-in-the-loop/human_in_the_loop_basic",
+		IsDeterministic: true,
+		Inputs:          inputLines("50", "25", "40", "45", "42"),
+		InputDelay:      time.Second,
+		MustContain:     []string{"42 found in 5 tries!"},
 	},
 	{
 		Name:            "03_workflows_human_in_the_loop_imperative_request",