From 8138d37aecac35fde0bccd1df54a0b5f838eba37 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 8 Oct 2026 10:08:22 -0700 Subject: [PATCH 1/5] docs: add CITATION.cff and a Citation section to the README Co-Authored-By: Claude Opus 5.5 --- CITATION.cff | 20 ++++++++++++++++++++ README.md | 14 ++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..d14b0e62 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,20 @@ +cff-version: 1.2.0 +message: "If you use AgentEnv Framework in your work, please cite it as below." +type: software +title: "AgentEnv Framework" +abstract: "AgentEnv Framework is a Python SDK and CLI for building, deploying and running agentic environments and the tasks that grade agents inside them." +authors: + - name: "Scale AI" +repository-code: "https://github.com/scaleapi/agentenv-framework" +license: Apache-2.0 +date-released: 2026-09-30 +keywords: + - reinforcement-learning + - rl-environments + - ai-agents + - llm-agents + - agent-evaluation + - mcp + - model-context-protocol + - a2a + - sandbox diff --git a/README.md b/README.md index 162aca0c..c517763e 100644 --- a/README.md +++ b/README.md @@ -355,3 +355,17 @@ A release is a version bump in both `pyproject.toml` files plus a `vX.Y.Z` tag. ### Support, security, license Ask questions in [Discussions Q&A](https://github.com/scaleapi/agentenv-framework/discussions/categories/q-a), and report bugs with the [bug report form](https://github.com/scaleapi/agentenv-framework/issues/new?template=bug_report.yml). Report vulnerabilities privately through the contact in [SECURITY.md](https://github.com/scaleapi/agentenv-framework/blob/main/SECURITY.md), not in public issues; only the latest release is supported, so reproduce against it first. Contributors follow [CONTRIBUTING.md](https://github.com/scaleapi/agentenv-framework/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/scaleapi/agentenv-framework/blob/main/CODE_OF_CONDUCT.md); every pull request needs a code-owner review. agent-env and agentenv-framework-protocol are licensed under the Apache License 2.0; see [`LICENSE`](https://github.com/scaleapi/agentenv-framework/blob/main/LICENSE) and [`NOTICE`](https://github.com/scaleapi/agentenv-framework/blob/main/NOTICE). Their third-party dependencies and those dependencies' licenses are listed in [`THIRD_PARTY_NOTICES.md`](https://github.com/scaleapi/agentenv-framework/blob/main/THIRD_PARTY_NOTICES.md). + +## Citation + +To cite AgentEnv Framework, use the metadata in [`CITATION.cff`](https://github.com/scaleapi/agentenv-framework/blob/main/CITATION.cff). GitHub's **Cite this repository** button in the repository sidebar exports it as APA or BibTeX: + +```bibtex +@software{agentenv_framework, + author = {{Scale AI}}, + title = {AgentEnv Framework}, + year = {2026}, + url = {https://github.com/scaleapi/agentenv-framework}, + license = {Apache-2.0} +} +``` From 2c3f563b116cfa1db104d222e86dacb6a53644a2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 8 Oct 2026 10:16:56 -0700 Subject: [PATCH 2/5] docs: move the plugin contract from the README to PLUGINS.md Co-Authored-By: Claude Opus 5.5 --- .github/scripts/check_plugin_api.py | 4 +- .github/workflows/local-backends.yml | 2 +- .github/workflows/plugin-api.yml | 2 +- AGENTS.md | 6 +- CONTRIBUTING.md | 2 +- PLUGINS.md | 282 +++++++++++++++++++++++ README.md | 286 +----------------------- src/agent_env/cli/plugin.py | 2 +- src/agent_env/plugins/__init__.py | 2 +- src/agent_env/plugins/_cli.py | 2 +- src/agent_env/plugins/_registration.py | 2 +- src/agent_env/plugins/_report.py | 2 +- src/agent_env/plugins/_requirements.py | 2 +- tst/installer/test_container_journey.py | 12 +- tst/unit/cli/plugin_report_test.py | 6 +- tst/unit/config/test_unread_keys.py | 4 +- tst/unit/test_check_plugin_api.py | 8 +- 17 files changed, 313 insertions(+), 313 deletions(-) create mode 100644 PLUGINS.md diff --git a/.github/scripts/check_plugin_api.py b/.github/scripts/check_plugin_api.py index aa20ab55..a58faac6 100644 --- a/.github/scripts/check_plugin_api.py +++ b/.github/scripts/check_plugin_api.py @@ -1,6 +1,6 @@ """Fail a change that breaks the plugin surface, unless its title marks the break with ``!``. -The surface is ``BASES`` and ``USED``, the list in README.md "Plugin compatibility"; a unit test +The surface is ``BASES`` and ``USED``, the list in PLUGINS.md "Plugin compatibility"; a unit test holds the two together. Both trees are read statically with griffe, which reports what breaks a caller, and the subclass rules below add what breaks a plugin's subclass of a base. A title in the form ``type(scope)!: summary`` marks the break deliberate: the breaks are printed and the check passes. @@ -617,7 +617,7 @@ def main() -> int: print("plugin API: the title has a ! but not in the form type(scope)!: summary, with the type one of " + ", ".join(TYPES)) print("plugin API: if the break is deliberate, mark it with ! in the pull request title, as in " - "feat(plugins)!: …; see \"Plugin compatibility\" in README.md") + "feat(plugins)!: …; see \"Plugin compatibility\" in PLUGINS.md") _report("error", f"{count}; mark it with ! in the title if it is deliberate", found) return 1 diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index d931ef8e..a6445240 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -186,7 +186,7 @@ jobs: # moves; diffing against base.sha would count the base's own later commits as changes. BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }} run: | - paths='^(src/agent_env/cli/(__init__|__main__|_installers|_plugin_changes|plugin)\.py|src/agent_env/(plugins|config)/|src/agent_env/py\.typed|packages/agentenv-protocol/|tst/installer/|tst/util/capabilities\.py|pyproject\.toml|uv\.lock|Makefile|README\.md|\.github/(workflows/local-backends\.yml|scripts/check_skip_policy\.py))' + paths='^(src/agent_env/cli/(__init__|__main__|_installers|_plugin_changes|plugin)\.py|src/agent_env/(plugins|config)/|src/agent_env/py\.typed|packages/agentenv-protocol/|tst/installer/|tst/util/capabilities\.py|pyproject\.toml|uv\.lock|Makefile|PLUGINS\.md|\.github/(workflows/local-backends\.yml|scripts/check_skip_policy\.py))' if ! changed=$(git diff --name-only "$BASE" HEAD 2>/dev/null); then echo "plugins=true" >> "$GITHUB_OUTPUT" elif grep -qE "$paths" <<< "$changed"; then diff --git a/.github/workflows/plugin-api.yml b/.github/workflows/plugin-api.yml index d431ad97..816e9360 100644 --- a/.github/workflows/plugin-api.yml +++ b/.github/workflows/plugin-api.yml @@ -1,6 +1,6 @@ name: Plugin API -# Fails a change that breaks the plugin surface (README "Plugin compatibility") unless its title +# Fails a change that breaks the plugin surface (PLUGINS.md "Plugin compatibility") unless its title # marks the break with `!`. A workflow of its own, so that a title edit re-runs only this check. # On main it checks the squash commit against its parent, a record of what reached history. diff --git a/AGENTS.md b/AGENTS.md index 9cfca05b..f591a04f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -183,7 +183,7 @@ skips for the allowed capabilities pass, and the fast job allows no skips under touches the plugin CLI, `agent_env/plugins`, the tier itself or the lockfile, and allows no skips; a `changes` job decides. `.github/workflows/plugin-api.yml` runs the `plugin-api` job on every pull request, title edits included, and on `main`: `.github/scripts/check_plugin_api.py` compares the -plugin surface (its `BASES` and `USED`, which a unit test holds to the README "Plugin compatibility" +plugin surface (its `BASES` and `USED`, which a unit test holds to the PLUGINS.md "Plugin compatibility" list) between `HEAD^1` and `HEAD` with griffe, pinned in the `dev` extra, and fails on a break the title does not mark with `!`. `.github/workflows/clean-install.yml` runs the `clean-install` job on every pull request and on `main`: `.github/scripts/clean_install.py` builds both distributions as the @@ -215,5 +215,5 @@ Actions are pinned to commit SHAs. ## Docs `README.md` is the user-facing front door (configuration, custom envs, steps, artifacts and -providers, the CLI). `packages/agentenv-protocol/README.md` documents the -protocol and the A2A agent framework. +providers, the CLI). `PLUGINS.md` is the plugin contract. `packages/agentenv-protocol/README.md` +documents the protocol and the A2A agent framework. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3ee3ff19..a6a33051 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ must name it (`agentenv-capability-missing: `, see `tst/util/capabilities. target as received`. Types in use: `feat`, `fix`, `refactor`, `docs`, `test`, `build`, `ci`, `chore`, `perf`, `security`. The title becomes the commit title. - Mark a breaking change with `!` after the scope: `feat(plugins)!: summary`. The `plugin-api` job - fails a pull request that breaks the plugin surface (README "Plugin compatibility") without it, + fails a pull request that breaks the plugin surface (PLUGINS.md "Plugin compatibility") without it, lists what broke, and runs again when you edit the title. To run it before you push: `.venv/bin/python .github/scripts/check_plugin_api.py --base origin/main --title ""`. - Commit with an email address you are comfortable publishing; the history of this repository is diff --git a/PLUGINS.md b/PLUGINS.md new file mode 100644 index 00000000..a61d4282 --- /dev/null +++ b/PLUGINS.md @@ -0,0 +1,282 @@ +# Plugin contract + +What a package that extends agent-env relies on. Unit tests and the plugin-API check in CI hold the code to these sections, so a pull request that changes the contract changes this file too. + +## Register types from an installed package + +An installed distribution registers envs, task steps, artifacts, sandbox, state and environment providers and explorer plugins by declaring entry points, with no config. The group says what kind of thing it is, the entry-point name is the registry key, and the value is the class: + +```toml +[project.entry-points."agent_env.envs"] +browser = "agentenv_browser.env:BrowserEnv" + +[project.entry-points."agent_env.task_steps"] +browser_navigate = "agentenv_browser.steps:NavigateTaskStep" +``` + +| Group | Value | Name | +|---|---|---| +| `agent_env.envs` | `Env` subclass with its own `type`, implementing `from_dict` | must equal the class's `type`, the spelling its documents are written under | +| `agent_env.task_steps` | `TaskStep` subclass with its own `type`, implementing `execute` and `from_dict` | must equal the class's `type` | +| `agent_env.artifacts` | `Artifact` subclass with its own `type` field default | the registry key; a class may also register under extra names, such as a legacy spelling, as long as its own `type` default resolves to it and its `type` field accepts each extra name | +| `agent_env.sandbox_providers` | `SandboxProvider` subclass implementing `create_sandbox` | the registry key; it must equal the `.type` of the sandboxes the provider produces, checked on the first `create_*` call | +| `agent_env.state_providers` | `EnvStateProvider` subclass implementing `acquire` and `_teardown` | must equal the class's `type`, checked at registration | +| `agent_env.env_providers` | `EnvironmentProvider` subclass implementing `deploy` and `close` | must equal the class's `type`, which its records carry as `env_provider_type`; checked at registration | +| `agent_env.explorer_plugins` | `ExplorerPlugin` subclass with its own `type`, implementing `router` | must equal the class's `type` | + +- The class must implement every abstract method of its base, and, for envs and task steps, `from_dict` as a classmethod, which the base defines only to raise. A plugin that does not is `failed` with `invalid-plugin`, naming what is missing, for example `GradeTaskStep must implement execute and from_dict`. An `impls` entry or provider table naming such a class is a `ConfigError`. +- Each registry takes the built-ins first, then plugins, then config. A plugin cannot replace a built-in: it is skipped with a warning, however many distributions claim that name. +- A name that two installed distributions register in one group, or that one declares twice, is left out of the registry with a warning, and the rest of the group, built-ins included, loads. An explorer plugin under that name is not mounted, so its routes are absent. Resolving the name fails like a plugin that did not load, naming each claimant and its version, for example `Unknown env type: browser (2 installed plugins register 'browser' in agent_env.envs: …)`, and says to remove all but one with `agent-env plugin remove PACKAGE`, or, when one package declares the name twice, to report it to the package's author. Neither claimant is picked, because the order entry points are found in is not fixed. +- Config cannot settle a conflicted name. An `impls` entry, a provider table, an `[explorer.plugins]` impl or an `[artifacts] type_aliases` entry mapping to that name is checked as usual, then skipped with a warning. A `type_aliases` entry whose old spelling is that name is a `ConfigError`, as it is when one plugin registers it. +- A plugin that fails to import, fails the checks above, or (for explorer plugins) fails to construct is skipped with a warning and the others still load. So is one whose requirement on agent-env excludes the installed version, before it is imported (see [Plugin compatibility](#plugin-compatibility)). Resolving its name then reports the recorded error, for example `Unknown env type: browser (registered by 'browser' from agentenv-web 2.1.0 (…) but failed to load: ModuleNotFoundError(…))`. +- Config replaces a plugin's class with a warning, except under a conflicted name: an `impls` entry of the same `type`, or a `[sandbox.providers.<name>]` / `[state.providers.<name>]` table carrying `impl`. Naming the plugin's own class is silent. A provider table without `impl` configures the plugin's provider the way it configures a built-in's, which is where per-deployment settings such as a secret name belong. A config-only provider table, or an `[artifacts] type_aliases` entry, that points at a plugin which failed to load is skipped with a warning rather than failing the whole registry. +- `agent_env.plugins.load_failures()` lists every plugin that did not take effect in the registries the current `Config` has built (failed to load, validate or construct, clashed with a built-in, or claims a name another entry point claims), by group and name, with the reason. The record belongs to the `Config`, so `reset_config()` starts a new one. A deployment that ships its plugins can build its registries at startup and assert it is empty. +- `agent_env.plugins.inventory()` lists every installed distribution that declares entry points in these groups, with a status for each contribution and a `code` saying why (see [Plugin report format](#plugin-report-format)): `active`; `replaced` (config names a different class, and `replaced_by` says where); `failed`; `skipped` (a built-in owns the name); `conflict`; `blocked` (the group's real build fails, for example because its config table is invalid, so nothing in the group loads); or `unloaded`. It builds the registries on a throwaway `Config` that reads the same document, so the `Config` in use keeps its registries and its `load_failures()`. `inventory(load=False)` reads installed metadata only and runs no plugin code, so a status that needs a build is `unloaded`. With the default `load=True` the plugins are imported and the explorer plugins constructed, with whatever process-wide effects that has. `discovery_errors` names each group whose installed entry points could not be read at all, so none of its plugins is listed. +- Discovery reads installed metadata and imports nothing. When entry points are loaded is unspecified: today each group is imported when its registry is first built, but a plugin must not depend on that. +- Loading a plugin never changes which config a `Config` reads: the `Config` resolves its document before it imports any plugin. A plugin package that sets `AGENT_ENV_CONFIG` on import affects only configs built afterwards, such as after `reset_config()`. + +The distribution that declares the entry points is the plugin, so `pip uninstall` removes it completely, apart from any `[plugins.<package>]` table in the config file. CLI commands are a separate contribution, described next. + +## CLI plugins, root options, explorer routes + +Installed packages add commands through two entry-point groups: + +```toml +[project.entry-points."agent_env.cli_plugins"] +my-tools = "mycorp_demo.cli:my_tools" +[project.entry-points."agent_env.cli_root_options"] +tenant = "mycorp_demo.cli:tenant_option" +``` + +`agent_env.cli_plugins` entries are click commands or groups added next to the built-ins, under the entry-point name whatever the command object is called: `my-tools` above is `agent-env my-tools`. `agent_env.cli_root_options` entries are optional `click.Option` instances with `expose_value=False`; their callback runs before the subcommand, so it can set `AGENT_ENV_CONFIG` and call `agent_env.config.reset_config()` to select the config for the whole process. Plugins load when `agent_env.cli` is imported, after the built-ins. Clash rules: a plugin command whose entry-point name core already uses, or a flag core owns, is skipped with a warning on stderr, and core wins. Two different root options on the same flag are both left off with a warning, and `agent-env plugin list` reports each as a `conflict`; the CLI still starts, so `agent-env plugin remove` can settle it. The same `click.Option` object exported by two entry points, from one package or two, is attached once. Nothing grafts onto existing groups. + +Explorer routes: subclass `agent_env.explorer.plugin.ExplorerPlugin`, set the `type` ClassVar, and return a FastAPI `APIRouter` from the `router` property. List it under `[explorer.plugins] impls` or declare an `agent_env.explorer_plugins` entry point; `from_config()` takes no arguments, and a plugin reads its own settings with `agent_env.plugins.settings` (see [Plugin settings](#plugin-settings)). `agent-env up --no-bootstrap` mounts it before the core routers and needs no Docker. An entity id can contain `/` (a bundle's ids are `@local/<package>/<name>`, and so are the instance ids of its runs), so in an explorer URL an id is one percent-encoded path segment, with `/` sent as `%2F`. Under the core prefixes `/api/v1/{artifacts,envs,tasks,agents,evals,task-instances}/` the explorer routes on the encoded path, so a plugin route there receives the segment still encoded and decodes it with `urllib.parse.unquote`. + +## Bundles from installed packages + +A package ships bundles by naming the package that holds their folders: + +```toml +[project.entry-points."agent_env.bundles"] +triage = "mycorp_demo.bundles" +``` + +- The bundle `triage` is the folder `mycorp_demo/bundles/triage/`. The entry-point name is both the bundle's name and its folder's name, so it may contain `-`. +- The value names a package, never an object. The folder is found without importing any of the package's code, so listing bundles runs nothing. +- The folder ships in the wheel as package data, and the package must be installed unpacked. +- An installed bundle's ids are rooted at its distribution, wherever the package is installed: `triage` above writes `@local/mycorp-demo/triage/...`. +- `agent-env run triage` runs it, and `agent-env run` with no argument lists the installed bundles, each with its folder. +- When two packages install bundles of one name, run each as `<package>/<name>`, using the canonical distribution name, for example `mycorp-demo/triage`. agent-env's own bundles keep their bare names, so a package's `hello` runs only as `<package>/hello`. +- `agent-env plugin list` shows each bundle as a contribution of its package, agent-env's own `hello` as `agentenv-framework … bundle hello`. `plugin check` fails when a bundle doesn't resolve to a folder, fails a check `agent-env run` makes before it reads a store (it isn't a valid bundle, a type it names doesn't resolve, or a step's fields don't build), is registered twice by one package (`conflict`), or comes from a package whose agent-env requirement isn't met (`incompatible-core`). `list --no-load` and `show --no-load` only parse it. + +## Plugin settings + +A plugin that needs settings of its own reads them from `[plugins.<package>]`, where `<package>` is its distribution name: the name `uv add` and `pip install` take and `agent-env plugin list` prints. That table belongs to the plugin. agent-env reads nothing in it and checks none of its keys. Every other top-level table belongs to agent-env, which warns about one it does not read in `agent-env config show`, so a plugin keeps nothing of its own anywhere else. + +```toml +[plugins.acme-agentenv-browser] +endpoint = "env:BROWSER_URL?http://localhost:9222" +timeout = 30 + +[plugins.acme-agentenv-browser.viewport] +width = 1280 +``` + +```python +from agent_env.plugins import settings + +def browser_timeout() -> int: + browser = settings("acme-agentenv-browser") # {} when the file has no such table + return browser.get("timeout", 10) # the plugin keeps its own defaults +``` + +- Pass the distribution name, not the module's `__name__` or `__package__`: `acme-agentenv-browser` may import as `acme_browser`. +- Call `settings()` where the value is used, in `from_config`, `execute` or a command's body, not at import. Entry points are imported before a root option can select the config file, so a value read at import may come from another file. +- The key is matched by canonical name (PEP 503: lowercase, with each run of `-`, `_` and `.` read as one `-`), so `[plugins.acme_agentenv_browser]` is the same table. Write the canonical form. It never needs quoting, whereas a dotted name written bare (`[plugins.acme.browser]`) nests. Two tables that name one package are a `ConfigError`, not a guess. +- `settings(name, *, config=None)` returns a copy of the table, read from `config`'s document (default: the process `Config`), with `env:` and `secret:` references resolved as in every other table. It raises `agent_env.config.ConfigError` when `[plugins]` or the plugin's entry is not a table, when two keys name the package, or when a reference without a `?default` cannot be resolved; the message names the table. agent-env never reads the table itself, so a broken entry fails only its own plugin's read, and `config show` reports it in place. +- No environment variable overrides a plugin setting. For a value that differs per deployment, write an `env:` reference in the table. +- `agent-env config show` lists each table under its package: with the installed version, with `(declares no agent_env entry point)` when the distribution is installed but is not a plugin, which usually means a misspelled entry-point group, with `(agent-env itself)` for `agentenv-framework`, whose table agent-env does not read, or with `(not installed)`. `config explain plugins.<package>.<key>` says whose table holds the key; agent-env does not read or check it, so it cannot tell whether the plugin reads that key, and a misspelled key is still shown. A table for a plugin that is not installed is reported, never an error, because one config file is often shared by processes that install different plugins. +- Both commands mask a plugin's table the way they mask agent-env's, and a service may log what they report. A literal value is printed as `***` when its key, or a key above it in the table, contains `password`, `passwd`, `passphrase`, `secret`, `token`, `credential`, `api_key`, `private_key`, `auth` or `bearer` in any case, and a connection URI's userinfo is masked. Any other literal is printed, so keep credentials behind `secret:` or `env:` references. A reference prints as written, with any `?default` masked like a literal. +- A provider the plugin registers is still configured in `[sandbox.providers.<name>]` or `[state.providers.<name>]`, under the provider's name, and an explorer plugin class is still listed in `[explorer.plugins]`. agent-env reads those tables itself. `[plugins.<package>]` holds only what the plugin reads. +- Uninstalling a plugin leaves its table in the config file. +- A top-level table one letter from `[plugins]`, such as `[plugin]`, gets a warning in `config show`: nothing reads it, so the plugin would get no settings. + +## Manage plugins + +<!-- tst/installer/test_container_journey.py runs the commands in the first bash block below. --> + +`agent-env plugin` shows what the installed plugins contribute and whether each piece took effect, and adds or removes them. + +```bash +agent-env plugin list # every plugin package, what it provides, and its status +agent-env plugin show PACKAGE # each contribution and why it is in that state +agent-env plugin check # exit 1 if any contribution did not take effect +agent-env plugin add SPEC... # install through this environment's installer, then check +agent-env plugin remove PACKAGE # uninstall through the same installer, after safety checks +``` + +``` +agent-env 0.9.1193 · uv tool at ~/.local/share/uv/tools/agentenv-framework +config: (none) + +PACKAGE VERSION PROVIDES STATUS +agentenv-browser 1.0.0 env browser, task step browser_navigate ok +agentenv-framework 0.9.1193 bundle hello ok +agentenv-grader 0.3.1 task step grade_essay 1 failed +``` + +| Status | Meaning | +|---|---| +| `active` | registered and in use | +| `replaced` | config names a different class for the name; `show` says where. Not a failure | +| `failed` | failed to import, validate or construct | +| `skipped` | a built-in, a core command or root option, or (for CLI commands) a plugin loaded first owns the name | +| `conflict` | another entry point, from another package or this one, claims the same name, so none of them is used. For a bundle, only a second claim from the same package is a conflict; another package's bundle of that name is `qualified-only` | +| `blocked` | the group cannot load at all, for example because its config table is invalid, so this contribution does not either | +| `unloaded` | not loaded (`--no-load`). When loading, it means the status could not be determined, and `check` fails on it | + +A contribution that is not `active` also has a code saying why, shown in brackets after its reason; the codes are listed under [Plugin report format](#plugin-report-format). + +- The header says how agent-env is installed (uv tool, pipx, uv project, virtualenv or system Python) and where: a plugin has to be installed into that same environment. +- `list` and `show` take `--json` and `--no-load`. `--no-load` reads installed metadata only and imports no type plugin; CLI plugins are already loaded, because the CLI loads them when it starts. Without it, `list`, `show` and `check` import every type plugin and construct the explorer plugins, so that plugin code runs. +- `show` also reports whether importing the package sets `AGENT_ENV_CONFIG`, checked in a fresh interpreter. +- `check` builds every registry the way a process does, adds the CLI's own plugins, and fails on `failed`, `skipped`, `conflict`, `blocked` or `unloaded`, or when the config file or the installed entry points cannot be read (a malformed `entry_points.txt` hides every plugin, so it fails rather than passing empty). `check --json` prints the `list --json` report plus `ok` and `problems`. A `replaced` contribution passes: config chose it. +- `plugin` is a core command: a CLI plugin that names a command `plugin` is skipped, like any clash with a core command. +- The Python equivalent is `agent_env.plugins.inventory()` (see [Register types from an installed package](#register-types-from-an-installed-package)). + +### Plugin report format + +`plugin list --json`, `plugin show --json` and `plugin check --json`, and the `agent_env.plugins` types they are built from, are the interface for scripts, CI and image builds. The human output of these commands is not: parse the JSON. + +```json +{ + "format_version": 1, + "agent_env": {"version": "0.9.1217", "environment": "uv tool", "location": "/home/me/.local/share/uv/tools/agentenv-framework"}, + "config": {"path": "/work/.agentenv/config.toml", "error": null}, + "loaded": true, + "group_errors": {}, + "discovery_errors": {}, + "plugins": [ + {"name": "agentenv-grader", "version": "0.3.1", "contributions": [ + {"group": "agent_env.task_steps", "name": "grade_essay", "value": "agentenv_grader.steps:GradeEssay", + "status": "failed", "code": "load-failed", "reason": "failed to load: ModuleNotFoundError(\"No module named 'openai'\")", + "replaced_by": null, "conflicts_with": []} + ]} + ] +} +``` + +- Every key is always present; an empty value is `null`, `[]` or `{}`. `group_errors` and `discovery_errors` map an entry-point group to `{code, reason}`, and `config.error` is `{code, reason}` or `null`. +- A distribution whose metadata cannot be read is listed under the label `(unreadable metadata: DIR)`, where `DIR` is its `.dist-info` directory. The label is not a package name, so `plugin remove` cannot take it: reinstall the package with its installer, or delete that directory. +- A contribution's `status` says what happened to it (the table above) and its `code` says why. `code` and `reason` are set together: on every contribution that is not `active`, and on an `active` one only for information. `replaced_by` is set exactly when the status is `replaced`: `{file, table, impl}`, where `table` is the config table naming the other class (`envs`, `task_steps`, `artifacts`, `explorer.plugins`, `sandbox.providers.NAME` or `state.providers.NAME`). `conflicts_with` lists the other entry points that claim the name, as `{package, version, value}`. +- `show --json` adds `config_effect`, a sentence saying whether importing the package sets `AGENT_ENV_CONFIG` (`null` with `--no-load`). `check --json` adds `ok` and `problems`, each a contribution with its `package` and `version`. +- `check` exits 0 when `ok` and 1 otherwise; a usage error exits 2. `show` exits 1 with no report when no installed plugin package has that name. A run that does not print exactly one JSON document on stdout, such as a crash while agent-env starts, is not a report: count it as a failure whatever its exit status. + +`format_version` changes only for a change a consumer cannot ignore: a key removed, renamed or given another type, a new status, a code redefined or reused, or a new meaning for an exit status. A new key, a new code, reworded `reason` or `config_effect` text, and a change in list order leave it as it is. So a consumer should ignore keys it does not know, handle a code it does not know by its `status`, refuse a `format_version` it does not know, and never parse `reason`. Which situation gets which status and code, and which group error is reported when several apply, is behaviour rather than format: a release that changes one says so in its notes. A code is never removed or reused. + +| Code | Where | Meaning | What to do | +|---|---|---|---| +| `load-failed` | `failed` | The plugin's code raised: when it was imported (`SystemExit` included), when an explorer plugin was constructed, or in a root option's callback with the flag absent | Install what it needs, or report it to the plugin's author | +| `invalid-plugin` | `failed` | It imported but does not fit its group: not a subclass of the group's base class; a `type` that is missing, inherited or not the entry-point name; a method its base requires left unimplemented; not a `click.Command` or `click.Option`; a root option that is required or exposes a value; a command click refused; an extra artifact name that reads no document. A bundle is never imported: it is invalid when its value isn't an installed, unpacked package holding that folder, when its metadata can't be read, or when the folder isn't a valid bundle (with plugins loading, one whose step, env or artifact types don't resolve) | Report it to the plugin's author; for a bundle whose package is installed zipped, reinstall it unpacked. For an extra name whose class's own type is in conflict, settle that conflict | +| `incompatible-core` | `failed` | The plugin's requirement on `agentenv-framework` or `agentenv-framework-protocol` excludes the installed version, so it was not imported. See [Plugin compatibility](#plugin-compatibility) | Upgrade agent-env, or install a version of the plugin that fits | +| `builtin-name` | `skipped` | agent-env owns the name: a built-in type, a core command or a core root option | The plugin has to rename it | +| `name-conflict` | `conflict`, `skipped` | More than one entry point claims the name, from two packages or twice from one; for bundles, only twice from one package (see `qualified-only`). In a type group none of them is registered, and the rest of the group loads; two different root options on one flag are each a `conflict`, and neither is attached. A CLI command whose name a plugin loaded earlier took is `skipped`, and the earlier one stays | Remove all but one: `agent-env plugin remove PACKAGE`. A package that declares a name twice has to be fixed by its author | +| `replaced-by-config` | `replaced` | The config registers another class under the name | Nothing, unless you did not mean it | +| `already-attached` | `active` | A second entry point, from the same package or another, exports the same `click.Option` object, which is attached once | Nothing | +| `not-loaded` | `unloaded` | `--no-load`, or `inventory(load=False)`: whether it takes effect needs an import | Run without `--no-load` | +| `status-unknown` | `unloaded` | It was loaded, but nothing was recorded for it. This should not happen | Report it as an agent-env bug | +| `config-not-found` | config error, group error, `blocked` | `AGENT_ENV_CONFIG` names a file that does not exist | Fix or unset `AGENT_ENV_CONFIG` | +| `config-unreadable` | config error, group error, `blocked` | The config file could not be read or parsed | Fix the file | +| `config-invalid` | group error, `blocked` | The file parsed, but what it says for this group is invalid | Fix the table the reason names | +| `group-build-failed` | group error, `blocked` | Building the group raised something else | See the reason | +| `entry-points-unreadable` | discovery error | The group's installed entry points could not be read, so none of its plugins is listed | Reinstall the package the reason names | +| `qualified-only` | `active` | Another package installs a bundle of the same name, so this one runs only as `<package>/<name>`. agent-env's own bundles keep the bare name | Run it by its qualified name | + +A `blocked` contribution carries its group error's code, so it says what to fix rather than only that the group failed. + +### Add and remove plugins + +agent-env never installs anything itself. `add` and `remove` run the installer that owns the environment agent-env runs in, so that installer's next rebuild keeps the change: + +| agent-env installed as | `add` runs | `remove` runs | +|---|---|---| +| uv tool | `uv tool install agentenv-framework --with ... --with SPEC`, passing back the receipt's requirements (editable and git ones included), Python and options | the same, without the package | +| pipx | `pipx inject VENV SPEC`, with the venv's own pip arguments (and `--force` to upgrade a package it already has) | `pipx uninject --leave-deps VENV PACKAGE` | +| uv project | `uv add --no-sync --project ROOT SPEC` and `uv sync --inexact`, then shows the `pyproject.toml` diff | `uv remove --no-sync --project ROOT PACKAGE` and `uv pip uninstall PACKAGE` | +| virtualenv | `python -m pip install SPEC` (or `uv pip install` when the venv has no pip) | `python -m pip uninstall -y PACKAGE` | +| Poetry or PDM project | prints `poetry add` / `pdm add` to run yourself | prints `poetry remove` / `pdm remove` | +| Hatch environment | says to add it to the environment's dependencies | the same, for removal | +| system Python with the PEP 668 marker, or read-only site-packages | refuses, with guidance | refuses | + +`SPEC` is anything the installer accepts, so a plugin can come from an index, a git repository or a file: + +| From | `SPEC` | +|---|---| +| PyPI, or the index the installer is configured with | `agentenv-grader`, `'agentenv-grader==0.3.0'`, `'agentenv-grader>=0.3,<0.4'` | +| a git tag, commit or branch | `'agentenv-grader @ git+https://github.com/mycorp/agentenv-grader@v0.3.0'` | +| one package in a monorepo | `'agentenv-grader @ git+https://github.com/mycorp/plugins@v1#subdirectory=agentenv-grader'` | +| a private repository | `'agentenv-grader @ git+ssh://git@github.com/mycorp/agentenv-grader.git@v0.3.0'` | +| a release asset | `https://github.com/mycorp/agentenv-grader/releases/download/v0.3.0/agentenv_grader-0.3.0-py3-none-any.whl` | +| a local wheel or checkout | `./dist/agentenv_grader-0.3.0-py3-none-any.whl`, `./agentenv-grader` | + +- Write a git URL as `NAME @ URL`, and a local checkout as `NAME @ file:///absolute/path`. agent-env cannot read the name from a bare `git+https://…` or a directory, so it cannot tell pipx or a uv tool which installed plugin the spec replaces: pipx leaves the installed one as it is, and `add` says so; a uv tool reports conflicting URLs, and the add is rolled back. A git install records the commit it resolved to, so a rollback reinstalls that commit. +- `add` never replaces agent-env itself. A spec that names `agentenv-framework`, as a name, a wheel or `NAME @ URL`, is refused before anything runs. A bare URL or directory that turns out to build it is rolled back after the install: agent-env may move to a newer release from the index when a plugin requires one, but not to a URL, git or directory source. Upgrade agent-env with the installer that owns the environment. +- Credentials stay with git and the installer: an SSH key or a git credential helper (`gh auth setup-git`) for a private repository, and the installer's own config for a private index. A token written into `SPEC` is kept in the installer's record of the environment. +- `--installer` overrides the detected installer, `--index-url` passes an index through to it (a uv tool keeps it as its default index), `--dry-run` shows what would run, and `--yes` skips the prompt. +- uv takes a pre-release or a yanked version only when you name it. If the installer reports that a plugin needs one, name that exact version in the same command, for example `agent-env plugin add agentenv-grader 'agentenv-grader-core==0.2.0b1'`; `plugin remove` removes it later the same way. +- A uv project whose environment is set by `UV_PROJECT_ENVIRONMENT` is found from the current directory, as uv finds it, so run `add` and `remove` inside the project. In a uv workspace whose root has no `[project]` table, `add` goes to the one member that declares `agentenv-framework` (`uv add --package MEMBER`). `remove` drops a package from every member and group that declares it (`--package`, `--group`, `--optional`, `--dev`). +- A uv project is never synced exactly, so packages from extras and anything installed outside the lock, agent-env included, stay installed. +- pipx and a virtualenv leave a removed plugin's own dependencies installed, as `pip uninstall` does. Left to itself, `pipx uninject` would also uninstall everything nothing else requires, agent-env included. +- A uv tool installed with constraints, overrides or executables from another package is refused: `uv tool install` cannot take those back from agent-env, so make that change with uv. +- One `add` or `remove` runs at a time per environment; a second one is refused until the first finishes. The lock is a file in `$XDG_STATE_HOME/agent-env/locks` (by default `~/.local/state/agent-env/locks`). If the installer's record changes some other way while a change waits for confirmation, the change is refused rather than run from its outdated plan. +- A plain virtualenv first shows the installer's own dry run of what would change. Every mode reports what did change afterwards, calling out a change to agent-env itself. +- `add` then checks each new plugin package in a fresh interpreter: it must declare at least one `agent_env.*` entry point, and every contribution must be `active` or `replaced`. `show` reports whether importing it sets `AGENT_ENV_CONFIG`. If the check fails, the installer fails or removes another plugin, or the change is interrupted (Ctrl-C or SIGTERM), `add` puts the environment back as it was: every package at its old version from its old source, and the uv tool receipt, pipx metadata, or `pyproject.toml` and `uv.lock`, byte for byte. The installer runs with Ctrl-C ignored, since one stopped halfway leaves packages no installer can read: it finishes, and then the change is rolled back. A second Ctrl-C stops it at once. It keeps the terminal, so its own prompts, such as git's for credentials, still work. When the environment cannot be put back exactly, `add` says which packages differ and the installer command that reinstalls them. Adding what is already installed changes nothing and succeeds, and a bare name that a uv tool already lists keeps its version pin. `--keep` keeps a change that failed the check instead. +- `remove` refuses, unless `--force`: + - while another installed package requires the package; + - while the config file names something only it provides (`[sandbox] default` or `agent_default`, a provider table, a `type_aliases` target), or an `impls` list, provider, store or runner `impl` names one of its modules; + - while stored documents use env, artifact or task-step types only it provides. This is checked for a local document store, and for a remote one only with `--check-usage`; a store that cannot be read also blocks, until `--force`. + + It never removes agent-env itself. A package that declares no `agent_env.*` entry point is removed only when the installer records it on its own (a uv tool's `--with`, a pipx injection, a project dependency), as it does for a package named next to a plugin in `add`; after removing a plugin, `remove` names any such package that only the plugin needed. In a uv tool or a uv project, removing a plugin also removes the plugins that only it needed, as the installer's rebuild would, so `remove` names them first and checks them like the package itself. If the installer fails, removes a plugin it did not name, or the change is interrupted, `remove` puts the environment back the same way `add` does. +- Plugins are trusted code: nothing installs one implicitly, from a task document, a config file or anywhere else. +- A plugin that breaks CLI startup does not lock you out. A root option whose callback raises or exits while its flag is absent is skipped, with a warning, and reported as `failed`; a flag you pass still fails, naming the plugin. A CLI plugin that exits while being imported is skipped like any load failure. For a plugin that hangs or kills the interpreter while being imported, uninstall it with the installer directly: the command `plugin remove --dry-run` would print. + +## Building a platform plugin + +One installable package can combine all of the above: bundled config files, a root option that selects one per invocation, store and provider classes, a `Runner`, custom steps, and explorer routers. The `--tenant` demo above is that pattern in miniature. Its callback points `AGENT_ENV_CONFIG` at a bundled `tenants/<name>.toml`; a name that does not exist fails loud with `ConfigError` on first use, while `--help` still works. A hosted control plane serves the explorer app from its own server process and lists its public hostnames under `[explorer] allowed_hosts` (see the comments in `.agentenv/config.example.toml`). A durable runner is the plugin's responsibility: this repository ships only `LocalRunner`. + +## Plugin compatibility + +What a plugin can build on, how to declare the agent-env it needs, and what agent-env does when the installed one does not fit. + +**The plugin surface.** A plugin may rely on these. Anything else, including every underscored name other than a method a subclass must implement (`EnvStateProvider._teardown`), is internal and can change in any release. + +- The entry-point groups and their rules, in [Register types from an installed package](#register-types-from-an-installed-package) and [CLI plugins, root options, explorer routes](#cli-plugins-root-options-explorer-routes). +- The base class each group names, with its public methods and attributes: `agent_env.env.env.Env`, `agent_env.task_step.task_step.TaskStep` and the `agent_env.task_step.context.TaskStepContext` a step runs with, `agent_env.artifact.artifact.Artifact`, `agent_env.providers.sandbox_providers.sandbox_provider.SandboxProvider`, `agent_env.providers.env_state.env_state_provider.EnvStateProvider`, `agent_env.providers.env_providers.env_provider.EnvironmentProvider`, and `agent_env.explorer.plugin.ExplorerPlugin`. +- The two functions an env that uses an environment provider calls: `agent_env.providers.env_providers.env_provider.build_env_provider` and `agent_env.env.store.register_env_instance`. +- What an environment provider reads to deploy a built-in env: `agent_env.env.envs.mcp_server.MCPServerEnv.docker_image_artifact` and `agent_env.env.envs.mcp_server.MCPServerEnv.environment_name`; `agent_env.env.envs.website.WebsiteEnv.backend_docker_image_artifact`, `agent_env.env.envs.website.WebsiteEnv.frontend_docker_image_artifact` and `agent_env.env.envs.website.WebsiteEnv.environment_name`; `agent_env.env.envs.multi_env.MultiEnv.mcp_server_envs`, `agent_env.env.envs.multi_env.MultiEnv.website_envs` and `agent_env.env.envs.multi_env.MultiEnv.name`; and each image's `agent_env.artifact.artifacts.docker_image.DockerImageArtifact.image_name`. +- The top level of `agent_env.plugins`, including `settings`, and the `[plugins.<package>]` table it reads ([Plugin settings](#plugin-settings)). +- The `plugin --json` output, which has its own rules: [Plugin report format](#plugin-report-format). + +The classes a config `impl` names, such as stores and runners, are not on the list yet. + +**Changes before 1.0.** Every merged change can ship as a release, several a day. A change that breaks the plugin surface is marked with `!` after the scope in its pull request title, which becomes its commit title, as in `feat(plugins)!: …`. Where the old behaviour can be kept for a while, it is deprecated first: it keeps working and emits a `DeprecationWarning` that names what replaces it. How long that lasts is not fixed before 1.0. + +The `plugin-api` CI job holds pull requests to this. It compares the listed base classes, `TaskStepContext`, the two functions, the env attributes and the top level of `agent_env.plugins` with the pull request's base (`.github/scripts/check_plugin_api.py`), and fails on a break the title does not mark; with the `!`, it lists the breaks and passes, and editing the title re-runs it. A break is what fails code written against the old surface: + +- for a caller, a name, parameter or `__all__` entry that is removed or renamed, a new required parameter, a parameter that can no longer be passed as before, or a changed default or constant, including the group-name constants `agent_env.plugins` exports, compared by value; +- for a subclass of a base class, a new abstract or required method, a method that becomes abstract or required, a new `ClassVar` with no value, a method that changes between plain, `async`, `classmethod`, `staticmethod` and property, and a base-class method that accepts more than before: a new parameter, even an optional one, a parameter that stops being required, a new `*args` or `**kwargs`, or a keyword-only parameter that can now be passed by position, since an override written for the old signature fails when core passes it; +- for construction, a changed `@dataclass(...)` option or pydantic `model_config`, a new metaclass, or a new `__init_subclass__` that can raise, which refuses a subclass. + +A member every subclass must implement is declared `@abstractmethod`, or listed in its registry's `_MUST_IMPLEMENT`, so that the check enforces it; a new method that only raises `NotImplementedError` gets a note but does not fail the job. + +agent-env ships `py.typed`, so mypy and pyright check a plugin against its annotations. Import each name from the module the list above gives, such as `TaskStepContext` from `agent_env.task_step.context`: pyright, and mypy under `--strict`, treat a name a module only imports as private to that module. Type annotations are not compared, so a release can correct one without a `!`. The check can report a change no plugin notices, such as a new parameter core never passes to an override; mark it all the same. It does not see a change in behaviour, in a type the surface only names in a signature, such as `DeployedEnv`, in a pydantic field's requirements or the order of dataclass fields, in an abstract method a new base from outside agent-env brings, or in a registration rule other than abstract methods and `_MUST_IMPLEMENT`; review covers those. A `!` can also mark a break outside the plugin surface. + +**Declaring the agent-env your plugin needs.** Declare a floor, `agentenv-framework>=X`, where `X` is the oldest release you test against. Leave out a ceiling such as `<1`: before 1.0 it would not guard against a change in a 0.9 release, and it would keep your plugin from installing with the next major one. Pin exact versions in the application or image that installs your plugin, not in the plugin. If your plugin imports `agentenv_protocol` itself, declare that as well. + +**What agent-env checks.** Before it imports a plugin, agent-env compares the plugin's requirements on `agentenv-framework` and `agentenv-framework-protocol` with the versions installed. A plugin they exclude is not loaded: + +- `plugin list`, `show` and `check` report each of its contributions as `failed` with the code `incompatible-core`, with or without `--no-load`, and `check` exits 1; +- using one of its types names the requirement, as in `needs agentenv-framework>=0.9.1220 (installed: 0.9.1218)`; +- it claims none of its names, so a plugin that can load and registers the same name does not conflict with it. + +An installer that resolves dependencies never gets you there; `pip install --no-deps` or a forced install can. There is no override: upgrade agent-env, or install a version of the plugin that fits. A requirement under an extra, or whose environment marker is false, does not count, and an agent-env with no installed metadata, such as a source tree on `sys.path`, is not checked. Requirements between plugins are the installer's to check; `pip check` lists any that are unmet. diff --git a/README.md b/README.md index c517763e..16f916c9 100644 --- a/README.md +++ b/README.md @@ -44,289 +44,6 @@ uv sync --extra dev source .venv/bin/activate ``` -## Plugin contract - -What a package that extends agent-env relies on. Unit tests and the plugin-API check in CI hold the code to these sections, so a pull request that changes the contract changes this README too. - -### Register types from an installed package - -An installed distribution registers envs, task steps, artifacts, sandbox, state and environment providers and explorer plugins by declaring entry points, with no config. The group says what kind of thing it is, the entry-point name is the registry key, and the value is the class: - -```toml -[project.entry-points."agent_env.envs"] -browser = "agentenv_browser.env:BrowserEnv" - -[project.entry-points."agent_env.task_steps"] -browser_navigate = "agentenv_browser.steps:NavigateTaskStep" -``` - -| Group | Value | Name | -|---|---|---| -| `agent_env.envs` | `Env` subclass with its own `type`, implementing `from_dict` | must equal the class's `type`, the spelling its documents are written under | -| `agent_env.task_steps` | `TaskStep` subclass with its own `type`, implementing `execute` and `from_dict` | must equal the class's `type` | -| `agent_env.artifacts` | `Artifact` subclass with its own `type` field default | the registry key; a class may also register under extra names, such as a legacy spelling, as long as its own `type` default resolves to it and its `type` field accepts each extra name | -| `agent_env.sandbox_providers` | `SandboxProvider` subclass implementing `create_sandbox` | the registry key; it must equal the `.type` of the sandboxes the provider produces, checked on the first `create_*` call | -| `agent_env.state_providers` | `EnvStateProvider` subclass implementing `acquire` and `_teardown` | must equal the class's `type`, checked at registration | -| `agent_env.env_providers` | `EnvironmentProvider` subclass implementing `deploy` and `close` | must equal the class's `type`, which its records carry as `env_provider_type`; checked at registration | -| `agent_env.explorer_plugins` | `ExplorerPlugin` subclass with its own `type`, implementing `router` | must equal the class's `type` | - -- The class must implement every abstract method of its base, and, for envs and task steps, `from_dict` as a classmethod, which the base defines only to raise. A plugin that does not is `failed` with `invalid-plugin`, naming what is missing, for example `GradeTaskStep must implement execute and from_dict`. An `impls` entry or provider table naming such a class is a `ConfigError`. -- Each registry takes the built-ins first, then plugins, then config. A plugin cannot replace a built-in: it is skipped with a warning, however many distributions claim that name. -- A name that two installed distributions register in one group, or that one declares twice, is left out of the registry with a warning, and the rest of the group, built-ins included, loads. An explorer plugin under that name is not mounted, so its routes are absent. Resolving the name fails like a plugin that did not load, naming each claimant and its version, for example `Unknown env type: browser (2 installed plugins register 'browser' in agent_env.envs: …)`, and says to remove all but one with `agent-env plugin remove PACKAGE`, or, when one package declares the name twice, to report it to the package's author. Neither claimant is picked, because the order entry points are found in is not fixed. -- Config cannot settle a conflicted name. An `impls` entry, a provider table, an `[explorer.plugins]` impl or an `[artifacts] type_aliases` entry mapping to that name is checked as usual, then skipped with a warning. A `type_aliases` entry whose old spelling is that name is a `ConfigError`, as it is when one plugin registers it. -- A plugin that fails to import, fails the checks above, or (for explorer plugins) fails to construct is skipped with a warning and the others still load. So is one whose requirement on agent-env excludes the installed version, before it is imported (see [Plugin compatibility](#plugin-compatibility)). Resolving its name then reports the recorded error, for example `Unknown env type: browser (registered by 'browser' from agentenv-web 2.1.0 (…) but failed to load: ModuleNotFoundError(…))`. -- Config replaces a plugin's class with a warning, except under a conflicted name: an `impls` entry of the same `type`, or a `[sandbox.providers.<name>]` / `[state.providers.<name>]` table carrying `impl`. Naming the plugin's own class is silent. A provider table without `impl` configures the plugin's provider the way it configures a built-in's, which is where per-deployment settings such as a secret name belong. A config-only provider table, or an `[artifacts] type_aliases` entry, that points at a plugin which failed to load is skipped with a warning rather than failing the whole registry. -- `agent_env.plugins.load_failures()` lists every plugin that did not take effect in the registries the current `Config` has built (failed to load, validate or construct, clashed with a built-in, or claims a name another entry point claims), by group and name, with the reason. The record belongs to the `Config`, so `reset_config()` starts a new one. A deployment that ships its plugins can build its registries at startup and assert it is empty. -- `agent_env.plugins.inventory()` lists every installed distribution that declares entry points in these groups, with a status for each contribution and a `code` saying why (see [Plugin report format](#plugin-report-format)): `active`; `replaced` (config names a different class, and `replaced_by` says where); `failed`; `skipped` (a built-in owns the name); `conflict`; `blocked` (the group's real build fails, for example because its config table is invalid, so nothing in the group loads); or `unloaded`. It builds the registries on a throwaway `Config` that reads the same document, so the `Config` in use keeps its registries and its `load_failures()`. `inventory(load=False)` reads installed metadata only and runs no plugin code, so a status that needs a build is `unloaded`. With the default `load=True` the plugins are imported and the explorer plugins constructed, with whatever process-wide effects that has. `discovery_errors` names each group whose installed entry points could not be read at all, so none of its plugins is listed. -- Discovery reads installed metadata and imports nothing. When entry points are loaded is unspecified: today each group is imported when its registry is first built, but a plugin must not depend on that. -- Loading a plugin never changes which config a `Config` reads: the `Config` resolves its document before it imports any plugin. A plugin package that sets `AGENT_ENV_CONFIG` on import affects only configs built afterwards, such as after `reset_config()`. - -The distribution that declares the entry points is the plugin, so `pip uninstall` removes it completely, apart from any `[plugins.<package>]` table in the config file. CLI commands are a separate contribution, described next. - -### CLI plugins, root options, explorer routes - -Installed packages add commands through two entry-point groups: - -```toml -[project.entry-points."agent_env.cli_plugins"] -my-tools = "mycorp_demo.cli:my_tools" -[project.entry-points."agent_env.cli_root_options"] -tenant = "mycorp_demo.cli:tenant_option" -``` - -`agent_env.cli_plugins` entries are click commands or groups added next to the built-ins, under the entry-point name whatever the command object is called: `my-tools` above is `agent-env my-tools`. `agent_env.cli_root_options` entries are optional `click.Option` instances with `expose_value=False`; their callback runs before the subcommand, so it can set `AGENT_ENV_CONFIG` and call `agent_env.config.reset_config()` to select the config for the whole process. Plugins load when `agent_env.cli` is imported, after the built-ins. Clash rules: a plugin command whose entry-point name core already uses, or a flag core owns, is skipped with a warning on stderr, and core wins. Two different root options on the same flag are both left off with a warning, and `agent-env plugin list` reports each as a `conflict`; the CLI still starts, so `agent-env plugin remove` can settle it. The same `click.Option` object exported by two entry points, from one package or two, is attached once. Nothing grafts onto existing groups. - -Explorer routes: subclass `agent_env.explorer.plugin.ExplorerPlugin`, set the `type` ClassVar, and return a FastAPI `APIRouter` from the `router` property. List it under `[explorer.plugins] impls` or declare an `agent_env.explorer_plugins` entry point; `from_config()` takes no arguments, and a plugin reads its own settings with `agent_env.plugins.settings` (see [Plugin settings](#plugin-settings)). `agent-env up --no-bootstrap` mounts it before the core routers and needs no Docker. An entity id can contain `/` (a bundle's ids are `@local/<package>/<name>`, and so are the instance ids of its runs), so in an explorer URL an id is one percent-encoded path segment, with `/` sent as `%2F`. Under the core prefixes `/api/v1/{artifacts,envs,tasks,agents,evals,task-instances}/` the explorer routes on the encoded path, so a plugin route there receives the segment still encoded and decodes it with `urllib.parse.unquote`. - -### Bundles from installed packages - -A package ships bundles by naming the package that holds their folders: - -```toml -[project.entry-points."agent_env.bundles"] -triage = "mycorp_demo.bundles" -``` - -- The bundle `triage` is the folder `mycorp_demo/bundles/triage/`. The entry-point name is both the bundle's name and its folder's name, so it may contain `-`. -- The value names a package, never an object. The folder is found without importing any of the package's code, so listing bundles runs nothing. -- The folder ships in the wheel as package data, and the package must be installed unpacked. -- An installed bundle's ids are rooted at its distribution, wherever the package is installed: `triage` above writes `@local/mycorp-demo/triage/...`. -- `agent-env run triage` runs it, and `agent-env run` with no argument lists the installed bundles, each with its folder. -- When two packages install bundles of one name, run each as `<package>/<name>`, using the canonical distribution name, for example `mycorp-demo/triage`. agent-env's own bundles keep their bare names, so a package's `hello` runs only as `<package>/hello`. -- `agent-env plugin list` shows each bundle as a contribution of its package, agent-env's own `hello` as `agentenv-framework … bundle hello`. `plugin check` fails when a bundle doesn't resolve to a folder, fails a check `agent-env run` makes before it reads a store (it isn't a valid bundle, a type it names doesn't resolve, or a step's fields don't build), is registered twice by one package (`conflict`), or comes from a package whose agent-env requirement isn't met (`incompatible-core`). `list --no-load` and `show --no-load` only parse it. - -### Plugin settings - -A plugin that needs settings of its own reads them from `[plugins.<package>]`, where `<package>` is its distribution name: the name `uv add` and `pip install` take and `agent-env plugin list` prints. That table belongs to the plugin. agent-env reads nothing in it and checks none of its keys. Every other top-level table belongs to agent-env, which warns about one it does not read in `agent-env config show`, so a plugin keeps nothing of its own anywhere else. - -```toml -[plugins.acme-agentenv-browser] -endpoint = "env:BROWSER_URL?http://localhost:9222" -timeout = 30 - -[plugins.acme-agentenv-browser.viewport] -width = 1280 -``` - -```python -from agent_env.plugins import settings - -def browser_timeout() -> int: - browser = settings("acme-agentenv-browser") # {} when the file has no such table - return browser.get("timeout", 10) # the plugin keeps its own defaults -``` - -- Pass the distribution name, not the module's `__name__` or `__package__`: `acme-agentenv-browser` may import as `acme_browser`. -- Call `settings()` where the value is used, in `from_config`, `execute` or a command's body, not at import. Entry points are imported before a root option can select the config file, so a value read at import may come from another file. -- The key is matched by canonical name (PEP 503: lowercase, with each run of `-`, `_` and `.` read as one `-`), so `[plugins.acme_agentenv_browser]` is the same table. Write the canonical form. It never needs quoting, whereas a dotted name written bare (`[plugins.acme.browser]`) nests. Two tables that name one package are a `ConfigError`, not a guess. -- `settings(name, *, config=None)` returns a copy of the table, read from `config`'s document (default: the process `Config`), with `env:` and `secret:` references resolved as in every other table. It raises `agent_env.config.ConfigError` when `[plugins]` or the plugin's entry is not a table, when two keys name the package, or when a reference without a `?default` cannot be resolved; the message names the table. agent-env never reads the table itself, so a broken entry fails only its own plugin's read, and `config show` reports it in place. -- No environment variable overrides a plugin setting. For a value that differs per deployment, write an `env:` reference in the table. -- `agent-env config show` lists each table under its package: with the installed version, with `(declares no agent_env entry point)` when the distribution is installed but is not a plugin, which usually means a misspelled entry-point group, with `(agent-env itself)` for `agentenv-framework`, whose table agent-env does not read, or with `(not installed)`. `config explain plugins.<package>.<key>` says whose table holds the key; agent-env does not read or check it, so it cannot tell whether the plugin reads that key, and a misspelled key is still shown. A table for a plugin that is not installed is reported, never an error, because one config file is often shared by processes that install different plugins. -- Both commands mask a plugin's table the way they mask agent-env's, and a service may log what they report. A literal value is printed as `***` when its key, or a key above it in the table, contains `password`, `passwd`, `passphrase`, `secret`, `token`, `credential`, `api_key`, `private_key`, `auth` or `bearer` in any case, and a connection URI's userinfo is masked. Any other literal is printed, so keep credentials behind `secret:` or `env:` references. A reference prints as written, with any `?default` masked like a literal. -- A provider the plugin registers is still configured in `[sandbox.providers.<name>]` or `[state.providers.<name>]`, under the provider's name, and an explorer plugin class is still listed in `[explorer.plugins]`. agent-env reads those tables itself. `[plugins.<package>]` holds only what the plugin reads. -- Uninstalling a plugin leaves its table in the config file. -- A top-level table one letter from `[plugins]`, such as `[plugin]`, gets a warning in `config show`: nothing reads it, so the plugin would get no settings. - -### Manage plugins - -<!-- tst/installer/test_container_journey.py runs the commands in the first bash block below. --> - -`agent-env plugin` shows what the installed plugins contribute and whether each piece took effect, and adds or removes them. - -```bash -agent-env plugin list # every plugin package, what it provides, and its status -agent-env plugin show PACKAGE # each contribution and why it is in that state -agent-env plugin check # exit 1 if any contribution did not take effect -agent-env plugin add SPEC... # install through this environment's installer, then check -agent-env plugin remove PACKAGE # uninstall through the same installer, after safety checks -``` - -``` -agent-env 0.9.1193 · uv tool at ~/.local/share/uv/tools/agentenv-framework -config: (none) - -PACKAGE VERSION PROVIDES STATUS -agentenv-browser 1.0.0 env browser, task step browser_navigate ok -agentenv-framework 0.9.1193 bundle hello ok -agentenv-grader 0.3.1 task step grade_essay 1 failed -``` - -| Status | Meaning | -|---|---| -| `active` | registered and in use | -| `replaced` | config names a different class for the name; `show` says where. Not a failure | -| `failed` | failed to import, validate or construct | -| `skipped` | a built-in, a core command or root option, or (for CLI commands) a plugin loaded first owns the name | -| `conflict` | another entry point, from another package or this one, claims the same name, so none of them is used. For a bundle, only a second claim from the same package is a conflict; another package's bundle of that name is `qualified-only` | -| `blocked` | the group cannot load at all, for example because its config table is invalid, so this contribution does not either | -| `unloaded` | not loaded (`--no-load`). When loading, it means the status could not be determined, and `check` fails on it | - -A contribution that is not `active` also has a code saying why, shown in brackets after its reason; the codes are listed under [Plugin report format](#plugin-report-format). - -- The header says how agent-env is installed (uv tool, pipx, uv project, virtualenv or system Python) and where: a plugin has to be installed into that same environment. -- `list` and `show` take `--json` and `--no-load`. `--no-load` reads installed metadata only and imports no type plugin; CLI plugins are already loaded, because the CLI loads them when it starts. Without it, `list`, `show` and `check` import every type plugin and construct the explorer plugins, so that plugin code runs. -- `show` also reports whether importing the package sets `AGENT_ENV_CONFIG`, checked in a fresh interpreter. -- `check` builds every registry the way a process does, adds the CLI's own plugins, and fails on `failed`, `skipped`, `conflict`, `blocked` or `unloaded`, or when the config file or the installed entry points cannot be read (a malformed `entry_points.txt` hides every plugin, so it fails rather than passing empty). `check --json` prints the `list --json` report plus `ok` and `problems`. A `replaced` contribution passes: config chose it. -- `plugin` is a core command: a CLI plugin that names a command `plugin` is skipped, like any clash with a core command. -- The Python equivalent is `agent_env.plugins.inventory()` (see [Register types from an installed package](#register-types-from-an-installed-package)). - -#### Plugin report format - -`plugin list --json`, `plugin show --json` and `plugin check --json`, and the `agent_env.plugins` types they are built from, are the interface for scripts, CI and image builds. The human output of these commands is not: parse the JSON. - -```json -{ - "format_version": 1, - "agent_env": {"version": "0.9.1217", "environment": "uv tool", "location": "/home/me/.local/share/uv/tools/agentenv-framework"}, - "config": {"path": "/work/.agentenv/config.toml", "error": null}, - "loaded": true, - "group_errors": {}, - "discovery_errors": {}, - "plugins": [ - {"name": "agentenv-grader", "version": "0.3.1", "contributions": [ - {"group": "agent_env.task_steps", "name": "grade_essay", "value": "agentenv_grader.steps:GradeEssay", - "status": "failed", "code": "load-failed", "reason": "failed to load: ModuleNotFoundError(\"No module named 'openai'\")", - "replaced_by": null, "conflicts_with": []} - ]} - ] -} -``` - -- Every key is always present; an empty value is `null`, `[]` or `{}`. `group_errors` and `discovery_errors` map an entry-point group to `{code, reason}`, and `config.error` is `{code, reason}` or `null`. -- A distribution whose metadata cannot be read is listed under the label `(unreadable metadata: DIR)`, where `DIR` is its `.dist-info` directory. The label is not a package name, so `plugin remove` cannot take it: reinstall the package with its installer, or delete that directory. -- A contribution's `status` says what happened to it (the table above) and its `code` says why. `code` and `reason` are set together: on every contribution that is not `active`, and on an `active` one only for information. `replaced_by` is set exactly when the status is `replaced`: `{file, table, impl}`, where `table` is the config table naming the other class (`envs`, `task_steps`, `artifacts`, `explorer.plugins`, `sandbox.providers.NAME` or `state.providers.NAME`). `conflicts_with` lists the other entry points that claim the name, as `{package, version, value}`. -- `show --json` adds `config_effect`, a sentence saying whether importing the package sets `AGENT_ENV_CONFIG` (`null` with `--no-load`). `check --json` adds `ok` and `problems`, each a contribution with its `package` and `version`. -- `check` exits 0 when `ok` and 1 otherwise; a usage error exits 2. `show` exits 1 with no report when no installed plugin package has that name. A run that does not print exactly one JSON document on stdout, such as a crash while agent-env starts, is not a report: count it as a failure whatever its exit status. - -`format_version` changes only for a change a consumer cannot ignore: a key removed, renamed or given another type, a new status, a code redefined or reused, or a new meaning for an exit status. A new key, a new code, reworded `reason` or `config_effect` text, and a change in list order leave it as it is. So a consumer should ignore keys it does not know, handle a code it does not know by its `status`, refuse a `format_version` it does not know, and never parse `reason`. Which situation gets which status and code, and which group error is reported when several apply, is behaviour rather than format: a release that changes one says so in its notes. A code is never removed or reused. - -| Code | Where | Meaning | What to do | -|---|---|---|---| -| `load-failed` | `failed` | The plugin's code raised: when it was imported (`SystemExit` included), when an explorer plugin was constructed, or in a root option's callback with the flag absent | Install what it needs, or report it to the plugin's author | -| `invalid-plugin` | `failed` | It imported but does not fit its group: not a subclass of the group's base class; a `type` that is missing, inherited or not the entry-point name; a method its base requires left unimplemented; not a `click.Command` or `click.Option`; a root option that is required or exposes a value; a command click refused; an extra artifact name that reads no document. A bundle is never imported: it is invalid when its value isn't an installed, unpacked package holding that folder, when its metadata can't be read, or when the folder isn't a valid bundle (with plugins loading, one whose step, env or artifact types don't resolve) | Report it to the plugin's author; for a bundle whose package is installed zipped, reinstall it unpacked. For an extra name whose class's own type is in conflict, settle that conflict | -| `incompatible-core` | `failed` | The plugin's requirement on `agentenv-framework` or `agentenv-framework-protocol` excludes the installed version, so it was not imported. See [Plugin compatibility](#plugin-compatibility) | Upgrade agent-env, or install a version of the plugin that fits | -| `builtin-name` | `skipped` | agent-env owns the name: a built-in type, a core command or a core root option | The plugin has to rename it | -| `name-conflict` | `conflict`, `skipped` | More than one entry point claims the name, from two packages or twice from one; for bundles, only twice from one package (see `qualified-only`). In a type group none of them is registered, and the rest of the group loads; two different root options on one flag are each a `conflict`, and neither is attached. A CLI command whose name a plugin loaded earlier took is `skipped`, and the earlier one stays | Remove all but one: `agent-env plugin remove PACKAGE`. A package that declares a name twice has to be fixed by its author | -| `replaced-by-config` | `replaced` | The config registers another class under the name | Nothing, unless you did not mean it | -| `already-attached` | `active` | A second entry point, from the same package or another, exports the same `click.Option` object, which is attached once | Nothing | -| `not-loaded` | `unloaded` | `--no-load`, or `inventory(load=False)`: whether it takes effect needs an import | Run without `--no-load` | -| `status-unknown` | `unloaded` | It was loaded, but nothing was recorded for it. This should not happen | Report it as an agent-env bug | -| `config-not-found` | config error, group error, `blocked` | `AGENT_ENV_CONFIG` names a file that does not exist | Fix or unset `AGENT_ENV_CONFIG` | -| `config-unreadable` | config error, group error, `blocked` | The config file could not be read or parsed | Fix the file | -| `config-invalid` | group error, `blocked` | The file parsed, but what it says for this group is invalid | Fix the table the reason names | -| `group-build-failed` | group error, `blocked` | Building the group raised something else | See the reason | -| `entry-points-unreadable` | discovery error | The group's installed entry points could not be read, so none of its plugins is listed | Reinstall the package the reason names | -| `qualified-only` | `active` | Another package installs a bundle of the same name, so this one runs only as `<package>/<name>`. agent-env's own bundles keep the bare name | Run it by its qualified name | - -A `blocked` contribution carries its group error's code, so it says what to fix rather than only that the group failed. - -#### Add and remove plugins - -agent-env never installs anything itself. `add` and `remove` run the installer that owns the environment agent-env runs in, so that installer's next rebuild keeps the change: - -| agent-env installed as | `add` runs | `remove` runs | -|---|---|---| -| uv tool | `uv tool install agentenv-framework --with ... --with SPEC`, passing back the receipt's requirements (editable and git ones included), Python and options | the same, without the package | -| pipx | `pipx inject VENV SPEC`, with the venv's own pip arguments (and `--force` to upgrade a package it already has) | `pipx uninject --leave-deps VENV PACKAGE` | -| uv project | `uv add --no-sync --project ROOT SPEC` and `uv sync --inexact`, then shows the `pyproject.toml` diff | `uv remove --no-sync --project ROOT PACKAGE` and `uv pip uninstall PACKAGE` | -| virtualenv | `python -m pip install SPEC` (or `uv pip install` when the venv has no pip) | `python -m pip uninstall -y PACKAGE` | -| Poetry or PDM project | prints `poetry add` / `pdm add` to run yourself | prints `poetry remove` / `pdm remove` | -| Hatch environment | says to add it to the environment's dependencies | the same, for removal | -| system Python with the PEP 668 marker, or read-only site-packages | refuses, with guidance | refuses | - -`SPEC` is anything the installer accepts, so a plugin can come from an index, a git repository or a file: - -| From | `SPEC` | -|---|---| -| PyPI, or the index the installer is configured with | `agentenv-grader`, `'agentenv-grader==0.3.0'`, `'agentenv-grader>=0.3,<0.4'` | -| a git tag, commit or branch | `'agentenv-grader @ git+https://github.com/mycorp/agentenv-grader@v0.3.0'` | -| one package in a monorepo | `'agentenv-grader @ git+https://github.com/mycorp/plugins@v1#subdirectory=agentenv-grader'` | -| a private repository | `'agentenv-grader @ git+ssh://git@github.com/mycorp/agentenv-grader.git@v0.3.0'` | -| a release asset | `https://github.com/mycorp/agentenv-grader/releases/download/v0.3.0/agentenv_grader-0.3.0-py3-none-any.whl` | -| a local wheel or checkout | `./dist/agentenv_grader-0.3.0-py3-none-any.whl`, `./agentenv-grader` | - -- Write a git URL as `NAME @ URL`, and a local checkout as `NAME @ file:///absolute/path`. agent-env cannot read the name from a bare `git+https://…` or a directory, so it cannot tell pipx or a uv tool which installed plugin the spec replaces: pipx leaves the installed one as it is, and `add` says so; a uv tool reports conflicting URLs, and the add is rolled back. A git install records the commit it resolved to, so a rollback reinstalls that commit. -- `add` never replaces agent-env itself. A spec that names `agentenv-framework`, as a name, a wheel or `NAME @ URL`, is refused before anything runs. A bare URL or directory that turns out to build it is rolled back after the install: agent-env may move to a newer release from the index when a plugin requires one, but not to a URL, git or directory source. Upgrade agent-env with the installer that owns the environment. -- Credentials stay with git and the installer: an SSH key or a git credential helper (`gh auth setup-git`) for a private repository, and the installer's own config for a private index. A token written into `SPEC` is kept in the installer's record of the environment. -- `--installer` overrides the detected installer, `--index-url` passes an index through to it (a uv tool keeps it as its default index), `--dry-run` shows what would run, and `--yes` skips the prompt. -- uv takes a pre-release or a yanked version only when you name it. If the installer reports that a plugin needs one, name that exact version in the same command, for example `agent-env plugin add agentenv-grader 'agentenv-grader-core==0.2.0b1'`; `plugin remove` removes it later the same way. -- A uv project whose environment is set by `UV_PROJECT_ENVIRONMENT` is found from the current directory, as uv finds it, so run `add` and `remove` inside the project. In a uv workspace whose root has no `[project]` table, `add` goes to the one member that declares `agentenv-framework` (`uv add --package MEMBER`). `remove` drops a package from every member and group that declares it (`--package`, `--group`, `--optional`, `--dev`). -- A uv project is never synced exactly, so packages from extras and anything installed outside the lock, agent-env included, stay installed. -- pipx and a virtualenv leave a removed plugin's own dependencies installed, as `pip uninstall` does. Left to itself, `pipx uninject` would also uninstall everything nothing else requires, agent-env included. -- A uv tool installed with constraints, overrides or executables from another package is refused: `uv tool install` cannot take those back from agent-env, so make that change with uv. -- One `add` or `remove` runs at a time per environment; a second one is refused until the first finishes. The lock is a file in `$XDG_STATE_HOME/agent-env/locks` (by default `~/.local/state/agent-env/locks`). If the installer's record changes some other way while a change waits for confirmation, the change is refused rather than run from its outdated plan. -- A plain virtualenv first shows the installer's own dry run of what would change. Every mode reports what did change afterwards, calling out a change to agent-env itself. -- `add` then checks each new plugin package in a fresh interpreter: it must declare at least one `agent_env.*` entry point, and every contribution must be `active` or `replaced`. `show` reports whether importing it sets `AGENT_ENV_CONFIG`. If the check fails, the installer fails or removes another plugin, or the change is interrupted (Ctrl-C or SIGTERM), `add` puts the environment back as it was: every package at its old version from its old source, and the uv tool receipt, pipx metadata, or `pyproject.toml` and `uv.lock`, byte for byte. The installer runs with Ctrl-C ignored, since one stopped halfway leaves packages no installer can read: it finishes, and then the change is rolled back. A second Ctrl-C stops it at once. It keeps the terminal, so its own prompts, such as git's for credentials, still work. When the environment cannot be put back exactly, `add` says which packages differ and the installer command that reinstalls them. Adding what is already installed changes nothing and succeeds, and a bare name that a uv tool already lists keeps its version pin. `--keep` keeps a change that failed the check instead. -- `remove` refuses, unless `--force`: - - while another installed package requires the package; - - while the config file names something only it provides (`[sandbox] default` or `agent_default`, a provider table, a `type_aliases` target), or an `impls` list, provider, store or runner `impl` names one of its modules; - - while stored documents use env, artifact or task-step types only it provides. This is checked for a local document store, and for a remote one only with `--check-usage`; a store that cannot be read also blocks, until `--force`. - - It never removes agent-env itself. A package that declares no `agent_env.*` entry point is removed only when the installer records it on its own (a uv tool's `--with`, a pipx injection, a project dependency), as it does for a package named next to a plugin in `add`; after removing a plugin, `remove` names any such package that only the plugin needed. In a uv tool or a uv project, removing a plugin also removes the plugins that only it needed, as the installer's rebuild would, so `remove` names them first and checks them like the package itself. If the installer fails, removes a plugin it did not name, or the change is interrupted, `remove` puts the environment back the same way `add` does. -- Plugins are trusted code: nothing installs one implicitly, from a task document, a config file or anywhere else. -- A plugin that breaks CLI startup does not lock you out. A root option whose callback raises or exits while its flag is absent is skipped, with a warning, and reported as `failed`; a flag you pass still fails, naming the plugin. A CLI plugin that exits while being imported is skipped like any load failure. For a plugin that hangs or kills the interpreter while being imported, uninstall it with the installer directly: the command `plugin remove --dry-run` would print. - -### Building a platform plugin - -One installable package can combine all of the above: bundled config files, a root option that selects one per invocation, store and provider classes, a `Runner`, custom steps, and explorer routers. The `--tenant` demo above is that pattern in miniature. Its callback points `AGENT_ENV_CONFIG` at a bundled `tenants/<name>.toml`; a name that does not exist fails loud with `ConfigError` on first use, while `--help` still works. A hosted control plane serves the explorer app from its own server process and lists its public hostnames under `[explorer] allowed_hosts` (see the comments in `.agentenv/config.example.toml`). A durable runner is the plugin's responsibility: this repository ships only `LocalRunner`. - -### Plugin compatibility - -What a plugin can build on, how to declare the agent-env it needs, and what agent-env does when the installed one does not fit. - -**The plugin surface.** A plugin may rely on these. Anything else, including every underscored name other than a method a subclass must implement (`EnvStateProvider._teardown`), is internal and can change in any release. - -- The entry-point groups and their rules, in [Register types from an installed package](#register-types-from-an-installed-package) and [CLI plugins, root options, explorer routes](#cli-plugins-root-options-explorer-routes). -- The base class each group names, with its public methods and attributes: `agent_env.env.env.Env`, `agent_env.task_step.task_step.TaskStep` and the `agent_env.task_step.context.TaskStepContext` a step runs with, `agent_env.artifact.artifact.Artifact`, `agent_env.providers.sandbox_providers.sandbox_provider.SandboxProvider`, `agent_env.providers.env_state.env_state_provider.EnvStateProvider`, `agent_env.providers.env_providers.env_provider.EnvironmentProvider`, and `agent_env.explorer.plugin.ExplorerPlugin`. -- The two functions an env that uses an environment provider calls: `agent_env.providers.env_providers.env_provider.build_env_provider` and `agent_env.env.store.register_env_instance`. -- What an environment provider reads to deploy a built-in env: `agent_env.env.envs.mcp_server.MCPServerEnv.docker_image_artifact` and `agent_env.env.envs.mcp_server.MCPServerEnv.environment_name`; `agent_env.env.envs.website.WebsiteEnv.backend_docker_image_artifact`, `agent_env.env.envs.website.WebsiteEnv.frontend_docker_image_artifact` and `agent_env.env.envs.website.WebsiteEnv.environment_name`; `agent_env.env.envs.multi_env.MultiEnv.mcp_server_envs`, `agent_env.env.envs.multi_env.MultiEnv.website_envs` and `agent_env.env.envs.multi_env.MultiEnv.name`; and each image's `agent_env.artifact.artifacts.docker_image.DockerImageArtifact.image_name`. -- The top level of `agent_env.plugins`, including `settings`, and the `[plugins.<package>]` table it reads ([Plugin settings](#plugin-settings)). -- The `plugin --json` output, which has its own rules: [Plugin report format](#plugin-report-format). - -The classes a config `impl` names, such as stores and runners, are not on the list yet. - -**Changes before 1.0.** Every merged change can ship as a release, several a day. A change that breaks the plugin surface is marked with `!` after the scope in its pull request title, which becomes its commit title, as in `feat(plugins)!: …`. Where the old behaviour can be kept for a while, it is deprecated first: it keeps working and emits a `DeprecationWarning` that names what replaces it. How long that lasts is not fixed before 1.0. - -The `plugin-api` CI job holds pull requests to this. It compares the listed base classes, `TaskStepContext`, the two functions, the env attributes and the top level of `agent_env.plugins` with the pull request's base (`.github/scripts/check_plugin_api.py`), and fails on a break the title does not mark; with the `!`, it lists the breaks and passes, and editing the title re-runs it. A break is what fails code written against the old surface: - -- for a caller, a name, parameter or `__all__` entry that is removed or renamed, a new required parameter, a parameter that can no longer be passed as before, or a changed default or constant, including the group-name constants `agent_env.plugins` exports, compared by value; -- for a subclass of a base class, a new abstract or required method, a method that becomes abstract or required, a new `ClassVar` with no value, a method that changes between plain, `async`, `classmethod`, `staticmethod` and property, and a base-class method that accepts more than before: a new parameter, even an optional one, a parameter that stops being required, a new `*args` or `**kwargs`, or a keyword-only parameter that can now be passed by position, since an override written for the old signature fails when core passes it; -- for construction, a changed `@dataclass(...)` option or pydantic `model_config`, a new metaclass, or a new `__init_subclass__` that can raise, which refuses a subclass. - -A member every subclass must implement is declared `@abstractmethod`, or listed in its registry's `_MUST_IMPLEMENT`, so that the check enforces it; a new method that only raises `NotImplementedError` gets a note but does not fail the job. - -agent-env ships `py.typed`, so mypy and pyright check a plugin against its annotations. Import each name from the module the list above gives, such as `TaskStepContext` from `agent_env.task_step.context`: pyright, and mypy under `--strict`, treat a name a module only imports as private to that module. Type annotations are not compared, so a release can correct one without a `!`. The check can report a change no plugin notices, such as a new parameter core never passes to an override; mark it all the same. It does not see a change in behaviour, in a type the surface only names in a signature, such as `DeployedEnv`, in a pydantic field's requirements or the order of dataclass fields, in an abstract method a new base from outside agent-env brings, or in a registration rule other than abstract methods and `_MUST_IMPLEMENT`; review covers those. A `!` can also mark a break outside the plugin surface. - -**Declaring the agent-env your plugin needs.** Declare a floor, `agentenv-framework>=X`, where `X` is the oldest release you test against. Leave out a ceiling such as `<1`: before 1.0 it would not guard against a change in a 0.9 release, and it would keep your plugin from installing with the next major one. Pin exact versions in the application or image that installs your plugin, not in the plugin. If your plugin imports `agentenv_protocol` itself, declare that as well. - -**What agent-env checks.** Before it imports a plugin, agent-env compares the plugin's requirements on `agentenv-framework` and `agentenv-framework-protocol` with the versions installed. A plugin they exclude is not loaded: - -- `plugin list`, `show` and `check` report each of its contributions as `failed` with the code `incompatible-core`, with or without `--no-load`, and `check` exits 1; -- using one of its types names the requirement, as in `needs agentenv-framework>=0.9.1220 (installed: 0.9.1218)`; -- it claims none of its names, so a plugin that can load and registers the same name does not conflict with it. - -An installer that resolves dependencies never gets you there; `pip install --no-deps` or a forced install can. There is no override: upgrade agent-env, or install a version of the plugin that fits. A requirement under an extra, or whose environment marker is false, does not count, and an agent-env with no installed metadata, such as a source tree on `sys.path`, is not checked. Requirements between plugins are the installer's to check; `pip check` lists any that are unmet. - ## Contribute, release, license [CONTRIBUTING.md](https://github.com/scaleapi/agentenv-framework/blob/main/CONTRIBUTING.md) is the contributor guide: open an issue before anything larger than a bug fix, one logical change per pull request with tests, pull request titles in the form `type(scope): summary`, an approving review from a code owner (`CODEOWNERS`) and green CI. [AGENTS.md](https://github.com/scaleapi/agentenv-framework/blob/main/AGENTS.md) is the repository map and conventions file for contributors and coding agents. @@ -338,6 +55,7 @@ An installer that resolves dependencies never gets you there; `pip install --no- | [www.agentenvframework.com/docs](https://www.agentenvframework.com/docs) | the user guide: environments, artifacts, agents, tasks, the registry and plugins | | [`packages/agentenv-protocol/README.md`](https://github.com/scaleapi/agentenv-framework/blob/main/packages/agentenv-protocol/README.md) | wire contract, environment server SDK, A2A agent framework | | [`CONTRIBUTING.md`](https://github.com/scaleapi/agentenv-framework/blob/main/CONTRIBUTING.md) | development setup, test tiers, CI jobs, pull request rules | +| [`PLUGINS.md`](https://github.com/scaleapi/agentenv-framework/blob/main/PLUGINS.md) | the plugin contract: entry-point groups, plugin settings, the `agent-env plugin` commands and their report format, and what a plugin may rely on across releases | | [`AGENTS.md`](https://github.com/scaleapi/agentenv-framework/blob/main/AGENTS.md) | repository map, configuration and extension-point summary, conventions for contributors and coding agents | | [`SECURITY.md`](https://github.com/scaleapi/agentenv-framework/blob/main/SECURITY.md) | private vulnerability reporting and supported versions | | [`CODE_OF_CONDUCT.md`](https://github.com/scaleapi/agentenv-framework/blob/main/CODE_OF_CONDUCT.md) | Contributor Covenant | @@ -346,7 +64,7 @@ An installer that resolves dependencies never gets you there; `pip install --no- ### Versioning and compatibility -The `agentenv-framework` distribution and `agentenv-framework-protocol` are versioned separately (`0.9.x` and `0.1.x` today), both in their `pyproject.toml`; agent-env releases carry a `vX.Y.Z` tag, and agentenv-framework-protocol is bumped in the same commit and has no separate tag today. There is no `agent_env.__version__` attribute; `agent-env --version` prints the installed version. Protocol extensions carry their version in the URI (`urn:agentenv:clock/v1`, `urn:agentenv:agent-config/v1`, `urn:agentenv:trajectory/v1`). One version can take more than one request shape: under `v1` the skill, trajectory, snapshot and changelog extensions accept object-transfer requests next to their older shapes, and the request field lists on an agent's card say which ones that agent takes. Renamed CLI commands are removed outright; no deprecated aliases exist at this version. What plugins may rely on, and how changes to it are made, is in [Plugin compatibility](#plugin-compatibility); the plugin commands' `--json` output has its own format version and rules ([Plugin report format](#plugin-report-format)). Beyond those, a written compatibility and deprecation policy does not exist yet. +The `agentenv-framework` distribution and `agentenv-framework-protocol` are versioned separately (`0.9.x` and `0.1.x` today), both in their `pyproject.toml`; agent-env releases carry a `vX.Y.Z` tag, and agentenv-framework-protocol is bumped in the same commit and has no separate tag today. There is no `agent_env.__version__` attribute; `agent-env --version` prints the installed version. Protocol extensions carry their version in the URI (`urn:agentenv:clock/v1`, `urn:agentenv:agent-config/v1`, `urn:agentenv:trajectory/v1`). One version can take more than one request shape: under `v1` the skill, trajectory, snapshot and changelog extensions accept object-transfer requests next to their older shapes, and the request field lists on an agent's card say which ones that agent takes. Renamed CLI commands are removed outright; no deprecated aliases exist at this version. What plugins may rely on, and how changes to it are made, is in [Plugin compatibility](https://github.com/scaleapi/agentenv-framework/blob/main/PLUGINS.md#plugin-compatibility); the plugin commands' `--json` output has its own format version and rules ([Plugin report format](https://github.com/scaleapi/agentenv-framework/blob/main/PLUGINS.md#plugin-report-format)). Beyond those, a written compatibility and deprecation policy does not exist yet. ### Releases diff --git a/src/agent_env/cli/plugin.py b/src/agent_env/cli/plugin.py index f69f834a..83429e55 100644 --- a/src/agent_env/cli/plugin.py +++ b/src/agent_env/cli/plugin.py @@ -179,7 +179,7 @@ def _bundle_contribution(bundle: InstalledBundle, bundles: tuple[InstalledBundle def _as_dict(report: Report) -> dict: - """The document `list --json` prints; `show` and `check` add to it. See "Plugin report format".""" + """The document `list --json` prints; `show` and `check` add to it. See PLUGINS.md "Plugin report format".""" error = report.inventory.config_error return { "format_version": FORMAT_VERSION, diff --git a/src/agent_env/plugins/__init__.py b/src/agent_env/plugins/__init__.py index 0e99378d..4d033ba4 100644 --- a/src/agent_env/plugins/__init__.py +++ b/src/agent_env/plugins/__init__.py @@ -12,7 +12,7 @@ This package's top level is the public surface: the group names, ``load_failures``, ``settings``, and ``inventory`` with the types it returns; its underscored modules are internal. The full contract -is in the README ("Register types from an installed package", "Plugin settings"), and the codes the +is in PLUGINS.md ("Register types from an installed package", "Plugin settings"), and the codes the inventory reports are in "Plugin report format": codes are stable, reason text is not. """ diff --git a/src/agent_env/plugins/_cli.py b/src/agent_env/plugins/_cli.py index ddfacd18..1f0d8a77 100644 --- a/src/agent_env/plugins/_cli.py +++ b/src/agent_env/plugins/_cli.py @@ -4,7 +4,7 @@ ``agent_env.cli_root_options`` entries to ``click.Option`` instances that attach to the root ``agent-env`` group. Broken plugins warn and are skipped; core names win over plugins; two different root options on the same flag are both left off and reported as a conflict. -Full contract in the README ("Extending the CLI (plugins)"). +Full contract in PLUGINS.md ("CLI plugins, root options, explorer routes"). """ import sys diff --git a/src/agent_env/plugins/_registration.py b/src/agent_env/plugins/_registration.py index 2093138f..0bb23088 100644 --- a/src/agent_env/plugins/_registration.py +++ b/src/agent_env/plugins/_registration.py @@ -1,6 +1,6 @@ """Loading a group's plugins into its registry, and recording each one that did not take effect. -Internal: the public contract is ``agent_env.plugins`` and the README section "Register types +Internal: the public contract is ``agent_env.plugins`` and the PLUGINS.md section "Register types from an installed package". Each registry calls ``merge`` when it is built, so a group's plugins are imported then and not before. """ diff --git a/src/agent_env/plugins/_report.py b/src/agent_env/plugins/_report.py index db24a65e..f944daea 100644 --- a/src/agent_env/plugins/_report.py +++ b/src/agent_env/plugins/_report.py @@ -1,6 +1,6 @@ """The plugin report's format version and reason codes. -Internal: the contract is the README section "Plugin report format". A code says why a +Internal: the contract is the PLUGINS.md section "Plugin report format". A code says why a contribution has its status, or why a group, the config or discovery failed. Codes are never removed or reused; a new one needs no format bump. """ diff --git a/src/agent_env/plugins/_requirements.py b/src/agent_env/plugins/_requirements.py index 7009a2aa..f49abe39 100644 --- a/src/agent_env/plugins/_requirements.py +++ b/src/agent_env/plugins/_requirements.py @@ -1,6 +1,6 @@ """Whether an installed plugin's requirements admit the agent-env installed next to it. -Internal: the contract is the README section "Plugin compatibility". Reads installed metadata only, +Internal: the contract is the PLUGINS.md section "Plugin compatibility". Reads installed metadata only, so it runs before a plugin is imported, and for ``inventory(load=False)``. """ diff --git a/tst/installer/test_container_journey.py b/tst/installer/test_container_journey.py index 6c0c45ab..ed7f40f1 100644 --- a/tst/installer/test_container_journey.py +++ b/tst/installer/test_container_journey.py @@ -1,7 +1,7 @@ """The open-source user journey, replayed in fresh containers with no network. A new user on `python:3.12-slim` installs agent-env as a uv tool, adds two plugins, runs a task -that uses their steps, and removes them; the README's plugin commands run on the way, so the +that uses their steps, and removes them; the plugin commands in PLUGINS.md run on the way, so the docs cannot drift from the CLI. A second container uses Ubuntu's system Python, which carries the PEP 668 marker, and `plugin add` must refuse it. Linux wheels for the dependencies are downloaded once, inside a container, and cached; the journey itself runs with `--network none`. @@ -124,9 +124,9 @@ def journey(tmp_path, checkout_wheels, linux_wheels) -> Journey: return Journey(mounts, work, f"/plugins/{grader.name}", f"/plugins/{browser.name}") -def _readme_plugin_commands() -> list[list[str]]: - """The commands in the README's "Manage plugins" block, as a user would type them.""" - section = (REPO / "README.md").read_text().split("### Manage plugins", 1)[1] +def _documented_plugin_commands() -> list[list[str]]: + """The commands in the "Manage plugins" block of PLUGINS.md, as a user would type them.""" + section = (REPO / "PLUGINS.md").read_text().split("## Manage plugins", 1)[1] block = section.split("```bash\n", 1)[1].split("```", 1)[0] lines = [line.split("#", 1)[0].split() for line in block.splitlines() if line.startswith("agent-env plugin")] assert [line[2] for line in lines] == ["list", "show", "check", "add", "remove"], lines @@ -148,8 +148,8 @@ def test_a_new_user_installs_agent_env_adds_plugins_runs_a_task_and_removes_them added = container.ok("agent-env", "plugin", "add", browser, grader, "--yes") assert "toy_navigate active" in added.stdout and "toy_grade active" in added.stdout - # The README's commands, verbatim but for their placeholders; --yes because there is no terminal. - for line in _readme_plugin_commands(): + # The documented commands, verbatim but for their placeholders; --yes because there is no terminal. + for line in _documented_plugin_commands(): command = [{"PACKAGE": GRADER, "SPEC...": grader}.get(word, word) for word in line] container.ok(*command, *(["--yes"] if line[2] in ("add", "remove") else [])) assert GRADER not in container.ok("agent-env", "plugin", "list").stdout diff --git a/tst/unit/cli/plugin_report_test.py b/tst/unit/cli/plugin_report_test.py index 997bafbd..3a9e61aa 100644 --- a/tst/unit/cli/plugin_report_test.py +++ b/tst/unit/cli/plugin_report_test.py @@ -4,7 +4,7 @@ code. Reason text is replaced by a placeholder: it may change in any release, and a reword should not touch the goldens. ``_SCHEMA`` is the format's keys and types, so a key added to the report fails here until it is added there too; that edit is where to decide whether the format version -moves (README "Plugin report format"). After an intended change, regenerate and review the diff: +moves (PLUGINS.md "Plugin report format"). After an intended change, regenerate and review the diff: AGENT_ENV_REGENERATE_GOLDENS=1 python -m pytest tst/unit/cli/plugin_report_test.py """ @@ -34,7 +34,7 @@ _THIS = "tst.unit.cli.plugin_report_test" _GOLDEN = Path(__file__).parent / "golden" / "plugin_report" _REGENERATE = os.environ.get("AGENT_ENV_REGENERATE_GOLDENS") == "1" -_README = Path(__file__).parents[3] / "README.md" +_PLUGINS_MD = Path(__file__).parents[3] / "PLUGINS.md" _COMMANDS = cli_plugins_mod.CLI_PLUGINS_GROUP _OPTIONS = cli_plugins_mod.CLI_ROOT_OPTIONS_GROUP @@ -286,7 +286,7 @@ def test_codes_are_kebab_case(): def _readme_section() -> str: - return _README.read_text().split("#### Plugin report format", 1)[1].split("\n#### ", 1)[0] + return _PLUGINS_MD.read_text().split("### Plugin report format", 1)[1].split("\n### ", 1)[0] def test_the_readme_lists_every_code(): diff --git a/tst/unit/config/test_unread_keys.py b/tst/unit/config/test_unread_keys.py index c49e7304..185101e9 100644 --- a/tst/unit/config/test_unread_keys.py +++ b/tst/unit/config/test_unread_keys.py @@ -236,8 +236,8 @@ def test_a_table_is_named_as_the_file_writes_it(tmp_path, monkeypatch, body, war def _documented_configs() -> list[str]: - readme = (_REPO / "README.md").read_text() - blocks = re.findall(r"```toml\n(.*?)```", readme, re.S) + docs = "".join((_REPO / name).read_text() for name in ("README.md", "PLUGINS.md")) + blocks = re.findall(r"```toml\n(.*?)```", docs, re.S) # the pyproject.toml blocks declare entry points, and a block headed by a bundle path # (`# agents/solver/agent.toml`) is an entity's toml; neither is a config file entity_tomls = tuple(f"# {kind.value}/" for kind in CONFIG_FILES) diff --git a/tst/unit/test_check_plugin_api.py b/tst/unit/test_check_plugin_api.py index ee043249..36fc9eda 100644 --- a/tst/unit/test_check_plugin_api.py +++ b/tst/unit/test_check_plugin_api.py @@ -252,8 +252,8 @@ def test_a_removed_surface_class_is_a_break(tmp_path): assert "agent_env.providers.env_state.env_state_provider.EnvStateProvider: removed" in result.stdout -def _readme_surface() -> set[str]: - section = (REPO / "README.md").read_text().split("**The plugin surface.**", 1)[1].split("\n\n", 2)[1] +def _documented_surface() -> set[str]: + section = (REPO / "PLUGINS.md").read_text().split("**The plugin surface.**", 1)[1].split("\n\n", 2)[1] return set(re.findall(r"`(agent_env(?:\.\w+)+)`", section)) @@ -268,8 +268,8 @@ def test_the_title_types_are_the_documented_ones(path, lead): assert _documented_types(path, lead) == set(check_plugin_api.TYPES) -def test_the_readme_lists_the_checked_surface(): - assert _readme_surface() == set(check_plugin_api.SURFACE) +def test_the_plugin_contract_lists_the_checked_surface(): + assert _documented_surface() == set(check_plugin_api.SURFACE) def test_the_surface_is_found_in_this_checkout(): From c0356127b1f9bc4738394dc6dcd099d92e198dfa Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan <edgar.arakelyan@scale.com> Date: Thu, 8 Oct 2026 10:28:52 -0700 Subject: [PATCH 3/5] docs: drop PLUGINS.md and link the README to the plugin docs Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .github/scripts/check_plugin_api.py | 10 +- .github/workflows/local-backends.yml | 2 +- .github/workflows/plugin-api.yml | 2 +- AGENTS.md | 7 +- CONTRIBUTING.md | 2 +- PLUGINS.md | 282 ------------------------ README.md | 7 +- src/agent_env/cli/plugin.py | 2 +- src/agent_env/plugins/__init__.py | 5 +- src/agent_env/plugins/_cli.py | 1 - src/agent_env/plugins/_registration.py | 3 +- src/agent_env/plugins/_report.py | 2 +- src/agent_env/plugins/_requirements.py | 2 +- tst/installer/test_container_journey.py | 21 +- tst/unit/cli/plugin_report_test.py | 19 +- tst/unit/config/test_unread_keys.py | 4 +- tst/unit/test_check_plugin_api.py | 9 - 17 files changed, 30 insertions(+), 350 deletions(-) delete mode 100644 PLUGINS.md diff --git a/.github/scripts/check_plugin_api.py b/.github/scripts/check_plugin_api.py index a58faac6..141ad50d 100644 --- a/.github/scripts/check_plugin_api.py +++ b/.github/scripts/check_plugin_api.py @@ -1,9 +1,9 @@ """Fail a change that breaks the plugin surface, unless its title marks the break with ``!``. -The surface is ``BASES`` and ``USED``, the list in PLUGINS.md "Plugin compatibility"; a unit test -holds the two together. Both trees are read statically with griffe, which reports what breaks a -caller, and the subclass rules below add what breaks a plugin's subclass of a base. A title in the -form ``type(scope)!: summary`` marks the break deliberate: the breaks are printed and the check passes. +The surface is ``BASES`` and ``USED``. Both trees are read statically with griffe, which reports what +breaks a caller, and the subclass rules below add what breaks a plugin's subclass of a base. A title in +the form ``type(scope)!: summary`` marks the break deliberate: the breaks are printed and the check +passes. """ from __future__ import annotations @@ -617,7 +617,7 @@ def main() -> int: print("plugin API: the title has a ! but not in the form type(scope)!: summary, with the type one of " + ", ".join(TYPES)) print("plugin API: if the break is deliberate, mark it with ! in the pull request title, as in " - "feat(plugins)!: …; see \"Plugin compatibility\" in PLUGINS.md") + "feat(plugins)!: …") _report("error", f"{count}; mark it with ! in the title if it is deliberate", found) return 1 diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index a6445240..3dcd03ef 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -186,7 +186,7 @@ jobs: # moves; diffing against base.sha would count the base's own later commits as changes. BASE: ${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }} run: | - paths='^(src/agent_env/cli/(__init__|__main__|_installers|_plugin_changes|plugin)\.py|src/agent_env/(plugins|config)/|src/agent_env/py\.typed|packages/agentenv-protocol/|tst/installer/|tst/util/capabilities\.py|pyproject\.toml|uv\.lock|Makefile|PLUGINS\.md|\.github/(workflows/local-backends\.yml|scripts/check_skip_policy\.py))' + paths='^(src/agent_env/cli/(__init__|__main__|_installers|_plugin_changes|plugin)\.py|src/agent_env/(plugins|config)/|src/agent_env/py\.typed|packages/agentenv-protocol/|tst/installer/|tst/util/capabilities\.py|pyproject\.toml|uv\.lock|Makefile|\.github/(workflows/local-backends\.yml|scripts/check_skip_policy\.py))' if ! changed=$(git diff --name-only "$BASE" HEAD 2>/dev/null); then echo "plugins=true" >> "$GITHUB_OUTPUT" elif grep -qE "$paths" <<< "$changed"; then diff --git a/.github/workflows/plugin-api.yml b/.github/workflows/plugin-api.yml index 816e9360..4ece4b4a 100644 --- a/.github/workflows/plugin-api.yml +++ b/.github/workflows/plugin-api.yml @@ -1,6 +1,6 @@ name: Plugin API -# Fails a change that breaks the plugin surface (PLUGINS.md "Plugin compatibility") unless its title +# Fails a change that breaks the plugin surface unless its title # marks the break with `!`. A workflow of its own, so that a title edit re-runs only this check. # On main it checks the squash commit against its parent, a record of what reached history. diff --git a/AGENTS.md b/AGENTS.md index f591a04f..173c7658 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -183,8 +183,7 @@ skips for the allowed capabilities pass, and the fast job allows no skips under touches the plugin CLI, `agent_env/plugins`, the tier itself or the lockfile, and allows no skips; a `changes` job decides. `.github/workflows/plugin-api.yml` runs the `plugin-api` job on every pull request, title edits included, and on `main`: `.github/scripts/check_plugin_api.py` compares the -plugin surface (its `BASES` and `USED`, which a unit test holds to the PLUGINS.md "Plugin compatibility" -list) between `HEAD^1` and `HEAD` with griffe, pinned in the `dev` extra, and fails on a break the +plugin surface (its `BASES` and `USED`) between `HEAD^1` and `HEAD` with griffe, pinned in the `dev` extra, and fails on a break the title does not mark with `!`. `.github/workflows/clean-install.yml` runs the `clean-install` job on every pull request and on `main`: `.github/scripts/clean_install.py` builds both distributions as the release does, fails if the wheel leaves out a file tracked under `src/agent_env/examples` or a @@ -215,5 +214,5 @@ Actions are pinned to commit SHAs. ## Docs `README.md` is the user-facing front door (configuration, custom envs, steps, artifacts and -providers, the CLI). `PLUGINS.md` is the plugin contract. `packages/agentenv-protocol/README.md` -documents the protocol and the A2A agent framework. +providers, the CLI). `packages/agentenv-protocol/README.md` documents the +protocol and the A2A agent framework. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6a33051..0e0a5460 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ must name it (`agentenv-capability-missing: <name>`, see `tst/util/capabilities. target as received`. Types in use: `feat`, `fix`, `refactor`, `docs`, `test`, `build`, `ci`, `chore`, `perf`, `security`. The title becomes the commit title. - Mark a breaking change with `!` after the scope: `feat(plugins)!: summary`. The `plugin-api` job - fails a pull request that breaks the plugin surface (PLUGINS.md "Plugin compatibility") without it, + fails a pull request that breaks the plugin surface without it, lists what broke, and runs again when you edit the title. To run it before you push: `.venv/bin/python .github/scripts/check_plugin_api.py --base origin/main --title "<title>"`. - Commit with an email address you are comfortable publishing; the history of this repository is diff --git a/PLUGINS.md b/PLUGINS.md deleted file mode 100644 index a61d4282..00000000 --- a/PLUGINS.md +++ /dev/null @@ -1,282 +0,0 @@ -# Plugin contract - -What a package that extends agent-env relies on. Unit tests and the plugin-API check in CI hold the code to these sections, so a pull request that changes the contract changes this file too. - -## Register types from an installed package - -An installed distribution registers envs, task steps, artifacts, sandbox, state and environment providers and explorer plugins by declaring entry points, with no config. The group says what kind of thing it is, the entry-point name is the registry key, and the value is the class: - -```toml -[project.entry-points."agent_env.envs"] -browser = "agentenv_browser.env:BrowserEnv" - -[project.entry-points."agent_env.task_steps"] -browser_navigate = "agentenv_browser.steps:NavigateTaskStep" -``` - -| Group | Value | Name | -|---|---|---| -| `agent_env.envs` | `Env` subclass with its own `type`, implementing `from_dict` | must equal the class's `type`, the spelling its documents are written under | -| `agent_env.task_steps` | `TaskStep` subclass with its own `type`, implementing `execute` and `from_dict` | must equal the class's `type` | -| `agent_env.artifacts` | `Artifact` subclass with its own `type` field default | the registry key; a class may also register under extra names, such as a legacy spelling, as long as its own `type` default resolves to it and its `type` field accepts each extra name | -| `agent_env.sandbox_providers` | `SandboxProvider` subclass implementing `create_sandbox` | the registry key; it must equal the `.type` of the sandboxes the provider produces, checked on the first `create_*` call | -| `agent_env.state_providers` | `EnvStateProvider` subclass implementing `acquire` and `_teardown` | must equal the class's `type`, checked at registration | -| `agent_env.env_providers` | `EnvironmentProvider` subclass implementing `deploy` and `close` | must equal the class's `type`, which its records carry as `env_provider_type`; checked at registration | -| `agent_env.explorer_plugins` | `ExplorerPlugin` subclass with its own `type`, implementing `router` | must equal the class's `type` | - -- The class must implement every abstract method of its base, and, for envs and task steps, `from_dict` as a classmethod, which the base defines only to raise. A plugin that does not is `failed` with `invalid-plugin`, naming what is missing, for example `GradeTaskStep must implement execute and from_dict`. An `impls` entry or provider table naming such a class is a `ConfigError`. -- Each registry takes the built-ins first, then plugins, then config. A plugin cannot replace a built-in: it is skipped with a warning, however many distributions claim that name. -- A name that two installed distributions register in one group, or that one declares twice, is left out of the registry with a warning, and the rest of the group, built-ins included, loads. An explorer plugin under that name is not mounted, so its routes are absent. Resolving the name fails like a plugin that did not load, naming each claimant and its version, for example `Unknown env type: browser (2 installed plugins register 'browser' in agent_env.envs: …)`, and says to remove all but one with `agent-env plugin remove PACKAGE`, or, when one package declares the name twice, to report it to the package's author. Neither claimant is picked, because the order entry points are found in is not fixed. -- Config cannot settle a conflicted name. An `impls` entry, a provider table, an `[explorer.plugins]` impl or an `[artifacts] type_aliases` entry mapping to that name is checked as usual, then skipped with a warning. A `type_aliases` entry whose old spelling is that name is a `ConfigError`, as it is when one plugin registers it. -- A plugin that fails to import, fails the checks above, or (for explorer plugins) fails to construct is skipped with a warning and the others still load. So is one whose requirement on agent-env excludes the installed version, before it is imported (see [Plugin compatibility](#plugin-compatibility)). Resolving its name then reports the recorded error, for example `Unknown env type: browser (registered by 'browser' from agentenv-web 2.1.0 (…) but failed to load: ModuleNotFoundError(…))`. -- Config replaces a plugin's class with a warning, except under a conflicted name: an `impls` entry of the same `type`, or a `[sandbox.providers.<name>]` / `[state.providers.<name>]` table carrying `impl`. Naming the plugin's own class is silent. A provider table without `impl` configures the plugin's provider the way it configures a built-in's, which is where per-deployment settings such as a secret name belong. A config-only provider table, or an `[artifacts] type_aliases` entry, that points at a plugin which failed to load is skipped with a warning rather than failing the whole registry. -- `agent_env.plugins.load_failures()` lists every plugin that did not take effect in the registries the current `Config` has built (failed to load, validate or construct, clashed with a built-in, or claims a name another entry point claims), by group and name, with the reason. The record belongs to the `Config`, so `reset_config()` starts a new one. A deployment that ships its plugins can build its registries at startup and assert it is empty. -- `agent_env.plugins.inventory()` lists every installed distribution that declares entry points in these groups, with a status for each contribution and a `code` saying why (see [Plugin report format](#plugin-report-format)): `active`; `replaced` (config names a different class, and `replaced_by` says where); `failed`; `skipped` (a built-in owns the name); `conflict`; `blocked` (the group's real build fails, for example because its config table is invalid, so nothing in the group loads); or `unloaded`. It builds the registries on a throwaway `Config` that reads the same document, so the `Config` in use keeps its registries and its `load_failures()`. `inventory(load=False)` reads installed metadata only and runs no plugin code, so a status that needs a build is `unloaded`. With the default `load=True` the plugins are imported and the explorer plugins constructed, with whatever process-wide effects that has. `discovery_errors` names each group whose installed entry points could not be read at all, so none of its plugins is listed. -- Discovery reads installed metadata and imports nothing. When entry points are loaded is unspecified: today each group is imported when its registry is first built, but a plugin must not depend on that. -- Loading a plugin never changes which config a `Config` reads: the `Config` resolves its document before it imports any plugin. A plugin package that sets `AGENT_ENV_CONFIG` on import affects only configs built afterwards, such as after `reset_config()`. - -The distribution that declares the entry points is the plugin, so `pip uninstall` removes it completely, apart from any `[plugins.<package>]` table in the config file. CLI commands are a separate contribution, described next. - -## CLI plugins, root options, explorer routes - -Installed packages add commands through two entry-point groups: - -```toml -[project.entry-points."agent_env.cli_plugins"] -my-tools = "mycorp_demo.cli:my_tools" -[project.entry-points."agent_env.cli_root_options"] -tenant = "mycorp_demo.cli:tenant_option" -``` - -`agent_env.cli_plugins` entries are click commands or groups added next to the built-ins, under the entry-point name whatever the command object is called: `my-tools` above is `agent-env my-tools`. `agent_env.cli_root_options` entries are optional `click.Option` instances with `expose_value=False`; their callback runs before the subcommand, so it can set `AGENT_ENV_CONFIG` and call `agent_env.config.reset_config()` to select the config for the whole process. Plugins load when `agent_env.cli` is imported, after the built-ins. Clash rules: a plugin command whose entry-point name core already uses, or a flag core owns, is skipped with a warning on stderr, and core wins. Two different root options on the same flag are both left off with a warning, and `agent-env plugin list` reports each as a `conflict`; the CLI still starts, so `agent-env plugin remove` can settle it. The same `click.Option` object exported by two entry points, from one package or two, is attached once. Nothing grafts onto existing groups. - -Explorer routes: subclass `agent_env.explorer.plugin.ExplorerPlugin`, set the `type` ClassVar, and return a FastAPI `APIRouter` from the `router` property. List it under `[explorer.plugins] impls` or declare an `agent_env.explorer_plugins` entry point; `from_config()` takes no arguments, and a plugin reads its own settings with `agent_env.plugins.settings` (see [Plugin settings](#plugin-settings)). `agent-env up --no-bootstrap` mounts it before the core routers and needs no Docker. An entity id can contain `/` (a bundle's ids are `@local/<package>/<name>`, and so are the instance ids of its runs), so in an explorer URL an id is one percent-encoded path segment, with `/` sent as `%2F`. Under the core prefixes `/api/v1/{artifacts,envs,tasks,agents,evals,task-instances}/` the explorer routes on the encoded path, so a plugin route there receives the segment still encoded and decodes it with `urllib.parse.unquote`. - -## Bundles from installed packages - -A package ships bundles by naming the package that holds their folders: - -```toml -[project.entry-points."agent_env.bundles"] -triage = "mycorp_demo.bundles" -``` - -- The bundle `triage` is the folder `mycorp_demo/bundles/triage/`. The entry-point name is both the bundle's name and its folder's name, so it may contain `-`. -- The value names a package, never an object. The folder is found without importing any of the package's code, so listing bundles runs nothing. -- The folder ships in the wheel as package data, and the package must be installed unpacked. -- An installed bundle's ids are rooted at its distribution, wherever the package is installed: `triage` above writes `@local/mycorp-demo/triage/...`. -- `agent-env run triage` runs it, and `agent-env run` with no argument lists the installed bundles, each with its folder. -- When two packages install bundles of one name, run each as `<package>/<name>`, using the canonical distribution name, for example `mycorp-demo/triage`. agent-env's own bundles keep their bare names, so a package's `hello` runs only as `<package>/hello`. -- `agent-env plugin list` shows each bundle as a contribution of its package, agent-env's own `hello` as `agentenv-framework … bundle hello`. `plugin check` fails when a bundle doesn't resolve to a folder, fails a check `agent-env run` makes before it reads a store (it isn't a valid bundle, a type it names doesn't resolve, or a step's fields don't build), is registered twice by one package (`conflict`), or comes from a package whose agent-env requirement isn't met (`incompatible-core`). `list --no-load` and `show --no-load` only parse it. - -## Plugin settings - -A plugin that needs settings of its own reads them from `[plugins.<package>]`, where `<package>` is its distribution name: the name `uv add` and `pip install` take and `agent-env plugin list` prints. That table belongs to the plugin. agent-env reads nothing in it and checks none of its keys. Every other top-level table belongs to agent-env, which warns about one it does not read in `agent-env config show`, so a plugin keeps nothing of its own anywhere else. - -```toml -[plugins.acme-agentenv-browser] -endpoint = "env:BROWSER_URL?http://localhost:9222" -timeout = 30 - -[plugins.acme-agentenv-browser.viewport] -width = 1280 -``` - -```python -from agent_env.plugins import settings - -def browser_timeout() -> int: - browser = settings("acme-agentenv-browser") # {} when the file has no such table - return browser.get("timeout", 10) # the plugin keeps its own defaults -``` - -- Pass the distribution name, not the module's `__name__` or `__package__`: `acme-agentenv-browser` may import as `acme_browser`. -- Call `settings()` where the value is used, in `from_config`, `execute` or a command's body, not at import. Entry points are imported before a root option can select the config file, so a value read at import may come from another file. -- The key is matched by canonical name (PEP 503: lowercase, with each run of `-`, `_` and `.` read as one `-`), so `[plugins.acme_agentenv_browser]` is the same table. Write the canonical form. It never needs quoting, whereas a dotted name written bare (`[plugins.acme.browser]`) nests. Two tables that name one package are a `ConfigError`, not a guess. -- `settings(name, *, config=None)` returns a copy of the table, read from `config`'s document (default: the process `Config`), with `env:` and `secret:` references resolved as in every other table. It raises `agent_env.config.ConfigError` when `[plugins]` or the plugin's entry is not a table, when two keys name the package, or when a reference without a `?default` cannot be resolved; the message names the table. agent-env never reads the table itself, so a broken entry fails only its own plugin's read, and `config show` reports it in place. -- No environment variable overrides a plugin setting. For a value that differs per deployment, write an `env:` reference in the table. -- `agent-env config show` lists each table under its package: with the installed version, with `(declares no agent_env entry point)` when the distribution is installed but is not a plugin, which usually means a misspelled entry-point group, with `(agent-env itself)` for `agentenv-framework`, whose table agent-env does not read, or with `(not installed)`. `config explain plugins.<package>.<key>` says whose table holds the key; agent-env does not read or check it, so it cannot tell whether the plugin reads that key, and a misspelled key is still shown. A table for a plugin that is not installed is reported, never an error, because one config file is often shared by processes that install different plugins. -- Both commands mask a plugin's table the way they mask agent-env's, and a service may log what they report. A literal value is printed as `***` when its key, or a key above it in the table, contains `password`, `passwd`, `passphrase`, `secret`, `token`, `credential`, `api_key`, `private_key`, `auth` or `bearer` in any case, and a connection URI's userinfo is masked. Any other literal is printed, so keep credentials behind `secret:` or `env:` references. A reference prints as written, with any `?default` masked like a literal. -- A provider the plugin registers is still configured in `[sandbox.providers.<name>]` or `[state.providers.<name>]`, under the provider's name, and an explorer plugin class is still listed in `[explorer.plugins]`. agent-env reads those tables itself. `[plugins.<package>]` holds only what the plugin reads. -- Uninstalling a plugin leaves its table in the config file. -- A top-level table one letter from `[plugins]`, such as `[plugin]`, gets a warning in `config show`: nothing reads it, so the plugin would get no settings. - -## Manage plugins - -<!-- tst/installer/test_container_journey.py runs the commands in the first bash block below. --> - -`agent-env plugin` shows what the installed plugins contribute and whether each piece took effect, and adds or removes them. - -```bash -agent-env plugin list # every plugin package, what it provides, and its status -agent-env plugin show PACKAGE # each contribution and why it is in that state -agent-env plugin check # exit 1 if any contribution did not take effect -agent-env plugin add SPEC... # install through this environment's installer, then check -agent-env plugin remove PACKAGE # uninstall through the same installer, after safety checks -``` - -``` -agent-env 0.9.1193 · uv tool at ~/.local/share/uv/tools/agentenv-framework -config: (none) - -PACKAGE VERSION PROVIDES STATUS -agentenv-browser 1.0.0 env browser, task step browser_navigate ok -agentenv-framework 0.9.1193 bundle hello ok -agentenv-grader 0.3.1 task step grade_essay 1 failed -``` - -| Status | Meaning | -|---|---| -| `active` | registered and in use | -| `replaced` | config names a different class for the name; `show` says where. Not a failure | -| `failed` | failed to import, validate or construct | -| `skipped` | a built-in, a core command or root option, or (for CLI commands) a plugin loaded first owns the name | -| `conflict` | another entry point, from another package or this one, claims the same name, so none of them is used. For a bundle, only a second claim from the same package is a conflict; another package's bundle of that name is `qualified-only` | -| `blocked` | the group cannot load at all, for example because its config table is invalid, so this contribution does not either | -| `unloaded` | not loaded (`--no-load`). When loading, it means the status could not be determined, and `check` fails on it | - -A contribution that is not `active` also has a code saying why, shown in brackets after its reason; the codes are listed under [Plugin report format](#plugin-report-format). - -- The header says how agent-env is installed (uv tool, pipx, uv project, virtualenv or system Python) and where: a plugin has to be installed into that same environment. -- `list` and `show` take `--json` and `--no-load`. `--no-load` reads installed metadata only and imports no type plugin; CLI plugins are already loaded, because the CLI loads them when it starts. Without it, `list`, `show` and `check` import every type plugin and construct the explorer plugins, so that plugin code runs. -- `show` also reports whether importing the package sets `AGENT_ENV_CONFIG`, checked in a fresh interpreter. -- `check` builds every registry the way a process does, adds the CLI's own plugins, and fails on `failed`, `skipped`, `conflict`, `blocked` or `unloaded`, or when the config file or the installed entry points cannot be read (a malformed `entry_points.txt` hides every plugin, so it fails rather than passing empty). `check --json` prints the `list --json` report plus `ok` and `problems`. A `replaced` contribution passes: config chose it. -- `plugin` is a core command: a CLI plugin that names a command `plugin` is skipped, like any clash with a core command. -- The Python equivalent is `agent_env.plugins.inventory()` (see [Register types from an installed package](#register-types-from-an-installed-package)). - -### Plugin report format - -`plugin list --json`, `plugin show --json` and `plugin check --json`, and the `agent_env.plugins` types they are built from, are the interface for scripts, CI and image builds. The human output of these commands is not: parse the JSON. - -```json -{ - "format_version": 1, - "agent_env": {"version": "0.9.1217", "environment": "uv tool", "location": "/home/me/.local/share/uv/tools/agentenv-framework"}, - "config": {"path": "/work/.agentenv/config.toml", "error": null}, - "loaded": true, - "group_errors": {}, - "discovery_errors": {}, - "plugins": [ - {"name": "agentenv-grader", "version": "0.3.1", "contributions": [ - {"group": "agent_env.task_steps", "name": "grade_essay", "value": "agentenv_grader.steps:GradeEssay", - "status": "failed", "code": "load-failed", "reason": "failed to load: ModuleNotFoundError(\"No module named 'openai'\")", - "replaced_by": null, "conflicts_with": []} - ]} - ] -} -``` - -- Every key is always present; an empty value is `null`, `[]` or `{}`. `group_errors` and `discovery_errors` map an entry-point group to `{code, reason}`, and `config.error` is `{code, reason}` or `null`. -- A distribution whose metadata cannot be read is listed under the label `(unreadable metadata: DIR)`, where `DIR` is its `.dist-info` directory. The label is not a package name, so `plugin remove` cannot take it: reinstall the package with its installer, or delete that directory. -- A contribution's `status` says what happened to it (the table above) and its `code` says why. `code` and `reason` are set together: on every contribution that is not `active`, and on an `active` one only for information. `replaced_by` is set exactly when the status is `replaced`: `{file, table, impl}`, where `table` is the config table naming the other class (`envs`, `task_steps`, `artifacts`, `explorer.plugins`, `sandbox.providers.NAME` or `state.providers.NAME`). `conflicts_with` lists the other entry points that claim the name, as `{package, version, value}`. -- `show --json` adds `config_effect`, a sentence saying whether importing the package sets `AGENT_ENV_CONFIG` (`null` with `--no-load`). `check --json` adds `ok` and `problems`, each a contribution with its `package` and `version`. -- `check` exits 0 when `ok` and 1 otherwise; a usage error exits 2. `show` exits 1 with no report when no installed plugin package has that name. A run that does not print exactly one JSON document on stdout, such as a crash while agent-env starts, is not a report: count it as a failure whatever its exit status. - -`format_version` changes only for a change a consumer cannot ignore: a key removed, renamed or given another type, a new status, a code redefined or reused, or a new meaning for an exit status. A new key, a new code, reworded `reason` or `config_effect` text, and a change in list order leave it as it is. So a consumer should ignore keys it does not know, handle a code it does not know by its `status`, refuse a `format_version` it does not know, and never parse `reason`. Which situation gets which status and code, and which group error is reported when several apply, is behaviour rather than format: a release that changes one says so in its notes. A code is never removed or reused. - -| Code | Where | Meaning | What to do | -|---|---|---|---| -| `load-failed` | `failed` | The plugin's code raised: when it was imported (`SystemExit` included), when an explorer plugin was constructed, or in a root option's callback with the flag absent | Install what it needs, or report it to the plugin's author | -| `invalid-plugin` | `failed` | It imported but does not fit its group: not a subclass of the group's base class; a `type` that is missing, inherited or not the entry-point name; a method its base requires left unimplemented; not a `click.Command` or `click.Option`; a root option that is required or exposes a value; a command click refused; an extra artifact name that reads no document. A bundle is never imported: it is invalid when its value isn't an installed, unpacked package holding that folder, when its metadata can't be read, or when the folder isn't a valid bundle (with plugins loading, one whose step, env or artifact types don't resolve) | Report it to the plugin's author; for a bundle whose package is installed zipped, reinstall it unpacked. For an extra name whose class's own type is in conflict, settle that conflict | -| `incompatible-core` | `failed` | The plugin's requirement on `agentenv-framework` or `agentenv-framework-protocol` excludes the installed version, so it was not imported. See [Plugin compatibility](#plugin-compatibility) | Upgrade agent-env, or install a version of the plugin that fits | -| `builtin-name` | `skipped` | agent-env owns the name: a built-in type, a core command or a core root option | The plugin has to rename it | -| `name-conflict` | `conflict`, `skipped` | More than one entry point claims the name, from two packages or twice from one; for bundles, only twice from one package (see `qualified-only`). In a type group none of them is registered, and the rest of the group loads; two different root options on one flag are each a `conflict`, and neither is attached. A CLI command whose name a plugin loaded earlier took is `skipped`, and the earlier one stays | Remove all but one: `agent-env plugin remove PACKAGE`. A package that declares a name twice has to be fixed by its author | -| `replaced-by-config` | `replaced` | The config registers another class under the name | Nothing, unless you did not mean it | -| `already-attached` | `active` | A second entry point, from the same package or another, exports the same `click.Option` object, which is attached once | Nothing | -| `not-loaded` | `unloaded` | `--no-load`, or `inventory(load=False)`: whether it takes effect needs an import | Run without `--no-load` | -| `status-unknown` | `unloaded` | It was loaded, but nothing was recorded for it. This should not happen | Report it as an agent-env bug | -| `config-not-found` | config error, group error, `blocked` | `AGENT_ENV_CONFIG` names a file that does not exist | Fix or unset `AGENT_ENV_CONFIG` | -| `config-unreadable` | config error, group error, `blocked` | The config file could not be read or parsed | Fix the file | -| `config-invalid` | group error, `blocked` | The file parsed, but what it says for this group is invalid | Fix the table the reason names | -| `group-build-failed` | group error, `blocked` | Building the group raised something else | See the reason | -| `entry-points-unreadable` | discovery error | The group's installed entry points could not be read, so none of its plugins is listed | Reinstall the package the reason names | -| `qualified-only` | `active` | Another package installs a bundle of the same name, so this one runs only as `<package>/<name>`. agent-env's own bundles keep the bare name | Run it by its qualified name | - -A `blocked` contribution carries its group error's code, so it says what to fix rather than only that the group failed. - -### Add and remove plugins - -agent-env never installs anything itself. `add` and `remove` run the installer that owns the environment agent-env runs in, so that installer's next rebuild keeps the change: - -| agent-env installed as | `add` runs | `remove` runs | -|---|---|---| -| uv tool | `uv tool install agentenv-framework --with ... --with SPEC`, passing back the receipt's requirements (editable and git ones included), Python and options | the same, without the package | -| pipx | `pipx inject VENV SPEC`, with the venv's own pip arguments (and `--force` to upgrade a package it already has) | `pipx uninject --leave-deps VENV PACKAGE` | -| uv project | `uv add --no-sync --project ROOT SPEC` and `uv sync --inexact`, then shows the `pyproject.toml` diff | `uv remove --no-sync --project ROOT PACKAGE` and `uv pip uninstall PACKAGE` | -| virtualenv | `python -m pip install SPEC` (or `uv pip install` when the venv has no pip) | `python -m pip uninstall -y PACKAGE` | -| Poetry or PDM project | prints `poetry add` / `pdm add` to run yourself | prints `poetry remove` / `pdm remove` | -| Hatch environment | says to add it to the environment's dependencies | the same, for removal | -| system Python with the PEP 668 marker, or read-only site-packages | refuses, with guidance | refuses | - -`SPEC` is anything the installer accepts, so a plugin can come from an index, a git repository or a file: - -| From | `SPEC` | -|---|---| -| PyPI, or the index the installer is configured with | `agentenv-grader`, `'agentenv-grader==0.3.0'`, `'agentenv-grader>=0.3,<0.4'` | -| a git tag, commit or branch | `'agentenv-grader @ git+https://github.com/mycorp/agentenv-grader@v0.3.0'` | -| one package in a monorepo | `'agentenv-grader @ git+https://github.com/mycorp/plugins@v1#subdirectory=agentenv-grader'` | -| a private repository | `'agentenv-grader @ git+ssh://git@github.com/mycorp/agentenv-grader.git@v0.3.0'` | -| a release asset | `https://github.com/mycorp/agentenv-grader/releases/download/v0.3.0/agentenv_grader-0.3.0-py3-none-any.whl` | -| a local wheel or checkout | `./dist/agentenv_grader-0.3.0-py3-none-any.whl`, `./agentenv-grader` | - -- Write a git URL as `NAME @ URL`, and a local checkout as `NAME @ file:///absolute/path`. agent-env cannot read the name from a bare `git+https://…` or a directory, so it cannot tell pipx or a uv tool which installed plugin the spec replaces: pipx leaves the installed one as it is, and `add` says so; a uv tool reports conflicting URLs, and the add is rolled back. A git install records the commit it resolved to, so a rollback reinstalls that commit. -- `add` never replaces agent-env itself. A spec that names `agentenv-framework`, as a name, a wheel or `NAME @ URL`, is refused before anything runs. A bare URL or directory that turns out to build it is rolled back after the install: agent-env may move to a newer release from the index when a plugin requires one, but not to a URL, git or directory source. Upgrade agent-env with the installer that owns the environment. -- Credentials stay with git and the installer: an SSH key or a git credential helper (`gh auth setup-git`) for a private repository, and the installer's own config for a private index. A token written into `SPEC` is kept in the installer's record of the environment. -- `--installer` overrides the detected installer, `--index-url` passes an index through to it (a uv tool keeps it as its default index), `--dry-run` shows what would run, and `--yes` skips the prompt. -- uv takes a pre-release or a yanked version only when you name it. If the installer reports that a plugin needs one, name that exact version in the same command, for example `agent-env plugin add agentenv-grader 'agentenv-grader-core==0.2.0b1'`; `plugin remove` removes it later the same way. -- A uv project whose environment is set by `UV_PROJECT_ENVIRONMENT` is found from the current directory, as uv finds it, so run `add` and `remove` inside the project. In a uv workspace whose root has no `[project]` table, `add` goes to the one member that declares `agentenv-framework` (`uv add --package MEMBER`). `remove` drops a package from every member and group that declares it (`--package`, `--group`, `--optional`, `--dev`). -- A uv project is never synced exactly, so packages from extras and anything installed outside the lock, agent-env included, stay installed. -- pipx and a virtualenv leave a removed plugin's own dependencies installed, as `pip uninstall` does. Left to itself, `pipx uninject` would also uninstall everything nothing else requires, agent-env included. -- A uv tool installed with constraints, overrides or executables from another package is refused: `uv tool install` cannot take those back from agent-env, so make that change with uv. -- One `add` or `remove` runs at a time per environment; a second one is refused until the first finishes. The lock is a file in `$XDG_STATE_HOME/agent-env/locks` (by default `~/.local/state/agent-env/locks`). If the installer's record changes some other way while a change waits for confirmation, the change is refused rather than run from its outdated plan. -- A plain virtualenv first shows the installer's own dry run of what would change. Every mode reports what did change afterwards, calling out a change to agent-env itself. -- `add` then checks each new plugin package in a fresh interpreter: it must declare at least one `agent_env.*` entry point, and every contribution must be `active` or `replaced`. `show` reports whether importing it sets `AGENT_ENV_CONFIG`. If the check fails, the installer fails or removes another plugin, or the change is interrupted (Ctrl-C or SIGTERM), `add` puts the environment back as it was: every package at its old version from its old source, and the uv tool receipt, pipx metadata, or `pyproject.toml` and `uv.lock`, byte for byte. The installer runs with Ctrl-C ignored, since one stopped halfway leaves packages no installer can read: it finishes, and then the change is rolled back. A second Ctrl-C stops it at once. It keeps the terminal, so its own prompts, such as git's for credentials, still work. When the environment cannot be put back exactly, `add` says which packages differ and the installer command that reinstalls them. Adding what is already installed changes nothing and succeeds, and a bare name that a uv tool already lists keeps its version pin. `--keep` keeps a change that failed the check instead. -- `remove` refuses, unless `--force`: - - while another installed package requires the package; - - while the config file names something only it provides (`[sandbox] default` or `agent_default`, a provider table, a `type_aliases` target), or an `impls` list, provider, store or runner `impl` names one of its modules; - - while stored documents use env, artifact or task-step types only it provides. This is checked for a local document store, and for a remote one only with `--check-usage`; a store that cannot be read also blocks, until `--force`. - - It never removes agent-env itself. A package that declares no `agent_env.*` entry point is removed only when the installer records it on its own (a uv tool's `--with`, a pipx injection, a project dependency), as it does for a package named next to a plugin in `add`; after removing a plugin, `remove` names any such package that only the plugin needed. In a uv tool or a uv project, removing a plugin also removes the plugins that only it needed, as the installer's rebuild would, so `remove` names them first and checks them like the package itself. If the installer fails, removes a plugin it did not name, or the change is interrupted, `remove` puts the environment back the same way `add` does. -- Plugins are trusted code: nothing installs one implicitly, from a task document, a config file or anywhere else. -- A plugin that breaks CLI startup does not lock you out. A root option whose callback raises or exits while its flag is absent is skipped, with a warning, and reported as `failed`; a flag you pass still fails, naming the plugin. A CLI plugin that exits while being imported is skipped like any load failure. For a plugin that hangs or kills the interpreter while being imported, uninstall it with the installer directly: the command `plugin remove --dry-run` would print. - -## Building a platform plugin - -One installable package can combine all of the above: bundled config files, a root option that selects one per invocation, store and provider classes, a `Runner`, custom steps, and explorer routers. The `--tenant` demo above is that pattern in miniature. Its callback points `AGENT_ENV_CONFIG` at a bundled `tenants/<name>.toml`; a name that does not exist fails loud with `ConfigError` on first use, while `--help` still works. A hosted control plane serves the explorer app from its own server process and lists its public hostnames under `[explorer] allowed_hosts` (see the comments in `.agentenv/config.example.toml`). A durable runner is the plugin's responsibility: this repository ships only `LocalRunner`. - -## Plugin compatibility - -What a plugin can build on, how to declare the agent-env it needs, and what agent-env does when the installed one does not fit. - -**The plugin surface.** A plugin may rely on these. Anything else, including every underscored name other than a method a subclass must implement (`EnvStateProvider._teardown`), is internal and can change in any release. - -- The entry-point groups and their rules, in [Register types from an installed package](#register-types-from-an-installed-package) and [CLI plugins, root options, explorer routes](#cli-plugins-root-options-explorer-routes). -- The base class each group names, with its public methods and attributes: `agent_env.env.env.Env`, `agent_env.task_step.task_step.TaskStep` and the `agent_env.task_step.context.TaskStepContext` a step runs with, `agent_env.artifact.artifact.Artifact`, `agent_env.providers.sandbox_providers.sandbox_provider.SandboxProvider`, `agent_env.providers.env_state.env_state_provider.EnvStateProvider`, `agent_env.providers.env_providers.env_provider.EnvironmentProvider`, and `agent_env.explorer.plugin.ExplorerPlugin`. -- The two functions an env that uses an environment provider calls: `agent_env.providers.env_providers.env_provider.build_env_provider` and `agent_env.env.store.register_env_instance`. -- What an environment provider reads to deploy a built-in env: `agent_env.env.envs.mcp_server.MCPServerEnv.docker_image_artifact` and `agent_env.env.envs.mcp_server.MCPServerEnv.environment_name`; `agent_env.env.envs.website.WebsiteEnv.backend_docker_image_artifact`, `agent_env.env.envs.website.WebsiteEnv.frontend_docker_image_artifact` and `agent_env.env.envs.website.WebsiteEnv.environment_name`; `agent_env.env.envs.multi_env.MultiEnv.mcp_server_envs`, `agent_env.env.envs.multi_env.MultiEnv.website_envs` and `agent_env.env.envs.multi_env.MultiEnv.name`; and each image's `agent_env.artifact.artifacts.docker_image.DockerImageArtifact.image_name`. -- The top level of `agent_env.plugins`, including `settings`, and the `[plugins.<package>]` table it reads ([Plugin settings](#plugin-settings)). -- The `plugin --json` output, which has its own rules: [Plugin report format](#plugin-report-format). - -The classes a config `impl` names, such as stores and runners, are not on the list yet. - -**Changes before 1.0.** Every merged change can ship as a release, several a day. A change that breaks the plugin surface is marked with `!` after the scope in its pull request title, which becomes its commit title, as in `feat(plugins)!: …`. Where the old behaviour can be kept for a while, it is deprecated first: it keeps working and emits a `DeprecationWarning` that names what replaces it. How long that lasts is not fixed before 1.0. - -The `plugin-api` CI job holds pull requests to this. It compares the listed base classes, `TaskStepContext`, the two functions, the env attributes and the top level of `agent_env.plugins` with the pull request's base (`.github/scripts/check_plugin_api.py`), and fails on a break the title does not mark; with the `!`, it lists the breaks and passes, and editing the title re-runs it. A break is what fails code written against the old surface: - -- for a caller, a name, parameter or `__all__` entry that is removed or renamed, a new required parameter, a parameter that can no longer be passed as before, or a changed default or constant, including the group-name constants `agent_env.plugins` exports, compared by value; -- for a subclass of a base class, a new abstract or required method, a method that becomes abstract or required, a new `ClassVar` with no value, a method that changes between plain, `async`, `classmethod`, `staticmethod` and property, and a base-class method that accepts more than before: a new parameter, even an optional one, a parameter that stops being required, a new `*args` or `**kwargs`, or a keyword-only parameter that can now be passed by position, since an override written for the old signature fails when core passes it; -- for construction, a changed `@dataclass(...)` option or pydantic `model_config`, a new metaclass, or a new `__init_subclass__` that can raise, which refuses a subclass. - -A member every subclass must implement is declared `@abstractmethod`, or listed in its registry's `_MUST_IMPLEMENT`, so that the check enforces it; a new method that only raises `NotImplementedError` gets a note but does not fail the job. - -agent-env ships `py.typed`, so mypy and pyright check a plugin against its annotations. Import each name from the module the list above gives, such as `TaskStepContext` from `agent_env.task_step.context`: pyright, and mypy under `--strict`, treat a name a module only imports as private to that module. Type annotations are not compared, so a release can correct one without a `!`. The check can report a change no plugin notices, such as a new parameter core never passes to an override; mark it all the same. It does not see a change in behaviour, in a type the surface only names in a signature, such as `DeployedEnv`, in a pydantic field's requirements or the order of dataclass fields, in an abstract method a new base from outside agent-env brings, or in a registration rule other than abstract methods and `_MUST_IMPLEMENT`; review covers those. A `!` can also mark a break outside the plugin surface. - -**Declaring the agent-env your plugin needs.** Declare a floor, `agentenv-framework>=X`, where `X` is the oldest release you test against. Leave out a ceiling such as `<1`: before 1.0 it would not guard against a change in a 0.9 release, and it would keep your plugin from installing with the next major one. Pin exact versions in the application or image that installs your plugin, not in the plugin. If your plugin imports `agentenv_protocol` itself, declare that as well. - -**What agent-env checks.** Before it imports a plugin, agent-env compares the plugin's requirements on `agentenv-framework` and `agentenv-framework-protocol` with the versions installed. A plugin they exclude is not loaded: - -- `plugin list`, `show` and `check` report each of its contributions as `failed` with the code `incompatible-core`, with or without `--no-load`, and `check` exits 1; -- using one of its types names the requirement, as in `needs agentenv-framework>=0.9.1220 (installed: 0.9.1218)`; -- it claims none of its names, so a plugin that can load and registers the same name does not conflict with it. - -An installer that resolves dependencies never gets you there; `pip install --no-deps` or a forced install can. There is no override: upgrade agent-env, or install a version of the plugin that fits. A requirement under an extra, or whose environment marker is false, does not count, and an agent-env with no installed metadata, such as a source tree on `sys.path`, is not checked. Requirements between plugins are the installer's to check; `pip check` lists any that are unmet. diff --git a/README.md b/README.md index 16f916c9..d5e206d6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,10 @@ uv sync --extra dev source .venv/bin/activate ``` +## Plugins + +A package of your own can add env types, providers, task steps and stores to agent-env, and `agent-env plugin` lists, checks, adds and removes installed plugins. The [plugin docs](https://www.agentenvframework.com/docs/plugins) show how. + ## Contribute, release, license [CONTRIBUTING.md](https://github.com/scaleapi/agentenv-framework/blob/main/CONTRIBUTING.md) is the contributor guide: open an issue before anything larger than a bug fix, one logical change per pull request with tests, pull request titles in the form `type(scope): summary`, an approving review from a code owner (`CODEOWNERS`) and green CI. [AGENTS.md](https://github.com/scaleapi/agentenv-framework/blob/main/AGENTS.md) is the repository map and conventions file for contributors and coding agents. @@ -55,7 +59,6 @@ source .venv/bin/activate | [www.agentenvframework.com/docs](https://www.agentenvframework.com/docs) | the user guide: environments, artifacts, agents, tasks, the registry and plugins | | [`packages/agentenv-protocol/README.md`](https://github.com/scaleapi/agentenv-framework/blob/main/packages/agentenv-protocol/README.md) | wire contract, environment server SDK, A2A agent framework | | [`CONTRIBUTING.md`](https://github.com/scaleapi/agentenv-framework/blob/main/CONTRIBUTING.md) | development setup, test tiers, CI jobs, pull request rules | -| [`PLUGINS.md`](https://github.com/scaleapi/agentenv-framework/blob/main/PLUGINS.md) | the plugin contract: entry-point groups, plugin settings, the `agent-env plugin` commands and their report format, and what a plugin may rely on across releases | | [`AGENTS.md`](https://github.com/scaleapi/agentenv-framework/blob/main/AGENTS.md) | repository map, configuration and extension-point summary, conventions for contributors and coding agents | | [`SECURITY.md`](https://github.com/scaleapi/agentenv-framework/blob/main/SECURITY.md) | private vulnerability reporting and supported versions | | [`CODE_OF_CONDUCT.md`](https://github.com/scaleapi/agentenv-framework/blob/main/CODE_OF_CONDUCT.md) | Contributor Covenant | @@ -64,7 +67,7 @@ source .venv/bin/activate ### Versioning and compatibility -The `agentenv-framework` distribution and `agentenv-framework-protocol` are versioned separately (`0.9.x` and `0.1.x` today), both in their `pyproject.toml`; agent-env releases carry a `vX.Y.Z` tag, and agentenv-framework-protocol is bumped in the same commit and has no separate tag today. There is no `agent_env.__version__` attribute; `agent-env --version` prints the installed version. Protocol extensions carry their version in the URI (`urn:agentenv:clock/v1`, `urn:agentenv:agent-config/v1`, `urn:agentenv:trajectory/v1`). One version can take more than one request shape: under `v1` the skill, trajectory, snapshot and changelog extensions accept object-transfer requests next to their older shapes, and the request field lists on an agent's card say which ones that agent takes. Renamed CLI commands are removed outright; no deprecated aliases exist at this version. What plugins may rely on, and how changes to it are made, is in [Plugin compatibility](https://github.com/scaleapi/agentenv-framework/blob/main/PLUGINS.md#plugin-compatibility); the plugin commands' `--json` output has its own format version and rules ([Plugin report format](https://github.com/scaleapi/agentenv-framework/blob/main/PLUGINS.md#plugin-report-format)). Beyond those, a written compatibility and deprecation policy does not exist yet. +The `agentenv-framework` distribution and `agentenv-framework-protocol` are versioned separately (`0.9.x` and `0.1.x` today), both in their `pyproject.toml`; agent-env releases carry a `vX.Y.Z` tag, and agentenv-framework-protocol is bumped in the same commit and has no separate tag today. There is no `agent_env.__version__` attribute; `agent-env --version` prints the installed version. Protocol extensions carry their version in the URI (`urn:agentenv:clock/v1`, `urn:agentenv:agent-config/v1`, `urn:agentenv:trajectory/v1`). One version can take more than one request shape: under `v1` the skill, trajectory, snapshot and changelog extensions accept object-transfer requests next to their older shapes, and the request field lists on an agent's card say which ones that agent takes. Renamed CLI commands are removed outright; no deprecated aliases exist at this version. A pull request that breaks the classes and entry points plugins build on fails CI unless its title marks the break with `!`; beyond that, a written compatibility and deprecation policy does not exist yet. ### Releases diff --git a/src/agent_env/cli/plugin.py b/src/agent_env/cli/plugin.py index 83429e55..ac6f76bd 100644 --- a/src/agent_env/cli/plugin.py +++ b/src/agent_env/cli/plugin.py @@ -179,7 +179,7 @@ def _bundle_contribution(bundle: InstalledBundle, bundles: tuple[InstalledBundle def _as_dict(report: Report) -> dict: - """The document `list --json` prints; `show` and `check` add to it. See PLUGINS.md "Plugin report format".""" + """The document `list --json` prints; `show` and `check` add to it.""" error = report.inventory.config_error return { "format_version": FORMAT_VERSION, diff --git a/src/agent_env/plugins/__init__.py b/src/agent_env/plugins/__init__.py index 4d033ba4..88b509e9 100644 --- a/src/agent_env/plugins/__init__.py +++ b/src/agent_env/plugins/__init__.py @@ -11,9 +11,8 @@ ``settings`` reads them. This package's top level is the public surface: the group names, ``load_failures``, ``settings``, -and ``inventory`` with the types it returns; its underscored modules are internal. The full contract -is in PLUGINS.md ("Register types from an installed package", "Plugin settings"), and the codes the -inventory reports are in "Plugin report format": codes are stable, reason text is not. +and ``inventory`` with the types it returns; its underscored modules are internal. The codes the +inventory reports are stable; reason text is not. """ from agent_env.config.plugin_tables import settings diff --git a/src/agent_env/plugins/_cli.py b/src/agent_env/plugins/_cli.py index 1f0d8a77..88c03416 100644 --- a/src/agent_env/plugins/_cli.py +++ b/src/agent_env/plugins/_cli.py @@ -4,7 +4,6 @@ ``agent_env.cli_root_options`` entries to ``click.Option`` instances that attach to the root ``agent-env`` group. Broken plugins warn and are skipped; core names win over plugins; two different root options on the same flag are both left off and reported as a conflict. -Full contract in PLUGINS.md ("CLI plugins, root options, explorer routes"). """ import sys diff --git a/src/agent_env/plugins/_registration.py b/src/agent_env/plugins/_registration.py index 0bb23088..8ba3c4da 100644 --- a/src/agent_env/plugins/_registration.py +++ b/src/agent_env/plugins/_registration.py @@ -1,7 +1,6 @@ """Loading a group's plugins into its registry, and recording each one that did not take effect. -Internal: the public contract is ``agent_env.plugins`` and the PLUGINS.md section "Register types -from an installed package". Each registry calls ``merge`` when it is built, so a group's plugins +Internal: the public contract is ``agent_env.plugins``. Each registry calls ``merge`` when it is built, so a group's plugins are imported then and not before. """ diff --git a/src/agent_env/plugins/_report.py b/src/agent_env/plugins/_report.py index f944daea..050c9b37 100644 --- a/src/agent_env/plugins/_report.py +++ b/src/agent_env/plugins/_report.py @@ -1,6 +1,6 @@ """The plugin report's format version and reason codes. -Internal: the contract is the PLUGINS.md section "Plugin report format". A code says why a +Internal. A code says why a contribution has its status, or why a group, the config or discovery failed. Codes are never removed or reused; a new one needs no format bump. """ diff --git a/src/agent_env/plugins/_requirements.py b/src/agent_env/plugins/_requirements.py index f49abe39..354393bf 100644 --- a/src/agent_env/plugins/_requirements.py +++ b/src/agent_env/plugins/_requirements.py @@ -1,6 +1,6 @@ """Whether an installed plugin's requirements admit the agent-env installed next to it. -Internal: the contract is the PLUGINS.md section "Plugin compatibility". Reads installed metadata only, +Internal. Reads installed metadata only, so it runs before a plugin is imported, and for ``inventory(load=False)``. """ diff --git a/tst/installer/test_container_journey.py b/tst/installer/test_container_journey.py index ed7f40f1..549b29d2 100644 --- a/tst/installer/test_container_journey.py +++ b/tst/installer/test_container_journey.py @@ -1,9 +1,8 @@ """The open-source user journey, replayed in fresh containers with no network. A new user on `python:3.12-slim` installs agent-env as a uv tool, adds two plugins, runs a task -that uses their steps, and removes them; the plugin commands in PLUGINS.md run on the way, so the -docs cannot drift from the CLI. A second container uses Ubuntu's system Python, which carries -the PEP 668 marker, and `plugin add` must refuse it. Linux wheels for the dependencies are +that uses their steps, and removes them; every `agent-env plugin` command runs on the way. A second +container uses Ubuntu's system Python, which carries the PEP 668 marker, and `plugin add` must refuse it. Linux wheels for the dependencies are downloaded once, inside a container, and cached; the journey itself runs with `--network none`. """ @@ -124,15 +123,6 @@ def journey(tmp_path, checkout_wheels, linux_wheels) -> Journey: return Journey(mounts, work, f"/plugins/{grader.name}", f"/plugins/{browser.name}") -def _documented_plugin_commands() -> list[list[str]]: - """The commands in the "Manage plugins" block of PLUGINS.md, as a user would type them.""" - section = (REPO / "PLUGINS.md").read_text().split("## Manage plugins", 1)[1] - block = section.split("```bash\n", 1)[1].split("```", 1)[0] - lines = [line.split("#", 1)[0].split() for line in block.splitlines() if line.startswith("agent-env plugin")] - assert [line[2] for line in lines] == ["list", "show", "check", "add", "remove"], lines - return lines - - def test_a_new_user_installs_agent_env_adds_plugins_runs_a_task_and_removes_them(journey): grader, browser = journey.grader, journey.browser offline = {"UV_OFFLINE": "1", "UV_FIND_LINKS": "/checkout,/wheels", "UV_TOOL_BIN_DIR": "/usr/local/bin"} @@ -148,10 +138,9 @@ def test_a_new_user_installs_agent_env_adds_plugins_runs_a_task_and_removes_them added = container.ok("agent-env", "plugin", "add", browser, grader, "--yes") assert "toy_navigate active" in added.stdout and "toy_grade active" in added.stdout - # The documented commands, verbatim but for their placeholders; --yes because there is no terminal. - for line in _documented_plugin_commands(): - command = [{"PACKAGE": GRADER, "SPEC...": grader}.get(word, word) for word in line] - container.ok(*command, *(["--yes"] if line[2] in ("add", "remove") else [])) + # Every plugin command; --yes because there is no terminal. + for command in (["list"], ["show", GRADER], ["check"], ["add", grader, "--yes"], ["remove", GRADER, "--yes"]): + container.ok("agent-env", "plugin", *command) assert GRADER not in container.ok("agent-env", "plugin", "list").stdout container.ok("agent-env", "plugin", "add", grader, "--yes") diff --git a/tst/unit/cli/plugin_report_test.py b/tst/unit/cli/plugin_report_test.py index 3a9e61aa..26c2b9ef 100644 --- a/tst/unit/cli/plugin_report_test.py +++ b/tst/unit/cli/plugin_report_test.py @@ -4,7 +4,7 @@ code. Reason text is replaced by a placeholder: it may change in any release, and a reword should not touch the goldens. ``_SCHEMA`` is the format's keys and types, so a key added to the report fails here until it is added there too; that edit is where to decide whether the format version -moves (PLUGINS.md "Plugin report format"). After an intended change, regenerate and review the diff: +moves. After an intended change, regenerate and review the diff: AGENT_ENV_REGENERATE_GOLDENS=1 python -m pytest tst/unit/cli/plugin_report_test.py """ @@ -34,7 +34,6 @@ _THIS = "tst.unit.cli.plugin_report_test" _GOLDEN = Path(__file__).parent / "golden" / "plugin_report" _REGENERATE = os.environ.get("AGENT_ENV_REGENERATE_GOLDENS") == "1" -_PLUGINS_MD = Path(__file__).parents[3] / "PLUGINS.md" _COMMANDS = cli_plugins_mod.CLI_PLUGINS_GROUP _OPTIONS = cli_plugins_mod.CLI_ROOT_OPTIONS_GROUP @@ -285,22 +284,6 @@ def test_codes_are_kebab_case(): assert all(re.fullmatch(r"[a-z]+(-[a-z]+)*", code) for code in CODES) -def _readme_section() -> str: - return _PLUGINS_MD.read_text().split("### Plugin report format", 1)[1].split("\n### ", 1)[0] - - -def test_the_readme_lists_every_code(): - listed = re.findall(r"^\| `([a-z-]+)` \|", _readme_section(), flags=re.MULTILINE) - - assert sorted(listed) == sorted(CODES) - - -def test_the_readme_example_has_the_formats_keys_and_types(): - example = json.loads(_readme_section().split("```json", 1)[1].split("```", 1)[0]) - - assert _mismatches(example, _SCHEMA["list"]) == [] - - @pytest.mark.parametrize(("body", "code"), [ ("[envs\n", "config-unreadable"), ('[envs]\nimpls = "x"\n', "config-invalid"), diff --git a/tst/unit/config/test_unread_keys.py b/tst/unit/config/test_unread_keys.py index 185101e9..c49e7304 100644 --- a/tst/unit/config/test_unread_keys.py +++ b/tst/unit/config/test_unread_keys.py @@ -236,8 +236,8 @@ def test_a_table_is_named_as_the_file_writes_it(tmp_path, monkeypatch, body, war def _documented_configs() -> list[str]: - docs = "".join((_REPO / name).read_text() for name in ("README.md", "PLUGINS.md")) - blocks = re.findall(r"```toml\n(.*?)```", docs, re.S) + readme = (_REPO / "README.md").read_text() + blocks = re.findall(r"```toml\n(.*?)```", readme, re.S) # the pyproject.toml blocks declare entry points, and a block headed by a bundle path # (`# agents/solver/agent.toml`) is an entity's toml; neither is a config file entity_tomls = tuple(f"# {kind.value}/" for kind in CONFIG_FILES) diff --git a/tst/unit/test_check_plugin_api.py b/tst/unit/test_check_plugin_api.py index 36fc9eda..84fc4c5b 100644 --- a/tst/unit/test_check_plugin_api.py +++ b/tst/unit/test_check_plugin_api.py @@ -252,11 +252,6 @@ def test_a_removed_surface_class_is_a_break(tmp_path): assert "agent_env.providers.env_state.env_state_provider.EnvStateProvider: removed" in result.stdout -def _documented_surface() -> set[str]: - section = (REPO / "PLUGINS.md").read_text().split("**The plugin surface.**", 1)[1].split("\n\n", 2)[1] - return set(re.findall(r"`(agent_env(?:\.\w+)+)`", section)) - - def _documented_types(path: str, lead: str) -> set[str]: text = " ".join((REPO / path).read_text().split()) listed = text.split(lead, 1)[1].split(".", 1)[0].split(";", 1)[0] @@ -268,10 +263,6 @@ def test_the_title_types_are_the_documented_ones(path, lead): assert _documented_types(path, lead) == set(check_plugin_api.TYPES) -def test_the_plugin_contract_lists_the_checked_surface(): - assert _documented_surface() == set(check_plugin_api.SURFACE) - - def test_the_surface_is_found_in_this_checkout(): pkg = check_plugin_api.load(REPO / "src") found = {path: check_plugin_api.lookup(pkg, path) for path in check_plugin_api.SURFACE} From c1526f0ba396b188b62ca17831baa66b0909416b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan <edgar.arakelyan@scale.com> Date: Thu, 8 Oct 2026 11:14:13 -0700 Subject: [PATCH 4/5] docs: cite Edgar Arakelyan as the author Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CITATION.cff | 4 +++- README.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index d14b0e62..c3dd521e 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,7 +4,9 @@ type: software title: "AgentEnv Framework" abstract: "AgentEnv Framework is a Python SDK and CLI for building, deploying and running agentic environments and the tasks that grade agents inside them." authors: - - name: "Scale AI" + - family-names: "Arakelyan" + given-names: "Edgar" + affiliation: "Scale AI" repository-code: "https://github.com/scaleapi/agentenv-framework" license: Apache-2.0 date-released: 2026-09-30 diff --git a/README.md b/README.md index d5e206d6..3e919a9e 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ To cite AgentEnv Framework, use the metadata in [`CITATION.cff`](https://github. ```bibtex @software{agentenv_framework, - author = {{Scale AI}}, + author = {Arakelyan, Edgar}, title = {AgentEnv Framework}, year = {2026}, url = {https://github.com/scaleapi/agentenv-framework}, From f73b3b944a87265ac20b0cf4e7aaf0eee3284cc4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan <edgar.arakelyan@scale.com> Date: Thu, 8 Oct 2026 11:28:40 -0700 Subject: [PATCH 5/5] docs: add Tejas Polakam and Pratyush Singhal as citation authors Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- CITATION.cff | 6 ++++++ README.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index c3dd521e..18c8c93a 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -7,6 +7,12 @@ authors: - family-names: "Arakelyan" given-names: "Edgar" affiliation: "Scale AI" + - family-names: "Polakam" + given-names: "Tejas" + affiliation: "Scale AI" + - family-names: "Singhal" + given-names: "Pratyush" + affiliation: "Scale AI" repository-code: "https://github.com/scaleapi/agentenv-framework" license: Apache-2.0 date-released: 2026-09-30 diff --git a/README.md b/README.md index 3e919a9e..64adcc2d 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ To cite AgentEnv Framework, use the metadata in [`CITATION.cff`](https://github. ```bibtex @software{agentenv_framework, - author = {Arakelyan, Edgar}, + author = {Arakelyan, Edgar and Polakam, Tejas and Singhal, Pratyush}, title = {AgentEnv Framework}, year = {2026}, url = {https://github.com/scaleapi/agentenv-framework},