diff --git a/AGENTS.md b/AGENTS.md index a80343d0..a46f644a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,7 +86,7 @@ and deserialized through a registry. | Package | Contents | |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset` and `load_file_artifact_universe`. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`). `env/gateway/gateway.py` is the env-side gateway server, container-only code. | -| `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Built-ins: `CliArtifact` (`cli`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`; legacy `service`), `EnvironmentUniverseArtifact` (`environment_universe`; legacy `service_universe`), `SkillArtifact` (`skill`). | +| `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Built-ins: `CliArtifact` (`cli`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`; legacy `service`), `EnvironmentUniverseArtifact` (`environment_universe`; legacy `service_universe`), `SkillArtifact` (`skill`). A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, and more). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | diff --git a/src/agent_env/artifact/artifacts/docker_image.py b/src/agent_env/artifact/artifacts/docker_image.py index 6b13687d..34c6a844 100644 --- a/src/agent_env/artifact/artifacts/docker_image.py +++ b/src/agent_env/artifact/artifacts/docker_image.py @@ -23,7 +23,9 @@ from agent_env.artifact.artifact import Artifact, _write_twin from agent_env.store.ids import fs_safe, image_repository, is_local_id +from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import is_loopback_host, names_registry, registry_host_from_ref +from agent_env.store.image_store.registry_api import pin_digest from agent_env.utils.deprecation import OMITTED, renamed_keyword logger = logging.getLogger(__name__) @@ -208,6 +210,31 @@ def put_tar( ) return store.put_document(instance) + @classmethod + def put_ref(cls, id: str, *, description: str, image_name: str) -> "DockerImageArtifact": + """Register ``image_name``, an image already in a registry, writing this document only: nothing is pushed, + saved or uploaded, and a sandbox pulls the image. A tag is pinned to the digest the registry serves for it + now, recorded as ``name:tag@sha256:...``, so every run gets the same image; a digest is checked to exist. + + ``image_name`` must name its registry. The registry is read with the configured image store's credentials + where it holds that registry, anonymously otherwise. One on this machine is refused unless ``id``'s image + store is the local registry, since nothing else could pull from it.""" + from agent_env.artifact.store import get_artifact_store + from agent_env.config import get_config + + if not names_registry(image_name): + raise ValueError(f"{image_name!r} doesn't name its registry; spell it out, as in " + f"docker.io/library/{image_name}") + config = get_config() + if (is_loopback_host(registry_host_from_ref(image_name)) + and not isinstance(config.get_image_store_for(id), LocalRegistryImageStore)): + raise ValueError(f"{image_name} is in a registry on this machine, which only an image in the local " + "registry's store, such as an @local one, can name") + pinned = pin_digest(image_name, config.get_image_store().auth(image_name)) + store = get_artifact_store() + version = store.next_version(id) + return store.put_document(cls(id=id, version=version, description=description, image_name=pinned)) + def load(self) -> bytes: from agent_env.artifact.store import get_artifact_store if not self.tar_gz_object_url: diff --git a/src/agent_env/cli/a2a_agent/put.py b/src/agent_env/cli/a2a_agent/put.py index a2786649..ae52ce4f 100644 --- a/src/agent_env/cli/a2a_agent/put.py +++ b/src/agent_env/cli/a2a_agent/put.py @@ -7,13 +7,16 @@ from agent_env.artifact import DockerImageArtifact from agent_env.cli.utils import build_platform_option, detect_env_metadata, refuse_unwritable_ids from agent_env.store.ids import derive_id, image_repository -from agent_env.utils.docker_build import build_image +from agent_env.utils.docker_build import DEFAULT_BUILD_PLATFORM, build_image @click.command() @click.option("--id", "agent_id", required=True, help="A2A agent id") -@click.option("--dockerfile", required=True, type=click.Path(exists=True), help="Path to Dockerfile") +@click.option("--dockerfile", default=None, type=click.Path(exists=True), help="Path to Dockerfile") @click.option("--context", "context_path", default=None, type=click.Path(exists=True), help="Docker build context (defaults to Dockerfile's directory)") +@click.option("--image-ref", "image_ref", default=None, + help="An image already in a registry, e.g. ghcr.io/org/agent:v1: registered as it is, with its tag pinned " + "to the digest the registry serves now, and pulled by sandboxes (instead of --dockerfile)") @click.option("--env-var", "env_var_pairs", multiple=True, help="Default env var KEY=VALUE (repeatable)") @click.option("--metadata", "metadata_pairs", multiple=True, help="Metadata key=value pair (repeatable)") @click.option("--min-disk-size-gb", type=float, default=None, help="Minimum disk size in GB for agent deployment") @@ -21,8 +24,15 @@ @click.option("--skip-validation", is_flag=True, default=False, help="Skip A2A agent validation after registration") @click.option("--litellm-api-key", type=str, default=None, help="LiteLLM API key for validation prompts") @build_platform_option -def put(agent_id: str, dockerfile: str, context_path: str | None, env_var_pairs: tuple[str, ...], metadata_pairs: tuple[str, ...], min_disk_size_gb: float | None, default_model: str | None, skip_validation: bool, litellm_api_key: str | None, build_platform: str): - """Build a Docker image and register an A2A agent.""" +def put(agent_id: str, dockerfile: str | None, context_path: str | None, image_ref: str | None, env_var_pairs: tuple[str, ...], metadata_pairs: tuple[str, ...], min_disk_size_gb: float | None, default_model: str | None, skip_validation: bool, litellm_api_key: str | None, build_platform: str): + """Build a Docker image and register an A2A agent, or register one over an image already in a registry.""" + + if bool(dockerfile) == bool(image_ref): + click.echo("Error: exactly one of --dockerfile and --image-ref is required", err=True) + sys.exit(1) + if context_path and not dockerfile: + click.echo("Error: --context needs --dockerfile", err=True) + sys.exit(1) default_env_vars = {} for pair in env_var_pairs: @@ -40,26 +50,32 @@ def put(agent_id: str, dockerfile: str, context_path: str | None, env_var_pairs: key, value = pair.split("=", 1) user_metadata[key] = value - dockerfile_path = Path(dockerfile) - context = Path(context_path) if context_path else dockerfile_path.parent image_id = derive_id(agent_id, "agent_image") - image_tag = image_repository(image_id) refuse_unwritable_ids(agent_id, image_id) + if image_ref: + if build_platform != DEFAULT_BUILD_PLATFORM: + click.echo(f"Warning: --platform {build_platform!r} is ignored for --image-ref, which builds nothing", err=True) + artifact = DockerImageArtifact.put_ref(image_id, description=f"A2A agent image for {agent_id}", image_name=image_ref) + click.echo(f"Registered image: id={artifact.id} version={artifact.version} image={artifact.image_name}") + metadata = {} + else: + dockerfile_path = Path(dockerfile) + context = Path(context_path) if context_path else dockerfile_path.parent + image_tag = image_repository(image_id) - click.echo(f"Building Docker image...") - build_image(dockerfile_path, context, image_tag, platform=build_platform) - - click.echo(f"Creating DockerImageArtifact...") - artifact = DockerImageArtifact.put( - id=image_id, - description=f"A2A agent image for {agent_id}", - image_name=image_tag, - build_context_path=str(context), - dockerfile_path=str(dockerfile_path), - ) - click.echo(f"Created artifact: id={artifact.id} version={artifact.version}") + click.echo(f"Building Docker image...") + build_image(dockerfile_path, context, image_tag, platform=build_platform) - metadata = detect_env_metadata(dockerfile_path, context) + click.echo(f"Creating DockerImageArtifact...") + artifact = DockerImageArtifact.put( + id=image_id, + description=f"A2A agent image for {agent_id}", + image_name=image_tag, + build_context_path=str(context), + dockerfile_path=str(dockerfile_path), + ) + click.echo(f"Created artifact: id={artifact.id} version={artifact.version}") + metadata = detect_env_metadata(dockerfile_path, context) metadata.update(user_metadata) if min_disk_size_gb is not None: metadata["min_disk_size_gb"] = min_disk_size_gb diff --git a/src/agent_env/cli/env/mcp_server.py b/src/agent_env/cli/env/mcp_server.py index 516e1a85..f8fef4ca 100644 --- a/src/agent_env/cli/env/mcp_server.py +++ b/src/agent_env/cli/env/mcp_server.py @@ -111,6 +111,9 @@ def _gate_release(env, report_dirs, override: bool) -> None: @click.option("--context", "context_path", default=None, type=click.Path(exists=True), help="Docker build context (defaults to Dockerfile's directory)") @click.option("--dockerfile-github-url", "dockerfile_github_url", default=None, help="GitHub URL to Dockerfile (e.g. https://github.com/owner/repo/tree/main/path/Dockerfile)") @click.option("--docker-context-github-url", "docker_context_github_url", default=None, help="GitHub URL to build context directory (defaults to Dockerfile's parent)") +@click.option("--image-ref", "image_ref", default=None, + help="An image already in a registry, e.g. ghcr.io/org/server:v1: registered as it is, with its tag pinned " + "to the digest the registry serves now, and pulled by sandboxes (needs --environment-name)") @environment_name_options @env_provider_type_option("What deploys the env: 'gateway' (a gateway and service database in front of the server), 'server' (the " "server on its own), or the type of an installed agent_env.env_providers plugin") @@ -120,19 +123,21 @@ def _gate_release(env, report_dirs, override: bool) -> None: @click.option("--override", "override", is_flag=True, default=False, help="Run the release gate but publish even if it fails (records an audited override); implies --validate") @build_platform_option -def put(env_id: str, dockerfile: str | None, context_path: str | None, dockerfile_github_url: str | None, docker_context_github_url: str | None, environment_name: str | None, env_provider_type: str, metadata_pairs: tuple[str, ...], run_validation: bool, override: bool, build_platform: str): - """Build and upload an MCP server environment.""" +def put(env_id: str, dockerfile: str | None, context_path: str | None, dockerfile_github_url: str | None, docker_context_github_url: str | None, image_ref: str | None, environment_name: str | None, env_provider_type: str, metadata_pairs: tuple[str, ...], run_validation: bool, override: bool, build_platform: str): + """Build and upload an MCP server environment, or register one over an image already in a registry.""" environment_name = resolve_environment_name(environment_name, allow_missing=True) - if dockerfile and dockerfile_github_url: - click.echo("Error: --dockerfile and --dockerfile-github-url are mutually exclusive", err=True) + sources = [flag for flag, value in (("--dockerfile", dockerfile), ("--dockerfile-github-url", dockerfile_github_url), + ("--image-ref", image_ref)) if value] + if len(sources) > 1: + click.echo(f"Error: {' and '.join(sources)} are mutually exclusive", err=True) sys.exit(1) - if not dockerfile and not dockerfile_github_url: - click.echo("Error: either --dockerfile or --dockerfile-github-url is required", err=True) + if not sources: + click.echo("Error: one of --dockerfile, --dockerfile-github-url or --image-ref is required", err=True) sys.exit(1) - if context_path and dockerfile_github_url: - click.echo("Error: --context cannot be used with --dockerfile-github-url", err=True) + if context_path and not dockerfile: + click.echo("Error: --context needs --dockerfile", err=True) sys.exit(1) if docker_context_github_url and not dockerfile_github_url: click.echo("Error: --docker-context-github-url requires --dockerfile-github-url", err=True) @@ -182,32 +187,43 @@ def _on_progress(step: str, message: str, percent: int) -> None: _gate_release(env, [], override) return - dockerfile_path = Path(dockerfile) - context = Path(context_path) if context_path else dockerfile_path.parent - if environment_name is None: - environment_name = card_name_from_source(str(dockerfile_path), str(context)) - if not environment_name: - click.echo("Error: no @environment_card(name=...) found in the build source; pass --environment-name.", err=True) - sys.exit(1) - click.echo(f"Derived environment_name={environment_name!r} from the environment card.") image_id = derive_id(env_id, "env_image") - image_tag = image_repository(image_id) refuse_unwritable_ids(env_id, image_id) - - click.echo(f"Building MCP server Docker image...") - build_image(dockerfile_path, context, image_tag, platform=build_platform) - - click.echo(f"Creating DockerImageArtifact...") - artifact = DockerImageArtifact.put( - id=image_id, - description="Created from agent-env CLI", - image_name=image_tag, - build_context_path=str(context), - dockerfile_path=str(dockerfile_path), - ) - click.echo(f"Created artifact: id={artifact.id} version={artifact.version}") - - metadata = detect_env_metadata(dockerfile_path, context) + if image_ref: + if environment_name is None: + click.echo("Error: --image-ref needs --environment-name: an image in a registry has no environment card to " + "read it from", err=True) + sys.exit(1) + if build_platform != DEFAULT_BUILD_PLATFORM: + click.echo(f"Warning: --platform {build_platform!r} is ignored for --image-ref, which builds nothing", err=True) + artifact = DockerImageArtifact.put_ref(image_id, description="Registered from agent-env CLI", image_name=image_ref) + click.echo(f"Registered image: id={artifact.id} version={artifact.version} image={artifact.image_name}") + metadata, report_dirs = {}, [] + else: + dockerfile_path = Path(dockerfile) + context = Path(context_path) if context_path else dockerfile_path.parent + if environment_name is None: + environment_name = card_name_from_source(str(dockerfile_path), str(context)) + if not environment_name: + click.echo("Error: no @environment_card(name=...) found in the build source; pass --environment-name.", err=True) + sys.exit(1) + click.echo(f"Derived environment_name={environment_name!r} from the environment card.") + image_tag = image_repository(image_id) + click.echo(f"Building MCP server Docker image...") + build_image(dockerfile_path, context, image_tag, platform=build_platform) + + click.echo(f"Creating DockerImageArtifact...") + artifact = DockerImageArtifact.put( + id=image_id, + description="Created from agent-env CLI", + image_name=image_tag, + build_context_path=str(context), + dockerfile_path=str(dockerfile_path), + ) + click.echo(f"Created artifact: id={artifact.id} version={artifact.version}") + metadata = detect_env_metadata(dockerfile_path, context) + # Local build: look for the env-build handoff report next to the build context. + report_dirs = [str(context), str(dockerfile_path.parent)] metadata.update(user_metadata) click.echo(f"Creating MCPServerEnv...") @@ -220,8 +236,7 @@ def _on_progress(step: str, message: str, percent: int) -> None: ) click.echo(f"Created MCPServerEnv: id={env.id} version={env.version} environment_name={env.environment_name} env_provider_type={env.env_provider_type}") if run_validation or override: - # Local build: look for the env-build handoff report next to the build context. - _gate_release(env, [str(context), str(dockerfile_path.parent)], override) + _gate_release(env, report_dirs, override) _SPEC_FINDING_LIMIT = 20 diff --git a/src/agent_env/store/image_store/oci_registry_credentials.py b/src/agent_env/store/image_store/oci_registry_credentials.py index 6d3e4d97..7ae25895 100644 --- a/src/agent_env/store/image_store/oci_registry_credentials.py +++ b/src/agent_env/store/image_store/oci_registry_credentials.py @@ -91,11 +91,12 @@ def registry_host_from_ref(ref: str) -> str | None: def is_loopback_host(host: str | None) -> bool: - """Whether a registry ``host``, with or without its port, names this machine.""" + """Whether a registry ``host``, with or without its port, names this machine: a loopback address, the + unspecified ``0.0.0.0``, or ``host.docker.internal``, which only this machine's containers resolve.""" if not host: return False - name = host[1:host.find("]")] if host.startswith("[") else host.rsplit(":", 1)[0] - return name == "localhost" or name.startswith("127.") or name == "::1" + name = (host[1:host.find("]")] if host.startswith("[") else host.rsplit(":", 1)[0]).lower() + return name in ("localhost", "::1", "0.0.0.0", "host.docker.internal") or name.startswith("127.") class SecretStoreCredentials(OciRegistryCredentials): diff --git a/src/agent_env/store/image_store/registry_api.py b/src/agent_env/store/image_store/registry_api.py new file mode 100644 index 00000000..62b3feec --- /dev/null +++ b/src/agent_env/store/image_store/registry_api.py @@ -0,0 +1,127 @@ +"""Read an image's digest from its registry over the OCI distribution API, to pin a tag to the image it names now.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass + +import httpx + +from agent_env.store.image_store.oci_registry_credentials import RegistryAuth, is_loopback_host, normalize_registry_host + +# An index (multi-platform) or a single manifest, OCI or Docker: the digest is of whichever the registry serves. +_MANIFEST_TYPES = ", ".join([ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +]) +# The digests Docker pulls by: the algorithm, then its full-length lowercase hex. +_DIGEST = re.compile(r"^(?:sha256:[a-f0-9]{64}|sha384:[a-f0-9]{96}|sha512:[a-f0-9]{128})$") +_DOCKER_HUB_API = "registry-1.docker.io" +_TIMEOUT_SECONDS = 30 + + +@dataclass(frozen=True) +class _ImageRef: + name: str # the reference without its tag or digest, spelled as given + host: str + repository: str + tag: str | None + digest: str | None + + +def pin_digest(ref: str, auth: RegistryAuth | None) -> str: + """``ref`` pinned to the digest its registry serves for it now, as ``name:tag@sha256:...``. + + A tag (``latest`` when none is given) is resolved; a digest is checked to exist; a ref giving both must agree, + since a pull goes by the digest and ignores the tag. ``auth`` is used where the registry asks for credentials; + without it, a registry's anonymous token is. Raises ValueError naming what the registry answered. + """ + image = _parse(ref) + if image.digest is None: + tag = image.tag or "latest" + return f"{image.name}:{tag}@{_check(image, tag, auth)}" + served = _check(image, image.tag or image.digest, auth) + if image.tag and served != image.digest: + raise ValueError(f"{ref}: the tag {image.tag!r} names {served} now, not {image.digest}; give the tag or the " + "digest alone") + return ref + + +def _parse(ref: str) -> _ImageRef: + name, _, digest = ref.partition("@") + if digest and not _DIGEST.match(digest): + raise ValueError(f"{ref}: {digest!r} isn't a sha256, sha384 or sha512 digest") + head, slash, tail = name.rpartition("/") + tag = None + if ":" in tail: + tail, tag = tail.split(":", 1) + name = f"{head}{slash}{tail}" + host, _, repository = name.partition("/") + host = normalize_registry_host(host) + if host == "docker.io": + host = _DOCKER_HUB_API + if "/" not in repository: + repository = f"library/{repository}" + return _ImageRef(name=name, host=host, repository=repository, tag=tag, digest=digest or None) + + +def _check(image: _ImageRef, reference: str, auth: RegistryAuth | None) -> str: + """The digest the registry serves for ``reference`` (a tag or a digest) in ``image``'s repository.""" + # A registry on this machine serves plain HTTP, as Docker allows for a loopback one. + scheme = "http" if is_loopback_host(image.host) else "https" + url = f"{scheme}://{image.host}/v2/{image.repository}/manifests/{reference}" + what = f"{image.name}{'@' if _DIGEST.match(reference) else ':'}{reference}" + basic = (auth.username, auth.password) if auth is not None else None + try: + with httpx.Client(timeout=_TIMEOUT_SECONDS, follow_redirects=True) as client: + headers = {"Accept": _MANIFEST_TYPES} + # Basic credentials go with every request until a Bearer token replaces them. + credentials = basic + response = client.head(url, headers=headers, auth=credentials) + if response.status_code == 401 and (token := _bearer_token(client, response, basic, what)) is not None: + headers["Authorization"] = f"Bearer {token}" + credentials = None + response = client.head(url, headers=headers) + if response.status_code == 200 and not response.headers.get("docker-content-digest"): + response = client.get(url, headers=headers, auth=credentials) + if response.status_code == 200: + return f"sha256:{hashlib.sha256(response.content).hexdigest()}" + except httpx.HTTPError as e: + raise ValueError(f"{what}: couldn't read its manifest from {image.host}: {type(e).__name__}: {e}") from e + if response.status_code == 200: + served = response.headers["docker-content-digest"] + if not _DIGEST.match(served): + raise ValueError(f"{what}: {image.host} answered with {served!r}, which isn't a digest") + return served + if response.status_code == 404: + raise ValueError(f"{what}: {image.host} has no such image") + if response.status_code in (401, 403): + raise ValueError(f"{what}: {image.host} refused to serve it (HTTP {response.status_code}); agent-env reaches a " + "private registry only with the configured image store's credentials, for its own registry") + raise ValueError(f"{what}: {image.host} answered HTTP {response.status_code}") + + +def _bearer_token(client: httpx.Client, challenged: httpx.Response, basic: tuple[str, str] | None, + what: str) -> str | None: + """A token from the realm a ``Bearer`` challenge names, asked for with ``basic`` when given. None for any other + challenge, or when the realm refuses; a ValueError when it answers 200 without a token, as a sign-in page does.""" + challenge = challenged.headers.get("www-authenticate", "") + if not challenge.lower().startswith("bearer "): + return None + params = dict(re.findall(r'(\w+)="([^"]*)"', challenge)) + realm = params.pop("realm", None) + if not realm: + return None + response = client.get(realm, params=params, auth=basic) + if response.status_code != 200: + return None + try: + body = response.json() + except ValueError: + body = None + if not isinstance(body, dict) or not (token := body.get("token") or body.get("access_token")): + raise ValueError(f"{what}: the token service at {realm} answered without a token") + return token diff --git a/tst/unit/artifact/test_docker_image_put_ref.py b/tst/unit/artifact/test_docker_image_put_ref.py new file mode 100644 index 00000000..3ec15635 --- /dev/null +++ b/tst/unit/artifact/test_docker_image_put_ref.py @@ -0,0 +1,89 @@ +"""`DockerImageArtifact.put_ref` registers an image already in a registry: a document with no tar.gz, its tag pinned +to a digest, refused when no sandbox elsewhere could pull it. The registry API is patched out.""" + +import pytest + +from agent_env.artifact.artifacts import docker_image +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.store import ImageStore, OciRegistryImageStore, RegistryAuth +from agent_env.store.base import NotFoundError +from agent_env.store.routing import namespace_routing + +DIGEST = "sha256:" + "c" * 64 + + +@pytest.fixture +def resolved(monkeypatch): + """Each ref put_ref resolves and the credentials it resolves it with; every tag resolves to DIGEST.""" + calls = [] + + def pin(ref, auth): + calls.append((ref, auth)) + return ref if "@" in ref else f"{ref}@{DIGEST}" + + monkeypatch.setattr(docker_image, "pin_digest", pin) + return calls + + +def test_a_ref_is_registered_with_its_tag_pinned_and_no_tarball(local_stores, resolved): + image = DockerImageArtifact.put_ref("tool-image", description="d", image_name="ghcr.io/org/tool:v1") + + stored = DockerImageArtifact.get("tool-image", image.version) + assert (stored.version, stored.image_name, stored.tar_gz_object_url) == (1, f"ghcr.io/org/tool:v1@{DIGEST}", None) + assert stored.load_problem() is None + assert DockerImageArtifact.put_ref("tool-image", description="d", image_name="ghcr.io/org/tool:v1").version == 2 + + +def test_the_configured_store_lends_its_credentials_only_for_its_own_registry(local_stores, resolved): + class _Registry(ImageStore): + def image_ref(self, repository, tag): + return f"registry.example/{repository}:{tag}" + + def auth(self, ref): + return RegistryAuth("registry.example", "user", "pw") if ref.startswith("registry.example/") else None + + local_stores.set_image_store(_Registry()) + + DockerImageArtifact.put_ref("own", description="d", image_name="registry.example/team/app:v1") + DockerImageArtifact.put_ref("public", description="d", image_name="ghcr.io/org/tool:v1") + + assert resolved == [("registry.example/team/app:v1", RegistryAuth("registry.example", "user", "pw")), + ("ghcr.io/org/tool:v1", None)] + + +def test_a_name_without_its_registry_is_refused_before_any_request(local_stores, resolved): + with pytest.raises(ValueError, match="'tool:v1' doesn't name its registry; spell it out, as in docker.io/library/tool:v1"): + DockerImageArtifact.put_ref("tool-image", description="d", image_name="tool:v1") + + assert resolved == [] + with pytest.raises(NotFoundError): + DockerImageArtifact.get("tool-image") + + +@pytest.mark.parametrize("ref", ["localhost:5000/team/img:v1", "127.0.0.1:5000/img", "host.docker.internal:5000/img", + "0.0.0.0:5000/img"]) +def test_a_registry_on_this_machine_is_refused_for_a_shared_store(local_stores, resolved, ref): + local_stores.set_image_store(OciRegistryImageStore("registry.example")) + + with pytest.raises(ValueError, match="is in a registry on this machine"): + DockerImageArtifact.put_ref("img", description="d", image_name=ref) + assert resolved == [] + + +def test_a_registry_on_this_machine_is_fine_for_an_image_in_the_local_registrys_store(local_stores, resolved): + with namespace_routing(): + image = DockerImageArtifact.put_ref("@local/team/img", description="d", image_name="localhost:5000/team/img:v1") + + assert image.image_name == f"localhost:5000/team/img:v1@{DIGEST}" + + +def test_a_ref_the_registry_cant_serve_writes_nothing(local_stores, monkeypatch): + def unresolvable(ref, auth): + raise ValueError(f"{ref}: ghcr.io has no such image") + + monkeypatch.setattr(docker_image, "pin_digest", unresolvable) + + with pytest.raises(ValueError, match="ghcr.io has no such image"): + DockerImageArtifact.put_ref("tool-image", description="d", image_name="ghcr.io/org/tool:v9") + assert get_artifact_store().next_version("tool-image") == 1 diff --git a/tst/unit/cli/image_ref_put_test.py b/tst/unit/cli/image_ref_put_test.py new file mode 100644 index 00000000..16fbaa36 --- /dev/null +++ b/tst/unit/cli/image_ref_put_test.py @@ -0,0 +1,78 @@ +"""`env mcp-server put --image-ref` and `a2a-agent put --image-ref` register an env or agent over an image already in a +registry, through `DockerImageArtifact.put_ref`, on the local stores; the registry API is patched out.""" + +import pytest +from click.testing import CliRunner + +from agent_env.a2a_agent import A2AAgent +from agent_env.artifact.artifacts import docker_image +from agent_env.cli import cli +from agent_env.env import Env + +DIGEST = "sha256:" + "d" * 64 +REF = "ghcr.io/org/server:v1" + + +@pytest.fixture +def pinned(monkeypatch): + monkeypatch.setattr(docker_image, "pin_digest", lambda ref, auth: f"{ref}@{DIGEST}") + + +def _run(*args): + return CliRunner().invoke(cli, list(args), catch_exceptions=False) + + +def test_an_mcp_server_is_registered_over_an_image_in_a_registry(local_stores, pinned): + result = _run("env", "mcp-server", "put", "--id", "server", "--image-ref", REF, "--environment-name", "items") + + assert result.exit_code == 0, result.output + env = Env.get("server") + image = env.docker_image_artifact + assert (env.environment_name, image.id, image.image_name, image.tar_gz_object_url) == ( + "items", "server__env_image", f"{REF}@{DIGEST}", None) + assert f"Registered image: id=server__env_image version=1 image={REF}@{DIGEST}" in result.output + + +def test_an_mcp_server_over_an_image_ref_needs_its_environment_name(local_stores, pinned): + result = _run("env", "mcp-server", "put", "--id", "server", "--image-ref", REF) + + assert result.exit_code == 1 + assert "--image-ref needs --environment-name" in result.output + + +@pytest.mark.parametrize("extra, message", [ + (["--dockerfile", "Dockerfile"], "--dockerfile and --image-ref are mutually exclusive"), + (["--dockerfile-github-url", "https://github.com/o/r/tree/main/Dockerfile"], + "--dockerfile-github-url and --image-ref are mutually exclusive"), + (["--context", "."], "--context needs --dockerfile"), +]) +def test_an_mcp_server_takes_one_image_source(local_stores, pinned, tmp_path, extra, message): + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + + result = _run("env", "mcp-server", "put", "--id", "server", "--image-ref", REF, "--environment-name", "items", *extra) + + assert result.exit_code == 1 and message in result.output + + +def test_an_agent_is_registered_over_an_image_in_a_registry(local_stores, pinned): + result = _run("a2a-agent", "put", "--id", "solver", "--image-ref", REF, "--default-model", "claude-sonnet-4-6", + "--skip-validation") + + assert result.exit_code == 0, result.output + agent = A2AAgent.get("solver") + assert (agent.docker_image_artifact.id, agent.docker_image_artifact.image_name) == ("solver__agent_image", + f"{REF}@{DIGEST}") + assert agent.metadata == {"default_model": "claude-sonnet-4-6"} + + +@pytest.mark.parametrize("extra, message", [ + ([], "exactly one of --dockerfile and --image-ref is required"), + (["--image-ref", REF, "--dockerfile", "Dockerfile"], "exactly one of --dockerfile and --image-ref is required"), + (["--image-ref", REF, "--context", "."], "--context needs --dockerfile"), +]) +def test_an_agent_takes_one_image_source(local_stores, pinned, tmp_path, extra, message): + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + + result = _run("a2a-agent", "put", "--id", "solver", "--skip-validation", *extra) + + assert result.exit_code == 1 and message in result.output diff --git a/tst/unit/store/oci_registry_image_store_test.py b/tst/unit/store/oci_registry_image_store_test.py index acd41259..9fae15a3 100644 --- a/tst/unit/store/oci_registry_image_store_test.py +++ b/tst/unit/store/oci_registry_image_store_test.py @@ -4,6 +4,7 @@ from agent_env.store import ImageStore, OciRegistryCredentials, OciRegistryImageStore, RegistryAuth from agent_env.store.image_store import names_registry, registry_host_from_ref +from agent_env.store.image_store.oci_registry_credentials import is_loopback_host class _Credentials(OciRegistryCredentials): @@ -70,6 +71,15 @@ def test_a_reference_names_its_registry_only_as_docker_reads_one(ref, named): assert names_registry(ref) is named +@pytest.mark.parametrize("host, here", [ + ("localhost", True), ("localhost:5000", True), ("LOCALHOST:5000", True), ("127.0.0.2:5000", True), + ("[::1]:5000", True), ("0.0.0.0:5000", True), ("host.docker.internal:5000", True), ("Host.Docker.Internal", True), + ("registry.example", False), ("10.0.0.1:5000", False), ("ghcr.io", False), (None, False), ("", False), +]) +def test_a_registry_host_on_this_machine_is_one_only_this_machine_can_reach(host, here): + assert is_loopback_host(host) is here + + @pytest.mark.parametrize("host", ["", " ", "https://ghcr.io", "ghcr.io/team"]) def test_rejects_invalid_registry_host(host): with pytest.raises(ValueError, match="registry_host"): diff --git a/tst/unit/store/registry_api_test.py b/tst/unit/store/registry_api_test.py new file mode 100644 index 00000000..f47c575a --- /dev/null +++ b/tst/unit/store/registry_api_test.py @@ -0,0 +1,186 @@ +"""Pinning an image ref to the digest its registry serves, over a fake registry's OCI distribution API.""" + +import base64 +import hashlib + +import httpx +import pytest + +from agent_env.store import RegistryAuth +from agent_env.store.image_store import registry_api +from agent_env.store.image_store.registry_api import pin_digest + +DIGEST = "sha256:" + "a" * 64 +OTHER = "sha256:" + "b" * 64 + + +@pytest.fixture +def fake_registry(monkeypatch): + """Route the resolver's HTTP client to ``handler`` and record each request it makes.""" + seen: list[httpx.Request] = [] + real = httpx.Client + + def route(handler): + def recorded(request): + seen.append(request) + return handler(request) + + monkeypatch.setattr(registry_api.httpx, "Client", lambda **kwargs: real(transport=httpx.MockTransport(recorded), + **kwargs)) + return seen + + return route + + +def _served(digest=DIGEST): + return httpx.Response(200, headers={"Docker-Content-Digest": digest}) + + +def test_a_tag_is_pinned_to_its_digest_through_the_registrys_anonymous_token(fake_registry): + def ghcr(request): + if request.url.path == "/token": + assert request.url.params["scope"] == "repository:org/tool:pull" + return httpx.Response(200, json={"token": "anon"}) + if request.headers.get("authorization") == "Bearer anon": + return _served() + return httpx.Response(401, headers={"WWW-Authenticate": 'Bearer realm="https://ghcr.io/token",service="ghcr.io",' + 'scope="repository:org/tool:pull"'}) + + seen = fake_registry(ghcr) + + assert pin_digest("ghcr.io/org/tool:v1", None) == f"ghcr.io/org/tool:v1@{DIGEST}" + assert [r.method for r in seen] == ["HEAD", "GET", "HEAD"] + assert seen[0].url == "https://ghcr.io/v2/org/tool/manifests/v1" + assert "application/vnd.oci.image.index.v1+json" in seen[0].headers["accept"] + + +def test_the_image_stores_credentials_go_to_a_registry_that_takes_them_directly(fake_registry): + def ecr(request): + expected = "Basic " + base64.b64encode(b"AWS:token").decode() + return _served() if request.headers.get("authorization") == expected else httpx.Response(401) + + fake_registry(ecr) + + ref = "123456789012.dkr.ecr.us-west-2.amazonaws.com/team/app:v2" + assert pin_digest(ref, RegistryAuth("123456789012.dkr.ecr.us-west-2.amazonaws.com", "AWS", "token")) == f"{ref}@{DIGEST}" + + +def test_no_tag_means_latest_and_docker_hub_names_its_library(fake_registry): + seen = fake_registry(lambda request: _served()) + + assert pin_digest("docker.io/alpine", None) == f"docker.io/alpine:latest@{DIGEST}" + assert seen[0].url == "https://registry-1.docker.io/v2/library/alpine/manifests/latest" + + +def test_a_digest_is_kept_once_the_registry_serves_it(fake_registry): + seen = fake_registry(lambda request: _served()) + + assert pin_digest(f"ghcr.io/org/tool@{DIGEST}", None) == f"ghcr.io/org/tool@{DIGEST}" + assert seen[0].url == f"https://ghcr.io/v2/org/tool/manifests/{DIGEST}" + + +def test_a_tag_and_digest_must_agree_since_a_pull_goes_by_the_digest(fake_registry): + fake_registry(lambda request: _served(DIGEST if request.url.path.endswith(DIGEST) else OTHER)) + + with pytest.raises(ValueError, match=f"the tag 'v1' names {OTHER} now, not {DIGEST}"): + pin_digest(f"ghcr.io/org/tool:v1@{DIGEST}", None) + + +def test_a_registry_on_this_machine_is_read_over_plain_http(fake_registry): + seen = fake_registry(lambda request: _served()) + + assert pin_digest("localhost:5000/team/img:v1", None) == f"localhost:5000/team/img:v1@{DIGEST}" + assert seen[0].url == "http://localhost:5000/v2/team/img/manifests/v1" + + +def test_a_registry_that_omits_the_digest_header_is_hashed_from_the_manifest(fake_registry): + body = b'{"schemaVersion": 2}' + fake_registry(lambda request: httpx.Response(200, content=body if request.method == "GET" else b"")) + + assert pin_digest("ghcr.io/org/tool:v1", None) == f"ghcr.io/org/tool:v1@sha256:{hashlib.sha256(body).hexdigest()}" + + +def test_the_manifest_is_fetched_with_the_credentials_the_registry_took(fake_registry): + body = b'{"schemaVersion": 2}' + expected = "Basic " + base64.b64encode(b"AWS:token").decode() + + def private(request): + if request.headers.get("authorization") != expected: + return httpx.Response(401) + return httpx.Response(200, content=body if request.method == "GET" else b"") + + seen = fake_registry(private) + + ref = "123456789012.dkr.ecr.us-west-2.amazonaws.com/team/app:v2" + auth = RegistryAuth("123456789012.dkr.ecr.us-west-2.amazonaws.com", "AWS", "token") + assert pin_digest(ref, auth) == f"{ref}@sha256:{hashlib.sha256(body).hexdigest()}" + assert [r.method for r in seen] == ["HEAD", "GET"] + + +@pytest.mark.parametrize("header", ["sha256:xyz", "sha256:" + "a" * 32, "sha256:" + "A" * 64]) +def test_a_digest_header_docker_couldnt_pull_by_is_refused(fake_registry, header): + fake_registry(lambda request: _served(header)) + + with pytest.raises(ValueError, match=f"ghcr.io answered with '{header}', which isn't a digest"): + pin_digest("ghcr.io/org/tool:v1", None) + + +@pytest.mark.parametrize("answer", [ + httpx.Response(200, html="Sign in"), + httpx.Response(200, json=[]), + httpx.Response(200, json={"error": "denied"}), +]) +def test_a_token_service_that_answers_without_a_token_is_named(fake_registry, answer): + def registry(request): + if request.url.path == "/token": + return answer + return httpx.Response(401, headers={"WWW-Authenticate": 'Bearer realm="https://ghcr.io/token",service="ghcr.io"'}) + + fake_registry(registry) + + with pytest.raises(ValueError, match="ghcr.io/org/tool:v1: the token service at https://ghcr.io/token answered " + "without a token"): + pin_digest("ghcr.io/org/tool:v1", None) + + +def test_a_token_service_that_refuses_leaves_the_registrys_refusal(fake_registry): + def registry(request): + if request.url.path == "/token": + return httpx.Response(403) + return httpx.Response(401, headers={"WWW-Authenticate": 'Bearer realm="https://ghcr.io/token",service="ghcr.io"'}) + + fake_registry(registry) + + with pytest.raises(ValueError, match="ghcr.io refused to serve it \\(HTTP 401\\)"): + pin_digest("ghcr.io/org/tool:v1", None) + + +@pytest.mark.parametrize("status, message", [ + (404, "ghcr.io/org/tool:v9: ghcr.io has no such image"), + (401, "ghcr.io/org/tool:v9: ghcr.io refused to serve it \\(HTTP 401\\); agent-env reaches a private registry only"), + (500, "ghcr.io/org/tool:v9: ghcr.io answered HTTP 500"), +]) +def test_what_the_registry_answers_is_named(fake_registry, status, message): + fake_registry(lambda request: httpx.Response(status)) + + with pytest.raises(ValueError, match=message): + pin_digest("ghcr.io/org/tool:v9", None) + + +def test_an_unreachable_registry_is_named(fake_registry): + def down(request): + raise httpx.ConnectError("connection refused") + + fake_registry(down) + + with pytest.raises(ValueError, match="ghcr.io/org/tool:v1: couldn't read its manifest from ghcr.io: ConnectError"): + pin_digest("ghcr.io/org/tool:v1", None) + + +@pytest.mark.parametrize("digest", ["sha256:xyz", "sha256:" + "a" * 32, "sha256:" + "A" * 64, "md5:" + "a" * 32]) +def test_a_digest_docker_couldnt_pull_by_is_refused_before_any_request(fake_registry, digest): + seen = fake_registry(lambda request: _served()) + + with pytest.raises(ValueError, match=f"'{digest}' isn't a sha256, sha384 or sha512 digest"): + pin_digest(f"ghcr.io/org/tool@{digest}", None) + assert seen == []