From ee7f968b370a204262f4a97fbea597dce8e03304 Mon Sep 17 00:00:00 2001 From: Benjamin Swerdlow <38309438+theswerd@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:08:25 -0700 Subject: [PATCH 1/2] feat(sandbox): add Freestyle VM provider --- AGENTS.md | 2 +- README.md | 7 + src/agent_env/cli/a2a_agent/deploy.py | 2 +- src/agent_env/cli/env/deploy.py | 2 +- .../sandbox_providers/freestyle/README.md | 61 +++ .../sandbox_providers/freestyle/__init__.py | 8 + .../sandbox_providers/freestyle/client.py | 48 +++ .../sandbox_providers/freestyle/provider.py | 274 +++++++++++++ .../sandbox_providers/freestyle/sandbox.py | 100 +++++ .../sandbox_providers/sandbox_provider.py | 1 + .../freestyle_sandbox_smoke_test.py | 89 ++++ .../sandbox_providers/egress_hosts_test.py | 2 +- .../freestyle_sandbox_test.py | 387 ++++++++++++++++++ .../network_policy_stamping_test.py | 10 + tst/util/capabilities.py | 10 +- 15 files changed, 994 insertions(+), 9 deletions(-) create mode 100644 src/agent_env/providers/sandbox_providers/freestyle/README.md create mode 100644 src/agent_env/providers/sandbox_providers/freestyle/__init__.py create mode 100644 src/agent_env/providers/sandbox_providers/freestyle/client.py create mode 100644 src/agent_env/providers/sandbox_providers/freestyle/provider.py create mode 100644 src/agent_env/providers/sandbox_providers/freestyle/sandbox.py create mode 100644 tst/integration/providers/sandbox_providers/freestyle_sandbox_smoke_test.py create mode 100644 tst/unit/providers/sandbox_providers/freestyle_sandbox_test.py diff --git a/AGENTS.md b/AGENTS.md index 878b7712..e02bc43f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,7 +95,7 @@ and deserialized through a registry. | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, and more). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | -| `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store) and `EnvironmentGatewayProvider`, which renders a docker-compose for the gateway and its MCP servers inside the sandbox; `providers/env_state/` holds env-state providers (`local_postgres` built in). | +| `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `freestyle`; `[sandbox] default` and `agent_default` accept a comma-separated fallback chain. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store) and `EnvironmentGatewayProvider`, which renders a docker-compose for the gateway and its MCP servers inside the sandbox; `providers/env_state/` holds env-state providers (`local_postgres` built in). | | `a2a_agent/` | The `A2AAgent` entity (`a2a_agent`), its stores and the validator steps. The protocol package provides the agent-side framework. | | `runner/` | The `[runner]` seam: `Runner.submit()` returns `(run_id, instance_id)`; `LocalRunner` is built in. | | `explorer/` | Optional local web UI: `agent-env up`, needs the `explorer` extra, binds loopback `:8234`. | diff --git a/README.md b/README.md index 64adcc2d..765852c6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,13 @@ uv sync --extra dev source .venv/bin/activate ``` +## Freestyle sandboxes + +Freestyle is available as the `freestyle` sandbox backend for environments and agents. +It boots Docker-capable VM snapshots, exposes requested ports over HTTPS, and supports +file transfer, reconnection, and teardown. See the [Freestyle provider guide](src/agent_env/providers/sandbox_providers/freestyle/README.md) +for configuration, resource sizing, and command/network limits. + ## Plugins A package of your own can add env types, providers, task steps and stores to agent-env, and `agent-env plugin` lists, checks, adds and removes installed plugins. The [plugin docs](https://www.agentenvframework.com/docs/plugins) show how. diff --git a/src/agent_env/cli/a2a_agent/deploy.py b/src/agent_env/cli/a2a_agent/deploy.py index 5c6ae687..7dc365bb 100644 --- a/src/agent_env/cli/a2a_agent/deploy.py +++ b/src/agent_env/cli/a2a_agent/deploy.py @@ -18,7 +18,7 @@ @click.option("--ttl-seconds", type=click.IntRange(min=MIN_TTL_SECONDS, max=MAX_TTL_SECONDS), default=DEFAULT_TTL_SECONDS, help=f"VM lifetime in seconds (default {DEFAULT_TTL_SECONDS})") @click.option("--sandbox", default=None, - help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, local) " + help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, sail_vm, freestyle, local) " "or a name from [sandbox.providers] in .agentenv/config.toml; comma-separated for a " "fallback chain. Defaults to [sandbox].agent_default (else local) when omitted.") def deploy(agent_id: str, agent_version: int | None, env_var_pairs: tuple[str, ...], ttl_seconds: int, sandbox: str): diff --git a/src/agent_env/cli/env/deploy.py b/src/agent_env/cli/env/deploy.py index 770e5e38..e29e6022 100644 --- a/src/agent_env/cli/env/deploy.py +++ b/src/agent_env/cli/env/deploy.py @@ -19,7 +19,7 @@ @click.option("--gateway-mode", type=click.Choice([m.value for m in GatewayMode], case_sensitive=False), default=GatewayMode.PERFORMANCE.value, help="Gateway mode (performance or consistent)") @click.option("--sandbox", default=None, - help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, local) " + help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, sail_vm, freestyle, local) " "or a name from [sandbox.providers] in .agentenv/config.toml; comma-separated for a " "fallback chain. Defaults to [sandbox].default (else local) when omitted.") @click.option("--service-db", "service_db_env_id", default=None, diff --git a/src/agent_env/providers/sandbox_providers/freestyle/README.md b/src/agent_env/providers/sandbox_providers/freestyle/README.md new file mode 100644 index 00000000..35d02e57 --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/freestyle/README.md @@ -0,0 +1,61 @@ +# Freestyle VM sandboxes + +Select `freestyle` for environments, agents, or a fallback chain. The provider uses +the Freestyle v5 HTTP API through the existing `httpx` dependency; no extra SDK is needed. + +```toml +[sandbox] +default = "freestyle" +agent_default = "freestyle" + +[sandbox.providers.freestyle.config] +api_key = "env:FREESTYLE_API_KEY" # or secret:freestyle_api_key +snapshot_id = "freestyle/ubuntu" +# exec_timeout_seconds = 300 +# api_url = "https://beta-api.freestyle.sh" +``` + +Get a key through the [Freestyle dashboard](https://dash.freestyle.sh) or the +[Freestyle CLI](https://www.freestyle.sh/docs/cli). Set it in your environment or secret +store; the provider resolves it from config and never injects it into the guest. +Then use `agent-env env deploy --sandbox freestyle` or select the backend in config. + +The snapshot must contain Bash, Docker, Docker Compose v2, curl, and the usual GNU +utilities. `freestyle/ubuntu` supplies those. To pin a prepared environment, use its +`sh-…` snapshot id; public snapshot names can be updated by Freestyle. `create_vm(image=…)` +selects another snapshot, and `boot_mode` overrides are rejected. + +CPU requests round up to whole vCPUs, memory is MiB, and disk requests are converted +from GiB to MiB. A VM starts with its snapshot's resources; requested dimensions grow +it when larger. Requests below the snapshot's minimum retain that minimum, because +Freestyle resizing only grows resources. `timeout` becomes an absolute VM lifetime +(`ttlSeconds`), including provisioning. Always terminate completed sandboxes. + +Each requested port receives its own public HTTPS `style.dev` URL through a TLS +rule created with the VM. Deleting the VM deletes those rules. Reconnection reads +the VM and its TLS rules to recover published ports; it reports an unknown network +policy rather than assuming that the firewall has not changed. + +This adapter supports `allow_all` egress only. Freestyle's firewall matches addresses +and VM/network identities, while agent-env's allowlists include hostnames. Restricted +policies are rejected before provisioning; fallback chains can select another backend. +`*.style.dev` joins the framework's platform egress hosts so other restricted sandboxes +can reach a Freestyle environment. See the [firewall docs](https://www.freestyle.sh/docs/vms/network/firewall). + +Commands run as root through the buffered exec API. Each command has a maximum +five-minute runtime (`exec_timeout_seconds`, 1–300); a guest timeout reports exit 124 +and is not retried. Run longer services in Docker or as background guest processes. +Live stdout/stderr streaming is not supplied by this adapter. File writes use the +binary guest-filesystem API; shared VM helpers handle image artifacts and container files. +Attribution is forwarded as metadata and must fit Freestyle's 64-entry limit with +keys and values at most 63 characters; `freestyle.sh/` keys are reserved. + +With credentials configured, run the real provider smoke test: + +```bash +.venv/bin/python -m pytest tst/integration/providers/sandbox_providers/freestyle_sandbox_smoke_test.py -v +``` + +It creates a Docker container, checks HTTPS and container-file writes, verifies a +binary host-file checksum, reconnects, and confirms deletion. Its VM has a 900-second +TTL and is terminated in `finally`. diff --git a/src/agent_env/providers/sandbox_providers/freestyle/__init__.py b/src/agent_env/providers/sandbox_providers/freestyle/__init__.py new file mode 100644 index 00000000..f520a1a7 --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/freestyle/__init__.py @@ -0,0 +1,8 @@ +"""Freestyle sandbox backend.""" + +from agent_env.providers.sandbox_providers.freestyle.provider import ( + FreestyleSandboxProvider, +) +from agent_env.providers.sandbox_providers.freestyle.sandbox import FreestyleSandbox + +__all__ = ["FreestyleSandbox", "FreestyleSandboxProvider"] diff --git a/src/agent_env/providers/sandbox_providers/freestyle/client.py b/src/agent_env/providers/sandbox_providers/freestyle/client.py new file mode 100644 index 00000000..c1f0ccb7 --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/freestyle/client.py @@ -0,0 +1,48 @@ +"""The Freestyle v5 HTTP operations used by the sandbox adapter.""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import quote + +import httpx + + +class FreestyleClient: + def __init__(self, *, api_key: str, api_url: str): + url = httpx.URL(api_url) + if ( + url.scheme != "https" + or not url.host + or url.userinfo + or url.query + or url.fragment + ): + raise ValueError( + "Freestyle api_url must be an HTTPS origin without credentials, query or fragment" + ) + self._api_key = api_key + self._api_url = str(url).rstrip("/") + self._client: httpx.AsyncClient | None = None + + async def request(self, method: str, path: str, **kwargs: Any) -> dict: + if self._client is None: + self._client = httpx.AsyncClient( + base_url=self._api_url, + headers={"Authorization": f"Bearer {self._api_key}"}, + timeout=httpx.Timeout(330, connect=30), + ) + response = await self._client.request(method, path, **kwargs) + response.raise_for_status() + if response.status_code == 204 or not response.content: + return {} + return response.json() + + async def close(self) -> None: + if self._client is not None: + await self._client.aclose() + self._client = None + + +def vm_path(vm_id: str) -> str: + return f"/v5/vms/{quote(vm_id, safe='')}" diff --git a/src/agent_env/providers/sandbox_providers/freestyle/provider.py b/src/agent_env/providers/sandbox_providers/freestyle/provider.py new file mode 100644 index 00000000..61a7090a --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/freestyle/provider.py @@ -0,0 +1,274 @@ +"""Freestyle VM provisioning, resource sizing and reconnection.""" + +from __future__ import annotations + +import asyncio +import logging +import math +import uuid +from typing import Any, ClassVar, Self + +import httpx + +from agent_env.attribution import Attribution +from agent_env.config.errors import ConfigError +from agent_env.providers.sandbox_providers.freestyle.client import ( + FreestyleClient, + vm_path, +) +from agent_env.providers.sandbox_providers.freestyle.sandbox import FreestyleSandbox +from agent_env.providers.sandbox_providers.sandbox import ( + NetworkPolicy, + NetworkPolicyUnsupportedError, +) +from agent_env.providers.sandbox_providers.sandbox_provider import ( + SandboxProvider, + apply_default_attribution, +) + +logger = logging.getLogger(__name__) + + +class FreestyleSandboxProvider(SandboxProvider): + """Boot Docker-capable snapshots and publish the requested ports over HTTPS.""" + + EGRESS_HOSTS: ClassVar[tuple[str, ...]] = ("*.style.dev",) + + def __init__( + self, + *, + api_key: str, + snapshot_id: str = "freestyle/ubuntu", + api_url: str = "https://beta-api.freestyle.sh", + exec_timeout_seconds: int = 300, + ): + if not isinstance(api_key, str) or not api_key.strip(): + raise ConfigError( + "[sandbox.providers.freestyle.config] requires a non-empty 'api_key'" + ) + if not isinstance(snapshot_id, str) or not snapshot_id.strip(): + raise ConfigError( + "Freestyle snapshot_id must be a non-empty Docker-capable snapshot" + ) + if ( + type(exec_timeout_seconds) is not int + or not 1 <= exec_timeout_seconds <= 300 + ): + raise ConfigError( + "Freestyle exec_timeout_seconds must be an integer from 1 to 300" + ) + self._client = FreestyleClient(api_key=api_key, api_url=api_url) + self._snapshot_id = snapshot_id + self._exec_timeout_seconds = exec_timeout_seconds + self._cleanups: set[asyncio.Task] = set() + + @classmethod + def from_config(cls, **config: Any) -> Self: + if not config.get("api_key"): + raise ConfigError( + "[sandbox.providers.freestyle.config] requires a non-empty 'api_key'" + ) + return cls(**config) + + async def create_vm( + self, + *, + image: str | None = None, + boot_mode: str | None = None, + cpu: float = 1.0, + memory: int = 8192, + disk_size_gb: float = 10, + timeout: int = 3600 * 2, + exposed_ports: list[int] | None = None, + setup_for_gateway: bool = True, + attribution: Attribution | None = None, + network_policy: NetworkPolicy | None = None, + ) -> FreestyleSandbox: + policy = network_policy or NetworkPolicy() + if not self.supports_network_policy(policy): + raise NetworkPolicyUnsupportedError( + f"Freestyle cannot enforce network policy mode {policy.mode.value}; " + "hostname allowlists require a different sandbox backend" + ) + if boot_mode is not None: + raise ValueError( + "Freestyle boot_mode overrides are unsupported; select a snapshot instead" + ) + if ( + not math.isfinite(cpu) + or cpu <= 0 + or not math.isfinite(disk_size_gb) + or disk_size_gb <= 0 + ): + raise ValueError( + "Freestyle CPU and disk requests must be finite and positive" + ) + if ( + type(memory) is not int + or memory <= 0 + or type(timeout) is not int + or timeout <= 0 + ): + raise ValueError("Freestyle memory and timeout must be positive integers") + ports = list(dict.fromkeys(exposed_ports or [])) + if any(type(port) is not int or not 1 <= port <= 65535 for port in ports): + raise ValueError("Freestyle exposed ports must be integers from 1 to 65535") + metadata = { + key: value + for key, value in apply_default_attribution(dict(attribution or {})).items() + if value is not None + } + if len(metadata) > 64 or any( + not isinstance(key, str) + or not key + or len(key) > 63 + or key.startswith("freestyle.sh/") + or not isinstance(value, str) + or len(value) > 63 + for key, value in metadata.items() + ): + raise ValueError( + "Freestyle attribution supports 64 entries, keys and values up to 63 characters, and no freestyle.sh/ keys" + ) + slug = f"agentenv-{uuid.uuid4().hex}" + tunnels = {port: f"https://{slug}-{port}.style.dev" for port in ports} + request = { + "snapshotId": image or self._snapshot_id, + "slug": slug, + "ttlSeconds": timeout, + "metadata": metadata, + "firewall": { + "rules": [ + {"action": "allow", "source": {}, "destination": {"public": True}} + ] + }, + "tls": { + "rules": [ + { + "action": "allow", + "domain": url.removeprefix("https://"), + "source": {"public": True}, + "destination": {"port": port}, + } + for port, url in tunnels.items() + ] + }, + } + creation = asyncio.create_task( + self._client.request("POST", "/v5/vms", json=request, timeout=60) + ) + try: + data = await asyncio.shield(creation) + vm_id = data["id"] + resources = data["resources"] + requested = { + "cpu": math.ceil(cpu), + "memory": memory, + "storage": math.ceil(disk_size_gb * 1024), + } + resize = { + key: value for key, value in requested.items() if value > resources[key] + } + if resize: + await self._client.request( + "POST", f"{vm_path(vm_id)}/resize", json=resize, timeout=60 + ) + sandbox = FreestyleSandbox( + self._client, + vm_id, + tunnel_urls=tunnels, + exec_timeout_seconds=self._exec_timeout_seconds, + network_policy=policy, + ) + if setup_for_gateway: + await sandbox.setup_vm_for_gateway(ports) + return sandbox + except BaseException: + cleanup = asyncio.create_task(self._reap_creation(creation, slug)) + self._cleanups.add(cleanup) + cleanup.add_done_callback(self._cleanups.discard) + await asyncio.shield(cleanup) + raise + + async def _reap_creation(self, creation: asyncio.Task, slug: str) -> None: + """Wait for a cancelled create's response, or find its VM by the unique slug after a lost response.""" + try: + data = await creation + vm_id = data["id"] + except (httpx.HTTPError, ValueError, KeyError, TypeError): + vm_id = slug + for attempt in range(3): + try: + await self._client.request("DELETE", vm_path(vm_id), timeout=30) + return + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return + if exc.response.status_code < 500 and exc.response.status_code != 429: + break + except httpx.TransportError: + pass + if attempt < 2: + await asyncio.sleep(2**attempt) + logger.error( + "Could not delete Freestyle VM %s after provisioning failure; its TTL bounds its lifetime", + vm_id, + ) + + async def create_sandbox( + self, + *, + image_name: str, + port: int, + env: dict[str, str], + cpu: float = 1.0, + memory: int = 8192, + disk_size_gb: float = 10, + timeout: int = 3600 * 2, + attribution: Attribution | None = None, + network_policy: NetworkPolicy | None = None, + ) -> FreestyleSandbox: + return await self.create_vm( + cpu=cpu, + memory=memory, + disk_size_gb=disk_size_gb, + timeout=timeout, + exposed_ports=[port], + attribution=attribution, + network_policy=network_policy, + ) + + async def get_sandbox(self, sandbox_id: str) -> FreestyleSandbox: + data = await self._client.request("GET", vm_path(sandbox_id)) + vm_id = data["id"] + tunnels: dict[int, str] = {} + offset = 0 + while True: + page = await self._client.request( + "GET", "/v5/tls", params={"vmId": vm_id, "limit": 100, "offset": offset} + ) + for rule in page["rules"]: + destination = rule["destination"] + port = destination.get("port") + if ( + rule["protocol"] == "http" + and rule["source"] == {"public": True} + and destination.get("vmId") == vm_id + and port is not None + and rule["domain"] == f"{data['slug']}-{port}.style.dev" + ): + tunnels[port] = f"https://{rule['domain']}" + offset += len(page["rules"]) + if not page["rules"] or offset >= page["totalCount"]: + break + return FreestyleSandbox( + self._client, + vm_id, + tunnel_urls=tunnels, + exec_timeout_seconds=self._exec_timeout_seconds, + ) + + async def close(self) -> None: + if self._cleanups: + await asyncio.gather(*self._cleanups) + await self._client.close() diff --git a/src/agent_env/providers/sandbox_providers/freestyle/sandbox.py b/src/agent_env/providers/sandbox_providers/freestyle/sandbox.py new file mode 100644 index 00000000..84c4964e --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/freestyle/sandbox.py @@ -0,0 +1,100 @@ +"""A Docker-capable Freestyle VM implementing the shared sandbox contract.""" + +from __future__ import annotations + +import shlex +from collections.abc import AsyncIterator + +import httpx + +from agent_env.providers.sandbox_providers.freestyle.client import ( + FreestyleClient, + vm_path, +) +from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, VmSandbox +from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM + + +class _BytesReader: + def __init__(self, value: str | None): + self._value = (value or "").encode() + + async def read(self) -> bytes: + return self._value + + async def _chunks(self) -> AsyncIterator[bytes]: + if self._value: + yield self._value + + def __aiter__(self) -> AsyncIterator[bytes]: + return self._chunks() + + +class _CompletedProcess: + def __init__(self, result: dict): + self.stdout = _BytesReader(result.get("stdout")) + self.stderr = _BytesReader(result.get("stderr")) + status = result.get("statusCode") + self._exit_code = 124 if status is None else int(status) + + async def wait(self) -> int: + return self._exit_code + + +class FreestyleSandbox(VmSandbox): + type = "freestyle" + + def __init__( + self, + client: FreestyleClient, + sandbox_id: str, + *, + tunnel_urls: dict[int, str], + exec_timeout_seconds: int = 300, + network_policy: NetworkPolicy | None = None, + ): + self._client = client + self.sandbox_id = sandbox_id + self.tunnel_urls = tunnel_urls + self._exec_timeout_seconds = exec_timeout_seconds + self.network_policy = network_policy + self.mode = SANDBOX_MODE_VM + self.vnc_url = None + + async def exec(self, *command: str) -> _CompletedProcess: + """Run argv as root; Freestyle buffers output and caps each command at five minutes.""" + argv = command[1:] if command[:1] == ("sudo",) else command + result = await self._client.request( + "POST", + f"{vm_path(self.sandbox_id)}/exec-await", + json={ + "command": shlex.join(argv), + "linuxUser": "root", + "timeoutMs": self._exec_timeout_seconds * 1000, + }, + ) + return _CompletedProcess(result) + + async def setup_vm_for_gateway( + self, exposed_ports: list[int] | None = None + ) -> None: + await self.wait_for_vm() + await self.exec_script( + "docker info > /dev/null && docker compose version > /dev/null" + ) + + async def _write_bytes_to_vm_path(self, data: bytes, vm_path_: str) -> None: + await self._client.request( + "PUT", + f"{vm_path(self.sandbox_id)}/fs/write", + params={"path": vm_path_}, + content=data, + headers={"Content-Type": "application/octet-stream"}, + ) + + async def terminate(self) -> None: + try: + await self._client.request("DELETE", vm_path(self.sandbox_id), timeout=30) + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise diff --git a/src/agent_env/providers/sandbox_providers/sandbox_provider.py b/src/agent_env/providers/sandbox_providers/sandbox_provider.py index b0a8fdd0..8089219b 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox_provider.py +++ b/src/agent_env/providers/sandbox_providers/sandbox_provider.py @@ -201,6 +201,7 @@ def effective_network_policy(cls, policy: Optional[NetworkPolicy]) -> NetworkPol "modal_vm": "agent_env.providers.sandbox_providers.modal_vm_sandbox:ModalVmSandboxProvider", "e2b": "agent_env.providers.sandbox_providers.e2b:E2BSandboxProvider", "sail_vm": "agent_env.providers.sandbox_providers.sail_vm.provider:SailVmSandboxProvider", + "freestyle": "agent_env.providers.sandbox_providers.freestyle:FreestyleSandboxProvider", "local": "agent_env.providers.sandbox_providers.local_sandbox:LocalSandboxProvider", } diff --git a/tst/integration/providers/sandbox_providers/freestyle_sandbox_smoke_test.py b/tst/integration/providers/sandbox_providers/freestyle_sandbox_smoke_test.py new file mode 100644 index 00000000..6eb11b74 --- /dev/null +++ b/tst/integration/providers/sandbox_providers/freestyle_sandbox_smoke_test.py @@ -0,0 +1,89 @@ +"""Real Freestyle VM/container lifecycle; enabled by resolved provider credentials.""" + +import asyncio +import hashlib + +import httpx +import pytest +import pytest_asyncio + +from agent_env.providers.sandbox_providers.sandbox_provider import ( + build_sandbox_provider, +) +from tst.util.capabilities import skip_without_remote_sandbox + +pytestmark = [ + pytest.mark.integration, + pytest.mark.int_test_slow, + pytest.mark.asyncio, + skip_without_remote_sandbox("freestyle"), +] + + +@pytest_asyncio.fixture(scope="module") +async def freestyle_provider(): + provider = build_sandbox_provider("freestyle") + try: + yield provider + finally: + await provider.close() + + +async def test_container_https_files_reconnect_and_delete(freestyle_provider): + sandbox = await freestyle_provider.create_container( + image_name="public.ecr.aws/docker/library/nginx:alpine", + port=80, + env={"AGENTENV_SMOKE": "freestyle"}, + cpu=3, + memory=5120, + disk_size_gb=17, + timeout=900, + ) + vm_id = sandbox.sandbox_id + try: + assert (sandbox.type, sandbox.mode) == ("freestyle", "container") + data = await freestyle_provider._client.request("GET", f"/v5/vms/{vm_id}") + assert data["resources"]["cpu"] >= 3 + assert data["resources"]["memory"] >= 5120 + assert data["resources"]["storage"] >= 17 * 1024 + assert data["ttlSeconds"] == 900 + url = sandbox.tunnel_urls[80] + async with httpx.AsyncClient(timeout=15) as client: + for _ in range(30): + response = await client.get(url) + if response.status_code == 200: + break + await asyncio.sleep(1) + assert response.status_code == 200 + assert "nginx" in response.text + + await sandbox.write_file_from_text( + "freestyle-smoke", "/usr/share/nginx/html/smoke.txt" + ) + async with httpx.AsyncClient(timeout=15) as client: + response = await client.get(f"{url}/smoke.txt") + assert response.status_code == 200 + assert response.text == "freestyle-smoke" + + data = bytes(range(256)) * 4096 + await sandbox.write_host_file(data, "/tmp/agentenv-smoke/blob.bin") + status, digest, stderr = await sandbox.exec_with_output( + "sha256sum", "/tmp/agentenv-smoke/blob.bin" + ) + assert status == 0, stderr + assert digest.split()[0] == hashlib.sha256(data).hexdigest() + + reconnected = await freestyle_provider.get_sandbox(vm_id) + assert reconnected.sandbox_id == vm_id + assert reconnected.tunnel_urls == sandbox.tunnel_urls + assert reconnected.network_policy is None + status, _, stderr = await reconnected.exec_with_output( + "docker", "compose", "version" + ) + assert status == 0, stderr + finally: + await sandbox.terminate() + with pytest.raises(httpx.HTTPStatusError) as error: + await freestyle_provider.get_sandbox(vm_id) + assert error.value.response.status_code == 404 + await sandbox.terminate() diff --git a/tst/unit/providers/sandbox_providers/egress_hosts_test.py b/tst/unit/providers/sandbox_providers/egress_hosts_test.py index 7024b69d..2d8e1d1f 100644 --- a/tst/unit/providers/sandbox_providers/egress_hosts_test.py +++ b/tst/unit/providers/sandbox_providers/egress_hosts_test.py @@ -95,4 +95,4 @@ def test_the_builtin_floor_is_pinned(builtin_registry): """Every restricted sandbox gets these, so an empty allowlist is not total denial. Pinned rather than counted: a new entry should be argued for in review. Platforms registered from config add theirs on top.""" - assert set(all_sandbox_egress_hosts()) == {"*.modal.host", "*.w.modal.host", "*.e2b.app", "*.sail.box"} + assert set(all_sandbox_egress_hosts()) == {"*.modal.host", "*.w.modal.host", "*.e2b.app", "*.sail.box", "*.style.dev"} diff --git a/tst/unit/providers/sandbox_providers/freestyle_sandbox_test.py b/tst/unit/providers/sandbox_providers/freestyle_sandbox_test.py new file mode 100644 index 00000000..a209e4f2 --- /dev/null +++ b/tst/unit/providers/sandbox_providers/freestyle_sandbox_test.py @@ -0,0 +1,387 @@ +"""Freestyle wire contract and lifecycle failures, with no network access.""" + +import asyncio +import json +import shlex +from unittest.mock import AsyncMock + +import httpx +import pytest +import pytest_asyncio + +from agent_env.config import ConfigError, reset_config +from agent_env.providers.sandbox_providers.freestyle import ( + FreestyleSandbox, + FreestyleSandboxProvider, +) +from agent_env.providers.sandbox_providers.sandbox import ( + NetworkMode, + NetworkPolicy, + NetworkPolicyUnsupportedError, +) +from agent_env.providers.sandbox_providers.sandbox_provider import ( + build_sandbox_provider, +) + + +@pytest_asyncio.fixture +async def backend(): + requests = [] + vm = {"id": "vm-test", "resources": {"cpu": 1, "memory": 2048, "storage": 10240}} + + async def respond(request): + requests.append(request) + path = request.url.path + if request.method == "POST" and path == "/v5/vms": + vm.update(json.loads(request.content)) + return httpx.Response(200, json=vm) + if request.method == "GET" and path == "/v5/tls": + rules = [ + { + **rule, + "protocol": "http", + "destination": {**rule["destination"], "vmId": vm["id"]}, + } + for rule in vm["tls"]["rules"] + ] + return httpx.Response(200, json={"rules": rules, "totalCount": len(rules)}) + if request.method == "GET": + return httpx.Response(200, json=vm) + if path.endswith("/exec-await"): + return httpx.Response( + 200, json={"statusCode": 0, "stdout": "ok\n", "stderr": ""} + ) + return httpx.Response(204) + + provider = FreestyleSandboxProvider(api_key="private-test-key") + provider._client._client = httpx.AsyncClient( + base_url="https://api.example.test", + transport=httpx.MockTransport(respond), + headers={"Authorization": "Bearer private-test-key"}, + ) + yield provider, requests, vm + await provider.close() + + +@pytest.mark.asyncio +async def test_create_sizes_publishes_ports_and_keeps_attribution(backend): + provider, requests, _ = backend + sandbox = await provider.create_vm( + cpu=1.5, + memory=4096, + disk_size_gb=12.25, + timeout=600, + exposed_ports=[8080, 8081, 8080], + attribution={"run_id": "run-123"}, + ) + request = json.loads(requests[0].content) + assert request["snapshotId"] == "freestyle/ubuntu" + assert request["ttlSeconds"] == 600 + assert request["metadata"] == {"run_id": "run-123"} + assert request["firewall"] == { + "rules": [{"action": "allow", "source": {}, "destination": {"public": True}}] + } + assert [rule["destination"] for rule in request["tls"]["rules"]] == [ + {"port": 8080}, + {"port": 8081}, + ] + assert json.loads(requests[1].content) == { + "cpu": 2, + "memory": 4096, + "storage": 12544, + } + assert sandbox.type == "freestyle" + assert sandbox.mode == "vm" + assert sandbox.network_policy == NetworkPolicy() + assert sandbox.tunnel_urls == { + port: f"https://{request['slug']}-{port}.style.dev" for port in (8080, 8081) + } + assert "private-test-key" not in repr(provider) + + +@pytest.mark.asyncio +async def test_snapshot_minimum_is_preserved_without_downsizing(backend): + provider, requests, _ = backend + await provider.create_vm( + image="sh-custom", memory=1024, disk_size_gb=1, setup_for_gateway=False + ) + assert len(requests) == 1 + assert json.loads(requests[0].content)["snapshotId"] == "sh-custom" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "kwargs", + [ + {"cpu": 0}, + {"cpu": float("nan")}, + {"disk_size_gb": float("inf")}, + {"memory": -1}, + {"memory": 1.5}, + {"timeout": 0}, + {"timeout": -1}, + {"exposed_ports": [0]}, + {"exposed_ports": [65536]}, + {"exposed_ports": [True]}, + {"boot_mode": "uefi"}, + {"attribution": {"freestyle.sh/reserved": "x"}}, + {"attribution": {"run_id": "x" * 64}}, + {"attribution": {"": "x"}}, + {"attribution": {str(i): "x" for i in range(65)}}, + ], +) +async def test_invalid_requests_fail_before_provisioning(backend, kwargs): + provider, requests, _ = backend + with pytest.raises(ValueError): + await provider.create_vm(**kwargs) + assert requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,kwargs", + [ + ("create_vm", {}), + ("create_sandbox", {"image_name": "img", "port": 8080, "env": {}}), + ("create_container", {"image_name": "img", "port": 8080, "env": {}}), + ], +) +async def test_restrictive_policy_is_rejected_on_every_entry_point( + backend, method, kwargs +): + provider, requests, _ = backend + policy = NetworkPolicy(mode=NetworkMode.ALLOWLIST, allow_hosts=("example.test",)) + assert not provider.supports_network_policy(policy) + with pytest.raises(NetworkPolicyUnsupportedError): + await getattr(provider, method)(network_policy=policy, **kwargs) + assert requests == [] + + +@pytest.mark.asyncio +async def test_setup_failure_deletes_the_vm(backend, monkeypatch): + provider, requests, _ = backend + monkeypatch.setattr( + FreestyleSandbox, + "setup_vm_for_gateway", + AsyncMock(side_effect=RuntimeError("Docker missing")), + ) + with pytest.raises(RuntimeError, match="Docker missing"): + await provider.create_vm() + assert requests[-1].method == "DELETE" + assert requests[-1].url.path == "/v5/vms/vm-test" + assert not provider._cleanups + + +@pytest.mark.asyncio +async def test_cancellation_waits_for_creation_then_deletes(backend, monkeypatch): + provider, _, _ = backend + started, release = asyncio.Event(), asyncio.Event() + calls = [] + + async def request(method, path, **kwargs): + calls.append((method, path)) + if method == "POST": + started.set() + await release.wait() + return {"id": "vm-late"} + return {} + + monkeypatch.setattr(provider._client, "request", request) + task = asyncio.create_task(provider.create_vm()) + await started.wait() + task.cancel() + await asyncio.sleep(0) + assert not task.done() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert calls == [("POST", "/v5/vms"), ("DELETE", "/v5/vms/vm-late")] + + +@pytest.mark.asyncio +async def test_lost_create_response_cleans_up_by_slug(backend, monkeypatch): + provider, _, _ = backend + calls = [] + + async def request(method, path, **kwargs): + calls.append((method, path, kwargs)) + if method == "POST": + raise httpx.ReadTimeout("lost response") + return {} + + monkeypatch.setattr(provider._client, "request", request) + with pytest.raises(httpx.ReadTimeout): + await provider.create_vm() + assert calls[1][:2] == ("DELETE", "/v5/vms/" + calls[0][2]["json"]["slug"]) + + +@pytest.mark.asyncio +async def test_reconnect_recovers_ingress_and_reports_unknown_policy(backend): + provider, requests, _ = backend + created = await provider.create_vm( + exposed_ports=[8080, 8081], setup_for_gateway=False + ) + reconnected = await provider.get_sandbox(created.sandbox_id) + assert reconnected.sandbox_id == created.sandbox_id + assert reconnected.tunnel_urls == created.tunnel_urls + assert reconnected.network_policy is None + assert requests[-1].url.params["vmId"] == created.sandbox_id + + +@pytest.mark.asyncio +async def test_exec_preserves_argv_and_nonzero_exit(backend, monkeypatch): + provider, _, _ = backend + request = AsyncMock( + return_value={"statusCode": 7, "stdout": "héllo", "stderr": "failed"} + ) + monkeypatch.setattr(provider._client, "request", request) + sandbox = FreestyleSandbox(provider._client, "vm-test", tunnel_urls={}) + argv = ("bash", "-c", "printf '%s' \"$1\"", "arg0", "a b; $(touch /tmp/bad)") + assert await sandbox.exec_with_output("sudo", *argv) == (7, "héllo", "failed") + payload = request.call_args.kwargs["json"] + assert shlex.split(payload["command"]) == list(argv) + assert payload["linuxUser"] == "root" + assert payload["timeoutMs"] == 300000 + process = await sandbox.exec("true") + assert b"".join([chunk async for chunk in process.stdout]) == "héllo".encode() + + +@pytest.mark.asyncio +async def test_guest_timeout_is_a_failure_without_transient_retry(backend, monkeypatch): + provider, _, _ = backend + request = AsyncMock( + return_value={"statusCode": None, "stdout": "partial", "stderr": ""} + ) + monkeypatch.setattr(provider._client, "request", request) + sandbox = FreestyleSandbox(provider._client, "vm-test", tunnel_urls={}) + with pytest.raises(RuntimeError, match="exit 124"): + await sandbox.exec_script("sleep 500", max_retries=2) + assert request.await_count == 1 + + +@pytest.mark.asyncio +async def test_host_file_write_uses_binary_filesystem_api(backend): + provider, requests, _ = backend + sandbox = FreestyleSandbox(provider._client, "vm-test", tunnel_urls={}) + data = bytes(range(256)) * 1024 + await sandbox.write_host_file(data, "/tmp/folder/file with spaces") + request = requests[-1] + assert request.method == "PUT" + assert request.url.path == "/v5/vms/vm-test/fs/write" + assert request.url.params["path"] == "/tmp/folder/file with spaces" + assert request.content == data + assert request.headers["content-type"] == "application/octet-stream" + + +def test_config_resolves_credentials_and_snapshot(tmp_path, monkeypatch): + config = tmp_path / "config.toml" + config.write_text( + '[sandbox.providers.freestyle.config]\napi_key = "env:TEST_FREESTYLE_KEY"\nsnapshot_id = "sh-custom"\n' + ) + monkeypatch.setenv("AGENT_ENV_CONFIG", str(config)) + monkeypatch.setenv("TEST_FREESTYLE_KEY", "resolved-key") + reset_config() + provider = build_sandbox_provider("freestyle") + assert isinstance(provider, FreestyleSandboxProvider) + assert provider._client._api_key == "resolved-key" + assert provider._snapshot_id == "sh-custom" + assert provider._client._client is None + + +def test_missing_config_is_actionable(): + with pytest.raises(ConfigError, match="requires a non-empty 'api_key'"): + FreestyleSandboxProvider.from_config() + + +@pytest.mark.parametrize( + "url", + [ + "http://api.example.test", + "https://user:pass@api.example.test", + "https://api.example.test?key=abc", + ], +) +def test_credentials_require_a_safe_https_api_url(url): + with pytest.raises(ValueError, match="HTTPS"): + FreestyleSandboxProvider(api_key="private-key", api_url=url) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status,attempts", [(503, 3), (429, 3), (401, 1), (404, 1)]) +async def test_provisioning_cleanup_retries_only_transient_errors( + backend, monkeypatch, status, attempts, caplog +): + provider, _, _ = backend + response = httpx.Response( + status, + request=httpx.Request("DELETE", "https://api.example.test/v5/vms/vm-test"), + ) + request = AsyncMock( + side_effect=httpx.HTTPStatusError( + "delete failed", request=response.request, response=response + ) + ) + monkeypatch.setattr(provider._client, "request", request) + monkeypatch.setattr(asyncio, "sleep", AsyncMock()) + creation = asyncio.get_running_loop().create_future() + creation.set_result({"id": "vm-test"}) + await provider._reap_creation(creation, "agentenv-test") + assert request.await_count == attempts + assert bool(caplog.records) is (status != 404) + + +@pytest.mark.asyncio +async def test_reconnect_pages_and_excludes_egress_rules(backend, monkeypatch): + provider, _, _ = backend + vm = {"id": "vm-test", "slug": "agentenv-test"} + egress = { + "protocol": "http", + "source": {"vmId": "vm-test"}, + "destination": {"public": True}, + "domain": "api.example.test", + } + ingress = { + "protocol": "http", + "source": {"public": True}, + "destination": {"vmId": "vm-test", "port": 8080}, + "domain": "agentenv-test-8080.style.dev", + } + request = AsyncMock( + side_effect=[ + vm, + {"rules": [egress], "totalCount": 2}, + {"rules": [ingress], "totalCount": 2}, + ] + ) + monkeypatch.setattr(provider._client, "request", request) + sandbox = await provider.get_sandbox("vm-test") + assert sandbox.tunnel_urls == {8080: "https://agentenv-test-8080.style.dev"} + assert request.call_args.kwargs["params"]["offset"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [204, 404, 403]) +async def test_termination_is_idempotent_and_preserves_permission_errors( + backend, monkeypatch, status +): + provider, _, _ = backend + response = httpx.Response( + status, + request=httpx.Request("DELETE", "https://api.example.test/v5/vms/vm-test"), + ) + error = ( + None + if status == 204 + else httpx.HTTPStatusError( + "delete failed", request=response.request, response=response + ) + ) + monkeypatch.setattr( + provider._client, "request", AsyncMock(return_value={}, side_effect=error) + ) + sandbox = FreestyleSandbox(provider._client, "vm-test", tunnel_urls={}) + if status == 403: + with pytest.raises(httpx.HTTPStatusError): + await sandbox.terminate() + else: + await sandbox.terminate() diff --git a/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py b/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py index 0b801a53..f0895eed 100644 --- a/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py +++ b/tst/unit/providers/sandbox_providers/network_policy_stamping_test.py @@ -14,6 +14,7 @@ import pytest from agent_env.providers.sandbox_providers.e2b.provider import E2BSandboxProvider +from agent_env.providers.sandbox_providers.freestyle.provider import FreestyleSandboxProvider from agent_env.providers.sandbox_providers.local_sandbox import LocalSandboxProvider from agent_env.providers.sandbox_providers.modal_sandbox import ModalSandboxProvider from agent_env.providers.sandbox_providers.modal_vm_sandbox import ModalVmSandboxProvider @@ -86,11 +87,20 @@ async def _sail() -> Sandbox: return await provider.create_vm(exposed_ports=[], setup_for_gateway=False) +async def _freestyle() -> Sandbox: + provider = FreestyleSandboxProvider(api_key="test-key") + provider._client.request = AsyncMock(return_value={ + "id": "vm-test", "resources": {"cpu": 1, "memory": 8192, "storage": 10240}, + }) + return await provider.create_vm(exposed_ports=[], setup_for_gateway=False) + + FACTORIES = { "modal": _modal, "modal_vm": _modal_vm, "e2b": _e2b, "sail_vm": _sail, + "freestyle": _freestyle, "local": _local, } diff --git a/tst/util/capabilities.py b/tst/util/capabilities.py index ef86c8f6..71c1d48e 100644 --- a/tst/util/capabilities.py +++ b/tst/util/capabilities.py @@ -19,7 +19,7 @@ #: Only the agent-driving paths need a model endpoint. MODEL_ENDPOINT = "model_endpoint_configured" -#: The remote sandbox providers (``modal``, ``modal_vm``, ``e2b``, ``sail_vm``) need credentials the resolved +#: The remote sandbox providers (``modal``, ``modal_vm``, ``e2b``, ``sail_vm``, ``freestyle``) need credentials the resolved #: config may not carry; the local default never does. REMOTE_SANDBOX = "remote_sandbox" @@ -75,16 +75,16 @@ def skip_without_model_endpoint() -> pytest.MarkDecorator: def remote_sandbox_is_available(provider: str) -> bool: - """Whether the resolved config can build the ``modal`` / ``modal_vm`` / ``e2b`` / ``sail_vm`` sandbox + """Whether the resolved config can build the ``modal`` / ``modal_vm`` / ``e2b`` / ``sail_vm`` / ``freestyle`` sandbox provider, credentials included. - Modal: any failure to resolve the credentials answers False. E2B and Sail: building the provider + Modal: any failure to resolve the credentials answers False. E2B, Sail and Freestyle: building the provider resolves ``[sandbox.providers..config]`` and its ``secret:`` references; only a ``ConfigError`` (absent or incomplete config) answers False, any other failure answers True and lets the test fail on the real problem, as ``model_endpoint_is_configured`` does.""" - if provider not in ("modal", "modal_vm", "e2b", "sail_vm"): + if provider not in ("modal", "modal_vm", "e2b", "sail_vm", "freestyle"): raise ValueError(f"unknown remote sandbox provider {provider!r}") - if provider in ("e2b", "sail_vm"): + if provider in ("e2b", "sail_vm", "freestyle"): from agent_env.config.errors import ConfigError from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider From a261b5b3690e18abf6b8a8f5a15bfdff42a7759d Mon Sep 17 00:00:00 2001 From: Benjamin Swerdlow <38309438+theswerd@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:11:44 -0700 Subject: [PATCH 2/2] docs(cli): keep sandbox help changes scoped to Freestyle --- src/agent_env/cli/a2a_agent/deploy.py | 2 +- src/agent_env/cli/env/deploy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent_env/cli/a2a_agent/deploy.py b/src/agent_env/cli/a2a_agent/deploy.py index 7dc365bb..0d554864 100644 --- a/src/agent_env/cli/a2a_agent/deploy.py +++ b/src/agent_env/cli/a2a_agent/deploy.py @@ -18,7 +18,7 @@ @click.option("--ttl-seconds", type=click.IntRange(min=MIN_TTL_SECONDS, max=MAX_TTL_SECONDS), default=DEFAULT_TTL_SECONDS, help=f"VM lifetime in seconds (default {DEFAULT_TTL_SECONDS})") @click.option("--sandbox", default=None, - help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, sail_vm, freestyle, local) " + help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, freestyle, local) " "or a name from [sandbox.providers] in .agentenv/config.toml; comma-separated for a " "fallback chain. Defaults to [sandbox].agent_default (else local) when omitted.") def deploy(agent_id: str, agent_version: int | None, env_var_pairs: tuple[str, ...], ttl_seconds: int, sandbox: str): diff --git a/src/agent_env/cli/env/deploy.py b/src/agent_env/cli/env/deploy.py index e29e6022..50f05e81 100644 --- a/src/agent_env/cli/env/deploy.py +++ b/src/agent_env/cli/env/deploy.py @@ -19,7 +19,7 @@ @click.option("--gateway-mode", type=click.Choice([m.value for m in GatewayMode], case_sensitive=False), default=GatewayMode.PERFORMANCE.value, help="Gateway mode (performance or consistent)") @click.option("--sandbox", default=None, - help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, sail_vm, freestyle, local) " + help="Sandbox backend(s): a built-in (modal, modal_vm, e2b, freestyle, local) " "or a name from [sandbox.providers] in .agentenv/config.toml; comma-separated for a " "fallback chain. Defaults to [sandbox].default (else local) when omitted.") @click.option("--service-db", "service_db_env_id", default=None,