diff --git a/.github/workflows/skills-tests.yml b/.github/workflows/skills-tests.yml new file mode 100644 index 0000000..a363b43 --- /dev/null +++ b/.github/workflows/skills-tests.yml @@ -0,0 +1,28 @@ +name: skills-tests +on: + push: + paths: ["src/webwright/skill_factory/**", "src/webwright/tools/skill_use.py", "tests/skill_factory/**"] + pull_request: + paths: ["src/webwright/skill_factory/**", "src/webwright/tools/skill_use.py", "tests/skill_factory/**"] +jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: "3.12" } + - run: pip install httpx pyyaml jinja2 pydantic + - name: skills unit tests (LLM-free) + run: | + set -e + for t in tests/skill_factory/test_*.py; do + echo "== $t"; PYTHONPATH=src python "$t" + done + - name: wrapper usage check (F6) + run: | + set +e + bash src/webwright/skill_factory/examples/solve_with_library.sh > /dev/null 2>&1 + [ $? -eq 1 ] && echo "usage-exit OK" || { echo "wrapper must exit 1 on missing args"; exit 1; } + bash -n src/webwright/skill_factory/examples/quickstart.sh || exit 1 + bash src/webwright/skill_factory/examples/quickstart.sh badmode > /dev/null 2>&1 + [ $? -eq 1 ] && echo "quickstart usage-exit OK" || { echo "quickstart must exit 1 on bad mode"; exit 1; } diff --git a/README.md b/README.md index 891bbd7..042c61f 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,25 @@ python assets/task_showcase/app.py \ --- +## 🧠 Skill Library (reuse solved tasks across tasks) + +[`webwright.skills`](src/webwright/skills/) turns solved tasks into **reusable, executable code +skills**, retrieves and judges them at solve time, gates what enters the library, and grows the +library incrementally β€” a self-evolving *store β†’ retrieve β†’ use/adapt β†’ gate β†’ evolve* loop on top +of Webwright's code-as-action solves. Plugs in with **no change to the agent loop**: + +- **Reuse** β€” the agent calls `python -m webwright.tools.skill_use --task "..." --library ...` + (like `self_reflection`/`image_qa`); it returns `{verdict: use|adapt|skip, source_path}`. +- **Grow** β€” `python -m webwright.skills.update --manifest batch.json --library ./library` + distills a batch of gate-passed solves into a parameterized, primitive-decomposed skill. + +Validated end-to-end on a real public website (read-only GitHub): solve two repos from scratch β†’ +`update` builds a parameterized skill β†’ a held-out repo is solved by reusing it (agent calls +`skill_use`, verdict `use`, answer correct); a wrong solve is kept out by the gate; a second batch +improves the existing skill in place. See [`src/webwright/skills/README.md`](src/webwright/skills/README.md). + +--- + ## πŸš€ Quick Start ### Prerequisites diff --git a/assets/skill_factory_demo.mp4 b/assets/skill_factory_demo.mp4 new file mode 100644 index 0000000..8cee6c2 Binary files /dev/null and b/assets/skill_factory_demo.mp4 differ diff --git a/assets/skill_factory_pipeline.png b/assets/skill_factory_pipeline.png new file mode 100644 index 0000000..a85a794 Binary files /dev/null and b/assets/skill_factory_pipeline.png differ diff --git a/assets/skill_factory_pipeline.svg b/assets/skill_factory_pipeline.svg new file mode 100644 index 0000000..f06dad6 --- /dev/null +++ b/assets/skill_factory_pipeline.svg @@ -0,0 +1,173 @@ + + + + + + + + + + + + Web Skill Factory β€” data flow & interfaces + + + + + + SOLVE TIME Β· agent + LIBRARY Β· disk + UPDATE TIME Β· offline batch + + + + task + start_url + with_skill_hint(prompt, task, library) β†’ reuse instructions + + + + + webwright agent (bash loop β€” unchanged) + + + $ skill_use --task "…" --library ROOT + + + + skill_use (one tool call; any error β†’ skip) + + + retrieve + in : catalog (meta only) + out: ≀3 Candidate{id,score} + + + + + decide + out: Decision{verdict: + use|adapt|skip, skill_id} + + + guards + hallucinated skill_id β†’ skip Β· empty library β†’ skip + warning + + + + + tool output β†’ agent observation (stdout JSON) + {verdict, skill_id, source_path, call, + how_to_reuse: "copy + fill params / reuse core"} + + + reads source, copies / adapts + + + final_script.py (skill code, THIS task's params) + run β†’ answer + evidence screenshots + + + + + run artifacts + agent_response.json {task_type, status, retrieved_data} + skill_decision.json {skill_id, verdict, reason} Β· trajectory + + + + library/<template-slug>/ Γ— N skills + + + meta.json β€” the catalog card + {template, site, summary, + signature:{params, call}, output_schema, + n_solves, revisions, verified, grade} + ← only THIS enters the retrieve prompt + + + skill.py β€” executable, standalone + primitives: login() Β· open_…() Β· extract_…() + task layer: solve(taskspec) + ← read as SOURCE at solve time; + runs anywhere via the CLI below + + + standalone interface (no model needed) + $ python skill.py taskspec.json + taskspec {params, start_url, credentials, output_schema} + writes β†’ agent_response.json {retrieved_data} + + + + + + + + gate(result, gold?, output_schema?, method) + method: gold | self_verify | none β†’ GateResult{admit} + admit=false β‡’ the solve NEVER enters the library + + + + + batch.json β€” the manifest + {template, ← skills are keyed by this string + runs:[{dir, admit(bool, REQUIRED), params, + verdict:skip|use|adapt, output_schema, answer?}]} + code ← dir/final_script.py (the solve's working script) + answer omitted β†’ read dir/agent_response.json + + + β†’ [Trace] + + + + evolve(traces, library) + traces bucketed by EXACT template string β€” one bucket, one skill: + + + β‘  template not in library β†’ ADD + _refine(N solves) β†’ parameters + primitives β†’ new skill + + + β‘‘ covered + verdict has adapt β†’ REFINE in place + LLM in : EXISTING skill.py + new solves (code+params) + contract: keep working functions, only widen / fix + meta : n_solves += N, revisions += 1 + + + β‘’ covered + all use-success β†’ KEEP (byte-identical) + + changelog {added, adapt_refined, use, reference, rejected, dropped_wrong} + + + + + replay-verify β€” nothing lands unproven + the candidate re-runs its OWN training taskspecs β€” no model + strict: the recorded answer Β· shape: schema β†’ executable / reference + + + + + library v_n β†’ next batch solves WITH it β†’ v_n+1 + grown, never rebuilt β€” untouched skills stay byte-identical + + + + + agent_response.json (+ gold on benchmarks) β†’ gate + + + + write skill.py + meta.json + diff --git a/docs/skill_factory/manual.md b/docs/skill_factory/manual.md new file mode 100644 index 0000000..04143ed --- /dev/null +++ b/docs/skill_factory/manual.md @@ -0,0 +1,192 @@ +# Manual mode β€” manifests, gold gates, full control + +[← back to the module README](../../src/webwright/skill_factory/README.md) + +The library grows **offline** from batches of solved tasks, and is consumed **at solve time** by +the agent. Tasks are provided **manually** today β€” you pick which tasks to solve and batch. The +current focus is **same-template generalization**, so feed several instances of the SAME template +(3+ instances with different parameter values works well): `refine` aligns them, and exactly what +differs between instances becomes the skill's parameters β€” more instances, wider generalization. +(Planned: bootstrap β€” automatically expand one seed task into multiple instances.) + +### 1. Solve a few instances of a template (normal Webwright runs) + +**Important:** stock Webwright does NOT write the answer to a machine-readable file by itself β€” +tell the agent to, by appending an output instruction to the task (the gate in step 2 reads it): + +```bash +ANSWER_SPEC='Additionally, write the final answer into $WORKSPACE_DIR/agent_response.json +as {"retrieved_data": }.' + +python -m webwright.run.cli main \ + -t "How many commits did kilian make to a11yproject on 3/1/2023? $ANSWER_SPEC" \ + --task-id t132_a --start-url http://gitlab.example.com -o outputs \ + -c base.yaml -c model_openai.yaml +``` + +> Custom OpenAI-compatible gateway? Copy `model_openai.yaml`, change `openai_endpoint` +> (and `model_name`), and stack your copy instead. + +Each run leaves a directory containing `final_script.py` (the executable solve) and β€” because of +the instruction above β€” `agent_response.json` (the answer). Repeat for 2–3 more instances of the +same template with different values (another user / repo / date). If you skip the output +instruction, fill each manifest run's `answer` field by hand in step 2 instead. + +### 2. Gate the solves, write the manifest + +Judge each run (`gate(result, method="gold")` against a known answer, or `method="self_verify"` +without one) and write one manifest per batch: + +```jsonc +// batch.json +{ + "template": "How many commits did {{user}} make to {{repo}} on {{date}}?", + "runs": [ + { + "dir": "outputs/t132_a_20260703_120000", // run dir; final_script.py is read from it + "admit": true, // gate verdict β€” false rows NEVER enter the library + "params": {"user": "kilian", "repo": "a11yproject", "date": "3/1/2023"}, + "verdict": "skip", // how the run used the library: skip = solved from + // scratch; use / adapt = reused a skill + // (adapt triggers refine-back into the skill) + "site": "gitlab", + "output_schema": {"type": "number"} // required shape of retrieved_data + // "answer": optional β€” read from the run dir's agent_response.json when omitted + }, + { "dir": "outputs/t132_b_20260703_121500", "admit": true, + "params": {"user": "gao", "repo": "2019", "date": "4/6/2023"}, + "verdict": "skip", "site": "gitlab", "output_schema": {"type": "number"} } + ] +} +``` + +Field by field: + +| field | required | meaning | +|---|---|---| +| `template` | yes | the template sentence with `{{param}}` placeholders. **Skills are keyed by it**: a manifest whose template already has a skill refines that skill in place; a new template adds a new skill. Use the same string across batches of the same template. | +| `runs[].dir` | yes | a Webwright run directory; `final_script.py` is read from it | +| `runs[].admit` | yes | the gate verdict; `false` rows are dropped and never enter the library | +| `runs[].params` | yes | this instance's concrete values β€” `refine` aligns the runs and exposes exactly these differing values as the skill's arguments (this is what powers generalization) | +| `runs[].verdict` | no (default `skip`) | how this run used the library: `skip` = solved from scratch; `use` = reused a skill as-is; `adapt` = reused + fixed the last step (**`adapt` is what triggers refining the fix back into the skill**) | +| `runs[].site` | no | site tag stored in the skill's meta (helps retrieval) | +| `runs[].output_schema` | no | required shape of `retrieved_data`, e.g. `{"type": "number"}` | +| `runs[].answer` | no | this run's answer; read from the run dir's `agent_response.json` when omitted | + +### 3. Build / evolve the library + +```bash +export OPENAI_API_KEY=... # backend key (never stored by the module) +# optional β€” defaults to OPENAI_MODEL / OPENAI_ENDPOINT: +export SKILL_MODEL_NAME=gpt-5.4 SKILL_MODEL_ENDPOINT=https://api.openai.com/v1/responses +python -m webwright.skill_factory.update --manifest batch.json --library ./library +``` + +Prints a changelog: `{"added": [...], "adapt_refined": [...], "use": [...], "dropped_wrong": n}`. +Re-run with later batches any time β€” a new template **adds** a skill, new solves for an existing +template **refine it in place** (keeps its working functions), templates with no new traces are +left untouched. Batches may mix templates. + +### 4. Reuse at solve time + +```python +from webwright.skill_factory import with_skill_hint +prompt = with_skill_hint(prompt, task=task_text, library="/abs/path/to/library") +``` + +```bash +python -m webwright.run.cli main -t "$prompt" ... +``` + +The hint tells the agent to query the library first; the agent runs the `skill_use` tool, gets +`{verdict, skill_id, source_path, how_to_reuse}`, reads the skill source, and reuses it +(use = as-is with new parameter values, adapt = reuse the core + change the last step, +skip = solve from scratch). + +Two path gotchas, both loud now but worth knowing: + +- **The library path ends up in a command that runs inside the agent's workspace** β€” + `with_skill_hint` resolves it to an absolute path for exactly that reason. If the tool is ever + pointed at a missing/empty library anyway, it answers `skip` with an explicit + `"warning": "library empty at "` instead of failing silently. +- **Precedence:** the hint bakes `--library` into the command, and `--library` beats the + `SKILL_LIBRARY_ROOT` env var (the env var is only the tool's default when `--library` is + omitted). Use one or the other, not both. + +### 5. Run a skill directly (optional) + +Every skill is also a standalone script: it reads a `taskspec.json` (parameters at run time) and +writes `agent_response.json`: + +```bash +cat > taskspec.json <<'EOF' +{"params": {"user": "byte", "repo": "empathy-prompts", "date": "4/2/2023"}, + "start_url": "http://gitlab.example.com", "credentials": null, + "output_schema": {"type": "number"}} +EOF +python library/how_many_commits_did_user_make_to_repo_on_date/skill.py taskspec.json +cat agent_response.json +``` + +### 6. The whole pipeline in one go (a batch of tasks) + +Steps 1–3 driven by a single task file. `tasks.json` β€” one entry per instance of the template +(`gold` is optional; with it the gate compares answers, without it it falls back to `self_verify`): + +```json +[ + {"id": "t132_a", "task": "How many commits did kilian make to a11yproject on 3/1/2023?", + "params": {"user": "kilian", "repo": "a11yproject", "date": "3/1/2023"}, "gold": 1}, + {"id": "t132_b", "task": "How many commits did gao make to 2019 on 4/6/2023?", + "params": {"user": "gao", "repo": "2019", "date": "4/6/2023"}, "gold": 0} +] +``` + +```bash +START_URL=http://gitlab.example.com + +# 1) solve every instance (sequential; add xargs -P N or & to parallelize) +# ANSWER_SPEC (from step 1 above) makes the agent write agent_response.json β€” the gate reads it +ANSWER_SPEC='Additionally, write the final answer into $WORKSPACE_DIR/agent_response.json +as {"retrieved_data": }.' +jq -c '.[]' tasks.json | while read -r row; do + python -m webwright.run.cli main -t "$(jq -r .task <<<"$row") $ANSWER_SPEC" \ + --task-id "$(jq -r .id <<<"$row")" --start-url "$START_URL" -o outputs \ + -c base.yaml -c model_openai.yaml +done + +# 2) gate each run + assemble the manifest +python - <<'PY' +import json, glob +from webwright.skill_factory import gate + +TEMPLATE = "How many commits did {{user}} make to {{repo}} on {{date}}?" +SCHEMA = {"type": "number"} +runs = [] +for t in json.load(open("tasks.json")): + d = sorted(glob.glob(f"outputs/{t['id']}_*"))[-1] # newest run dir of this task + answer = json.load(open(f"{d}/agent_response.json"))["retrieved_data"] + g = gate(answer, gold=t.get("gold"), output_schema=SCHEMA) # gold if present, else self_verify + runs.append({"dir": d, "admit": g.admit, "params": t["params"], "verdict": "skip", + "site": "gitlab", "output_schema": SCHEMA}) +json.dump({"template": TEMPLATE, "runs": runs}, open("batch.json", "w"), indent=2) +print(sum(r["admit"] for r in runs), "of", len(runs), "admitted") +PY + +# 3) evolve the library +python -m webwright.skill_factory.update --manifest batch.json --library ./library + +# 4) solve NEW instances of the template WITH the library: prepend the skill hint +# (the hint is what tells the agent to query; with_skill_hint resolves ./library +# to an absolute path against YOUR cwd, so the agent finds it from its workspace) +TASK="How many commits did byte make to empathy-prompts on 4/2/2023?" +PROMPT=$(python -c 'import sys; from webwright.skill_factory import with_skill_hint +print(with_skill_hint(sys.argv[1], task=sys.argv[1], library="./library"))' "$TASK") +python -m webwright.run.cli main -t "$PROMPT" \ + --task-id t132_new --start-url "$START_URL" -o outputs -c base.yaml -c model_openai.yaml +``` + +Repeat 1–3 whenever a new batch of solves lands β€” the library evolves in place (new templates are +added, existing skills are refined, untouched skills stay as they are). This is exactly the loop +our WebArena evaluation runs (train β†’ gate β†’ update β†’ held-out reuse). + diff --git a/docs/skill_factory/quickstart.md b/docs/skill_factory/quickstart.md new file mode 100644 index 0000000..36af4bb --- /dev/null +++ b/docs/skill_factory/quickstart.md @@ -0,0 +1,274 @@ +# Quickstart β€” the complete tutorial + +[← back to the module README](../../src/webwright/skill_factory/README.md) + +Three ways in, in the order you'd meet them: + +1. **[Run the checked-in skill](#1-run-the-checked-in-skill)** β€” no model, no key, ~40 s. +2. **[Watch the loop build it](#2-watch-the-loop-build-that-skill)** β€” the Google Flights + example, end to end, ~40 min. +3. **[Do it for your own task](#3-do-it-for-your-own-task)** β€” `init` β†’ fill β†’ `build`, or + `learn` if you already have runs. **This is the part that's yours.** + +Solves are long (10-30 min each). If your shell or tooling enforces command timeouts, run them +in the background β€” `--jobs` shortens the wall clock but the command still blocks until the last +one finishes. + +--- + +## 1. Run the checked-in skill + +```bash +cd src/webwright/skill_factory/examples +./quickstart.sh # SEA->DEN, ~40 s, no model, no API key +./quickstart.sh demo LAX ORD 2026-09-01 # your own route (codes + YYYY-MM-DD) +``` + +It prints the ten fixed steps it took β€” no model chose them, they're the skill's code β€” and +where it saved its screenshots, so "this is a program, not a model improvising" is something +you can check rather than take on faith. + +The example is *earliest nonstop flight* on Google Flights (the site from Webwright's own +README). That task was chosen carefully, and the reason matters more than the example: +a flight **schedule** is a fact the page states plainly, it doesn't move on its own, and it +reads the same on your machine as on ours. That's what lets the skill be replay-verified +(`--verify strict`) and reused standalone with a straight face. The **fare** on the same page +would fail all three. See [choosing a task](#choosing-a-task-that-can-be-verified). + +## 2. Watch the loop build that skill + +```bash +export OPENAI_API_KEY=... +./quickstart.sh ask # one LLM call: "can the library help here?" -> use / adapt / skip +./quickstart.sh solve # a full agent solve of an unseen route, reusing the checked-in skill +./quickstart.sh full # the whole loop from nothing: 3 solves -> learn -> reuse (~40 min) +``` + +`demo` (above) runs the skill itself. `ask` only **retrieves** β€” one round trip showing what the +agent gets told about the library, without solving anything. `solve` is the agent actually doing +a task with it. `full` rebuilds the library from scratch so you can watch it being made. + +What `full` does, spelled out β€” this is the loop, without the wrapper: + +```bash +# custom / OpenAI-compatible gateway? TWO knobs, both needed: +# 1. env vars for learn / skill_use (or reuse is silently off). The endpoint is the +# FULL request URL β€” ".../api" alone fails, ".../api/responses" works: +export OPENAI_ENDPOINT=https://your-gateway/api/responses OPENAI_MODEL=your-model +# 2. the AGENT's model in the solve steps reads its yaml, NOT these env vars β€” copy +# examples/model_gateway.example.yaml, fill in your endpoint/model, and use it +# below in place of `-c model_openai.yaml` (quickstart.sh: export MODEL_CFG=...). +cd src/webwright/skill_factory # commands below run from the module directory + +# 1. SOLVE a few instances of the same task type (library is empty β€” these run from scratch) +TASK='What is the earliest nonstop flight from %s to %s on 2026-08-15 (one-way)? Return the answer as a list: [flight_number, airline, departure_time], e.g. ["AS 336", "Alaska", "6:00 AM"].' +while IFS='|' read -r FROM TO; do + examples/solve_with_library.sh \ + "$(printf "$TASK" "$FROM" "$TO")" \ + https://www.google.com/flights "$PWD/library" -o outputs -c base.yaml -c model_openai.yaml +done <<'ROUTES' +Seattle (SEA)|New York (JFK) +San Francisco (SFO)|Boston (BOS) +Los Angeles (LAX)|Chicago (ORD) +ROUTES + +# 2. LEARN: distill everything you've solved into skills β€” no manifest, no fields to fill. +# --verify strict: the distilled skill must reproduce all three training answers standalone +# before it lands (a schedule is stable, so this is a fair bar). +python -m webwright.skill_factory learn outputs/ --library ./library --verify strict --verify-rounds 3 +# -> groups the 3 runs into ONE template and lifts FIVE parameters: +# origin city/code, destination city/code, date +# library/what_is_the_earliest_nonstop_flight_from_.../{skill.py, meta.json, replays.json} + +# 3. USE the library: same wrapper, an UNSEEN route β€” the agent finds and reuses the skill +examples/solve_with_library.sh \ + "$(printf "$TASK" 'Seattle (SEA)' 'Denver (DEN)')" \ + https://www.google.com/flights "$PWD/library" -o outputs -c base.yaml -c model_openai.yaml +# outputs//skill_decision.json -> {"verdict": "use", "skill_id": "what_is_the_earliest_nonstop_..."} +``` + +The library is also usable **without the agent** β€” this is the whole point of code skills: + +```bash +# ask it whether it can help a task (the same call the agent makes β€” one LLM round trip) +python -m webwright.tools.skill_use \ + --task "earliest nonstop flight from Portland (PDX) to Austin (AUS) on 2026-09-01" \ + --library ./library + +# or run the learned skill directly β€” no model in the loop, ~40 seconds +SKILL=$(ls "$PWD"/library/what_is_the_earliest_nonstop_flight_*/skill.py) +cd "$(mktemp -d)" # scratch dir: the skill writes its artifacts to the cwd +cat > taskspec.json <<'EOF' +{"params": {"origin_city": "Seattle", "origin_code": "SEA", "destination_city": "Denver", + "destination_code": "DEN", "date": "2026-08-15"}, + "output_schema": {"type": "array", "items": {"type": "string"}}} +EOF +python "$SKILL" taskspec.json +# -> {"retrieved_data": ["UA 2601", "United", "5:00 AM"]} (schedule may shift by season) +``` + +**What each way of running it actually costs** β€” measured on this machine: + +| | from scratch (3 training routes) | the skill, standalone (SEAβ†’DEN) | +|-------------|----------------------------------|---------------------------------| +| steps | 25 / 40 / 59 | **10** (fixed) | +| wall time | 10.8 / 25.7 / 32.0 min | **~40 s** | +| LLM calls | 29 / 45 / 65 | **0** | + +How to read it: from-scratch cost is high-variance β€” the *same* task type took 25, 40 and 59 +steps on three routes, because the agent re-derives the strategy each time (apply the nonstop +filter, sort by departure, expand the earliest row for its flight number). The learned skill +pins that strategy down to a fixed 10 steps, and the last column is the structural win: +**every run after the library exists uses no model at all** β€” a schedule watcher in cron pays +the agent exploration once, then ~40 s forever. + +**Verification, honestly:** because a schedule is a fixed, client-independent fact, this +family earns `--verify strict` β€” the distilled skill had to reproduce all three training +answers standalone before it landed (`meta.json`: `verified: true, grade: executable`). And +the standalone answer above, `["UA 2601", "United", "5:00 AM"]`, is byte-identical to what an +**independent** model-free probe reads off the page (`experiments/tools/earliest_nonstop_probe.py` +in the research repo) β€” two different code paths, one answer, so the check is real and not +self-confirming. This is the property the task selection buys you: pick a task whose truth the +page *states* and that doesn't drift by client, and admission becomes real verification without +hand-written golds. (For families whose answer genuinely changes between solve and replay β€” +live prices, inventory β€” use `--verify shape` instead, and pass `--golds` when you have them.) + +Exactly this loop, already run and checked in: `examples/learned_library/` (provenance in +`examples/README.md`). + +--- + +## 3. Do it for your own task + +The example above is ours. This is the part that's yours. + +```bash +# a task you keep repeating -> draft a spec +python -m webwright.skill_factory init "the cheapest on Amazon, for any product" +``` + +`init` makes one model call and writes `skill.yaml`: + +```yaml +task: Find the cheapest {product} on Amazon and return its brand and price. +start_url: https://www.amazon.com/ # guessed β€” check it opens the right page + +instances: # give a few real instances (3+ makes a verifiable skill) + - {product: "____"} + - {product: "____"} + - {product: "____"} + +build: + # this answer drifts (prices/stock/rankings change on their own), so replay only + # checks the shape β€” strict would reject a working skill when the value moved + verify: shape + verify_rounds: 2 + on_fail: reject + chunk: 25 +``` + +**It proposes the structure and leaves the values blank on purpose.** The template, the site and +the verify mode are guesses you can overrule; the values are your ground truth, and a value the +model invented would quietly train the skill on an answer nobody checked. Fill the `____`s, look +at the guessed `start_url`, then: + +```bash +python -m webwright.skill_factory build skill.yaml --library ./library --jobs 3 +``` + +`build` = **solve Γ— N + learn**. It fills the template with each instance, prints the tasks it's +about to solve and asks before spending agent time (`--dry-run` shows the plan and stops, +`--yes` skips the prompt), solves them, and hands the batch to `learn`. **An instance that +already produced an answer is never re-solved** β€” if a run dies halfway, re-running `build` only +pays for what's missing. + +### Already have runs? Skip the solving + +If you've been using Webwright anyway, the trajectories are already on disk β€” hand them straight +to `learn`, no spec to write: + +```bash +python -m webwright.skill_factory learn outputs/ --library ./library +``` + +That's the day-to-day path. The rest of this section is for a task you *haven't* solved yet. + +### `--jobs N`: solve N instances at once + +Each solve takes 10-30 minutes and they don't depend on each other, so they can overlap. `N` is +any number you like; the default is `1`, meaning one after another. + +```bash +--jobs 1 # the default β€” one at a time, output streams to your terminal +--jobs 3 # three at once β€” wall clock drops to roughly the slowest one +--jobs 10 # more than you have instances just means "all of them" +``` + +With `N > 1` each solve writes to `build_outputs/solve_NN.log` instead of interleaving on your +terminal, a progress line every 30 s shows the step each one is on, and each prints its result as +it finishes. + +The real ceiling isn't the flag β€” it's the site. Too many browsers from one IP and you get +throttled or soft-blocked, which shows up as *your* solves failing when it's the site pushing +back, and a throttled page can even poison a training answer. **3-5 is a safe place to start**; +this box runs 3 against Google Flights and Amazon without trouble. + +Parallelism only speeds up the solving half. `learn` β€” distil, then replay each instance in a +browser β€” is serial, so `--jobs` won't shorten those 5-13 minutes. + +### Vary the parameters, not just the count + +Distillation lifts a parameter from the **differences it observes**. A value that's identical in +every instance has no evidence behind it and may get baked in. So two instances that vary +everything you care about beat five that share a date: + +```yaml +# a spec with two params: two instances that move BOTH beat five that only move one +instances: + - {product: "makeup remover", max_price: "10"} + - {product: "usb mouse", max_price: "25"} +``` + +### Choosing a task that can be verified + +Four questions, learned the hard way. The example passes all four; "the cheapest X" fails three: + +1. **Does the page state the answer?** β€” or must you infer and compare it yourself? A skill can + anchor on what the site declares (a Cheapest tab's own label, a sort control); it can't + anchor on your judgement. +2. **Does the answer hold still?** β€” if it drifts on its own (prices, stock, rankings), `strict` + will reject a working skill for reporting today's truth. Use `shape`. `init` now guesses this + for you and writes the reason in the spec. +3. **Can each field be extracted reliably?** β€” truth being well-defined isn't enough. A flight + number is stated plainly and *still* sits glued to the aircraft type (`Airbus A321neo` `UA 729`) + next to look-alike tokens, so it's the field distillation gets wrong; the airline and the + departure time never were. +4. **Is the value the site declares the value you actually want?** β€” the subtle one. Sorting + Amazon by price ascending is the *right method* and faithfully returns `$0.00` placeholder + listings. The skill is correct and the answer is useless. A declarative anchor tells you + *where to read*; it can't tell you you're reading the right thing. + +### When a skill gets rejected + +Rejection is the gate working β€” nothing unproven lands, and the runs stay retryable (they're +kept out of `library/.learned.json`, so re-running `learn` retries without re-solving). Two kinds: + +| what you see | what it means | what to do | +|---|---|---| +| the diff is only in a value that moves (`$0.01` β†’ `$5.99`), other instances reproduced exactly | the **verify mode** is wrong for this task, the skill is fine | `--verify shape` | +| a crash (`Could not choose ...`), or an answer off in the wrong place | the distilled skill really is broken | **re-run `learn`** β€” distillation is stochastic, a fresh draw often lands; the failure prints the crash, and the last candidate is kept at `library/.rejected_.py` for a post-mortem | + +Distillation is one LLM call β€” a re-draw costs a rounding error next to the solves you already +paid for. `--verify-rounds N` bounds how many repair rounds one draw gets before it gives up. + +--- + +**Where this fits:** +- *recurring personal queries* β€” releases, commit counts, price checks: pay the exploration + once, every repeat is cheap (or free β€” run the skill standalone from cron, no model); +- *same-template batch jobs* β€” QA flows, report pulls: solve 3, learn, run the rest on skills; +- *a team library* β€” commit `./library` to your repo; everyone's agent reuses it. + +**Need every field under your control** β€” explicit manifests, benchmark-grade gold gates? +That's [manual mode](manual.md); you don't need it to get started. + diff --git a/docs/skill_factory/reference.md b/docs/skill_factory/reference.md new file mode 100644 index 0000000..63141fe --- /dev/null +++ b/docs/skill_factory/reference.md @@ -0,0 +1,97 @@ +# Reference β€” verification, parameters, components, backend + +[← back to the module README](../../src/webwright/skill_factory/README.md) + +## Verification and grades + +**Validation-gated β€” exactly as strong as the gate you give it.** Every solve passes an +admission gate before it can enter the library. With gold answers (benchmarks β€” this is what +our WebArena numbers used) the gate is real supervision: wrong answers never get in. The +default `self_verify` gate checks shape, non-emptiness, and the agent's **own final report** +(a run that reported `NOT_FOUND_ERROR` is rejected β€” the agent itself didn't believe it) β€” +it filters garbage and self-admitted failures, **not wrong-but-plausible answers the agent +believed**. Pass `--golds` to `learn`, or bring your own judge, when correctness matters. +The gate also has an **output side**: a skill must run **standalone** on its own training +taskspecs and reproduce the recorded answers before it may enter the library (no model in the +loop; up to `--verify-rounds` build attempts, then rejected). For task families whose answers +are live data (prices, listings), `--verify shape` relaxes the comparison to non-empty + +schema-shaped; `--verify off` skips replay entirely. + +**Verification decides a skill's grade, not just its existence** (`--on-fail reference`): + +| | `executable` (verified) | `reference` | +|-----------------|--------------------------------------------------|--------------------------------------| +| the bar | replays its training taskspecs standalone, reproduces the answers | failed that bar | +| cost to build | higher & slower: N replays + up to `--verify-rounds` distillation calls | one distillation call | +| what it buys | **run it directly** β€” plain python/playwright, no webwright, no model, cron-able | a **prior for the agent**: exact selectors, URLs, param shapes, fallbacks it reads and reuses | +| refining | incremental refines must pass **regression replay** of the stored training examples (`replays.json`); a verified skill is never overwritten by an unverified refine | refined freely β€” no execution promise to protect | + +Why code even at reference grade (vs. natural-language notes): the selectors, URLs and param +shapes are **verbatim-copyable** into the agent's next script, individual primitives often +still run even when the end-to-end skill doesn't, and a reference skill is one repair away +from executable β€” prose is none of these. + +Honest footnote: our WebArena numbers predate this gate β€” that library was effectively +all-reference (a later standalone audit: only 3/10 skills replayed clean), and it still +delivered **+15pp held-out accuracy**. That is the evidence that reference-grade priors help +an agent; the flights quickstart's three-way consistency is the evidence for the executable +grade. + +## Components + +| file | role | +|---|---| +| `library.py` | `Skill` + `Library(root)`: on-disk skills (`/skill.py` + `meta.json`) | +| `retrieve.py` | `retrieve(task, library)` β†’ ranked `Candidate`s (relevance) | +| `decide.py` | `decide(task, candidates)` β†’ `Decision(verdict, skill_id, reason)` (utility: use/adapt/skip) | +| `gate.py` | `gate(result, method=gold\|self_verify\|none)` β†’ admit? (keeps wrong solves out) | +| `update.py` | `evolve(traces, library)`: grow on the existing library β€” add / adapt-refine / keep; `_refine` parameterizes + decomposes into primitives, incrementally improving an existing skill | +| `llm.py` | `configure_llm(model)` + `llm()`: **backend-agnostic** via Webwright's `Model` abstraction; a bare CLI builds the model from `SKILL_MODEL_NAME`/`SKILL_MODEL_ENDPOINT` (or `OPENAI_*`) env β€” no hardcoded endpoint/key | +| `prompt.py` | `with_skill_hint(prompt, task, library)`: non-invasive task-prompt hint | + +## Backend + +Backend-agnostic. Either `configure_llm(model_config_or_Model)` once in-process, or set +`SKILL_MODEL_NAME` / `SKILL_MODEL_ENDPOINT` (falling back to `OPENAI_*`) so a bare tool invocation +uses the same backend as the running agent. No gateway or key is hardcoded. + +## All parameters + +### `python -m webwright.skill_factory learn ` + +| flag | default | meaning | +|---|---|---| +| `--library` | `library` | library directory to grow | +| `--golds` | β€” | JSON `{task_id: gold_answer}` β†’ gold gate instead of self_verify | +| `--chunk` | 25 | runs per LLM grouping call | +| `--dry-run` | off | print the grouping plan, change nothing | +| `--verify` | `strict` | replay bar: `strict` = reproduce recorded answers, `shape` = non-empty + schema-shaped (live data), `off` = skip | +| `--verify-rounds` | 2 | total build attempts (first + repairs) before giving up | +| `--on-fail` | `reject` | failed verification: `reject` (runs stay retryable) or `reference` (lands as a labeled prior; never overwrites an existing skill) | + +### `python -m webwright.skill_factory.update` + +| flag | default | meaning | +|---|---|---| +| `--manifest` | required | `{template, runs:[{dir, admit(bool, REQUIRED), params, verdict, site, output_schema, answer?, credentials?}]}` | +| `--library` | required | library directory | +| `--verify` / `--verify-rounds` / `--on-fail` | `off` / 2 / `reject` | as above (off by default: benchmark sites may need credentials) | + +### `python -m webwright.tools.skill_use` + +| flag | meaning | +|---|---| +| `--task` | the task text to match against the library | +| `--library` | library directory (or env `SKILL_LIBRARY_ROOT`) | +| `--output` | also write the JSON verdict to this file | + +### Environment variables + +| var | used by | meaning | +|---|---|---| +| `OPENAI_API_KEY` | all LLM calls | API key | +| `OPENAI_ENDPOINT` / `OPENAI_MODEL` | learn, skill_use | custom gateway; the endpoint is the FULL request URL (e.g. `https://gateway.example/api/responses`), not a base path | +| `SKILL_MODEL_NAME` / `SKILL_MODEL_ENDPOINT` / `SKILL_MODEL_CLASS` / `SKILL_MODEL_TIMEOUT` | module LLM | overrides for the module's model (fall back to `OPENAI_*`) | +| `SKILL_LIBRARY_ROOT` | skill_use | default library path | +| `WORKSPACE_DIR` | generated skills | where a skill writes its artifacts (default: cwd) | +| `MODEL_CFG` | examples/quickstart.sh | model yaml for the agent in solve/full modes | diff --git a/src/webwright/skill_factory/README.md b/src/webwright/skill_factory/README.md new file mode 100644 index 0000000..3da0823 --- /dev/null +++ b/src/webwright/skill_factory/README.md @@ -0,0 +1,228 @@ + +# Web Skill Factory + +**Most agent skills are context the model refers to. Ours are programs.** + +Every task Webwright solves leaves a working script behind. The Skill Factory turns those +scripts into a growing library of **reusable, verified, parameterized skills**: code you can +run without a model and compose into the next task instead of re-exploring the site. + +## πŸŽ₯ Demo + +https://github.com/user-attachments/assets/3f93fac4-bb93-4ea5-8b45-280ed1334feb + + +## ✨ Highlights + +- πŸƒ **Runs standalone, no model.** A learned skill is just code. It re-executes in ~30 s with zero tokens, so you can cron it to run every day, instead of having a model re-read a note and redo the work every time. +- πŸ› οΈ **Has a real software-engineering surface.** Because skills are code, they inherit code's tools and properties for free: inheritance, polymorphism, encapsulation, tests, versioning, and history. A skill is executable and verifiable, not prose the model has to interpret. +- βœ… **Verified twice before it lands.** First an input gate: a solve only becomes material if it got the task right, so a wrong answer never feeds a skill. Then the distilled skill must replay its own answers standalone, with no model, so a broken skill can't slip in and poison the library. +- 🌱 **Gets stronger the more you use it.** New solves widen a skill in place, self-evolving as you go. Regression-replay keeps old coverage from breaking, so a skill that's already been verified is never damaged by a later change. + + +## How it compares + +| | published `SKILL.md`| SkillOpt | OpenCLI | **Web Skill Factory (Ours)** | +|---|---|---|---|---| +| a skill **is** | a document the model reads | a document the model reads | one CLI command per site capability | **a parameterized Python program that runs on its own, no model** | +| **domain / whose need** | anything, but nothing runs | general agent tasks (ALFWorld, DocVQA, spreadsheets, math...) | web: a site's common capabilities, shared by all users | **web: the specific task *you* repeat, including private, cross-site, multi-step workflows no shared catalogue has** | +| **produced by** | a person writes and publishes it; you install it | edits to one document, driven by past runs | a person or agent writes one per site | **distilling several solves of the same task template** | +| **parameters come from** | whoever wrote it | none | the author declares them | **the differences actually observed between your solves** | +| **verified?** | no | one gate: scores higher on a held-out split | one gate: checked when it's first written, plus live tests | **two gates: a wrong answer never feeds a skill, *and* the skill must reproduce its own answers standalone, no model** | +| agent can **adapt** it | read-only | read-only | it edits the source only to repair the shared adapter when it breaks β€” never to fit the task in front of it | **yes, per task: the source is in hand, to copy or to rework as the task needs β€” the library is left alone** | +| **grows from your runs** | no, it's whatever its author last wrote | yes, but what grows is a document for a frozen agent, not a program | no; a broken adapter is patched back to what it did, and nothing accumulates from your runs | **yes: each new solve widens it in place, regression-replayed so old coverage can't break** | + +## πŸ—ΊοΈ How it works + +![data flow & interfaces](../../../assets/skill_factory_pipeline.png) + +``` +solve β†’ gate β†’ group by template β†’ distill β†’ replay-verify β†’ library β†’ next solve reuses +``` + +At solve time the agent asks the library once and gets `use` / `adapt` / `skip` β€” its stated +intent for the skill, after which it has the source and reuses it as the task needs. Reuse never +blocks solving. Two touch points into Webwright, no agent-loop changes: + +```bash +python -m webwright.tools.skill_use --task "" --library ./library # reuse at solve time +python -m webwright.skill_factory learn outputs/ --library ./library # grow it afterwards +``` + +## πŸš€ Quick Start + +### 1. Run a learned skill + +Task: *what is the earliest nonstop flight from A to B on this date?*, on the +live Google Flights. + +No model, no API key, about 40 seconds. The whole pitch in one command: + +```bash +cd src/webwright/skill_factory/examples +./quickstart.sh # the checked-in skill drives the live site +./quickstart.sh demo LAX ORD 2026-09-01 # ...on your own route +``` + +It prints the ten fixed steps it took and where it saved its screenshots. No model chose those +steps; they're the skill's code. + +--- + +### 2. Bring the agent in + +The same task family, now with the agent in the loop. Needs an API key: + +```bash +export OPENAI_API_KEY=... +./quickstart.sh ask # one LLM call: "can the library help here?" -> use / adapt / skip +./quickstart.sh solve # a full agent solve of an unseen route, reusing the checked-in skill +./quickstart.sh full # the whole loop from nothing: 3 solves -> learn -> reuse (~40 min) +``` + +- `demo` runs the skill directly +- `ask` only *retrieves*: one round trip that shows what the agent is told about the library, without solving anything +- `solve` is the agent actually doing a task with it +- `full` rebuilds the library from scratch, so you can watch it being made + +--- + +### 3. Do it for your task + +Two ways in, depending on whether you've solved the task yet. + +**3a. You have a task you keep repeating, but no runs yet.** + +Describe it, fill in your values, build: + +```bash +python -m webwright.skill_factory init "your task" +$EDITOR skill.yaml # fill the ____ values; check the guessed start_url +python -m webwright.skill_factory build skill.yaml --library ./library --jobs N # how many tasks you want to solve in parallel +``` + +`init` proposes the structure: a task template with `{holes}`, the site, and the verify mode that +fits your task. It leaves the values blank, because those are your ground truth, not the model's +guess. `build` fills the template with each instance, solves them, and hands the batch to `learn`. + +Nothing runs until you say so. `build` prints the tasks it's about to solve and asks first +(`--dry-run` just shows the plan), and a task that already has an answer is never re-solved. + +> If your answer moves on its own (a price, a ranking), the shape check can't tell right from +> wrong. Supply `--golds`, or plan to gate it with a judge (see Limitations). + +**3b. You already have a folder of webwright runs.** + +Skip the solving and distill what's there: + +```bash +python -m webwright.skill_factory learn outputs/ --library ./library +``` + +This is the day-to-day path once you're using Webwright anyway: the trajectories you produced +solving real work become the library, with no spec to write. + +
+On a custom OpenAI-compatible gateway +
+ +```bash +export OPENAI_ENDPOINT=... OPENAI_MODEL=... # for learn / init / skill_use +export MODEL_CFG=/abs/path/to/model.yaml # for the AGENT in solve / build +``` + +The endpoint is the full `.../responses` URL, not a base path. The agent reads its model from a +yaml, not from these env vars: copy `examples/model_gateway.example.yaml` and point `MODEL_CFG` at +it (or pass `-c` to `build`). +
+ +Full tutorial, with the loop spelled out, gateway setup, and running skills without the agent: +**[docs/skill_factory/quickstart.md](../../../docs/skill_factory/quickstart.md)** + +## πŸ“Š Results + +**Setting.** WebArena, 10 retrieve-type task templates across 3 self-hosted sites +(shopping-admin, gitlab, map). Each template contributes 3 train solves that build the library +(gated on ground-truth answers) and 2 held-out instances that measure reuse on unseen instances +of the same template. Every task is solved both with the library and from scratch. Model: +gpt-5.4. 100 runs in total. + +| | WITH library | from scratch | Ξ” | +|-------------------------|--------------|--------------|---------| +| held-out accuracy (20) | **70%** | 55% | **+15 pp** | +| held-out avg steps | **14.7** | 17.1 | βˆ’2.4 | +| train accuracy (30) | **86.7%** | 76.7% | +10 pp | +| train avg steps | **13.7** | 15.9 | βˆ’2.2 | + +- **Reuse helps most when solving from scratch is expensive.** Of 20 held-out tasks, 4 were + rescued (they failed from scratch and the library solved them) and 6 more solved in fewer + steps. The +15pp is measured on instances that never took part in building the library; the + same direction holds on training instances (86% vs 76%). +- **Biggest single win:** a task that took 33 steps from scratch ran in 10 with the library. +- **Wrong solves stay out.** 7 of 30 train solves failed the ground-truth gate and never + entered the library. +- **Retrieval stayed reliable as the library grew** to 10 skills: all 20 held-out solves + retrieved their own template's skill, including two near-duplicate gitlab commit skills. + +**How to read it.** In the agent-in-loop path (a `reference` skill the agent reads and reuses, +which is what the eval runs), step savings scale with how much the agent doesn't already know: +on a familiar site the cost of querying the library and reading the skill can outweigh what it +saves, so reuse pays off most on the hard tasks. From-scratch cost is also high-variance, and a +skill pins the strategy down. + +An `executable` skill skips that path entirely: it runs standalone, with no agent and no model +in the loop, in a fixed handful of steps, so every repeat after the first is essentially free. +(Results on this mode coming soon.) + +## 🚧 Limitations & Roadmap + +Here are some known rough edges, and directions we might take them. + +- **A skill can't outrun the agent that made it.** Everything is distilled from solves, so if the + agent never figured out a good way to do something, there's nothing to distill. Pooling a bunch + of failed attempts won't invent a strategy that was never there. The factory makes reuse cheap; + it doesn't make hard tasks solvable. + +- **The agent reaches for skills too eagerly.** Right now `decide` only asks "is there a relevant + skill?", not "is using it actually worth it?" On WebArena it said `use` 49 times and `skip` not + once, even on tasks that would've been quicker from scratch. It should weigh the two, and skip + when starting fresh is the cheaper bet. + +- **Reuse today is copy-and-edit, not a clean import.** The agent reuses a skill by reading the + source and editing a copy, which is why `use` and `adapt` blur together in practice, and even a + `use` can quietly rewrite half the code. A proper callable interface, where you import a skill + and just pass it parameters, would make reuse a lot cleaner. + +- **Verification is only as reliable as the reference answer it checks against.** On real websites, where no gold label is available, the LLM may misinterpret the task or produce an incorrect reference answer, and self-verification may fail to detect that error. For dynamic answers such as prices or rankings, verification often falls back to checking only the output format or structure. This can catch a skill that is broken or fails to execute, but not one that executes successfully and returns the wrong result. Achieving true correctness in these settings requires a stronger, independent judge, such as a WebJudge-style model. + +- **Distillation is stochastic.** A given attempt may produce a fragile skill that fails even its own replay. The gate filters out these failures, and rerunning distillation a few times usually succeeds. However, each retry consumes additional tokens, so improving the reliability of executable skill generationβ€”ideally succeeding on the first attemptβ€”remains an important direction to explore. + +- **Aggregation only groups by the literal template.** Ask the same task two different ways and + you get two skills, each thinner than one merged one would be. Letting a model recognize when + two wordings mean the same task would fix it. + +- **The library needs upkeep, like any package registry.** After a skill lands, a site can shift + under it with nothing re-checking, so it can quietly go stale and keep returning wrong answers. + Stale skills never retire, and near-duplicate ones never get merged. Health checks, a retirement + policy, and de-duplication are the obvious directions. + +## πŸ“š Documentation + +| doc | what's in it | +|---|---| +| [docs/skill_factory/quickstart.md](../../../docs/skill_factory/quickstart.md) | the complete tutorial: the flight-schedule loop, gateway knobs, standalone usage, measured costs | +| [docs/skill_factory/manual.md](../../../docs/skill_factory/manual.md) | manual mode: manifests field by field, gold gates, the batch pipeline | +| [docs/skill_factory/reference.md](../../../docs/skill_factory/reference.md) | verification & grades, every flag and env var, component map, backend | +| [examples/README.md](examples/README.md) | the checked-in skill and the example inputs | + +## πŸ“ Citation + +```bibtex +@misc{web_skill_factory, + title = {Web Skill Factory: Evolving Reusable, Verified, Code-Native Skills for Web Agents}, + author = {Demi Ruohan Wang, Yadong Lu}, + year = {2026}, + note = {Built on Webwright}, + url = {https://github.com/microsoft/Webwright} +} +``` diff --git a/src/webwright/skill_factory/__init__.py b/src/webwright/skill_factory/__init__.py new file mode 100644 index 0000000..cb91a5a --- /dev/null +++ b/src/webwright/skill_factory/__init__.py @@ -0,0 +1,26 @@ +"""webwright.skill_factory β€” a memory/skill library module for webwright. + +Store solved tasks as reusable, executable code skills; retrieve + judge (use/adapt/skip) at +solve time; admit via a gate; and grow the library incrementally (evolve). Plugs into webwright +as a built-in submodule: + - solve-time reuse : the `skill_use` tool (agent invokes it like self_reflection / image_qa) + - offline growth : `update.evolve` (run after solves to distill gate-passed solves into skills) + +Backend-agnostic: configure_llm(model) wires it to any webwright Model. +""" +from .library import Library, Skill +from .retrieve import retrieve, Candidate +from .decide import decide, Decision +from .gate import gate, GateResult +from .llm import configure_llm +from .prompt import with_skill_hint + +# NOTE: `update` (evolve / Trace) is deliberately NOT imported here. It is the module run as a +# CLI (`python -m webwright.skill_factory.update`); importing it eagerly makes runpy print a +# "found in sys.modules" RuntimeWarning on every CLI invocation. Import it directly: +# from webwright.skill_factory.update import evolve, Trace + +__all__ = [ + "Library", "Skill", "retrieve", "Candidate", "decide", "Decision", + "gate", "GateResult", "configure_llm", "with_skill_hint", +] diff --git a/src/webwright/skill_factory/__main__.py b/src/webwright/skill_factory/__main__.py new file mode 100644 index 0000000..f66dbb0 --- /dev/null +++ b/src/webwright/skill_factory/__main__.py @@ -0,0 +1,24 @@ +"""python -m webwright.skill_factory β€” friendly entry points.""" +import sys + +CMDS = { + "init": "webwright.skill_factory.init", + "build": "webwright.skill_factory.build", + "learn": "webwright.skill_factory.learn", + "update": "webwright.skill_factory.update", +} + +def main() -> int: + if len(sys.argv) > 1 and sys.argv[1] in CMDS: + import importlib + mod = importlib.import_module(CMDS[sys.argv[1]]) + return mod.main(sys.argv[2:]) + print("usage: python -m webwright.skill_factory …\n" + " init draft a skill.yaml skeleton from a one-line need (you fill the values)\n" + " build solve a spec's instances, then learn β€” for a task you haven't solved yet\n" + " learn distill a folder of finished runs into skills (no manifest needed)\n" + " update manual mode: distill from an explicit batch.json manifest") + return 1 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/webwright/skill_factory/build.py b/src/webwright/skill_factory/build.py new file mode 100644 index 0000000..3fe49a9 --- /dev/null +++ b/src/webwright/skill_factory/build.py @@ -0,0 +1,235 @@ +"""python -m webwright.skill_factory build β€” solve a template's instances, then learn. + +build = solve x N + learn. Give it a skill spec (a task template + a table of instances) and it +solves each instance with the webwright agent, then distills the solves into a verified library +skill. If you ALREADY have finished run directories, use `learn` instead β€” it skips solving. + +The spec is a single human-editable file (draft one with `init`): + + task: earliest nonstop flight from {origin} to {destination} on {date} (one-way)? ... + start_url: https://www.google.com/flights + instances: + - {origin: "Seattle (SEA)", destination: "New York (JFK)", date: "2026-08-15"} + - {origin: "San Francisco (SFO)", destination: "Boston (BOS)", date: "2026-08-15"} + - {origin: "Los Angeles (LAX)", destination: "Chicago (ORD)", date: "2026-08-15"} + build: # optional β€” verification / aggregation policy, all with defaults + verify: strict # strict | shape | off + verify_rounds: 3 + on_fail: reject # reject | reference + chunk: 25 + +Machine-specific settings stay OUT of the spec (so it stays committable): the agent's model +config is a CLI flag (-c model.yaml, repeatable), as are --library and --jobs (how many solves +to run in parallel β€” a property of your box and gateway, not of the skill). CLI flags override +the spec's `build:` block; the block overrides the built-in defaults. +""" +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +import yaml + +from .learn import learn + +_ANSWER_INSTR = ('Additionally, write the final answer into $WORKSPACE_DIR/agent_response.json ' + 'as {"retrieved_data": }.') + + +def _fill(template: str, params: dict) -> str: + """Substitute {name} holes in the template with this instance's values.""" + holes = set(re.findall(r"{(\w+)}", template)) + missing = holes - set(map(str, params)) + if missing: + raise SystemExit(f"build: instance {json.dumps(params, ensure_ascii=False)} is missing " + f"value(s) for {sorted(missing)} (holes in the task template)") + return re.sub(r"{(\w+)}", lambda m: str(params[m.group(1)]), template) + + +def _already_solved(outputs: Path, core_task: str) -> Path | None: + """Resume: a prior run whose task text contains this instance AND wrote an answer.""" + for tj in outputs.glob("*/task.json"): + try: + task = json.loads(tj.read_text(encoding="utf-8")).get("task", "") + except Exception: + continue + if core_task in task and (tj.parent / "agent_response.json").exists(): + return tj.parent + return None + + +def _solve(core_task: str, start_url: str, library: Path, outputs: Path, + task_id: str, cfg: list[str], log_path: Path | None = None) -> int: + """Run one agent solve. log_path captures its output (parallel mode); None streams it.""" + from .prompt import with_skill_hint + prompt = with_skill_hint(core_task + " " + _ANSWER_INSTR, task=core_task, library=str(library)) + cmd = [sys.executable, "-m", "webwright.run.cli", "main", "-t", prompt, + "--start-url", start_url, "-o", str(outputs), "--task-id", task_id] + for c in cfg: + cmd += ["-c", c] + if log_path is None: + return subprocess.run(cmd).returncode + with log_path.open("w", encoding="utf-8") as f: + return subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT).returncode + + +def _pick(cli, spec_val, default): + if cli is not None: + return cli + if spec_val is not None: + return spec_val + return default + + +def build(spec_path: str, library: str, cfg: list[str], *, verify=None, verify_rounds=None, + on_fail=None, chunk=None, golds=None, outputs_dir=None, dry_run=False, + assume_yes=False, jobs=1) -> int: + spec = yaml.safe_load(Path(spec_path).read_text(encoding="utf-8")) or {} + task = spec.get("task", "").strip() + start_url = spec.get("start_url", "").strip() + instances = spec.get("instances") or [] + policy = spec.get("build") or {} + if not task or not start_url or not instances: + raise SystemExit("build: spec needs non-empty 'task', 'start_url', and 'instances'.") + + verify = _pick(verify, policy.get("verify"), "strict") + verify_rounds = _pick(verify_rounds, policy.get("verify_rounds"), 2) + on_fail = _pick(on_fail, policy.get("on_fail"), "reject") + chunk = _pick(chunk, policy.get("chunk"), 25) + + lib = Path(library).resolve() + outputs = Path(outputs_dir).resolve() if outputs_dir else (Path(spec_path).resolve().parent / + "build_outputs") + outputs.mkdir(parents=True, exist_ok=True) + + concrete = [(_fill(task, p), p) for p in instances] + + print(f"build plan: {len(concrete)} instance(s) of\n {task}\n" + f"start_url: {start_url}\noutputs: {outputs}\n" + f"policy: verify={verify} rounds={verify_rounds} on_fail={on_fail} chunk={chunk}\n" + f"jobs: {jobs} solve(s) in parallel\n" + f"library: {lib}\n") + for i, (ct, _) in enumerate(concrete): + state = "already solved (will reuse)" if _already_solved(outputs, ct) else "will solve" + print(f" [{i}] {state}: {ct[:110]}") + + if dry_run: + print("\n--dry-run: nothing solved, nothing learned.") + return 0 + + to_solve = [c for c in concrete if not _already_solved(outputs, c[0])] + if to_solve and not assume_yes: + if not sys.stdin.isatty(): + raise SystemExit("\nbuild: solving costs real agent time. Re-run with --yes to proceed " + "(or --dry-run to just see the plan).") + ans = input(f"\nSolve {len(to_solve)} instance(s) with the agent? [y/N] ").strip().lower() + if ans not in ("y", "yes"): + print("aborted."); return 1 + + if to_solve and cfg == [] and __import__("os").environ.get("OPENAI_ENDPOINT"): + print("!! OPENAI_ENDPOINT is set but no -c model config was passed. The AGENT reads a yaml,\n" + "!! not env vars, and will hit api.openai.com. Pass -c your_model.yaml (FULL " + ".../responses endpoint).", file=sys.stderr) + + solved = len(concrete) - len(to_solve) # the resumed ones + failed = [] + + def _one(i_ct): + i, ct = i_ct + log = outputs / f"solve_{i:02d}.log" if jobs > 1 else None + rc = _solve(ct, start_url, lib, outputs, f"build_{i:02d}", cfg, log) + return i, ct, rc, log + + pending = [(i, ct) for i, (ct, _p) in enumerate(concrete) if not _already_solved(outputs, ct)] + + def _ticker(stop: threading.Event, idxs: list[int]) -> None: + """A solve is 10-60 min of silence otherwise, which reads as a hang. Report the step + each instance is on, so progress is visible without opening the logs.""" + while not stop.wait(30): + parts = [] + for i in idxs: + runs = sorted(outputs.glob(f"build_{i:02d}_*")) + steps = len(list((runs[-1] / "steps").glob("*"))) if runs and (runs[-1] / "steps").is_dir() else 0 + parts.append(f"[{i}] {steps} steps") + print(f" … {' '.join(parts)}", flush=True) + + if jobs > 1 and len(pending) > 1: + print(f"\n-- solving {len(pending)} instance(s), {jobs} at a time " + f"(output -> {outputs}/solve_NN.log; progress every 30s) --") + # as_completed, not map: map yields in submission order, so a finished instance + # stays invisible behind a slow one and the run looks hung when it isn't + stop = threading.Event() + tick = threading.Thread(target=_ticker, args=(stop, [i for i, _ in pending]), daemon=True) + tick.start() + with ThreadPoolExecutor(max_workers=jobs) as pool: + futures = [pool.submit(_one, p) for p in pending] + for done in as_completed(futures): + i, ct, rc, log = done.result() + # the answer file is the contract, not the exit code: a solve that wrote its + # answer and then exited non-zero (killed, late crash) is still usable β€” learn + # reads the artifact, so build must not disagree with it + if _already_solved(outputs, ct): + solved += 1 + note = "" if rc == 0 else f" (exited {rc}, but the answer was written)" + print(f" [{i}] done ({solved}/{len(pending)}){note}: {ct[:70]}", flush=True) + else: + failed.append((i, ct)) + print(f" ! [{i}] no answer (exit {rc}) β€” see {log}", flush=True) + stop.set() + else: + for i, ct in pending: + print(f"\n-- solving [{i}] {ct[:90]}") + _, _, rc, _ = _one((i, ct)) + if _already_solved(outputs, ct): + solved += 1 + else: + failed.append((i, ct)) + print(f" ! instance [{i}] did not produce an answer (exit {rc}); continuing") + + print(f"\nsolved {solved}/{len(concrete)} instance(s)" + + (f"; {len(failed)} failed" if failed else "")) + if solved == 0: + raise SystemExit("build: no instance solved β€” nothing to learn.") + + print("\n-- learning from the solves --") + learn(str(outputs), library, golds=golds, chunk=chunk, verify=verify, + rounds=verify_rounds, on_fail=on_fail) + return 0 + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="python -m webwright.skill_factory build", + description="Solve a template's instances, then learn a skill.") + p.add_argument("spec", help="skill.yaml: task template + start_url + instances (draft with init).") + p.add_argument("--library", default="library") + p.add_argument("-c", "--config", action="append", default=[], dest="cfg", + help="webwright model config for the agent (repeatable). Machine-specific β€” " + "keep it out of the spec.") + p.add_argument("--verify", choices=["off", "shape", "strict"], + help="Override the spec's build.verify (default strict).") + p.add_argument("--verify-rounds", type=int, help="Override build.verify_rounds (default 2).") + p.add_argument("--on-fail", choices=["reject", "reference"], help="Override build.on_fail.") + p.add_argument("--chunk", type=int, help="Override build.chunk (runs per grouping call).") + p.add_argument("--golds", default="", help="JSON file {task_id: gold_answer} for a gold gate.") + p.add_argument("--outputs", help="Where to write solves (default: /build_outputs).") + p.add_argument("--jobs", type=int, default=1, metavar="N", + help="Solve up to N instances in parallel (default 1). Machine-specific β€” " + "how much concurrency your box and gateway tolerate β€” so it is a flag, " + "not spec policy.") + p.add_argument("--dry-run", action="store_true", help="Print the plan; solve/learn nothing.") + p.add_argument("--yes", action="store_true", help="Skip the confirmation before solving.") + a = p.parse_args(argv) + golds = json.loads(Path(a.golds).read_text(encoding="utf-8")) if a.golds else None + return build(a.spec, a.library, a.cfg, verify=a.verify, verify_rounds=a.verify_rounds, + on_fail=a.on_fail, chunk=a.chunk, golds=golds, outputs_dir=a.outputs, + dry_run=a.dry_run, assume_yes=a.yes, jobs=a.jobs) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/webwright/skill_factory/decide.py b/src/webwright/skill_factory/decide.py new file mode 100644 index 0000000..a966adb --- /dev/null +++ b/src/webwright/skill_factory/decide.py @@ -0,0 +1,50 @@ +"""Decide whether to use: candidates + task -> use / adapt / skip (utility). + +Stable interface (swappable implementation): + decide(task, candidates, *, method="llm") -> Decision +Relevant != useful: retrieve gives "how similar", decide gives "whether and how to use it". +""" +from __future__ import annotations +from dataclasses import dataclass + +from .llm import llm_json + + +@dataclass +class Decision: + verdict: str # "use" | "adapt" | "skip" + skill_id: str | None + reason: str + + +def _decide_llm(task: str, candidates) -> Decision: + if not candidates: + return Decision("skip", None, "no candidate skills") + cat = "\n".join( + f"- skill_id: {c.skill.skill_id} | template: {c.skill.meta.get('template','')} | " + f"summary: {c.skill.summary} | params: {c.skill.signature.get('params', [])}" + for c in candidates + ) + sys = ( + "Decide whether a library skill is worth using for THIS task. Output STRICT JSON: " + '{"verdict":"use|adapt|skip","skill_id":"...","reason":"..."}.\n' + "- use = the skill fits the task as-is (just different parameter values).\n" + "- adapt = the skill's expensive core (login / navigation / extraction) is reusable, but the " + "FINAL step differs; the agent should reuse the front and add/adapt only the last step.\n" + "- skip = no candidate is worth it; solve from scratch (skill_id = null).\n" + "Relevance is not enough β€” only 'use'/'adapt' if it genuinely saves work." + ) + user = f"## Task\n{task}\n\n## Candidate skills (most relevant first)\n{cat}" + out = llm_json(sys, user) + verdict = out.get("verdict", "skip") + if verdict not in ("use", "adapt", "skip"): + verdict = "skip" + skill_id = out.get("skill_id") if verdict != "skip" else None + return Decision(verdict=verdict, skill_id=skill_id, reason=out.get("reason", "")) + + +_DECIDERS = {"llm": _decide_llm} + + +def decide(task: str, candidates, *, method: str = "llm") -> Decision: + return _DECIDERS[method](task, candidates) diff --git a/src/webwright/skill_factory/examples/README.md b/src/webwright/skill_factory/examples/README.md new file mode 100644 index 0000000..365dd47 --- /dev/null +++ b/src/webwright/skill_factory/examples/README.md @@ -0,0 +1,47 @@ +# Examples + +Everything the Quickstart runs lives here, and nothing is hand-written β€” the library is +verbatim `learn` output. + +``` +examples/ +β”œβ”€β”€ quickstart.sh # one command, every parameter pre-filled (demo, ask, solve, full) +β”œβ”€β”€ solve_with_library.sh # the solve wrapper: skill hint + answer-output instruction +β”œβ”€β”€ learned_library/ # the Quickstart's artifact, checked in (skill.py + meta.json + replays.json) +β”‚ └── what_is_the_earliest_nonstop_flight…/ +β”œβ”€β”€ tasks.example.json # manual mode: a filled task list (module README, step 6) +└── batch.example.json # manual mode: a filled manifest (module README, step 2) +``` + +## The checked-in skill + +Three from-scratch solves of "earliest nonstop flight" on Google Flights (SEAβ†’JFK, +SFOβ†’BOS, LAXβ†’ORD; 59, 25 and 40 agent steps) were grouped by +`python -m webwright.skill_factory learn --verify strict` into one template with **five** +lifted parameters, and the distilled skill reproduced all three training answers standalone +before it landed (`meta.json`: `verified: true, grade: executable`): + +```json +{ + "template": "What is the earliest nonstop flight from {{origin_city}} ({{origin_code}}) to {{destination_city}} ({{destination_code}}) on {{date}} (one-way)? Return the answer as a list: [flight_number, airline, departure_time], ...", + "signature": { "params": ["origin_city", "origin_code", "destination_city", "destination_code", "date"], + "call": "python skill.py taskspec.json" }, + "n_solves": 3, "verified": true, "grade": "executable" +} +``` + +Why this task: a flight *schedule* is a stable, client-independent fact the page states +plainly β€” so the answer is the same today, tomorrow, and on your machine, which is exactly +what lets `--verify strict` and standalone reuse mean something. On an unseen route +(SEAβ†’DEN): from scratch 25–59 steps (the training spread), standalone **~40 s with no model**, +answer `["UA 2601", "United", "5:00 AM"]` β€” identical to an independent model-free probe. + +## Run it + +```bash +./quickstart.sh # standalone, no API key, ~40 s +./quickstart.sh solve # the agent reuses this skill on a new route (needs a key) +``` + +Manual mode (explicit manifests, gold gates): see the module README; the two +`*.example.json` files here are filled-in versions of the inputs it asks you to write. diff --git a/src/webwright/skill_factory/examples/batch.example.json b/src/webwright/skill_factory/examples/batch.example.json new file mode 100644 index 0000000..f42f22e --- /dev/null +++ b/src/webwright/skill_factory/examples/batch.example.json @@ -0,0 +1,56 @@ +{ + "template": "How many commits did {{user}} make {{period}} in the current repository?", + "runs": [ + { + "dir": "outputs/commits_a_20260707_120000", + "admit": true, + "params": { + "user": "Jane Doe", + "period": "on January 5th 2023" + }, + "verdict": "skip", + "site": "gitlab", + "output_schema": { + "type": "array", + "items": { + "type": "number" + } + } + }, + { + "dir": "outputs/commits_b_20260707_121500", + "admit": true, + "params": { + "user": "John Smith", + "period": "on April 7th 2022" + }, + "verdict": "skip", + "site": "gitlab", + "output_schema": { + "type": "array", + "items": { + "type": "number" + } + }, + "answer": [ + 2 + ] + }, + { + "dir": "outputs/commits_c_20260707_123000", + "admit": false, + "params": { + "user": "Alex Lee", + "period": "between start of February 2023 and end of May 2023" + }, + "verdict": "skip", + "site": "gitlab", + "output_schema": { + "type": "array", + "items": { + "type": "number" + } + } + } + ] +} diff --git a/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/meta.json b/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/meta.json new file mode 100644 index 0000000..da423fd --- /dev/null +++ b/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/meta.json @@ -0,0 +1,26 @@ +{ + "template": "What is the earliest nonstop flight from {{origin_city}} ({{origin_code}}) to {{destination_city}} ({{destination_code}}) on {{date}} (one-way)? Return the answer as a list: [flight_number, airline, departure_time], e.g. [\"AS 336\", \"Alaska\", \"6:00 AM\"].", + "provenance": "update-refined", + "site": "www.google.com", + "summary": "Refined from 3 gate-passed solves; parameterized + primitives.", + "signature": { + "params": [ + "origin_city", + "origin_code", + "destination_city", + "destination_code", + "date" + ], + "call": "python skill.py taskspec.json" + }, + "output_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "n_solves": 3, + "revisions": 1, + "verified": true, + "grade": "executable" +} \ No newline at end of file diff --git a/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/replays.json b/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/replays.json new file mode 100644 index 0000000..e39b522 --- /dev/null +++ b/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/replays.json @@ -0,0 +1,65 @@ +[ + { + "params": { + "origin_city": "Los Angeles", + "origin_code": "LAX", + "destination_city": "Chicago", + "destination_code": "ORD", + "date": "2026-08-15" + }, + "start_url": "https://www.google.com/flights", + "output_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "answer": [ + "UA 729", + "United", + "12:10 AM" + ] + }, + { + "params": { + "origin_city": "Seattle", + "origin_code": "SEA", + "destination_city": "New York", + "destination_code": "JFK", + "date": "2026-08-15" + }, + "start_url": "https://www.google.com/flights", + "output_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "answer": [ + "AS26", + "Alaska", + "7:00 AM" + ] + }, + { + "params": { + "origin_city": "San Francisco", + "origin_code": "SFO", + "destination_city": "Boston", + "destination_code": "BOS", + "date": "2026-08-15" + }, + "start_url": "https://www.google.com/flights", + "output_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "answer": [ + "B6 434", + "JetBlue", + "6:00 AM" + ] + } +] \ No newline at end of file diff --git a/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/skill.py b/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/skill.py new file mode 100644 index 0000000..bd43fc4 --- /dev/null +++ b/src/webwright/skill_factory/examples/learned_library/what_is_the_earliest_nonstop_flight_from_2c8dab1/skill.py @@ -0,0 +1,736 @@ +import asyncio +import base64 +import json +import os +import re +import sys +from datetime import datetime +from pathlib import Path +from urllib.parse import parse_qs, unquote, urlparse + +from playwright.async_api import async_playwright + + +# ---------------------------- +# Workspace / IO +# ---------------------------- + +WORKSPACE = Path(os.environ.get("WORKSPACE_DIR", ".")).resolve() +TASKSPEC_PATH = Path(sys.argv[1]).resolve() +TASKSPEC = json.loads(TASKSPEC_PATH.read_text(encoding="utf-8")) +PARAMS = TASKSPEC.get("params", {}) or {} +START_URL = TASKSPEC.get("start_url") or "https://www.google.com/flights" +OUTPUT_PATH = WORKSPACE / "agent_response.json" + +RUNS_DIR = WORKSPACE / "runs" +RUNS_DIR.mkdir(parents=True, exist_ok=True) +existing = [ + int(p.name.split("_")[-1]) + for p in RUNS_DIR.glob("run_*") + if p.name.split("_")[-1].isdigit() +] +RUN_ID = max(existing, default=0) + 1 +RUN_DIR = RUNS_DIR / f"run_{RUN_ID:03d}" +RUN_DIR.mkdir(parents=True, exist_ok=False) +SCREENSHOTS_DIR = RUN_DIR / "screenshots" +SCREENSHOTS_DIR.mkdir(parents=True, exist_ok=True) +LOG_PATH = RUN_DIR / "skill_log.txt" + +step_num = 0 + + +# ---------------------------- +# Logging / helpers +# ---------------------------- + +def log(msg: str) -> None: + print(msg, flush=True) + with LOG_PATH.open("a", encoding="utf-8") as f: + f.write(msg + "\n") + + +def next_step(desc: str) -> None: + global step_num + step_num += 1 + log(f"step {step_num}: {desc}") + + +async def snap(page, name: str) -> None: + try: + await page.screenshot( + path=str(SCREENSHOTS_DIR / f"{step_num:02d}_{name}.png"), + full_page=True, + ) + except Exception as e: + log(f"screenshot warning: {e!r}") + + +def normalize_space(text: str) -> str: + return re.sub(r"\s+", " ", (text or "").replace("\u202f", " ").replace("\xa0", " ")).strip() + + +def date_aria_label(date_str: str) -> str: + dt = datetime.strptime(date_str, "%Y-%m-%d") + return dt.strftime("%A, %B ") + str(dt.day) + dt.strftime(", %Y") + + +def time_key(t: str) -> int: + s = normalize_space(t).upper() + m = re.match(r"(\d{1,2}):(\d{2})\s*([AP]M)", s) + if not m: + raise ValueError(f"Unparseable time: {t}") + h = int(m.group(1)) % 12 + minute = int(m.group(2)) + if m.group(3) == "PM": + h += 12 + return h * 60 + minute + + +def canonical_airline_name(name: str) -> str: + n = normalize_space(name).lower() + mapping = { + "alaska": "Alaska", + "american": "American", + "delta": "Delta", + "frontier": "Frontier", + "hawaiian": "Hawaiian", + "jetblue": "JetBlue", + "jet blue": "JetBlue", + "spirit": "Spirit", + "sun country": "Sun Country", + "southwest": "Southwest", + "united": "United", + } + return mapping.get(n, name.strip()) + + +def airline_code_map(): + return { + "Alaska": "AS", + "American": "AA", + "Delta": "DL", + "Frontier": "F9", + "Hawaiian": "HA", + "JetBlue": "B6", + "Spirit": "NK", + "Sun Country": "SY", + "Southwest": "WN", + "United": "UA", + } + + +def code_to_airline_map(): + return {v: k for k, v in airline_code_map().items()} + + +def known_airlines_pattern() -> str: + airlines = sorted(airline_code_map().keys(), key=len, reverse=True) + return "(" + "|".join(re.escape(a) for a in airlines) + ")" + + +def ensure_output_schema(answer): + if not isinstance(answer, list): + raise RuntimeError("retrieved_data must be a list") + if len(answer) != 3: + raise RuntimeError(f"retrieved_data must have length 3, got {len(answer)}") + if not all(isinstance(x, str) for x in answer): + raise RuntimeError("retrieved_data items must all be strings") + + +def normalize_flight_number(code: str, number: str) -> str: + code = normalize_space(code).upper() + number = normalize_space(number) + if code in {"AS"}: + return f"{code}{number}" + return f"{code} {number}" + + +# ---------------------------- +# Playwright primitives +# ---------------------------- + +async def open_homepage(page, start_url: str) -> None: + await page.goto(start_url, wait_until="domcontentloaded") + await page.wait_for_timeout(2000) + + +async def set_one_way(page) -> None: + candidates = [ + page.get_by_role("combobox", name=re.compile(r"ticket type|change ticket type", re.I)).first, + page.get_by_role("combobox").first, + ] + for combo in candidates: + try: + if await combo.count(): + await combo.click() + await page.wait_for_timeout(500) + opt = page.get_by_role("option", name=re.compile(r"^One way$", re.I)).first + if await opt.count(): + await opt.click() + await page.wait_for_timeout(800) + return + except Exception: + pass + raise RuntimeError("Could not set trip type to one-way") + + +async def choose_airport(page, field_label_regex: str, airport_code: str) -> None: + box = page.get_by_role("combobox", name=re.compile(field_label_regex, re.I)) + await box.click() + await page.keyboard.press("Control+A") + await page.keyboard.press("Backspace") + await page.keyboard.type(airport_code) + await page.wait_for_timeout(1500) + + option_sets = [ + page.locator('[role="option"]'), + page.locator("li"), + page.locator('[role="listbox"] [role="button"]'), + ] + for options in option_sets: + count = min(await options.count(), 40) + for i in range(count): + try: + txt = normalize_space(await options.nth(i).inner_text()) + except Exception: + continue + if airport_code.upper() in txt.upper(): + try: + await options.nth(i).click(timeout=3000) + await page.wait_for_timeout(1000) + log(f"selected airport {airport_code}: {txt}") + return + except Exception: + continue + raise RuntimeError(f"Could not select airport {airport_code}") + + +async def set_departure_date(page, date_str: str) -> None: + await page.get_by_role("textbox", name=re.compile(r"Departure", re.I)).click() + await page.wait_for_timeout(800) + label = date_aria_label(date_str) + + candidates = [ + page.get_by_role("button", name=re.compile(rf"^{re.escape(label)}$", re.I)).first, + page.get_by_label(re.compile(rf"^{re.escape(label)}$", re.I)).first, + page.locator(f'[aria-label="{label}"]').first, + page.locator(f'text="{label}"').first, + ] + clicked = False + for c in candidates: + try: + if await c.count(): + await c.click(timeout=4000) + clicked = True + break + except Exception: + pass + if not clicked: + raise RuntimeError(f"Could not select date {label}") + + await page.wait_for_timeout(800) + for done in [ + page.get_by_role("button", name=re.compile(r"^(Done|Apply)$", re.I)).first, + page.locator('[aria-label="Done"]').first, + page.locator("text=/^Done$/i").first, + ]: + try: + if await done.count(): + await done.click(timeout=3000) + await page.wait_for_timeout(1000) + return + except Exception: + pass + try: + await page.keyboard.press("Escape") + await page.wait_for_timeout(800) + except Exception: + pass + + +async def click_search(page) -> None: + for btn in [ + page.get_by_role("button", name=re.compile(r"^Search$", re.I)).first, + page.get_by_role("button", name=re.compile(r"Search for flights", re.I)).first, + ]: + try: + if await btn.count(): + await btn.click(timeout=5000) + return + except Exception: + pass + raise RuntimeError("Could not find Search button") + + +async def wait_for_results(page, timeout_ms: int = 60000) -> str: + waited = 0 + while waited < timeout_ms: + body = normalize_space(await page.locator("body").inner_text()) + url = page.url + if ( + "/travel/flights/search" in url + or "google.com/travel/flights" in url + or "google.com/flights" in url + ) and any( + token in body.lower() + for token in ["results", "search results", "nonstop", "best departing flights", "top flights"] + ): + return body + await page.wait_for_timeout(2000) + waited += 2000 + return normalize_space(await page.locator("body").inner_text()) + + +async def apply_nonstop_filter(page) -> None: + buttons = page.get_by_role("button") + stop_button = None + for i in range(min(await buttons.count(), 100)): + b = buttons.nth(i) + try: + blob = normalize_space(((await b.get_attribute("aria-label")) or "") + " " + (await b.inner_text())) + except Exception: + continue + if re.search(r"\b(stops?|nonstop)\b", blob, re.I): + stop_button = b + log(f"stops button candidate: {blob}") + break + if stop_button is None: + raise RuntimeError("Could not find Stops filter control") + + await stop_button.click() + await page.wait_for_timeout(1200) + + nonstop_candidates = [ + page.get_by_role("radio", name=re.compile(r"Nonstop only|Nonstop", re.I)).first, + page.get_by_role("checkbox", name=re.compile(r"Nonstop only|Nonstop", re.I)).first, + page.get_by_role("option", name=re.compile(r"Nonstop only|Nonstop", re.I)).first, + page.get_by_role("button", name=re.compile(r"Nonstop only|Nonstop", re.I)).first, + page.get_by_label(re.compile(r"Nonstop only|Nonstop", re.I)).first, + page.locator('[aria-label*="Nonstop"]').first, + page.locator("text=/\\bNonstop( only)?\\b/i").first, + ] + chosen = False + for item in nonstop_candidates: + try: + if await item.count(): + await item.click(timeout=4000) + chosen = True + await page.wait_for_timeout(2500) + break + except Exception: + pass + if not chosen: + raise RuntimeError("Could not choose Nonstop filter") + + for done in [ + page.get_by_role("button", name=re.compile(r"^(Done|Apply)$", re.I)).first, + ]: + try: + if await done.count(): + await done.click(timeout=2500) + await page.wait_for_timeout(1500) + break + except Exception: + pass + + try: + await page.keyboard.press("Escape") + await page.wait_for_timeout(500) + except Exception: + pass + + +async def sort_by_departure_time(page) -> None: + try: + sort_candidates = [ + page.get_by_role("button", name=re.compile(r"Sorted by|sort order|Sort", re.I)).first, + page.get_by_text(re.compile(r"Sorted by", re.I)).first, + ] + sort_btn = None + for cand in sort_candidates: + try: + if await cand.count(): + sort_btn = cand + break + except Exception: + pass + if sort_btn is None: + return + + await sort_btn.click(timeout=4000) + await page.wait_for_timeout(1000) + + for loc in [ + page.get_by_text(re.compile(r"^Departure time$", re.I)).first, + page.get_by_role("button", name=re.compile(r"^Departure time$", re.I)).first, + page.get_by_role("option", name=re.compile(r"^Departure time$", re.I)).first, + page.get_by_role("menuitem", name=re.compile(r"^Departure time$", re.I)).first, + page.locator("text=Departure time").first, + ]: + try: + if await loc.count(): + await loc.click(timeout=3000) + await page.wait_for_timeout(3000) + break + except Exception: + continue + try: + await page.keyboard.press("Escape") + except Exception: + pass + except Exception as e: + log(f"sort warning: {e!r}") + + +# ---------------------------- +# Extraction primitives +# ---------------------------- + +def parse_nonstop_candidates_from_text(text: str, origin_code: str, destination_code: str): + txt = normalize_space(text) + airline_pat = known_airlines_pattern() + route_pat = rf"{re.escape(origin_code)}\s*[–-]\s*{re.escape(destination_code)}" + patterns = [ + re.compile( + rf"(\d{{1,2}}:\d{{2}}\s*[AP]M)\s*[–-]\s*(\d{{1,2}}:\d{{2}}\s*[AP]M)(?:\+1)?\s*{airline_pat}.*?{route_pat}.*?Nonstop", + re.I | re.S, + ), + re.compile( + rf"(\d{{1,2}}:\d{{2}}\s*[AP]M)\s*[–-]\s*(\d{{1,2}}:\d{{2}}\s*[AP]M)(?:\+1)?\s*{airline_pat}.*?Nonstop.*?{route_pat}", + re.I | re.S, + ), + ] + rows = [] + for pat in patterns: + for m in pat.finditer(txt): + dep = normalize_space(m.group(1)).upper() + airline = canonical_airline_name(m.group(3)) + rows.append({"departure_time": dep, "airline": airline, "source": normalize_space(m.group(0))}) + dedup = [] + seen = set() + for row in sorted(rows, key=lambda r: time_key(r["departure_time"])): + key = (row["departure_time"], row["airline"]) + if key not in seen: + seen.add(key) + dedup.append(row) + return dedup + + +async def collect_page_blobs(page): + blobs = [] + try: + blobs.append(normalize_space(await page.locator("body").inner_text())) + except Exception: + pass + try: + blobs.append(await page.content()) + except Exception: + pass + blobs.append(page.url) + try: + aria = await page.locator("body").aria_snapshot(timeout=8000) + blobs.append(str(aria)) + except Exception: + pass + return blobs + + +def _decode_tfs_base64_chunks(tfs: str): + out = [] + for m in re.finditer(r'([A-Za-z0-9+/]{2,20})', tfs or ""): + token = m.group(1) + if len(token) < 2: + continue + try: + padded = token + "=" * (-len(token) % 4) + val = base64.b64decode(padded).decode("utf-8", errors="ignore") + if val: + out.append(val) + except Exception: + pass + return out + + +def extract_flight_number_from_tfs(tfs: str, expected_airline: str | None = None, expected_departure: str | None = None): + tfs = tfs or "" + code_map = airline_code_map() + reverse = code_to_airline_map() + + # 1) Explicit itinerary=-XX-123-YYYYMMDD + for code, airline in reverse.items(): + if expected_airline and airline != expected_airline: + continue + m = re.search(rf'itinerary=[^"\'<>]*?-{re.escape(code)}-(\d{{1,4}})-\d{{8}}', tfs, re.I) + if m: + return normalize_flight_number(code, m.group(1)) + + decoded = " ".join(_decode_tfs_base64_chunks(tfs)) + decoded_norm = normalize_space(decoded) + if decoded_norm: + for code, airline in reverse.items(): + if expected_airline and airline != expected_airline: + continue + m = re.search(rf"\b{re.escape(code)}\s?(\d{{1,4}})\b", decoded_norm, re.I) + if m: + return normalize_flight_number(code, m.group(1)) + + # 2) Specific marker pattern seen in Google Flights tfs URLs. + m = re.search(r'KgJ([A-Za-z0-9+/]{2,8})jID([A-Za-z0-9+/]{2,12})KAB', tfs) + if m: + airline_chunk = m.group(1) + number_chunk = m.group(2) + try: + airline_raw = base64.b64decode(airline_chunk + "=" * (-len(airline_chunk) % 4)).decode("utf-8", errors="ignore") + except Exception: + airline_raw = "" + try: + number_raw = base64.b64decode(number_chunk + "=" * (-len(number_chunk) % 4)).decode("utf-8", errors="ignore") + except Exception: + number_raw = "" + number = re.sub(r"[^\d]", "", number_raw) + airline_code = normalize_space(airline_raw).upper() + if airline_code not in reverse: + if expected_airline: + airline_code = code_map.get(expected_airline, airline_code) + if airline_code == "" or airline_code == "\x08": + if expected_airline: + airline_code = code_map.get(expected_airline, airline_code) + if airline_code in reverse and number: + return normalize_flight_number(airline_code, number) + + # 3) Encoded / unquoted fallback + uq = unquote(tfs) + for code, airline in reverse.items(): + if expected_airline and airline != expected_airline: + continue + m = re.search(rf"\b{re.escape(code)}\s?(\d{{1,4}})\b", uq, re.I) + if m: + return normalize_flight_number(code, m.group(1)) + + return None + + +def extract_flight_number_from_blobs(blobs, airline: str, departure_time: str | None = None): + code = airline_code_map().get(airline, airline[:2].upper()) + joined = "\n".join(str(b) for b in blobs if b) + joined_norm = normalize_space(joined) + + patterns = [ + re.compile(rf"\bFlight\s*{re.escape(code)}\s?(\d{{1,4}})\b", re.I), + re.compile(rf"\b{re.escape(code)}\s?(\d{{1,4}})\b"), + re.compile(rf'itinerary=[^"\'<>]*?-{re.escape(code)}-(\d{{1,4}})-\d{{8}}', re.I), + ] + + for pat in patterns: + m = pat.search(joined_norm) + if m: + return normalize_flight_number(code, m.group(1)) + + # Search around airline/departure anchor if present. + if departure_time: + departure_time = normalize_space(departure_time) + anchor_variants = [ + f"at {departure_time}", + departure_time, + f"{airline}. Leaves", + ] + for anchor in anchor_variants: + idx = joined_norm.find(anchor) + if idx != -1: + snippet = joined_norm[max(0, idx - 1200): idx + 8000] + for pat in patterns: + m = pat.search(snippet) + if m: + return normalize_flight_number(code, m.group(1)) + + # Parse tfs from any URLs in blobs. + for blob in blobs: + s = str(blob) + for match in re.finditer(r'https?://[^\s"\'<>]+', s): + url = match.group(0) + try: + tfs = parse_qs(urlparse(url).query).get("tfs", [""])[0] + except Exception: + tfs = "" + if tfs: + val = extract_flight_number_from_tfs(tfs, expected_airline=airline, expected_departure=departure_time) + if val: + return val + + # Also parse page.url-like raw text directly. + for blob in blobs: + s = str(blob) + try: + tfs = parse_qs(urlparse(s).query).get("tfs", [""])[0] + except Exception: + tfs = "" + if tfs: + val = extract_flight_number_from_tfs(tfs, expected_airline=airline, expected_departure=departure_time) + if val: + return val + + return None + + +async def find_and_open_earliest_row(page, earliest): + dep = earliest["departure_time"] + airline = earliest["airline"] + dep_nbsp = dep.replace(" ", "\u202f") + + locators = [ + page.locator(f'div[role="link"][aria-label*="{airline}"][aria-label*="{dep_nbsp}"]').first, + page.locator(f'div[role="link"][aria-label*="{airline}"][aria-label*="{dep}"]').first, + page.locator(f'[role="link"][aria-label*="{dep_nbsp}"]').first, + page.locator(f'[role="link"][aria-label*="{dep}"]').first, + page.get_by_text(re.compile(rf"^{re.escape(dep)}$", re.I)).first, + ] + + for idx, loc in enumerate(locators): + try: + if await loc.count(): + await loc.scroll_into_view_if_needed(timeout=2000) + await page.wait_for_timeout(300) + await loc.click(force=True, timeout=4000) + await page.wait_for_timeout(3500) + log(f"opened row via locator {idx}") + return True + except Exception as e: + log(f"row open locator {idx} failed: {e!r}") + + # JS fallback: click an element containing the departure time. + try: + clicked = await page.evaluate( + """(dep) => { + const norm = s => (s || '').replace(/\\s+/g, ' ').trim(); + const els = Array.from(document.querySelectorAll('*')); + for (const el of els) { + const txt = norm(el.innerText); + const aria = norm(el.getAttribute('aria-label')); + const role = el.getAttribute('role') || ''; + if ((txt === dep || aria.includes(dep)) && (role === 'link' || el.closest('[role="link"]'))) { + const target = role === 'link' ? el : el.closest('[role="link"]'); + if (target) { target.click(); return true; } + } + } + return false; + }""", + dep, + ) + if clicked: + await page.wait_for_timeout(3500) + log("opened row via JS fallback") + return True + except Exception as e: + log(f"row open JS fallback failed: {e!r}") + + return False + + +# ---------------------------- +# Thin task layer +# ---------------------------- + +async def retrieve_earliest_nonstop_flight(page, params): + next_step("open Google Flights homepage") + await open_homepage(page, START_URL) + await snap(page, "open_home") + + next_step("set trip to one-way") + await set_one_way(page) + await snap(page, "set_one_way") + + next_step("set route airports") + await choose_airport(page, r"Where from", params["origin_code"]) + await choose_airport(page, r"Where to", params["destination_code"]) + await snap(page, "set_route") + + next_step("set departure date") + await set_departure_date(page, params["date"]) + await snap(page, "set_date") + + next_step("search for flights") + await click_search(page) + body = await wait_for_results(page) + log("results url: " + page.url) + log("results body snippet: " + body[:5000]) + await snap(page, "results_loaded") + + next_step("apply nonstop filter") + await apply_nonstop_filter(page) + body = await wait_for_results(page, timeout_ms=20000) + log("filtered body snippet: " + body[:6000]) + await snap(page, "nonstop_applied") + + next_step("sort by departure time when available") + await sort_by_departure_time(page) + body = await wait_for_results(page, timeout_ms=15000) + log("post-sort body snippet: " + body[:6000]) + await snap(page, "sorted") + + next_step("parse visible nonstop rows and choose earliest") + rows = parse_nonstop_candidates_from_text(body, params["origin_code"], params["destination_code"]) + if not rows: + # fallback using broader body text after collecting fresh content + more = normalize_space(await page.locator("body").inner_text()) + rows = parse_nonstop_candidates_from_text(more, params["origin_code"], params["destination_code"]) + if not rows: + raise RuntimeError("Could not parse any nonstop rows from results text") + + earliest = sorted(rows, key=lambda r: time_key(r["departure_time"]))[0] + log("parsed nonstop candidates: " + json.dumps(rows[:10])) + log("earliest row parsed: " + json.dumps(earliest)) + + next_step("open earliest row to improve flight number extraction") + opened = await find_and_open_earliest_row(page, earliest) + log(f"opened earliest row: {opened}") + await snap(page, "earliest_row") + + next_step("extract flight number from detail/page blobs") + blobs = await collect_page_blobs(page) + flight_number = extract_flight_number_from_blobs(blobs, earliest["airline"], earliest["departure_time"]) + + # If opening row failed or no flight found, retry from results page blobs too. + if not flight_number: + try: + await page.goto(page.url, wait_until="domcontentloaded") + await page.wait_for_timeout(2500) + except Exception: + pass + retry_blobs = await collect_page_blobs(page) + flight_number = extract_flight_number_from_blobs(retry_blobs, earliest["airline"], earliest["departure_time"]) + + if not flight_number: + raise RuntimeError("Could not extract flight number from page details") + + answer = [flight_number, earliest["airline"], earliest["departure_time"]] + ensure_output_schema(answer) + return answer + + +async def main(): + LOG_PATH.write_text("", encoding="utf-8") + + required = ["origin_city", "origin_code", "destination_city", "destination_code", "date"] + missing = [k for k in required if not PARAMS.get(k)] + if missing: + raise RuntimeError(f"Missing required params: {missing}") + + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context( + viewport={"width": 1280, "height": 1800}, + locale="en-US", + ) + page = await context.new_page() + answer = await retrieve_earliest_nonstop_flight(page, PARAMS) + await browser.close() + + payload = {"retrieved_data": answer} + ensure_output_schema(payload["retrieved_data"]) + OUTPUT_PATH.write_text(json.dumps(payload, indent=2), encoding="utf-8") + log("final answer: " + json.dumps(answer)) + log("wrote: " + str(OUTPUT_PATH)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/webwright/skill_factory/examples/model_gateway.example.yaml b/src/webwright/skill_factory/examples/model_gateway.example.yaml new file mode 100644 index 0000000..106ed3a --- /dev/null +++ b/src/webwright/skill_factory/examples/model_gateway.example.yaml @@ -0,0 +1,12 @@ +# Agent model config for a custom OpenAI-compatible gateway. +# Copy this file, fill in your values, then: export MODEL_CFG=/abs/path/to/your_copy.yaml +# +# NOTE: openai_endpoint is the FULL request URL (including the /responses path), +# not a base URL. ".../api" alone will fail; ".../api/responses" works. +model: + model_class: openai + model_name: your-model-name + openai_endpoint: https://your-gateway.example.com/api/responses + # large final scripts need room; slow gateways need patience + max_output_tokens: 16000 + request_timeout_seconds: 600 diff --git a/src/webwright/skill_factory/examples/quickstart.sh b/src/webwright/skill_factory/examples/quickstart.sh new file mode 100755 index 0000000..e5e258f --- /dev/null +++ b/src/webwright/skill_factory/examples/quickstart.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# One-command Quickstart β€” every parameter pre-filled, nothing to write. +# +# ./quickstart.sh # instant: run the checked-in flight skill, NO model, no key +# ./quickstart.sh demo LAX ORD # ...on YOUR route (any airport codes, default date) +# ./quickstart.sh demo LAX ORD 2026-09-01 # ...and YOUR date +# ./quickstart.sh ask # ask the library about a new task (needs OPENAI_API_KEY) +# ./quickstart.sh solve # one agent solve that REUSES the checked-in skill (needs key) +# ./quickstart.sh full # the whole loop: 3 solves -> learn -> reuse (needs key, ~40 min) +# +# Custom / OpenAI-compatible gateway? Two knobs, both needed: +# export OPENAI_ENDPOINT=... OPENAI_MODEL=... (for learn / skill_use) +# export MODEL_CFG=/path/to/your_model.yaml (for the agent in solve/full β€” copy +# model_openai.yaml and set openai_endpoint/model_name; env vars do NOT reach it) +set -euo pipefail +SELF="$(readlink -f "$0")" +cd "$(dirname "$SELF")" +DATE=$(date -d "+30 days" +%Y-%m-%d 2>/dev/null || date -v+30d +%Y-%m-%d) +WORK="${QUICKSTART_WORKDIR:-$(mktemp -d /tmp/skills_quickstart.XXXX)}" +LIB="$PWD/learned_library" +CFG=(-c base.yaml -c "${MODEL_CFG:-model_openai.yaml}") + +need_key() { : "${OPENAI_API_KEY:?export OPENAI_API_KEY first (on a gateway also OPENAI_ENDPOINT / OPENAI_MODEL)}"; } + +warn_gateway_agent() { # solve/full: the AGENT reads its yaml, not the env vars + if [ -n "${OPENAI_ENDPOINT:-}" ] && [ -z "${MODEL_CFG:-}" ]; then + echo "!! OPENAI_ENDPOINT is set but MODEL_CFG is not." >&2 + echo "!! learn/ask will use your gateway, but the AGENT in this mode reads a yaml" >&2 + echo "!! and will hit api.openai.com. Copy model_gateway.example.yaml, fill in your" >&2 + echo "!! endpoint (the FULL .../responses URL), then: export MODEL_CFG=/abs/path.yaml" >&2 + fi +} + +flight_task() { # $1 "City (CODE)" $2 "City (CODE)" + echo "What is the earliest nonstop flight from $1 to $2 on $DATE (one-way)? Return the answer as a list: [flight_number, airline, departure_time], e.g. [\"AS 336\", \"Alaska\", \"6:00 AM\"]." +} + +spec() { # $1 code $2 code $3 date -> taskspec.json in $WORK. The skill drives the site by + # airport CODE; the *_city params are required by its signature but unused, so the + # code doubles as the city and you only ever type the codes. + cat > "$WORK/taskspec.json" <$TO on $ON (no model, ~40 s) ==" + # only pitch the custom-route form when the user hasn't already given one + [ $# -ge 3 ] || echo " (try your own route: $0 demo LAX ORD 2026-09-01)" + case "$ON" in + [0-9][0-9][0-9][0-9]-[0-9]*-[0-9]*) ;; + *) echo "!! date must be YYYY-MM-DD (e.g. 2026-09-01), got: $ON" >&2; exit 1 ;; + esac + spec "$FROM" "$TO" "$ON" + (cd "$WORK" && WORKSPACE_DIR="$WORK" python "$(ls -d "$LIB"/what_is_the_earliest_nonstop_flight_*)/skill.py" taskspec.json > run.log 2>&1) || { tail -5 "$WORK/run.log"; exit 1; } + echo + echo "-- what it did (no model chose these steps β€” they are the skill's code) --" + sed -n 's/^\(step [0-9]*:\)/ \1/p' "$WORK"/runs/run_*/skill_log.txt 2>/dev/null || true + echo + echo "answer: $(cat "$WORK/agent_response.json")" + SHOTS=$(ls "$WORK"/runs/run_*/screenshots/*.png 2>/dev/null | wc -l) + echo "evidence: $SHOTS screenshots + step log ->" + echo " $(ls -d "$WORK"/runs/run_* 2>/dev/null | head -1)" + echo "-> a learned skill just drove the live site with ZERO tokens. Next: $0 ask | solve | full" + ;; +ask) + need_key + echo "== asking the library about a route it has never seen (one LLM round trip) ==" + python -m webwright.tools.skill_use \ + --task "$(flight_task 'Portland (PDX)' 'Austin (AUS)')" --library "$LIB" + ;; +solve) + need_key + warn_gateway_agent + echo "== one agent solve on an UNSEEN route, reusing the checked-in skill ==" + ./solve_with_library.sh "$(flight_task 'Portland (PDX)' 'Austin (AUS)')" \ + https://www.google.com/flights "$LIB" -o "$WORK/outputs" --task-id qs_solve "${CFG[@]}" + echo "skill decision: $(cat "$WORK"/outputs/qs_solve_*/skill_decision.json 2>/dev/null || echo '(missing)')" + echo "answer: $(cat "$WORK"/outputs/qs_solve_*/agent_response.json 2>/dev/null || echo '(missing)')" + ;; +full) + need_key + warn_gateway_agent + echo "== full loop: 3 from-scratch solves -> learn -> reuse on an unseen route (~40 min) ==" + for r in "Seattle (SEA)|New York (JFK)" "San Francisco (SFO)|Boston (BOS)" "Los Angeles (LAX)|Chicago (ORD)"; do + FROM="${r%|*}"; TO="${r#*|}" + echo "-- solving $FROM -> $TO from scratch" + ./solve_with_library.sh "$(flight_task "$FROM" "$TO")" \ + https://www.google.com/flights "$WORK/library" -o "$WORK/outputs" "${CFG[@]}" + done + echo "-- learning (verify=strict: a schedule is stable, so each skill must reproduce + its own training answers standalone before it may land)" + python -m webwright.skill_factory learn "$WORK/outputs" --library "$WORK/library" --verify strict --verify-rounds 3 + echo "-- reusing on an unseen route" + ./solve_with_library.sh "$(flight_task 'Seattle (SEA)' 'Denver (DEN)')" \ + https://www.google.com/flights "$WORK/library" -o "$WORK/outputs" --task-id qs_heldout "${CFG[@]}" + echo "skill decision: $(cat "$WORK"/outputs/qs_heldout_*/skill_decision.json 2>/dev/null || echo '(missing)')" + echo "library now at: $WORK/library" + ;; +*) + # print the whole header comment β€” robust to edits, unlike a fixed line range + awk 'NR>1 && /^#/ {print; next} NR>1 {exit}' "$SELF"; exit 1 + ;; +esac +echo "(work dir: $WORK)" diff --git a/src/webwright/skill_factory/examples/solve_with_library.sh b/src/webwright/skill_factory/examples/solve_with_library.sh new file mode 100755 index 0000000..c44a683 --- /dev/null +++ b/src/webwright/skill_factory/examples/solve_with_library.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Solve a task WITH the library β€” prepends the skill hint and the answer-output +# instruction, then hands off to webwright. Usage: +# ./solve_with_library.sh "task text" START_URL /abs/path/to/library [webwright args...] +if [ $# -lt 3 ]; then + echo "usage: $0 \"task text\" START_URL /abs/path/to/library [webwright args...]" >&2 + exit 1 +fi +TASK="$1"; URL="$2"; LIB="$3"; shift 3 +SPEC='Additionally, write the final answer into $WORKSPACE_DIR/agent_response.json as {"retrieved_data": }.' +PROMPT=$(python -c 'import sys; from webwright.skill_factory import with_skill_hint +print(with_skill_hint(sys.argv[1] + " " + sys.argv[3], task=sys.argv[1], library=sys.argv[2]))' "$TASK" "$LIB" "$SPEC") +exec python -m webwright.run.cli main -t "$PROMPT" --start-url "$URL" "$@" diff --git a/src/webwright/skill_factory/examples/tasks.example.json b/src/webwright/skill_factory/examples/tasks.example.json new file mode 100644 index 0000000..898417a --- /dev/null +++ b/src/webwright/skill_factory/examples/tasks.example.json @@ -0,0 +1,35 @@ +[ + { + "id": "commits_a", + "task": "How many commits did Jane Doe make on January 5th 2023 in the current repository?", + "params": { + "user": "Jane Doe", + "period": "on January 5th 2023" + }, + "gold": [ + 1 + ] + }, + { + "id": "commits_b", + "task": "How many commits did John Smith make on April 7th 2022 in the current repository?", + "params": { + "user": "John Smith", + "period": "on April 7th 2022" + }, + "gold": [ + 2 + ] + }, + { + "id": "commits_c", + "task": "How many commits did Alex Lee make between start of February 2023 and end of May 2023 in the current repository?", + "params": { + "user": "Alex Lee", + "period": "between start of February 2023 and end of May 2023" + }, + "gold": [ + 7 + ] + } +] diff --git a/src/webwright/skill_factory/gate.py b/src/webwright/skill_factory/gate.py new file mode 100644 index 0000000..2740319 --- /dev/null +++ b/src/webwright/skill_factory/gate.py @@ -0,0 +1,73 @@ +"""Admission gate: only "correct" solves/skills enter the library, preventing correct-but-narrow / +regression pollution. The gate is an INDEPENDENT second eye β€” distinct from the solving agent's own +self_reflection (which is a solve-completion condition, not an admission check). + +Stable interface (swappable implementation), configurable method: + gate(result, *, gold=None, output_schema=None, method="auto", status="") -> GateResult + +- method="gold" : compare against gold (benchmarks like WebArena; truly independent, catches + mis-extracted solves). Recommended. +- method="self_verify" : invariants (result non-empty + shape matches output_schema) plus the + agent's OWN final report: a run whose agent_response.json says anything + but SUCCESS (e.g. NOT_FOUND_ERROR) is rejected β€” the agent itself did not + believe the answer. Costs nothing; no extra model call. + Limitation: still self-grading β€” a wrong answer the agent BELIEVED is + admitted anyway. + (Note: webwright's self_reflection is always predicted_label==1 due to + require_self_reflection_success, so it cannot serve as the gate β€” that is a + solve-completion condition, not independent admission.) +- method="none" : no gate (demo reuse only, no pollution protection). +- method="auto" : use gold if available, else self_verify. +Upgrade path (next step): for real websites use WebJudge (OM2W's official judge) or cross-source +consistency checks for a truly independent gate. +""" +from __future__ import annotations +from dataclasses import dataclass + + +@dataclass +class GateResult: + admit: bool + reason: str + + +def _shape_ok(result, output_schema) -> bool: + if not output_schema: + return True + t = output_schema.get("type") + if t == "array": + return isinstance(result, list) + if t == "object": + return isinstance(result, dict) + if t in ("string",): + return isinstance(result, str) + if t in ("number", "integer"): + return isinstance(result, (int, float)) and not isinstance(result, bool) + return True + + +def _self_verify(result, output_schema, status="") -> GateResult: + if status and status != "SUCCESS": + return GateResult(False, f"agent itself reported {status}") + if result is None: + return GateResult(False, "result is null") + if isinstance(result, (list, dict, str)) and len(result) == 0: + return GateResult(False, "result is empty") + if not _shape_ok(result, output_schema): + return GateResult(False, f"shape != output_schema ({output_schema.get('type')})") + return GateResult(True, "self-verify passed (non-empty, shape ok)") + + +def _gold(result, gold) -> GateResult: + if result == gold: + return GateResult(True, "matches gold") + return GateResult(False, "differs from gold") + + +def gate(result, *, gold=None, output_schema=None, method: str = "auto", + status: str = "") -> GateResult: + if method == "none": + return GateResult(True, "no gate (admit all)") + if method == "gold" or (method == "auto" and gold is not None): + return _gold(result, gold) + return _self_verify(result, output_schema, status=status) diff --git a/src/webwright/skill_factory/init.py b/src/webwright/skill_factory/init.py new file mode 100644 index 0000000..b85940c --- /dev/null +++ b/src/webwright/skill_factory/init.py @@ -0,0 +1,113 @@ +"""python -m webwright.skill_factory init "" β€” draft a skill spec you then fill in. + +One LLM call turns a natural-language need into a skill.yaml SKELETON: a task template with +{parameter} holes, a *guessed* start_url, and empty instance rows for YOU to fill with real +values. It deliberately does NOT invent the values β€” those are the ground truth you own, and a +wrong guessed value would quietly train the skill on the wrong answer. Review the file (the +guesses are marked), fill the rows, then run `build skill.yaml`. +""" +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +from .llm import llm_json + +_SYS = ( + "You turn a user's one-line description of a web task into a REUSABLE TEMPLATE. " + "Return STRICT JSON: {\"task\": \"\", " + "\"params\": [\"\", ...], \"start_url\": \"\"}.\n" + "The user almost always describes ONE CONCRETE INSTANCE of a task they will repeat with " + "different values ('the cheapest makeup remover on Amazon' means 'the cheapest {product} on " + "Amazon'). GENERALIZE: every concrete value they mention β€” a product, a place, a date, a " + "name, a count β€” becomes a {hole}. Do not echo their example values back; the task must " + "contain holes, not their specifics.\n" + "Keep genuinely site-fixed things literal (the site itself, the phrasing). Every {hole} in " + "task MUST appear in params and vice-versa. Ask for the answer in a stable, unambiguous form. " + "Only if the need truly has nothing that could vary, return params: [].\n" + "Also judge whether this task's ANSWER DRIFTS. The test is narrow: **run the same task again " + "tomorrow, changing nothing β€” is the correct answer still the same string?** Being fetched " + "live from a busy website does NOT make an answer drift; only the answer moving does.\n" + "Same site, both cases: 'the earliest nonstop flight from SEA to JFK on 2026-08-15' does NOT " + "drift β€” a schedule is published weeks ahead and reads the same tomorrow. 'the cheapest " + "flight on that route' DOES drift β€” the fare moves hourly. So: prices, fares, stock, " + "'today's top seller', anything ranked by a live number β†’ drifts. Schedules, specs, IDs, " + "counts of past things, published text β†’ does not.\n" + "Return \"drifts\": true|false, and \"drift_reason\": \"\"." +) + + +def _yaml_skeleton(task: str, params: list[str], start_url: str, rows: int, + drifts: bool = False) -> str: + cols = ", ".join(f"{p}: \"____\"" for p in params) + instance_lines = "\n".join(f" - {{{cols}}}" for _ in range(rows)) + # strict compares the replay against the recorded answer, so it is only fair when the + # answer holds still. On a drifting answer (a price, stock, a ranking) strict rejects a + # working skill for doing its job β€” pick shape up front rather than let the user find out + # after paying for the solves. + verify = "shape " if drifts else "strict" + why = ("# this answer drifts (prices/stock/rankings change on their own), so replay only\n" + " # checks the shape β€” strict would reject a working skill when the value moved\n " + if drifts else + "# this answer should hold still, so replay demands the recorded answer back\n ") + return ( + f"# Draft skill spec β€” fill the ____ values (your ground truth), then: build skill.yaml\n" + f"# The {{holes}} in `task` are the parameters; each is a column below.\n\n" + f"task: {task}\n" + f"start_url: {start_url} # guessed β€” check it opens the right page\n\n" + f"instances: # give a few real instances (3+ makes a verifiable skill)\n" + f"{instance_lines}\n\n" + f"build: # optional policy β€” CLI flags override these\n" + f" {why}" + f"verify: {verify} # strict (reproduce answers) | shape (drifting data) | off\n" + f" verify_rounds: 2\n" + f" on_fail: reject # reject | reference\n" + f" chunk: 25\n" + ) + + +def init(need: str, out_path: str, rows: int = 3) -> int: + out = Path(out_path) + if out.exists(): + raise SystemExit(f"init: {out} already exists β€” remove it or pass -o another path.") + data = llm_json(_SYS, f"Task need: {need}") + task = (data.get("task") or "").strip() + params = [str(p) for p in (data.get("params") or [])] + start_url = (data.get("start_url") or "").strip() + holes = set(re.findall(r"{(\w+)}", task)) + if not task or not holes: + # No holes means the need names ONE task, not a task TYPE β€” and a skill is only worth + # building for something you will repeat with different values. + raise SystemExit( + f"init: nothing in this need varies, so there is no reusable skill to build:\n" + f" {task or '(no task returned)'}\n\n" + f"Say what CHANGES between runs β€” name the varying part:\n" + f" instead of 'the cheapest makeup remover on Amazon'\n" + f" try 'the cheapest on Amazon, for any product'\n\n" + f"If you really only want this one answer once, you don't need a skill β€” solve it " + f"directly (webwright), or use /webwright:craft to get a re-runnable CLI for it.") + # trust the holes actually in the template over a possibly-mismatched params list + params = [p for p in params if p in holes] or sorted(holes) + drifts = bool(data.get("drifts")) + out.write_text(_yaml_skeleton(task, params, start_url, rows, drifts), encoding="utf-8") + print(f"wrote {out}\n\n task: {task}\n params: {params}\n start_url: {start_url}\n verify: {'shape (this answer drifts)' if drifts else 'strict (this answer should hold still)'}\n\n" + f"Next: fill the ____ values in {out}, then\n" + f" python -m webwright.skill_factory build {out} --library ./library -c your_model.yaml") + return 0 + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(prog="python -m webwright.skill_factory init", + description="Draft a skill.yaml skeleton from a one-line need.") + p.add_argument("need", help="One-line description of the repeatable task you want a skill for.") + p.add_argument("-o", "--out", default="skill.yaml", help="Where to write the spec.") + p.add_argument("--rows", type=int, default=3, help="Empty instance rows to leave (default 3).") + a = p.parse_args(argv) + return init(a.need, a.out, rows=a.rows) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/webwright/skill_factory/learn.py b/src/webwright/skill_factory/learn.py new file mode 100644 index 0000000..4a46d68 --- /dev/null +++ b/src/webwright/skill_factory/learn.py @@ -0,0 +1,207 @@ +"""learn β€” the friendly entry: turn a folder of finished runs into library skills. + + python -m webwright.skill_factory learn [--library ./library] [--golds golds.json] + [--chunk 25] [--dry-run] + +No manifest to write. For every run dir under it reads task.json (task text, +start_url) and agent_response.json (the answer), gates it (gold if --golds has this +task_id, else self_verify), asks the LLM ONCE per chunk to group tasks into templates and +extract per-task params, then feeds the groups to evolve. Idempotent: processed run dirs +are remembered in /.learned.json and skipped next time. The generated manifest +of each chunk is saved next to the ledger for auditing. +""" +from __future__ import annotations + +import json +from pathlib import Path +from urllib.parse import urlparse + +from .gate import gate +from .library import Library +from .llm import llm_json +from .update import Trace, evolve + + +def infer_schema(answer): + """Mechanically derive output_schema from the answer's shape.""" + if isinstance(answer, list): + item = answer[0] if answer else "" + t = ("number" if isinstance(item, (int, float)) and not isinstance(item, bool) + else "object" if isinstance(item, dict) else "string") + return {"type": "array", "items": {"type": t}} + if isinstance(answer, (int, float)) and not isinstance(answer, bool): + return {"type": "number"} + if isinstance(answer, dict): + return {"type": "object"} + return {"type": "string"} + + +def collect_runs(runs_dir: Path, ledger: dict): + """[{dir, task_id, task, start_url, answer}] for finished runs not yet learned.""" + out = [] + skipped_no_answer = 0 + for d in sorted(Path(runs_dir).iterdir()): + if not d.is_dir() or str(d.resolve()) in ledger["runs"]: + continue + tj, ar = d / "task.json", d / "agent_response.json" + if not tj.exists(): + continue + if not ar.exists(): + skipped_no_answer += 1 + print(f" skip {d.name}: no agent_response.json") + continue + try: + t = json.loads(tj.read_text(encoding="utf-8")) + resp = json.loads(ar.read_text(encoding="utf-8")) + answer, status = resp.get("retrieved_data"), resp.get("status", "") + except Exception as e: + print(f" skip {d.name}: unreadable ({e})") + continue + # strip pipeline text the wrapper may have carried into the prompt: the + # skill-library hint and the answer-output instruction must not leak into templates + task = t.get("task", "") + if "## Skill library" in task: + task = task.split("---", 1)[-1].strip() + if "Additionally, write the final answer into" in task: + task = task.split("Additionally, write the final answer into", 1)[0].strip() + out.append({"dir": str(d.resolve()), "task_id": t.get("task_id", d.name), + "task": task, "start_url": t.get("start_url", ""), + "answer": answer, "status": status}) + if skipped_no_answer: + print(f" ! {skipped_no_answer} run(s) had no answer file and were skipped β€” their solves " + f"cannot be aggregated. Solve via examples/solve_with_library.sh (it adds the " + f"answer-output instruction), or see README 'Manual mode' step 1.") + return out + + +_GROUP_SYS = ( + "You organize solved web tasks into task TEMPLATES. Tasks are instances of the same " + "template when they differ only in parameter values (names, dates, places, counts).\n" + "You are given existing template strings and a numbered task list. Return STRICT JSON:\n" + '{"groups": [{"template": "sentence with {{param}} placeholders", ' + '"members": [{"i": , "params": {"": "", ...}}]}]}\n' + "Rules: if a task matches an EXISTING template, use that exact template string verbatim. " + "Every task index appears in exactly one group. A group may have a single member. " + "Params must be the concrete values from the task text." +) + + +def group_chunk(runs, existing_templates): + listing = "\n".join(f"{i}: {r['task'][:220]}" for i, r in enumerate(runs)) + existing = "\n".join(f"- {t}" for t in existing_templates) or "(none yet)" + try: + out = llm_json(_GROUP_SYS, f"## Existing templates\n{existing}\n\n## Tasks\n{listing}") + except Exception as exc: + raise SystemExit( + f"learn: the grouping LLM call failed: {exc}\n" + f"Check OPENAI_API_KEY β€” and on a custom gateway also set " + f"OPENAI_ENDPOINT (and OPENAI_MODEL), or SKILL_MODEL_ENDPOINT/SKILL_MODEL_NAME. " + f"The endpoint is the FULL request URL (e.g. https://gateway.example/api/responses), " + f"not a base path.") + return out.get("groups", []) + + +def learn(runs_dir, library_root, golds=None, chunk=25, dry_run=False, verify="strict", + rounds=2, on_fail="reject"): + lib = Library(library_root) + ledger_path = Path(library_root) / ".learned.json" + ledger = json.loads(ledger_path.read_text(encoding="utf-8")) if ledger_path.exists() else {"runs": {}} + golds = golds or {} + + runs = collect_runs(Path(runs_dir), ledger) + if not runs: + print("nothing new to learn"); return + + # gate first β€” wrong solves never reach the grouping step + admitted = [] + for r in runs: + g = (gate(r["answer"], gold=golds[r["task_id"]], method="gold") + if r["task_id"] in golds + else gate(r["answer"], method="self_verify", status=r.get("status", ""))) + r["admit"] = g.admit + if g.admit: + admitted.append(r) + else: + print(f" gate βœ— {r['task_id']}: {g.reason}") + print(f"{len(admitted)}/{len(runs)} runs admitted by gate " + f"({'gold' if golds else 'self_verify'})") + if not golds: + print(" ! gate=self_verify: shape check + the agent's own SUCCESS report β€” an answer " + "the agent wrongly believed still PASSES. Pass --golds for real verification.") + if not admitted: + return + + for lo in range(0, len(admitted), chunk): + batch = admitted[lo:lo + chunk] + existing = [s.meta.get("template", "") for s in lib.list()] + groups = group_chunk(batch, existing) + print(f"\nchunk {lo // chunk + 1}: {len(batch)} runs -> {len(groups)} template(s)") + for g in groups: + members = [m for m in g.get("members", []) if 0 <= m.get("i", -1) < len(batch)] + print(f" {g.get('template', '?')[:90]} ({len(members)} solve(s))") + if dry_run: + continue + for g in groups: + tmpl = g.get("template", "") + traces = [] + for m in g.get("members", []): + if not (0 <= m.get("i", -1) < len(batch)): + continue + r = batch[m["i"]] + code_p = Path(r["dir"]) / "final_script.py" + traces.append(Trace( + template=tmpl, + code=code_p.read_text(encoding="utf-8") if code_p.exists() else "", + answer=r["answer"], correct=True, + # existing template -> mark adapt so evolve REFINES instead of ignoring + verdict="adapt" if tmpl in existing else "skip", + meta={"params": m.get("params", {}), + "site": urlparse(r["start_url"]).netloc, + "start_url": r["start_url"], + "output_schema": infer_schema(r["answer"])})) + if not traces: + continue + log = evolve(traces, lib, verify=verify, rounds=rounds, on_fail=on_fail) + print(f" evolve: {json.dumps(log)}") + if log.get("rejected"): + # skill did not land -> leave these runs OUT of the ledger so a later + # learn (fixed model/site/verify mode) can try them again + print(f" ! runs kept un-learned (skill rejected) β€” re-run learn to retry") + continue + for m in g.get("members", []): + if 0 <= m.get("i", -1) < len(batch): + ledger["runs"][batch[m["i"]]["dir"]] = {"template": tmpl} + # audit trail + idempotence, saved per chunk + ledger_path.write_text(json.dumps(ledger, indent=2), encoding="utf-8") + if not dry_run: + print(f"\nlibrary now has {len(lib.list())} skill(s); ledger -> {ledger_path}") + + +def main(argv=None) -> int: + import argparse + p = argparse.ArgumentParser(prog="python -m webwright.skill_factory learn", + description="Distill a folder of finished runs into library skills.") + p.add_argument("runs_dir", help="Folder containing webwright run directories.") + p.add_argument("--library", default="library") + p.add_argument("--golds", default="", help="JSON file {task_id: gold_answer} -> gold gate.") + p.add_argument("--chunk", type=int, default=25, help="Runs per LLM grouping call.") + p.add_argument("--dry-run", action="store_true", help="Show the grouping plan, change nothing.") + p.add_argument("--verify-rounds", type=int, default=2, + help="Total build attempts (first + repairs) before giving up. Default 2.") + p.add_argument("--on-fail", default="reject", choices=["reject", "reference"], + help="Failed verification: reject (default; runs stay retryable) or land as " + "grade=reference β€” readable prior for the agent, standalone NOT trusted.") + p.add_argument("--verify", default="strict", choices=["off", "shape", "strict"], + help="A skill must REPLAY its own training taskspecs standalone before it may " + "enter the library. strict (default): it must reproduce the recorded " + "answers; shape: any non-empty, schema-shaped answer passes β€” use this for " + "task families whose answers are live data (prices, listings); off: skip.") + a = p.parse_args(argv) + golds = json.loads(Path(a.golds).read_text(encoding="utf-8")) if a.golds else {} + learn(a.runs_dir, a.library, golds=golds, chunk=a.chunk, dry_run=a.dry_run, + verify=a.verify, rounds=a.verify_rounds, on_fail=a.on_fail) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/webwright/skill_factory/library.py b/src/webwright/skill_factory/library.py new file mode 100644 index 0000000..1a4286c --- /dev/null +++ b/src/webwright/skill_factory/library.py @@ -0,0 +1,60 @@ +"""Skill store. A skill = a directory under the library root holding skill.py + meta.json. + +Interface (stable β€” implementations behind it may change): + Library(root).list() -> [Skill] + Library(root).get(skill_id) -> Skill | None + Library(root).add(skill) # write skill.py + meta.json +""" +from __future__ import annotations +import json +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass +class Skill: + skill_id: str + code: str # source of skill.py + meta: dict = field(default_factory=dict) # {template, site, signature, summary, ...} + + @property + def summary(self) -> str: + return self.meta.get("summary", "") + + @property + def signature(self) -> dict: + return self.meta.get("signature", {}) + + +class Library: + def __init__(self, root: str | Path): + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + + def _dir(self, skill_id: str) -> Path: + return self.root / skill_id + + def list(self) -> list[Skill]: + out = [] + for d in sorted(self.root.iterdir()): + if (d / "meta.json").exists(): + out.append(self.get(d.name)) + return [s for s in out if s] + + def get(self, skill_id: str) -> Skill | None: + d = self._dir(skill_id) + if not (d / "meta.json").exists(): + return None + meta = json.loads((d / "meta.json").read_text(encoding="utf-8")) + code = (d / "skill.py").read_text(encoding="utf-8") if (d / "skill.py").exists() else "" + return Skill(skill_id=skill_id, code=code, meta=meta) + + def add(self, skill: Skill) -> None: + d = self._dir(skill.skill_id) + d.mkdir(parents=True, exist_ok=True) + (d / "skill.py").write_text(skill.code, encoding="utf-8") + (d / "meta.json").write_text(json.dumps(skill.meta, ensure_ascii=False, indent=2), + encoding="utf-8") + + def path(self, skill_id: str) -> Path: + return self._dir(skill_id) / "skill.py" diff --git a/src/webwright/skill_factory/llm.py b/src/webwright/skill_factory/llm.py new file mode 100644 index 0000000..57a8076 --- /dev/null +++ b/src/webwright/skill_factory/llm.py @@ -0,0 +1,72 @@ +"""LLM helper for the skills module β€” backend-agnostic, via webwright's own model abstraction. + +No hardcoded gateway/endpoint/key: the caller passes a webwright Model (or a model config dict), +so this works with any backend webwright supports (openai / anthropic / openrouter / custom). +""" +from __future__ import annotations + +import json +from typing import Any, Optional + +from webwright.models import get_model + +# Process-wide default model, set once via configure_llm() so retrieve/decide/update can call +# llm() without each caller threading a Model through. Falls back to env-configured openai. +_DEFAULT_MODEL: Optional[Any] = None + + +def configure_llm(model: Any) -> None: + """Register the Model (or model-config dict) the skills module should use.""" + global _DEFAULT_MODEL + _DEFAULT_MODEL = get_model(model) if isinstance(model, dict) else model + + +def _model() -> Any: + if _DEFAULT_MODEL is not None: + return _DEFAULT_MODEL + # default: build an openai-style model from env so a bare CLI invocation + # (e.g. `python -m webwright.tools.skill_use`) uses the SAME backend as the running agent. + # Honors SKILL_MODEL_NAME / SKILL_MODEL_ENDPOINT (or OPENAI_* fallbacks); no hardcoded gateway. + import os + # long request timeout: refine emits a large skill (~16k tokens) which is slow on a busy + # gateway; the model default (120s) truncates/ReadTimeouts. Overridable via SKILL_MODEL_TIMEOUT. + cfg = {"model_class": os.environ.get("SKILL_MODEL_CLASS", "openai"), + "request_timeout_seconds": int(os.environ.get("SKILL_MODEL_TIMEOUT", "600"))} + name = os.environ.get("SKILL_MODEL_NAME") or os.environ.get("OPENAI_MODEL") + endpoint = os.environ.get("SKILL_MODEL_ENDPOINT") or os.environ.get("OPENAI_ENDPOINT") + if name: + cfg["model_name"] = name + if endpoint: + cfg["openai_endpoint"] = endpoint + return get_model(cfg) + + +def llm(system: str, user: str, *, model: Any = None, max_tokens: int | None = None, **_: Any) -> str: + """Single-turn call. Returns raw text. `model` overrides the configured default. + max_tokens caps the reply length (the model default is small ~4000, which truncates a + refined skill); pass it through to the model so large code outputs aren't cut off.""" + m = model if model is not None else _model() + messages = [ + m.format_message(role="system", content=system), + m.format_message(role="user", content=user), + ] + if max_tokens is not None: + return m(messages, max_output_tokens=max_tokens) + return m(messages) + + +def llm_json(system: str, user: str, **kw: Any) -> dict: + """Call + parse the first valid {...} JSON object out of the reply (prose/fences around + it are fine; brace snippets that aren't valid JSON are skipped over).""" + txt = llm(system, user, **kw) + dec = json.JSONDecoder() + for i, ch in enumerate(txt): + if ch != "{": + continue + try: + obj, _ = dec.raw_decode(txt, i) + except ValueError: + continue + if isinstance(obj, dict): + return obj + return {} diff --git a/src/webwright/skill_factory/prompt.py b/src/webwright/skill_factory/prompt.py new file mode 100644 index 0000000..768a326 --- /dev/null +++ b/src/webwright/skill_factory/prompt.py @@ -0,0 +1,36 @@ +"""Prompt helper: prepend a SKILL-LIBRARY hint to a task prompt so the agent reuses the library. + +Kept at the prompt level (not system_template) because webwright merges system_template by +replacement; a task-prompt hint is non-invasive and leaves default behavior unchanged when unused. +""" +from __future__ import annotations + +import shlex +from pathlib import Path + +_HINT = """## Skill library (reuse before solving from scratch) +A library of previously-built executable code skills may contain one that helps this task. +BEFORE planning from scratch, query it ONCE from bash: + + python -m webwright.tools.skill_use --task {task_q} --library {library_q} + +It returns JSON {{verdict, skill_id, source_path, how_to_reuse}}: +- "use" : read source_path, copy it into your final_script, fill THIS task's params. +- "adapt" : read source_path, reuse its login/navigation/extraction core, change ONLY the last step. +- "skip" : no useful skill β€” solve from scratch. +Record your choice in skill_decision.json ({{skill_id, verdict, reason}}) before acting. + +--- +""" + + +def with_skill_hint(task_prompt: str, *, task: str, library: str) -> str: + """Prepend the skill-library hint to a task prompt. + + `library` is resolved to an ABSOLUTE path: the hint's command runs in the agent's + own workspace, so a relative path would silently point at a nonexistent (empty) + library there and every lookup would come back "skip". Both values are shell-quoted: + the agent executes this line in bash, where $VAR / `...` / $(...) would otherwise + expand even inside double quotes.""" + library = str(Path(library).resolve()) + return _HINT.format(task_q=shlex.quote(task), library_q=shlex.quote(library)) + task_prompt diff --git a/src/webwright/skill_factory/retrieve.py b/src/webwright/skill_factory/retrieve.py new file mode 100644 index 0000000..5f7a405 --- /dev/null +++ b/src/webwright/skill_factory/retrieve.py @@ -0,0 +1,77 @@ +"""Retrieve: task -> most relevant candidate skills (relevance only). + +Stable interface (swappable implementation): + retrieve(task, library, *, k=3, method="llm") -> [Candidate] +MVP: a single LLM call that lists the whole library as a flat catalog in the prompt and lets it +pick. Swap to embeddings when the library grows large β€” the interface stays the same. +""" +from __future__ import annotations +from dataclasses import dataclass + +from .library import Library, Skill +from .llm import llm_json + + +@dataclass +class Candidate: + skill: Skill + score: float # relevance 0..1 + + reason: str + + +def _catalog(library: Library) -> str: + lines = [] + for s in library.list(): + lines.append( + f"- skill_id: {s.skill_id}\n" + f" template: {s.meta.get('template','')}\n" + f" site: {s.meta.get('site','')}\n" + f" summary: {s.summary}\n" + f" params: {s.signature.get('params', [])}" + ) + return "\n".join(lines) + + +def _retrieve_llm(task: str, library: Library, k: int) -> list[Candidate]: + cat = _catalog(library) + if not cat: + return [] + sys = ( + "You match a web task to the most RELEVANT skills in a catalog (relevance only β€” not yet " + "whether to use them). Return STRICT JSON: " + '{"candidates":[{"skill_id":"...","score":<0..1>,"reason":"..."}]}, most relevant first, ' + f"at most {k}. score = how relevant. If nothing is relevant, return an empty list." + ) + user = f"## Task\n{task}\n\n## Skill catalog\n{cat}\n\nReturn at most {k} candidates." + out = llm_json(sys, user) + cands = [] + for c in (out.get("candidates") or [])[:k]: + sk = library.get(c.get("skill_id", "")) + if sk: + try: + score = float(c.get("score", 0)) + except Exception: + score = 0.0 + cands.append(Candidate(skill=sk, score=score, reason=c.get("reason", ""))) + return cands + + +def _retrieve_simple(task: str, library: Library, k: int) -> list[Candidate]: + """No-LLM fallback: rank by keyword overlap between task and template/summary.""" + toks = set(task.lower().split()) + scored = [] + for s in library.list(): + bag = (s.meta.get("template", "") + " " + s.summary).lower().split() + overlap = len(toks & set(bag)) + if overlap: + scored.append(Candidate(skill=s, score=overlap / (len(toks) or 1), reason="keyword overlap")) + scored.sort(key=lambda c: c.score, reverse=True) + return scored[:k] + + +_RETRIEVERS = {"llm": _retrieve_llm, "simple": _retrieve_simple} + + +def retrieve(task: str, library: Library, *, k: int = 3, method: str = "llm") -> list[Candidate]: + return _RETRIEVERS[method](task, library, k) diff --git a/src/webwright/skill_factory/update.py b/src/webwright/skill_factory/update.py new file mode 100644 index 0000000..c32b4e7 --- /dev/null +++ b/src/webwright/skill_factory/update.py @@ -0,0 +1,376 @@ +"""Sediment (intermittent): distill gate-passed solves back into the library so it grows from use. + +Stable interface (swappable implementation): + update(traces, library, *, method="grow") -> [added/updated skill_ids] + +- method="grow" : if the library does not yet cover this template, promote the successful solve + as-is into a skill (minimal form). +- method="refine" : batch distillation β€” align N gate-passed solves -> parameterize (generalize) + + factor out reusable primitives + a thin task layer -> one better library skill. + This is where update adds generalization + primitive reusability (one batched LLM call). +""" +from __future__ import annotations +import hashlib +import json +import re +from dataclasses import dataclass, field +from pathlib import Path + +from .library import Library, Skill +from .llm import llm + + +@dataclass +class Trace: + template: str + code: str # this task's final_script (already gate-passed = correct) + answer: object = None + meta: dict = field(default_factory=dict) # params / site / start_url / output_schema ... + # usage: how this task used the library (the signal that drives update) + used_skill_id: str | None = None + verdict: str | None = None # use | adapt | skip + correct: bool = True + + +def _slug(template: str) -> str: + s = re.sub(r"[^a-z0-9]+", "_", template.lower()).strip("_") + if not s: + return "skill" + if len(s) <= 48: + return s + # truncation could collide two templates that share a long prefix -> disambiguate with a hash + return f"{s[:40]}_{hashlib.md5(template.encode()).hexdigest()[:7]}" + + +def _extract_code(txt: str) -> str: + m = re.search(r"```(?:python)?[ \t]*\n", txt) + if m: + end = txt.rfind("```") + if end > m.end(): + return txt[m.end():end] + return txt[m.end():] # opening fence but no close (e.g. truncated) -> strip the fence anyway + return txt + + +def _norm(v): + """Scalar-normalize for replay comparison: 5 == "5" (type jitter between a solve's + string answer and a skill's numeric one is not a logic error; WebArena's own + evaluator normalizes the same way).""" + if isinstance(v, list): + return [_norm(x) for x in v] + if isinstance(v, dict): + return {k: _norm(x) for k, x in sorted(v.items())} + if isinstance(v, bool): + return v + if isinstance(v, (int, float)): + return str(v) + return v + + +def _replay(code: str, traces: list["Trace"], strict: bool = False) -> list[str]: + """Run the candidate skill on each source trace's OWN taskspec (no model in the loop). + PASS = exact answer match; in non-strict mode a non-empty, schema-shaped answer also + passes (live sites drift between solve time and replay time β€” prices, listings). + Catches what distillation can break: crashes, timeouts, empty/misshapen output.""" + import os + import subprocess + import sys + import tempfile + from .gate import gate + fails = [] + live = [t for t in traces if t.answer is not None] + for i, tr in enumerate(traces): + if tr.answer is None: + continue + # each replay drives a live site for up to 240s; without a line per instance the + # whole verify phase is minutes of silence that reads as a hang + print(f" replaying {live.index(tr) + 1}/{len(live)}: " + f"{json.dumps(tr.meta.get('params'), ensure_ascii=False)[:70]}", flush=True) + with tempfile.TemporaryDirectory() as td: + tdp = Path(td) + (tdp / "skill.py").write_text(code, encoding="utf-8") + (tdp / "taskspec.json").write_text(json.dumps( + {"params": tr.meta.get("params", {}), "start_url": tr.meta.get("start_url", ""), + "credentials": tr.meta.get("credentials"), + "output_schema": tr.meta.get("output_schema")}, ensure_ascii=False), + encoding="utf-8") + try: + proc = subprocess.run([sys.executable, "skill.py", "taskspec.json"], cwd=td, + env={**os.environ, "WORKSPACE_DIR": td}, + capture_output=True, text=True, timeout=240) + except subprocess.TimeoutExpired: + fails.append(f"instance {i} (params={json.dumps(tr.meta.get('params'), ensure_ascii=False)}): TIMEOUT") + continue + got = None + arp = tdp / "agent_response.json" + if arp.exists(): + try: + got = json.loads(arp.read_text(encoding="utf-8")).get("retrieved_data") + except Exception: + pass + if _norm(got) == _norm(tr.answer): + continue + if not strict and gate(got, output_schema=tr.meta.get("output_schema"), + method="self_verify").admit: + continue # tolerated: live-data drift (right shape, non-empty) + # a null answer means the skill crashed or never wrote output β€” without the + # subprocess's own words neither the human log nor the repair round can act + crash = "" + if got is None: + err = " ".join((proc.stderr or proc.stdout or "").split()) + crash = f"; stderr tail: {err[-400:] or '(empty)'}" + fails.append(f"instance {i} (params={json.dumps(tr.meta.get('params'), ensure_ascii=False)}): " + f"replay returned {json.dumps(got, ensure_ascii=False)[:120]}, " + f"the solve's answer was {json.dumps(tr.answer, ensure_ascii=False)[:120]}{crash}") + return fails + + +def _load_examples(library: Library, sid: str) -> list: + f = library.path(sid).parent / "replays.json" + try: + return json.loads(f.read_text(encoding="utf-8")) if f.exists() else [] + except Exception: + return [] + + +def _save_examples(library: Library, sid: str, traces: list["Trace"], old: list) -> None: + """Persist (params, start_url, output_schema, answer) per admitted solve so future + incremental refines can REGRESSION-replay old coverage. Credentials are never stored + (the library may be shared/committed); replay borrows them from the incoming batch.""" + ex = old + [{"params": t.meta.get("params", {}), "start_url": t.meta.get("start_url", ""), + "output_schema": t.meta.get("output_schema"), "answer": t.answer} + for t in traces if t.answer is not None] + (library.path(sid).parent / "replays.json").write_text( + json.dumps(ex[-12:], ensure_ascii=False, indent=1), encoding="utf-8") + + +_REFINE_SYS = ( + "You are given N working Python solutions that EACH solve one concrete instance of the SAME web-task " + "template (they already passed a correctness gate). Distill them into ONE better library skill.\n" + "Do TWO things:\n" + "1) GENERALIZE: align the N solutions; the parts that are IDENTICAL across them are the reusable " + "skeleton; the parts that DIFFER are parameters. Expose the differing values as function " + "arguments / taskspec params β€” do NOT hardcode any instance's specific values. Make extraction " + "robust (paginate/until-done, self-verify against any declared total).\n" + "2) DECOMPOSE INTO REUSABLE PRIMITIVES: factor the expensive, reusable core into clearly-named " + "primitive functions (e.g. login(), open_report(period), extract_rows()), and keep a THIN task " + "layer on top that calls them. This lets future tasks reuse the primitives even if the final step " + "differs.\n" + "Interface (fixed): the skill reads taskspec.json from sys.argv[1] " + "(taskspec = {params, start_url, credentials, output_schema}) and writes agent_response.json with " + "retrieved_data MATCHING output_schema exactly. ALL artifacts (answer, logs, screenshots) must go " + "under the WORKSPACE_DIR env var (default: the current working directory) β€” never next to " + "__file__: the skill file lives in a shared library. " + "Output ONLY the python code in one ```python block." +) + +_REFINE_INCREMENTAL = ( + "\n\nINCREMENTAL MODE: a CURRENT library skill for this template already exists (shown below). " + "Do NOT rewrite it from scratch. START from the current skill and IMPROVE it using the NEW solutions: " + "keep its working primitives and structure, only widen/fix what the new solutions reveal (handle a " + "param value it missed, make an extraction more robust, fix a bug). Preserve everything that already " + "works. Output the full improved skill in one ```python block." +) + + +def _refine(traces: list[Trace], library: Library, verify: str = "off", + rounds: int = 2, on_fail: str = "reject") -> list[str]: + """Batch distillation: align N gate-passed solves -> parameterize + primitives. + Incremental: if a skill for the same template already exists, improve/widen it on top of the + existing skill (rather than rewriting from the raw solves).""" + if not traces: + return [] + template = traces[0].template + schema = traces[0].meta.get("output_schema") + sid = _slug(template) + existing = library.get(sid) # skill already exists? -> incremental evolution + + blocks = [f"## Template\n{template}\n\n## Required output_schema for retrieved_data\n{json.dumps(schema)}\n"] + if existing and existing.code: + blocks.append(f"## CURRENT library skill (improve THIS, do not rewrite)\n```python\n{existing.code}\n```") + label = "NEW solutions" if existing else "Solutions" + for i, tr in enumerate(traces): + blocks.append( + f"## {label} {i} (params={json.dumps(tr.meta.get('params'), ensure_ascii=False)}, " + f"answer={json.dumps(tr.answer, ensure_ascii=False)[:120]})\n```python\n{tr.code}\n```" + ) + sys_prompt = _REFINE_SYS + (_REFINE_INCREMENTAL if existing else "") + user_msg = "\n\n".join(blocks) + print(f" distilling {len(traces)} solve(s) into {sid} …", flush=True) + code = _extract_code(llm(sys_prompt, user_msg, max_tokens=16000)) + verified = None + if verify != "off": # replay the candidate on its own training taskspecs before it may land + replay_set = list(traces) + if existing: + old_ex = _load_examples(library, sid) + if old_ex: + creds = traces[0].meta.get("credentials") # same template family, same site + replay_set += [Trace(template=template, code="", answer=e.get("answer"), + meta={"params": e.get("params", {}), + "start_url": e.get("start_url", ""), + "credentials": creds, + "output_schema": e.get("output_schema")}) + for e in old_ex] + elif existing.meta.get("verified"): + # a verified skill with no stored regression examples must not be touched: + # we could not prove the refine keeps its old coverage + print(f" βœ— {sid}: existing VERIFIED skill has no replays.json β€” refine skipped " + f"(old coverage can't be regression-checked); old skill kept") + return [] + verified, fails = False, [] + for attempt in range(1, max(rounds, 1) + 1): + print(f" verify ({verify}) round {attempt}/{max(rounds, 1)} β€” " + f"{len(replay_set)} instance(s), no model:", flush=True) + fails = _replay(code, replay_set, strict=(verify == "strict")) + if not fails: + verified = True + break + if attempt <= max(rounds, 1) - 1: # feedback rounds remaining + print(f" {len(fails)} failed β€” re-distilling with the failures as feedback", + flush=True) + feedback = ("\n\n## Replay failures of your previous attempt (fix the GENERAL " + "logic, do NOT hardcode answers)\n" + "\n".join(fails) + + "\n\n## Your previous attempt\n```python\n" + code + "\n```") + code = _extract_code(llm(sys_prompt, user_msg + feedback, max_tokens=16000)) + if not verified: + # NEVER overwrite an existing (possibly verified) skill with an unverified one + if on_fail == "reference" and not existing: + print(f" ! {sid}: replay verification failed after {rounds} round(s) β€” landing " + f"as grade=reference (a readable prior for the agent; standalone NOT trusted)") + else: + print(f" βœ— {sid}: replay verification failed after {rounds} round(s) β€” NOT written:") + for f in fails: + print(f" {f}") + if verify == "strict": + print(" (answers that legitimately change between solve and replay β€” " + "prices, live listings β€” need --verify shape)") + post_mortem = library.root / f".rejected_{sid}.py" + post_mortem.write_text(code, encoding="utf-8") + print(f" (last candidate kept for post-mortem: {post_mortem})") + return [] + n_prev = (existing.meta.get("n_solves", 0) if existing else 0) + meta = { + "template": template, + "provenance": "update-refined-incremental" if existing else "update-refined", + "site": traces[0].meta.get("site", ""), + "summary": f"Refined from {n_prev + len(traces)} gate-passed solves; parameterized + primitives.", + "signature": {"params": list((traces[0].meta.get("params") or {}).keys()), + "call": "python skill.py taskspec.json"}, + "output_schema": schema, + "n_solves": n_prev + len(traces), + "revisions": (existing.meta.get("revisions", 1) + 1) if existing else 1, + } + if verified is not None: + meta["verified"] = verified + meta["grade"] = "executable" if verified else "reference" + library.add(Skill(skill_id=sid, code=code, meta=meta)) + if verify != "off": + _save_examples(library, sid, traces, _load_examples(library, sid)) + return [sid] + + +def evolve(traces: list[Trace], library: Library, verify: str = "off", + rounds: int = 2, on_fail: str = "reject") -> dict: + """Unified update: evolve the EXISTING library, deciding per trace's usage (use/adapt/skip) how + to change it. This is the core of a continuously-growing library β€” not rebuilt from scratch each + time, but grown from v_{n-1} into v_n. + + - USE (successful) : the skill is good enough, leave it untouched (just reuse evidence). + - ADAPT (successful) : core reused, last step fixed -> refine this batch's fixed solves + back into the template's skill (widen/harden). This is how a fix + sediments into the library. + - SKIP / not yet covered : the template has no skill yet -> add one from this batch. + + Only consumes gate-passed (correct=True) traces (pollution protection). Returns a changelog. + """ + good = [t for t in traces if t.correct] + changelog = {"use": [], "adapt_refined": [], "added": [], "reference": [], "rejected": [], + "dropped_wrong": len(traces) - len(good)} + existing_templates = {s.meta.get("template"): s.skill_id for s in library.list()} + + # group by template (same-family solves are distilled/sedimented together) + by_tmpl: dict[str, list[Trace]] = {} + for t in good: + by_tmpl.setdefault(t.template, []).append(t) + + for tmpl, group in by_tmpl.items(): + verdicts = {t.verdict for t in group} + if tmpl not in existing_templates: + # not covered -> add (distill a skill from this batch) + added = _refine(group, library, verify=verify, rounds=rounds, on_fail=on_fail) + key = "added" if added else "rejected" + if added and library.get(added[0]) and library.get(added[0]).meta.get("grade") == "reference": + key = "reference" + changelog.setdefault(key, []).append(added[0] if added else _slug(tmpl)) + elif "adapt" in verdicts: + # a fix happened -> refine the fixed solves back into the skill (widen/harden) + added = _refine(group, library, verify=verify, rounds=rounds, on_fail=on_fail) + changelog["adapt_refined" if added else "rejected"].append(added[0] if added else _slug(tmpl)) + else: + # all use-success -> skill is good enough, leave it + changelog["use"].append(existing_templates[tmpl]) + return changelog + + +# ---------- CLI: batch update via a manifest ---------- +def traces_from_manifest(manifest: dict) -> list["Trace"]: + """manifest = {"template": str, "runs": [{"dir","admit","params","answer"?,"verdict"?}, ...]}. + Reads each run's final_script.py; builds a Trace. correct = the run's gate verdict (admit). + "admit" is REQUIRED per run β€” a missing gate verdict must fail loudly, not silently enter.""" + template = manifest.get("template", "") + out = [] + for r in manifest.get("runs", []): + if "admit" not in r: + raise KeyError(f"manifest run missing required 'admit' (gate verdict): {r.get('dir', r)}") + if not isinstance(r["admit"], bool): + # a hand-written manifest with "admit": "false" would otherwise be truthy -> admitted + raise TypeError(f"manifest 'admit' must be a JSON boolean, " + f"got {type(r['admit']).__name__} {r['admit']!r}: {r.get('dir', r)}") + d = Path(r["dir"]) + fs = d / "final_script.py" + code = fs.read_text(encoding="utf-8") if fs.exists() else "" + answer = r.get("answer") + if answer is None and (d / "agent_response.json").exists(): + try: + answer = json.loads((d / "agent_response.json").read_text(encoding="utf-8")).get("retrieved_data") + except Exception: + pass + out.append(Trace(template=template, code=code, answer=answer, + correct=r["admit"], + verdict=r.get("verdict", "skip"), + used_skill_id=r.get("used_skill_id"), + meta={"params": r.get("params", {}), "site": r.get("site", ""), + "start_url": r.get("start_url", ""), + "credentials": r.get("credentials"), + "output_schema": r.get("output_schema")})) + return out + + +def main(argv=None) -> int: + import argparse + p = argparse.ArgumentParser( + prog="python -m webwright.skill_factory.update", + description="Batch-update the skill library from a manifest of gate-judged solves.") + p.add_argument("--manifest", required=True, help="JSON: {template, runs:[{dir,admit,params,...}]}") + p.add_argument("--library", required=True, help="Path to the skill library directory.") + p.add_argument("--verify", default="off", choices=["off", "shape", "strict"], + help="Replay each new skill on its own training taskspecs before it enters " + "the library. shape: exact match OR well-formed non-empty (live data); " + "strict: exact match only. Needs the sites reachable from here.") + p.add_argument("--verify-rounds", type=int, default=2, + help="Total build attempts (first + repairs) before giving up. Default 2.") + p.add_argument("--on-fail", default="reject", choices=["reject", "reference"], + help="Failed verification: reject (default) or land as grade=reference β€” " + "readable prior for the agent, standalone NOT trusted. Never overwrites " + "an existing skill.") + a = p.parse_args(argv) + manifest = json.loads(Path(a.manifest).read_text(encoding="utf-8")) + traces = traces_from_manifest(manifest) + changelog = evolve(traces, Library(a.library), verify=a.verify, + rounds=a.verify_rounds, on_fail=a.on_fail) + print(json.dumps(changelog, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/webwright/tools/skill_use.py b/src/webwright/tools/skill_use.py new file mode 100644 index 0000000..4da2a5a --- /dev/null +++ b/src/webwright/tools/skill_use.py @@ -0,0 +1,93 @@ +"""skill_use β€” solve-time tool: query the skill library for a reusable skill for THIS task. + +Like self_reflection / image_qa, the agent invokes this from bash during solving: + + python -m webwright.tools.skill_use --task "Get the latest release version of facebook/react" \ + --library "$WORKSPACE_DIR/../library" + +It retrieves the most relevant skill (relevance) and judges utility (use / adapt / skip), then +prints a JSON recommendation telling the agent how to reuse it (and the path to read its source). +The agent decides: reuse as-is (use), reuse the core and change only the last step (adapt), or +solve from scratch (skip). Retrieval/judgement never block solving β€” on any error it prints skip. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +from webwright.skill_factory.library import Library +from webwright.skill_factory.retrieve import retrieve +from webwright.skill_factory.decide import decide + + +def recommend(task: str, library_root: str) -> dict: + root = Path(library_root).resolve() + # A missing/empty library is almost always a wrong path (relative paths resolve inside the + # agent's workspace). Say so LOUDLY instead of a silent skip β€” checked before Library(), + # whose constructor would mkdir the bogus path and hide the mistake. + if not root.is_dir() or not any((p / "meta.json").exists() for p in root.iterdir() if p.is_dir()): + return {"verdict": "skip", "skill_id": None, + "reason": f"skill library MISSING or EMPTY at {root} β€” check the --library path", + "warning": f"library empty at {root}"} + lib = Library(root) + cands = retrieve(task, lib) + if not cands: + return {"verdict": "skip", "skill_id": None, "reason": "library has no relevant skill"} + d = decide(task, cands) + # the decision must point at a RETRIEVED candidate β€” an LLM-hallucinated id (even one that + # happens to exist in the library) must not be recommended + if d.verdict != "skip" and d.skill_id not in {c.skill.skill_id for c in cands}: + return {"verdict": "skip", "skill_id": None, + "reason": f"decided skill '{d.skill_id}' is not among the retrieved candidates"} + out = {"verdict": d.verdict, "skill_id": d.skill_id, "reason": d.reason} + if d.verdict != "skip" and d.skill_id: + sk = lib.get(d.skill_id) + if sk: + out["summary"] = sk.summary + out["call"] = sk.signature.get("call", "") + out["source_path"] = str(lib.path(sk.skill_id)) + out["how_to_reuse"] = ( + "USE: copy the source into your final_script and fill THIS task's params; " + "ADAPT: reuse its login/navigation/extraction core, change ONLY the final step." + ) + return out + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="python -m webwright.tools.skill_use", + description="Query the skill library for a reusable skill for the current task.", + ) + p.add_argument("--task", required=True, help="The current task description / intent.") + p.add_argument("--library", default=os.environ.get("SKILL_LIBRARY_ROOT", "library"), + help="Path to the skill library dir (default: $SKILL_LIBRARY_ROOT or ./library).") + p.add_argument("--output", default="", help="Write JSON to this path instead of stdout.") + return p + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + result = recommend(args.task, args.library) + except Exception as exc: + # Degrade to skip so solving is never blocked β€” but say LOUDLY that the library + # was NOT consulted: a config/auth error here silently disables all reuse otherwise. + result = {"verdict": "skip", "skill_id": None, "error": str(exc), + "reason": "LOOKUP FAILED (library was NOT consulted) β€” this is an error, " + "not a no-match. Check OPENAI_API_KEY and, on a custom gateway, " + "OPENAI_ENDPOINT / SKILL_MODEL_ENDPOINT.", + } + print(f"skill_use ERROR: {exc}", file=sys.stderr) + payload = json.dumps(result, ensure_ascii=False, indent=2) + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + f.write(payload) + print(payload) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/skill_factory/test_build_init.py b/tests/skill_factory/test_build_init.py new file mode 100644 index 0000000..2d6588c --- /dev/null +++ b/tests/skill_factory/test_build_init.py @@ -0,0 +1,239 @@ +"""Unit tests: build (spec -> concrete tasks -> learn) and init (need -> spec skeleton). + +Everything here is LLM-free and site-free: the solve subprocess and the one LLM call in init +are stubbed, so what is under test is the logic those two commands actually own β€” template +substitution, policy precedence, resume detection, and the shape of the drafted spec. +""" +import json +import tempfile +from pathlib import Path + +import pytest +import yaml + +import webwright.skill_factory.build as B +import webwright.skill_factory.init as I + + +# ---------------------------------------------------------------- build: substitution + +def test_fill_substitutes_every_hole(): + got = B._fill("earliest flight from {origin} to {dest} on {date}", + {"origin": "SEA", "dest": "JFK", "date": "2026-08-15"}) + assert got == "earliest flight from SEA to JFK on 2026-08-15" + + +def test_fill_reports_the_missing_value_instead_of_guessing(): + # a hole with no value would otherwise be solved as the literal "{date}" + with pytest.raises(SystemExit) as e: + B._fill("flight on {date} from {origin}", {"origin": "SEA"}) + assert "date" in str(e.value) + + +def test_fill_ignores_extra_columns(): + assert B._fill("hi {a}", {"a": "x", "unused": "y"}) == "hi x" + + +# ---------------------------------------------------------------- build: policy precedence + +def test_policy_precedence_cli_over_spec_over_default(): + assert B._pick("shape", "strict", "strict") == "shape" # CLI wins + assert B._pick(None, "shape", "strict") == "shape" # spec beats the default + assert B._pick(None, None, "strict") == "strict" # default when nobody said + + +# ---------------------------------------------------------------- build: resume + +def _run_dir(outputs: Path, name: str, task: str, answer=True): + d = outputs / name + d.mkdir(parents=True) + (d / "task.json").write_text(json.dumps({"task": task}), encoding="utf-8") + if answer: + (d / "agent_response.json").write_text('{"retrieved_data": ["x"]}', encoding="utf-8") + return d + + +def test_resume_finds_a_prior_run_that_produced_an_answer(): + with tempfile.TemporaryDirectory() as d: + out = Path(d) + # the real prompt wraps the task in a skill hint + answer instruction, so the match + # must be a containment, not equality + _run_dir(out, "build_00_2026", "## Skill library ... --- cheapest widget on Acme. Also write...") + assert B._already_solved(out, "cheapest widget on Acme") is not None + assert B._already_solved(out, "cheapest gadget on Acme") is None + + +def test_resume_ignores_a_run_that_never_wrote_an_answer(): + with tempfile.TemporaryDirectory() as d: + out = Path(d) + _run_dir(out, "build_00_2026", "cheapest widget on Acme", answer=False) + assert B._already_solved(out, "cheapest widget on Acme") is None + + +# ---------------------------------------------------------------- build: spec validation + flow + +def _spec(tmp: Path, **over) -> Path: + spec = {"task": "cheapest {product} on Acme", + "start_url": "https://acme.example", + "instances": [{"product": "widget"}, {"product": "gadget"}]} + spec.update(over) + p = tmp / "skill.yaml" + p.write_text(yaml.safe_dump(spec), encoding="utf-8") + return p + + +@pytest.mark.parametrize("missing", ["task", "start_url", "instances"]) +def test_spec_must_have_the_three_things_build_cannot_invent(missing): + with tempfile.TemporaryDirectory() as d: + p = _spec(Path(d), **{missing: "" if missing != "instances" else []}) + with pytest.raises(SystemExit): + B.build(str(p), str(Path(d) / "lib"), [], dry_run=True) + + +def test_dry_run_neither_solves_nor_learns(): + calls = [] + with tempfile.TemporaryDirectory() as d: + p = _spec(Path(d)) + B._solve = lambda *a, **k: calls.append("solve") + B.learn = lambda *a, **k: calls.append("learn") + assert B.build(str(p), str(Path(d) / "lib"), [], dry_run=True) == 0 + assert calls == [] + + +def test_build_solves_each_instance_then_hands_the_batch_to_learn(monkeypatch): + seen, learned = [], {} + with tempfile.TemporaryDirectory() as d: + tmp = Path(d) + p = _spec(tmp) + outputs = tmp / "build_outputs" + + def fake_solve(core_task, start_url, library, out, task_id, cfg, log_path=None): + seen.append(core_task) + _run_dir(Path(out), f"{task_id}_2026", core_task) + return 0 + + def fake_learn(runs_dir, lib, **kw): + learned.update({"runs_dir": runs_dir, **kw}) + + monkeypatch.setattr(B, "_solve", fake_solve) + monkeypatch.setattr(B, "learn", fake_learn) + B.build(str(p), str(tmp / "lib"), [], assume_yes=True, verify="shape", verify_rounds=3) + + assert seen == ["cheapest widget on Acme", "cheapest gadget on Acme"] + assert learned["runs_dir"] == str(outputs) + # the flags the user chose must reach learn, not be silently dropped + assert learned["verify"] == "shape" and learned["rounds"] == 3 + + +def test_an_already_solved_instance_is_not_solved_again(monkeypatch): + """Solving is the expensive half; a re-run must not re-spend it.""" + seen = [] + with tempfile.TemporaryDirectory() as d: + tmp = Path(d) + p = _spec(tmp) + outputs = tmp / "build_outputs" + _run_dir(outputs, "build_00_2026", "cheapest widget on Acme") # widget already done + + monkeypatch.setattr(B, "_solve", lambda ct, *a, **k: (seen.append(ct), 0)[1]) + monkeypatch.setattr(B, "learn", lambda *a, **k: None) + B.build(str(p), str(tmp / "lib"), [], assume_yes=True) + + assert seen == ["cheapest gadget on Acme"], "the solved instance should have been skipped" + + +def test_a_solve_that_wrote_its_answer_counts_even_if_it_exited_non_zero(monkeypatch, capsys): + """learn reads the artifact, so build must not call it a failure.""" + with tempfile.TemporaryDirectory() as d: + tmp = Path(d) + p = _spec(tmp, instances=[{"product": "widget"}]) + + def killed_but_wrote(core_task, start_url, library, out, task_id, cfg, log_path=None): + _run_dir(Path(out), f"{task_id}_2026", core_task) + return -15 # SIGTERM + + monkeypatch.setattr(B, "_solve", killed_but_wrote) + monkeypatch.setattr(B, "learn", lambda *a, **k: None) + B.build(str(p), str(tmp / "lib"), [], assume_yes=True) + assert "solved 1/1" in capsys.readouterr().out + + +# ---------------------------------------------------------------- init: the drafted spec + +def _fake_llm(payload): + return lambda system, user, **kw: payload + + +def test_init_drafts_a_spec_whose_holes_match_its_instance_columns(monkeypatch): + monkeypatch.setattr(I, "llm_json", _fake_llm({ + "task": "cheapest {product} on Acme", "params": ["product"], + "start_url": "https://acme.example", "drifts": True})) + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "skill.yaml" + I.init("cheapest stuff on Acme", str(out)) + spec = yaml.safe_load(out.read_text(encoding="utf-8")) # must be valid YAML + assert set(spec["instances"][0]) == {"product"}, "columns must match the template's holes" + assert spec["start_url"] == "https://acme.example" + + +def test_init_drafts_valid_yaml_for_a_multi_param_task(monkeypatch): + """A one-column row needs no separator, so single-param specs cannot catch a broken + flow mapping. Most real tasks have several params β€” that is where it bites.""" + monkeypatch.setattr(I, "llm_json", _fake_llm({ + "task": "earliest flight from {origin} to {dest} on {date}", + "params": ["origin", "dest", "date"], + "start_url": "https://flights.example", "drifts": False})) + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "skill.yaml" + I.init("earliest flight between two cities", str(out)) + spec = yaml.safe_load(out.read_text(encoding="utf-8")) # would raise on a bad flow map + assert set(spec["instances"][0]) == {"origin", "dest", "date"} + # and the drafted spec must be something build can actually consume + B._fill(spec["task"], {"origin": "SEA", "dest": "JFK", "date": "2026-08-15"}) + + +def test_init_leaves_the_values_blank_for_the_user_to_fill(monkeypatch): + """The model proposes structure; the ground truth stays the user's.""" + monkeypatch.setattr(I, "llm_json", _fake_llm({ + "task": "cheapest {product} on Acme", "params": ["product"], + "start_url": "https://acme.example", "drifts": True})) + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "skill.yaml" + I.init("cheapest stuff on Acme", str(out), rows=3) + spec = yaml.safe_load(out.read_text(encoding="utf-8")) + assert len(spec["instances"]) == 3 + assert all(i["product"] == "____" for i in spec["instances"]) + + +@pytest.mark.parametrize("drifts,expected", [(True, "shape"), (False, "strict")]) +def test_init_picks_the_verify_mode_from_whether_the_answer_drifts(monkeypatch, drifts, expected): + """strict on a drifting answer rejects a working skill β€” the default must follow the task.""" + monkeypatch.setattr(I, "llm_json", _fake_llm({ + "task": "x {p}", "params": ["p"], "start_url": "https://a.example", "drifts": drifts})) + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "skill.yaml" + I.init("something", str(out)) + assert yaml.safe_load(out.read_text(encoding="utf-8"))["build"]["verify"] == expected + + +def test_init_refuses_a_need_with_nothing_varying(monkeypatch): + """No holes means one task, not a task type β€” there is no reusable skill in it.""" + monkeypatch.setattr(I, "llm_json", _fake_llm({ + "task": "today's top headline on Example News", "params": [], + "start_url": "https://news.example", "drifts": True})) + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "skill.yaml" + with pytest.raises(SystemExit) as e: + I.init("today's headline", str(out)) + assert "varies" in str(e.value) + assert not out.exists() + + +def test_init_will_not_clobber_an_existing_spec(monkeypatch): + monkeypatch.setattr(I, "llm_json", _fake_llm({ + "task": "x {p}", "params": ["p"], "start_url": "https://a.example"})) + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "skill.yaml" + out.write_text("# my filled-in values", encoding="utf-8") + with pytest.raises(SystemExit): + I.init("something", str(out)) + assert out.read_text(encoding="utf-8") == "# my filled-in values" diff --git a/tests/skill_factory/test_evolve.py b/tests/skill_factory/test_evolve.py new file mode 100644 index 0000000..c6faa3e --- /dev/null +++ b/tests/skill_factory/test_evolve.py @@ -0,0 +1,193 @@ +"""Unit test: evolve (growing library, usage-driven). Stubs _refine to stay LLM-free.""" +import sys, tempfile +from pathlib import Path +pass +import webwright.skill_factory.update as U +from webwright.skill_factory.library import Library, Skill + + +def run(): + # stub _refine: deterministically "build/widen" a skill for the group's template + def fake_refine(group, library, verify="off", rounds=2, on_fail="reject"): + from webwright.skill_factory.update import _slug + sid = _slug(group[0].template) + library.add(Skill(sid, f"# refined from {len(group)} solves\n", + {"template": group[0].template, "provenance": "test-refine"})) + return [sid] + U._refine = fake_refine + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + + # round 1: template T1 not in lib, skip verdict -> ADD + t1 = [U.Trace("T1", "code", verdict="skip", correct=True), + U.Trace("T1", "code", verdict="skip", correct=True)] + log1 = U.evolve(t1, lib) + assert log1["added"], f"new template should be added: {log1}" + assert len(lib.list()) == 1 + + # round 2: T1 now exists, all USE success -> library unchanged + t2 = [U.Trace("T1", "code", used_skill_id="t1", verdict="use", correct=True)] + log2 = U.evolve(t2, lib) + assert log2["use"] and not log2["added"] and not log2["adapt_refined"], log2 + assert len(lib.list()) == 1, "pure use must not change library" + + # round 3: T1 exists, an ADAPT happened -> refine back (widen) + t3 = [U.Trace("T1", "code2", used_skill_id="t1", verdict="adapt", correct=True)] + log3 = U.evolve(t3, lib) + assert log3["adapt_refined"], f"adapt should refine back: {log3}" + + # wrong solves are dropped (not fed to refine) + t4 = [U.Trace("T2", "bad", verdict="skip", correct=False)] + log4 = U.evolve(t4, lib) + assert log4["dropped_wrong"] == 1 and not log4["added"], log4 + + # manifest: a run missing the gate verdict must fail loudly, never default to admitted + try: + U.traces_from_manifest({"template": "T", "runs": [{"dir": "/nonexistent"}]}) + raise AssertionError("missing 'admit' must raise") + except KeyError: + pass + # ... and a hand-written string "false" (truthy!) must be rejected, not admitted + try: + U.traces_from_manifest({"template": "T", "runs": [{"dir": "/x", "admit": "false"}]}) + raise AssertionError("non-bool 'admit' must raise") + except TypeError: + pass + + # slug: two long templates sharing a 48-char prefix must NOT collide on one skill id + long_a = "get the value of " + "x" * 60 + " variant one" + long_b = "get the value of " + "x" * 60 + " variant two" + assert U._slug(long_a) != U._slug(long_b), "truncated slugs must be disambiguated" + assert U._slug(long_a) == U._slug(long_a), "slug must stay deterministic" + assert U._slug("Get the top-n best-selling entity") == "get_the_top_n_best_selling_entity", \ + "short templates keep the plain readable slug" + + # ---- replay verification (real _refine + _replay, only the LLM is faked) ---- + import importlib + importlib.reload(U) # drop the fake_refine stub + import json as _json + + BAD = 'import json\njson.dump({"retrieved_data": [7]}, open("agent_response.json", "w"))\n' + GOOD = 'import json\njson.dump({"retrieved_data": [42]}, open("agent_response.json", "w"))\n' + CRASH = 'raise RuntimeError("distillation bug")\n' + + def mktrace(): + return U.Trace("verify template", "solver code", answer=[42], verdict="skip", correct=True, + meta={"params": {"k": "v"}, "output_schema": {"type": "array", "items": {"type": "number"}}}) + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # strict: first attempt wrong -> repair returns good -> ADDED with the repaired code + replies = iter([BAD, GOOD]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([mktrace()], lib, verify="strict") + assert log["added"] and not log["rejected"], log + assert "[42]" in lib.list()[0].code, "repaired code must be what landed" + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # strict: wrong twice -> REJECTED, library stays empty + replies = iter([BAD, BAD]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([mktrace()], lib, verify="strict") + assert log["rejected"] and not log["added"], log + assert lib.list() == [], "rejected skill must not land" + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # shape: a non-empty, schema-shaped answer passes even if values drifted (live data) + replies = iter([BAD]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([mktrace()], lib, verify="shape") + assert log["added"], f"shape mode must tolerate value drift: {log}" + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # shape: a CRASHING skill is caught even in the tolerant mode + replies = iter([CRASH, CRASH]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([mktrace()], lib, verify="shape") + assert log["rejected"], f"crash must be caught: {log}" + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # rounds=3: two bad attempts, the third lands + replies = iter([BAD, BAD, GOOD]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([mktrace()], lib, verify="strict", rounds=3) + assert log["added"], log + assert lib.list()[0].meta.get("grade") == "executable" + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # on_fail=reference: failed verification lands as a labeled reference skill + replies = iter([BAD, BAD]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([mktrace()], lib, verify="strict", on_fail="reference") + assert log["reference"] and not log["rejected"], log + m = lib.list()[0].meta + assert m.get("verified") is False and m.get("grade") == "reference", m + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # guard: a failed refine must NEVER overwrite an existing skill, even with on_fail=reference + from webwright.skill_factory.library import Skill as _Skill + sid = U._slug("verify template") + lib.add(_Skill(sid, "GOOD OLD CODE", {"template": "verify template", "verified": True, + "grade": "executable"})) + tr = mktrace(); tr.verdict = "adapt" + replies = iter([BAD, BAD]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([tr], lib, verify="strict", on_fail="reference") + assert log["rejected"], log + assert lib.get(sid).code == "GOOD OLD CODE", "old verified skill must survive" + + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + # incremental refine must REGRESSION-replay old coverage: + # land v1 (answers [42]); then a refine whose code only satisfies the NEW instance + # ([43]) must be rejected because it breaks the stored old example + replies = iter([GOOD]) # v1: writes [42] + U.llm = lambda *a, **k: next(replies) + assert U.evolve([mktrace()], lib, verify="strict")["added"] + assert (Path(d) / U._slug("verify template") / "replays.json").exists() + GOOD43 = 'import json\njson.dump({"retrieved_data": [43]}, open("agent_response.json", "w"))\n' + t43 = mktrace(); t43.answer = [43]; t43.verdict = "adapt" + replies = iter([GOOD43, GOOD43]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([t43], lib, verify="strict") + assert log["rejected"], f"refine breaking old coverage must be rejected: {log}" + assert "[42]" in lib.list()[0].code, "v1 must survive" + # a refine that answers from the taskspec params passes BOTH old and new -> lands + PARAM = ('import json\nspec = json.load(open("taskspec.json"))\n' + 'json.dump({"retrieved_data": [int(spec["params"]["want"])]}, ' + 'open("agent_response.json", "w"))\n') + t42 = mktrace(); t42.meta["params"] = {"want": "42"} + # rebuild v1's example with params the general code can use + import json as _j + rp = Path(d) / U._slug("verify template") / "replays.json" + rp.write_text(_j.dumps([{"params": {"want": "42"}, "start_url": "", "output_schema": + {"type": "array", "items": {"type": "number"}}, "answer": [42]}])) + t43b = mktrace(); t43b.answer = [43]; t43b.verdict = "adapt"; t43b.meta["params"] = {"want": "43"} + replies = iter([PARAM]) + U.llm = lambda *a, **k: next(replies) + log = U.evolve([t43b], lib, verify="strict") + assert log["adapt_refined"], f"general refine passing old+new must land: {log}" + + # update CLI smoke: -m webwright.skill_factory.update must not NameError on Path (regression) + with tempfile.TemporaryDirectory() as d: + mf = Path(d) / "m.json" + mf.write_text(_json.dumps({"template": "T", "runs": []})) + assert U.main(["--manifest", str(mf), "--library", str(Path(d) / "lib")]) == 0 + + print("test_evolve OK") + + +# pytest entry point (CI also runs this file as a script) +def test_all(): + run() + + +if __name__ == "__main__": + run() diff --git a/tests/skill_factory/test_gate.py b/tests/skill_factory/test_gate.py new file mode 100644 index 0000000..cb5b0b4 --- /dev/null +++ b/tests/skill_factory/test_gate.py @@ -0,0 +1,48 @@ +"""Unit test: admission gate (deterministic, no external).""" +import sys +from pathlib import Path +pass +from webwright.skill_factory.gate import gate + +ARR = {"type": "array", "items": {"type": "string"}} + + +def run(): + # self_verify: reject null / empty, admit non-empty + assert gate(None, method="self_verify").admit is False + assert gate([], method="self_verify").admit is False + assert gate("", method="self_verify").admit is False + assert gate(["Sprite"], method="self_verify").admit is True + + # self_verify: shape must match output_schema + assert gate(["a", "b"], output_schema=ARR, method="self_verify").admit is True + assert gate({"x": 1}, output_schema=ARR, method="self_verify").admit is False, "dict != array schema" + + # gold: admit iff equal + assert gate(["Sprite"], gold=["Sprite"], method="gold").admit is True + assert gate(["Pepsi"], gold=["Sprite"], method="gold").admit is False + + # auto: gold present -> use gold; absent -> self_verify + assert gate(["Sprite"], gold=["Sprite"]).admit is True # auto+gold -> match + assert gate(["Pepsi"], gold=["Sprite"]).admit is False # auto+gold -> mismatch -> reject + assert gate(["anything"]).admit is True # auto, no gold -> self_verify pass + assert gate(None).admit is False # auto, no gold -> self_verify fail + + # self_verify folds in the agent's own final report (free signal, no LLM) + r = gate(["ok"], method="self_verify", status="NOT_FOUND_ERROR") + assert not r.admit and "reported NOT_FOUND_ERROR" in r.reason, r + assert gate(["ok"], method="self_verify", status="SUCCESS").admit + assert gate(["ok"], method="self_verify").admit, "no status -> unchanged behaviour" + # gold outranks the self-report path entirely + assert gate([1], gold=[1], method="auto", status="NOT_FOUND_ERROR").admit + + print("test_gate OK") + + +# pytest entry point (CI also runs this file as a script) +def test_all(): + run() + + +if __name__ == "__main__": + run() diff --git a/tests/skill_factory/test_learn.py b/tests/skill_factory/test_learn.py new file mode 100644 index 0000000..f2bb1a5 --- /dev/null +++ b/tests/skill_factory/test_learn.py @@ -0,0 +1,114 @@ +"""Unit test: learn's LLM-free plumbing (schema inference, run collection, ledger skip).""" +import json, tempfile +from pathlib import Path +from webwright.skill_factory.learn import infer_schema, collect_runs + + +def run(): + assert infer_schema([1, 2]) == {"type": "array", "items": {"type": "number"}} + assert infer_schema(["a"]) == {"type": "array", "items": {"type": "string"}} + assert infer_schema(7) == {"type": "number"} + assert infer_schema({"k": 1}) == {"type": "object"} + assert infer_schema("x") == {"type": "string"} + + with tempfile.TemporaryDirectory() as td: + td = Path(td) + d1 = td / "run_a"; d1.mkdir() + (d1 / "task.json").write_text(json.dumps( + {"task": "## Skill library\nblah\n---\nCount commits by Jane Additionally, " + "write the final answer into $WORKSPACE_DIR/agent_response.json as {...}.", + "task_id": "a", "start_url": "http://gitlab.example.com/x"})) + (d1 / "agent_response.json").write_text(json.dumps({"retrieved_data": [3]})) + d2 = td / "run_b"; d2.mkdir() # unfinished: no agent_response + (d2 / "task.json").write_text(json.dumps({"task": "t", "task_id": "b"})) + + runs = collect_runs(td, {"runs": {}}) + assert len(runs) == 1 and runs[0]["task_id"] == "a", runs + assert runs[0]["task"] == "Count commits by Jane", "hint AND answer-spec must be stripped" + assert runs[0]["answer"] == [3] + + # ledger makes it idempotent + runs2 = collect_runs(td, {"runs": {str(d1.resolve()): {}}}) + assert runs2 == [], "already-learned run must be skipped" + print("test_learn OK") + + +def run_status_gate(): + """END-TO-END: a run whose own agent reported failure must be rejected by learn's + gate (status read from agent_response.json), even when the answer is well-formed. + All runs rejected -> learn returns before any LLM call, so this stays offline.""" + import contextlib, io, json, tempfile + from webwright.skill_factory.learn import learn + with tempfile.TemporaryDirectory() as td: + run = Path(td) / "runs" / "r1_20260711_000000" + run.mkdir(parents=True) + (run / "task.json").write_text(json.dumps( + {"task": "find the thing", "task_id": "r1", "start_url": "https://example.com"})) + (run / "agent_response.json").write_text(json.dumps( + {"task_type": "RETRIEVE", "status": "NOT_FOUND_ERROR", + "retrieved_data": ["well-formed but self-admitted failure"]})) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + learn(str(run.parent), str(Path(td) / "lib")) + out = buf.getvalue() + assert "agent itself reported NOT_FOUND_ERROR" in out, out + assert "0/1 runs admitted" in out, out + assert not (Path(td) / "lib" / ".learned.json").exists() or \ + "r1" not in (Path(td) / "lib" / ".learned.json").read_text() + print("test_learn status-gate OK") + + +def run_regressions(): + """F3: grouping-LLM failure must exit with an actionable one-liner, not a traceback.""" + import webwright.skill_factory.learn as L + orig = L.llm_json + L.llm_json = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("401 unauthorized")) + try: + try: + L.group_chunk([{"task": "t"}], []) + raise AssertionError("must raise SystemExit") + except SystemExit as e: + msg = str(e) + assert "OPENAI_ENDPOINT" in msg and "401" in msg, msg + finally: + L.llm_json = orig + print("test_learn regressions OK") + + +# pytest entry point (CI also runs this file as a script) +def run_reject_ledger(): + """A rejected skill must NOT mark its runs as learned (they get another chance).""" + import contextlib, io, json, tempfile + from unittest import mock + import webwright.skill_factory.learn as L + with tempfile.TemporaryDirectory() as td: + run = Path(td) / "runs" / "r1_x" + run.mkdir(parents=True) + (run / "task.json").write_text(json.dumps( + {"task": "count things", "task_id": "r1", "start_url": "https://example.com"})) + (run / "agent_response.json").write_text(json.dumps( + {"status": "SUCCESS", "retrieved_data": [1]})) + groups = [{"template": "count {{x}}", "members": [{"i": 0, "params": {"x": "things"}}]}] + with mock.patch.object(L, "group_chunk", lambda *a: groups), \ + mock.patch.object(L, "evolve", lambda *a, **k: {"added": [], "rejected": ["count_x"]}): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + L.learn(str(run.parent), str(Path(td) / "lib")) + led = Path(td) / "lib" / ".learned.json" + assert "kept un-learned" in buf.getvalue() + assert not led.exists() or "r1_x" not in led.read_text(), "rejected runs must stay un-learned" + print("test_learn reject-ledger OK") + + +def test_all(): + run() + run_regressions() + run_status_gate() + run_reject_ledger() + + +if __name__ == "__main__": + run() + run_regressions() + run_status_gate() + run_reject_ledger() diff --git a/tests/skill_factory/test_learned_example.py b/tests/skill_factory/test_learned_example.py new file mode 100644 index 0000000..e5d16c1 --- /dev/null +++ b/tests/skill_factory/test_learned_example.py @@ -0,0 +1,41 @@ +"""Lock the flagship property: the checked-in learned skill was AGGREGATED from +multiple solves (n_solves >= 3) with real lifted parameters β€” not a single-solve copy.""" +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] / "src/webwright/skill_factory/examples/learned_library" + + +def run(): + dirs = [d for d in ROOT.iterdir() if (d / "meta.json").exists()] + assert dirs, "learned_library example missing" + for d in dirs: + meta = json.loads((d / "meta.json").read_text()) + assert meta["n_solves"] >= 3, f"{d.name}: must be aggregated from >=3 solves, got {meta['n_solves']}" + params = meta["signature"]["params"] + assert len(params) >= 2, f"{d.name}: parameters must be lifted, got {params}" + assert "{{" in meta["template"], "template must have {{param}} placeholders" + assert "Additionally, write" not in meta["template"], "pipeline text must not leak (F7)" + extras = {f.name for f in d.iterdir()} - {"skill.py", "meta.json", "replays.json"} + assert not extras, f"{d.name}: run artifacts must not be committed: {extras}" + code = (d / "skill.py").read_text() + compile(code, d.name, "exec") + # artifacts must never be written next to __file__ (shared library dir) + assert "Path(__file__).resolve().parent\n" not in code + for line in code.splitlines(): + if "__file__" in line and "=" in line: + assert "WORKSPACE_DIR" not in line.split("=")[0], line + assert not any(k in line for k in ("RUN_DIR", "SCREENSHOT", "LOG")), \ + f"artifact path anchored to __file__: {line.strip()}" + for p in params: + assert p in code, f"param {p} must appear in the skill code" + print("test_learned_example OK") + + +# pytest entry point (CI also runs this file as a script) +def test_all(): + run() + + +if __name__ == "__main__": + run() diff --git a/tests/skill_factory/test_library.py b/tests/skill_factory/test_library.py new file mode 100644 index 0000000..bab8583 --- /dev/null +++ b/tests/skill_factory/test_library.py @@ -0,0 +1,38 @@ +"""Unit test: library store (deterministic, no LLM).""" +import sys, tempfile +from pathlib import Path +pass +from webwright.skill_factory.library import Library, Skill + + +def run(): + with tempfile.TemporaryDirectory() as d: + lib = Library(d) + assert lib.list() == [], "empty library should list nothing" + assert lib.get("nope") is None, "missing skill -> None" + + sk = Skill(skill_id="s1", code="print('hi')\n", + meta={"template": "do {x}", "summary": "does x", "signature": {"params": ["x"]}}) + lib.add(sk) + + got = lib.get("s1") + assert got is not None and got.code == "print('hi')\n", "get returns added code" + assert got.meta["template"] == "do {x}" + assert got.summary == "does x" + assert got.signature["params"] == ["x"] + assert [s.skill_id for s in lib.list()] == ["s1"], "list shows added skill" + assert lib.path("s1").name == "skill.py" and lib.path("s1").exists() + + # re-open from disk -> persisted + lib2 = Library(d) + assert [s.skill_id for s in lib2.list()] == ["s1"], "persisted across re-open" + print("test_library OK") + + +# pytest entry point (CI also runs this file as a script) +def test_all(): + run() + + +if __name__ == "__main__": + run() diff --git a/tests/skill_factory/test_retrieve_decide.py b/tests/skill_factory/test_retrieve_decide.py new file mode 100644 index 0000000..37151fe --- /dev/null +++ b/tests/skill_factory/test_retrieve_decide.py @@ -0,0 +1,98 @@ +"""Unit test: deterministic parts of retrieve/decide (no LLM). +The LLM paths are smoke-tested in test_front.py.""" +import json +import sys, tempfile +from pathlib import Path +pass +from webwright.skill_factory.library import Library, Skill +from webwright.skill_factory.retrieve import retrieve, Candidate +from webwright.skill_factory.decide import decide, Decision + + +def _lib(d): + lib = Library(d) + lib.add(Skill("bestsellers", "x", {"template": "Get the top best-selling product in period", + "summary": "magento bestsellers report"})) + lib.add(Skill("reviews", "x", {"template": "Get reviewers who mention something", + "summary": "product page reviews"})) + return lib + + +def run(): + with tempfile.TemporaryDirectory() as d: + lib = _lib(d) + + # retrieve(method="simple"): keyword overlap, deterministic + cands = retrieve("top best-selling product", lib, method="simple") + assert cands, "simple retrieve should find the bestsellers skill" + assert cands[0].skill.skill_id == "bestsellers", "most-overlapping skill ranked first" + + cands2 = retrieve("zzz nonsense quux", lib, method="simple") + assert cands2 == [], "no overlap -> no candidates" + + # decide with no candidates -> skip (deterministic, no LLM) + d0 = decide("anything", []) + assert isinstance(d0, Decision) and d0.verdict == "skip" and d0.skill_id is None + + # skill_use.recommend: a decision pointing OUTSIDE the retrieved candidates (LLM + # hallucination β€” even an id that exists in the library) must downgrade to skip + import webwright.tools.skill_use as T + orig_retrieve, orig_decide = T.retrieve, T.decide + try: + T.retrieve = lambda task, lib: [Candidate(lib.get("bestsellers"), 0.9, "stub")] + T.decide = lambda task, cands: Decision("use", "reviews", "hallucinated: not a candidate") + r = T.recommend("top best-selling product", d) + assert r["verdict"] == "skip" and r["skill_id"] is None, r + T.decide = lambda task, cands: Decision("use", "bestsellers", "in candidates") + r2 = T.recommend("top best-selling product", d) + assert r2["verdict"] == "use" and r2["skill_id"] == "bestsellers", r2 + finally: + T.retrieve, T.decide = orig_retrieve, orig_decide + + # with_skill_hint must bake an ABSOLUTE library path into the hint (the command runs in + # the agent's workspace, where a relative path would point at nothing) + from webwright.skill_factory.prompt import with_skill_hint + hinted = with_skill_hint("solve it", task="t", library="./some_rel_lib") + import os, shlex + assert f'--library {os.path.abspath("./some_rel_lib")}' in hinted, hinted[:300] + + # ...and shell-quote the task: $VAR / $(...) / backticks expand in bash even inside + # double quotes, so a task containing them must land single-quoted in the hint + evil = 'count $(whoami) commits in `pwd` for $USER' + hinted2 = with_skill_hint("solve it", task=evil, library="./some_rel_lib") + assert f"--task {shlex.quote(evil)}" in hinted2, hinted2[:300] + + # recommend on a MISSING or EMPTY library -> loud skip with a warning (and no mkdir side effect) + import webwright.tools.skill_use as T2 + r = T2.recommend("anything", "/nonexistent/skill/lib/path") + assert r["verdict"] == "skip" and "warning" in r and "empty" in r["warning"], r + assert not os.path.exists("/nonexistent/skill/lib/path"), "must not mkdir a bogus path" + with tempfile.TemporaryDirectory() as d2: + r2 = T2.recommend("anything", d2) # exists but has no skills + assert r2["verdict"] == "skip" and "warning" in r2, r2 + + # F1: a hard failure inside recommend must surface as an ERROR (loud), not a quiet skip + import io, contextlib + import webwright.tools.skill_use as TU + orig_rec = TU.recommend + TU.recommend = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom 401")) + try: + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + TU.main(["--task", "x", "--library", "lib"]) + out = json.loads(buf.getvalue()) + assert out["verdict"] == "skip" and "error" in out, out + assert "NOT consulted" in out["reason"], out["reason"] + finally: + TU.recommend = orig_rec + + print("test_retrieve_decide OK") + + +# pytest entry point (CI also runs this file as a script) +def test_all(): + run() + + +if __name__ == "__main__": + run()