diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000000..57e6ce87c0 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,7 @@ +{ + "permissions": { + "allow": [ + "Bash(bash /tmp/claude-1000/-workspaces-ako-mxcli/e4512e16-4485-4216-afba-df4212243089/scratchpad/verify-rebuild.sh 2>&1)" + ] + } +} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl index ab26e0d61c..28f9dc5415 100644 --- a/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl +++ b/.claude/skills/fix-issue/findings/cmd-mxcli.jsonl @@ -95,3 +95,5 @@ {"area": "cmd/mxcli", "date": "2026-08-31", "symptom": "Every open PR goes red at once on `build-and-test` with a failure in a package none of them touched — `--- FAIL: TestSessionLog_PersistAndPrune`, \"after reload+prune: 0 records, want 1\" — and the same test fails on a clean checkout of `main`", "cause": "A **time bomb in the test**, not a regression: the fixture pinned `base := time.Date(2026, 8, 1, ...)` against a 30-day retention window, and `NewSessionLogFile` prunes inside `load()` — *before* the test can assign `log2.now`, so the reload prune runs on the real `time.Now()` whatever clock is injected afterwards. It passed for 30 days and then failed permanently, on every branch simultaneously", "file": "`cmd/mxcli/tunnelhub/sessions_test.go` (`TestSessionLog_PersistAndPrune`), `cmd/mxcli/tunnelhub/sessions.go` (`load` → `pruneLocked` → `clock`)", "insight": "**First establish it is not yours**: run the failing test on a clean `origin/main`. Several PRs failing on one unrelated test is the signature. Then make the fixture relative — `base := time.Now().UTC()` — so the record ages, not the calendar, decide the outcome; the other tests in the file keep their fixed base legitimately, because they use `NewSessionLog` and inject the clock before recording. **A date fixture is only safe where no code path reads the real clock**; the moment a constructor prunes, expires or compares against `time.Now()` before the seam is in place, an absolute date has a fuse on it. Control the repair: stub `pruneLocked` to a no-op and confirm the test still fails (2 records, want 1), or the fix is just a test that stopped testing"} {"area": "cmd/mxcli", "date": "2026-08-31", "symptom": "`mxcli test --local` cannot start a runtime for ANY project: `Error: local runtime: runtime admin API did not come up: runtime process exited during startup`, then `java.lang.IllegalArgumentException: Path '/.mxcli/deployment-test/model/bundles' cannot be resolved in base path '/.mxcli/deployment-test'`. The tree has `data/` and no `model/`", "cause": "**mxbuild writes the deployment to `/deployment` and has no option to move it** \u2014 measured, not inferred: `--target=deploy` on a project whose `deployment/` had just been deleted recreated it there, `mxbuild --help` lists no deployment-path flag, and `BuildRequest` carries none. So giving the test boot a `DeployDir` of its own moved where the RUNTIME reads and not where the BUILD writes, and it booted against an empty directory. `StartLocalApp` now refuses a `DeployDir` the build will not populate, naming mxbuild as the constraint. The \u00a762 blanking the scratch tree was meant to prevent is fixed the only way the constraint allows: `preserveWebClientBundle` copies `deployment/web/dist` aside before the boot and puts it back after (a few MB of copy, against the ~30s re-bundle that made warning the earlier choice), restoring only when the bundle is actually gone so a fresher one is never clobbered.", "file": "`cmd/mxcli/docker/localapp.go` (`checkDeployDirIsBuildable`, the `preserveWebClientBundle` call), `cmd/mxcli/docker/webclient_preserve.go` (new), `cmd/mxcli/docker/localapp_integration_test.go` (new), `cmd/mxcli/testrunner/localapp_options.go`, `cmd/mxcli/testrunner/runner_local.go`, `cmd/mxcli/testrunner/runner.go`", "insight": "**The lesson is in the tests that did not catch it**, and two reporting projects drew it independently (mxcli-ledger \u00a7150, mxcli-sudoku \u00a751): four unit tests asserted `DeployDir` was set, was under `.mxcli/`, was per-project and was not the dev loop's \u2014 all four passed against a build that could not start, because every one was about the OPTION while the symptom lived in what is on disk after mxbuild runs. A fix that redirects a consumer without redirecting the producer moves the failure rather than removing it. The guard is now an integration test asserting the artefact: after a real build, the directory the runtime boots against holds `model/bundles`. Note the reporters' prescribed fix \u2014 thread `DeployDir` through the build \u2014 is not available; mxbuild has no such parameter, which is why the shared tree is accepted and the bundle carried across instead. Controls, both end-to-end on a real 11.13 project: the pre-fix binary reproduces the reported JVM error byte for byte while the fixed one passes; and with `preserveWebClientBundle` stubbed the sentinel bundle is destroyed by a test run that still reports all tests passed. Reinstating the regression fails the new integration test naming both missing paths."} {"area": "cmd/mxcli", "date": "2026-09-01", "symptom": "After `mxcli test --local`, a live `mxcli run --local` serving the SAME project starts answering **HTTP 200 with a zero-byte body** on every microflow-backed resource \u2014 not a 500, not an error page \u2014 while source-backed ones keep working, so half the app looks fine. The runtime log shows `java.lang.NoClassDefFoundError` on a project class. In a two-app solution it surfaces as tests failing in the OTHER app.", "cause": "The test run recompiles the project's Java into `deployment/run/bin`, which is the classpath the running JVM is holding open. Measured on a real 11.13 project: after one test run all 134 class files have **new inodes and byte-identical content** \u2014 every one deleted and rewritten. A JVM loads classes lazily, so one it has not reached yet can fail permanently. mxcli cannot prevent this: mxbuild's Gradle pass owns the compile and the deployment directory cannot be moved (ledger \u00a7150). So it warns instead, which is what was missing.", "file": "`cmd/mxcli/devloop_recompile_warning.go` (new \u2014 `warnIfDevLoopServing`, `recompileWarning`), `cmd/mxcli/cmd_test_run.go`; reads the existing `cmd/mxcli/devloop_handshake.go`", "insight": "**The mechanism already existed and a duplicate would have broken it.** `mxcli run --local` publishes `devLoopHandshake` at `.mxcli/run-local.json` for `mxcli constant set --apply` \u2014 same path, same pid-liveness staleness check, plus the `adminPass` and `bootConfig` that `--apply` and `--attach` depend on. A second state file was written at that path before this was noticed; it parsed fine (JSON ignores unknown fields) but its WRITER would have silently dropped those two keys. Grep the path before inventing a file. **The liveness check is the feature**: a `run --local` killed or ended by its development licence (\u00a760, measured lifetimes under six hours) leaves the file behind, and a warning driven by the file alone fires forever \u2014 one that is always wrong teaches the reader to skip it. It **warns rather than refuses**, since the warm loop exists so an app can stay up while you work and the reporting project runs two apps that way; neither `--attach` nor `--skip-build` builds, so neither warns. The finding's cost was diagnosis, not breakage \u2014 108 log lines and a wrong hypothesis about a different app, for something whose remedy is one restart \u2014 so the warning names the symptom (HTTP 200, empty body), the part nobody guesses. Controls, end-to-end against a real `run --local`: the warning carries that app's actual pid and port and its handshake still has adminPass and 9 bootConfig keys afterwards; with the loop stopped, and with a stale dead-pid handshake, the same command is silent. Reported as mxcli-formula1 FINDINGS \u00a781."} +{"area": "cmd/mxcli", "date": "2026-09-03", "symptom": "`mxcli brain check` reports an entry as MISFILED even though the entry is correct and its anchor points at a real document — the anchor's target is simply of a document type the catalog's `objects` view does not index", "cause": "Misfiling was decided by comparing the shard against the modules of *resolved* anchors. An entry whose only anchor came back NotIndexable had an empty resolved-module list, so the comparison found no match and reported it misfiled — reintroducing, through the misfiling axis, exactly the false staleness that the NotIndexable state exists on the anchor axis to prevent", "file": "`cmd/mxcli/brain/entry.go` (`MisfiledIn`)", "insight": "When a check has two axes, an 'unknown' outcome on one of them must not be read as a negative on the other. The fix is to make misfiling *undecidable* rather than false when nothing resolved: with no resolved anchor there is no evidence about where the entry belongs, and an anchor that truly names nothing is already a failure on its own axis. Caught in development by a table test whose control stubbed the guard to `if false` — a control that deletes the block instead fails to compile on unused variables, which is not a control", "refs": ["ako/mxcli#385", "PROPOSAL_project_brain.md A1"]} +{"area": "cmd/mxcli", "date": "2026-09-03", "symptom": "`mxcli brain check` exits 1 on a requirement that is simply not built yet — the entry is correct and current, and the check reports its anchor as NOT FOUND", "cause": "Requirements were recorded as ordinary brain entries, but an entry's anchor was assumed to point BACKWARD at something that exists. A decision's unresolved anchor means the decision is stale; a requirement's unresolved anchor means the work is not done. Same syntax, opposite meaning, and the store had no way to tell them apart", "file": "`cmd/mxcli/brain/entry.go` (`Kind`), `cmd/mxcli/brain/check.go` (`checkSlice`)", "insight": "Before adding a record type to an existing store, ask what a FAILED validation means for it — not just what it looks like. Requirements and decisions share the anchor syntax exactly, which is what made them look like the same thing; they differ only in the direction the anchor points, and that difference is the whole lifecycle. Measured before designing: one unbuilt requirement filed as a decision took `brain check` to exit 1, which settled it in one command. The inversion then pays for itself — a requirement is 'built' when its anchors resolve, so `brain plan` reports progress derived from the model (measured 0/1 -> 1/0 after creating the microflow, with the plan file untouched) instead of a status column that goes stale silently", "refs": ["ako/mxcli#385"]} diff --git a/.claude/skills/fix-issue/findings/mdl-backend.jsonl b/.claude/skills/fix-issue/findings/mdl-backend.jsonl index bbdd546813..5278312a84 100644 --- a/.claude/skills/fix-issue/findings/mdl-backend.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-backend.jsonl @@ -80,4 +80,4 @@ {"area": "mdl/backend", "date": "2026-08-26", "symptom": "`ALTER PAGE … SET Layout` / `ALTER LAYOUT` cannot find any widget in a layout — \"widget X not found\" for a widget DESCRIBE LAYOUT clearly shows", "cause": "Two finder gaps compounding. `findBsonWidget` starts from the page's `FormCall`, which a `Forms$Layout` does not have (its tree hangs off `Content`), so every widget in every layout was unreachable; and `findInWidgetChildren` never descended into a `ScrollContainer`'s five named slots, so a layout's topbar (`Top`) and navigation (`Left`) were out of reach even once the root was found — as was anything inside a scroll container a *page* places", "file": "`mdl/backend/pagemutator/mutator.go` (`findBsonWidgetInLayout`, region descent in `findInWidgetChildren`), `mdl/backend/pagemutator/scrollregion.go`", "insight": "**A region is addressed by slot, not by name** — it has no `Name`, so `INSERT INTO layoutContainer.top` reuses the dotted widgetRef that also serves DataGrid2 columns, and which one it means is decided by the named widget's `$Type`. Only `INSERT INTO` takes a region: BEFORE/AFTER position a widget among siblings, and silently treating them as INTO would put widgets somewhere the script did not ask for. The MDL slot name is `center`; the stored key is `CenterRegion`. Controlled by emptying the slot list and by respelling the centre key"} {"area": "mdl/backend", "date": "2026-08-27", "symptom": "No export mapping mxcli writes matches its Studio Pro original, so every one stays in #260's silent-loss set even once its source kind is authorable", "cause": "The export writers hardcoded three properties the IMPORT twin already read off the element: `MinOccurs 0` (a schema root has **1**), `MaxLength 0`, and `IsKey` not written at all. `MaxLength` is the one that cannot be a constant — Studio Pro stores **0 for a string element and -1 for a numeric one**, mirroring the bound schema element exactly as `MaxOccurs` already did", "file": "`mdl/backend/modelsdk/mapping_write.go` (`exportValueElementToGen`, `exportObjectElementToGen`) + `mapping_read.go`, `sdk/mpr/writer_export_mapping.go`, `modelsdk/mpr/serialize_mappings.go`, `mdl/executor/cmd_export_mappings.go`, `model.ExportMappingElement`", "insight": "The export element model had **none** of these fields and the reader populated none of them, so this is four layers (model, reader, builder, three writers), not a writer patch — and a writer patch alone would have written zeros from an empty model. Clone them from the schema element where the builder already clones `MaxOccurs`. Check the import twin first when an export property looks wrong: this was a divergence between the two writers, not a considered decision, and the import side is the correct reference. Repro `mdl-examples/bug-tests/mapping-277-export-property-set.mdl`, tests `sdk/mpr/writer_export_mapping_properties_test.go`. Issue ako/mxcli#277", "refs": ["#260", "ako/mxcli#277"]} {"area": "mdl/backend", "date": "2026-08-27", "symptom": "A rebuilt mapping drops `MessageDefinition2`, so a document written by Studio Pro 11.10+ never round-trips", "cause": "The key is version-introduced — `modelsdk/gen/mappings/version.go` records `messageDefinition2` as `Introduced: \"11.10.0\"` — and gen generates **no accessor** for it, so nothing read or wrote it. A blank 11.13 app's own mappings carry it as `\"\"`; none of the older pinned fixtures has it at all", "file": "`model.ImportMapping`/`ExportMapping` (`MessageDefinition2 *string`), `mdl/backend/modelsdk/mapping_read.go` (`messageDefinition2FromRaw`) + `mapping_write.go`, `sdk/mpr/parser_*_mapping.go` + `writer_*_mapping.go`, `mdl/executor/cmd_import_mappings.go` + `cmd_export_mappings.go`", "insight": "**Carry it, do not derive it.** A pointer, because nil (absent) is NOT the same as present-and-empty — writing the key onto a pre-11.10 document is the overlay-rule mistake CLAUDE.md warns about, which mxbuild tolerates and Studio Pro refuses to open. The executor decides: carry `existing.MessageDefinition2` on an update, apply the version gate only on a CREATE where there is nothing to read it off. A plain version gate in the writer looks equivalent and is not — it re-adds the key to every older document a rewrite touches, which turned four previously-clean fixtures red. gen having no accessor means the read goes to raw BSON, the route `parameterEntityFromRaw` takes. **`MappingSourceReference` is the same family and deliberately NOT carried**: its gate (10.16) predates every project in the field, and the codec emits it through a package-level `TypeDefaults` registration, so making it conditional would touch every mapping write to preserve one pre-10.16 fixture. Issue ako/mxcli#279", "refs": ["ako/mxcli#279"]} -{"area": "mdl/backend", "date": "2026-08-31", "symptom": "`DESCRIBE NAVIGATION` prints `home page` and the menu but **silently omits `login page` and `not found page`**, so pasting its output back (the documented copy workflow) deletes both from the profile. The clauses are on disk and `MXCLI_ENGINE=legacy` prints them", "cause": "The **reader**, not the writer: `mdl/backend/modelsdk/navigation_read.go` type-asserted only the `$Type`s `modelsdk/gen` declares for those two slots, and neither is what the documents carry — `LoginPageSettings` is stored as `Forms$FormSettings` with the page under `Form` (gen expects `Navigation$NavigationProfileLoginFormSettings` / `LoginPage`), and `NotFoundHomepage` as `Navigation$HomePage` (gen and `generated/metamodel` both expect `Navigation$NotFoundHomePage`). A failed type assertion leaves the field empty, so the loss is silent", "file": "`mdl/backend/modelsdk/navigation_read.go` (`navLoginPageOf`, `navNotFoundPageOf`), cross-check `generated/metamodel/types.go` `NavigationNavigationProfile`", "insight": "**The other engine is the control.** Legacy read the same bytes correctly throughout, which is what identifies a reader bug: `describe navigation X` on both engines must agree, and a disagreement localises the defect to the one that reads through gen. Accept the `$Type` the documents actually carry and keep gen's as a fallback branch. Note the two slots fail in **opposite directions** and want opposite fixes: for the login page a real Studio Pro document and `generated/metamodel` agree with the writers, so **gen** is wrong; for the not-found page — Studio Pro's **\"Fallback page\"** — metamodel and gen agree with each other and the three mxcli **writers** are the odd one out, emitting `Navigation$HomePage` where Studio Pro stores `Navigation$NotFoundHomePage`. Only a reference document could tell those apart, since mxbuild accepts either; ako/TestApp supplied it. Keep reading both `$Type`s regardless: documents written before the writer fix carry the `HomePage` spelling and must keep round-tripping. Repro `mdl-examples/bug-tests/navigation-describe-profile-pages.mdl`"} +{"area": "mdl/backend", "cause": "The **reader**, not the writer: `mdl/backend/modelsdk/navigation_read.go` type-asserted only the `$Type`s `modelsdk/gen` declares for those two slots, and neither is what the documents carry \u2014 `LoginPageSettings` is stored as `Forms$FormSettings` with the page under `Form` (gen expects `Navigation$NavigationProfileLoginFormSettings` / `LoginPage`), and `NotFoundHomepage` as `Navigation$HomePage` (gen and `generated/metamodel` both expect `Navigation$NotFoundHomePage`). A failed type assertion leaves the field empty, so the loss is silent", "date": "2026-09-01", "file": "`mdl/backend/modelsdk/navigation_read.go` (`navLoginPageOf`, `navNotFoundPageOf`), cross-check `generated/metamodel/types.go` `NavigationNavigationProfile`", "insight": "**The other engine is the control.** Legacy read the same bytes correctly throughout, which is what identifies a reader bug: `describe navigation X` on both engines must agree, and a disagreement localises the defect to the one that reads through gen. Accept the `$Type` the documents actually carry and keep gen's as a fallback branch. Note the two slots fail in **opposite directions** and want opposite fixes: for the login page a real Studio Pro document and `generated/metamodel` agree with the writers, so **gen** is wrong; for the not-found page \u2014 Studio Pro's **\"Fallback page\"** \u2014 metamodel and gen agree with each other and the three mxcli **writers** are the odd one out, emitting `Navigation$HomePage` where Studio Pro stores `Navigation$NotFoundHomePage`. ako/TestApp supplied the reference document that settled it. **Correction (2026-09-01): mxbuild does NOT accept either.** Measured on 11.13 against a build emitting the old spelling, `mx check` and `mxbuild --target=deploy` both exit 1 with \"Object of type 'Mendix.Modeler.WebUI.Navigation.HomePage' cannot be converted to type '...NotFoundHomePage'\" -- the project will not LOAD, so every downstream check is lost. What actually let it through is that nothing ever BUILT a project with a fallback page set: the automated mx-check coverage is doctype-tests/ only and no script there sets one, so the first was added by the fix itself. **Generalisable: 'the build tolerates it' is a claim that needs the same control as the fix** -- revert the writer, rebuild, and run the tool, or the reason a bug escaped gets recorded backwards and sends the next reader looking in the wrong place. Keep reading both `$Type`s regardless, but for the repair path rather than round-tripping: a pre-fix project does not build, and mxcli reads BSON directly, so accepting the old spelling is what lets it open and fix one. Repro `mdl-examples/bug-tests/navigation-describe-profile-pages.mdl`", "symptom": "`DESCRIBE NAVIGATION` prints `home page` and the menu but **silently omits `login page` and `not found page`**, so pasting its output back (the documented copy workflow) deletes both from the profile. The clauses are on disk and `MXCLI_ENGINE=legacy` prints them"} diff --git a/.claude/skills/fix-issue/findings/mdl-executor.jsonl b/.claude/skills/fix-issue/findings/mdl-executor.jsonl index 08ea896a55..599e15f989 100644 --- a/.claude/skills/fix-issue/findings/mdl-executor.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-executor.jsonl @@ -496,3 +496,12 @@ {"area": "mdl/executor", "date": "2026-08-31", "symptom": "`main` goes red on a test that passed in **both** PRs that touched it — here `TestDescribeWorkflow_NoAnnotationEmitsNoComment`: `unexpected violations [MDL-WF05] for a plain jump`", "cause": "Two PRs merged in sequence. One added a validator rule (MDL-WF05, dangling `jump to` target); the other's control test asserted `len(violations) == 0` over a fixture that was **not a valid workflow** — a lone jump whose target did not exist. Each CI run was green because neither saw the other's change", "file": "`mdl/executor/issue1007_annotation_emit_test.go` (the fixture), `mdl/executor/validate_workflow_jump.go` (the rule, which is right)", "insight": "**A test fixture that is not a valid instance of the thing under test is a landmine for the next rule.** \"No violations at all\" is only meaningful over input that *should* have none; over an invalid fixture it silently asserts \"no rule has been written yet that notices this\". Fix the **fixture**, not the rule. Generalise: when a test asserts the ABSENCE of diagnostics, make the input something you would be happy to ship. Two green PRs can still merge to red and **neither PR's CI can detect it** — the only protection is a fixture that does not depend on which rules exist today. Found by running `make test` on an unrelated docs branch cut from the merged main, which is an argument for doing that on any branch cut after a batch merge.", "refs": ["#350", "#351"], "rules": ["MDL-WF04", "MDL-WF05"]} {"area": "mdl/executor", "date": "2026-08-31", "symptom": "`filter($L, Amount > 0)` — or any FILTER/FIND predicate with a bare attribute and an operator other than `=` — fails the build with CE0117 \"Error(s) in expression.\", while the same attribute with `=` builds fine. `mxcli check` and `check --references` both pass. Reported as a Mendix 11.13 regression.", "cause": "The unfinished half of bug #343. Mendix has two filter operations: `Microflows$Filter` takes a member NAME (filter by attribute) and `Microflows$FilterByExpression` takes an expression evaluated per item with the item bound to `$currentObject`. #343 rerouted only `attr = value` to the by-attribute form; every other predicate still fell through to the expression form, where mxcli stored the authored text verbatim — and a bare attribute is not a valid Mendix expression. So the split was on the OPERATOR and invisible to the author: `Status = 'x'` built, `Status != 'x'` did not.", "file": "`mdl/executor/cmd_microflows_builder_actions.go` (`qualifyIteratorAttributes`, `listElementEntity`, `iteratorMemberPath`, `qualifyNamesInSource`); `mdl/executor/validate_microflow_listop_iterator.go` (MDL-LISTOP01); syntax topic `cmd/mxcli/syntax/features_microflow.go`; repros `mdl-examples/bug-tests/1002-filter-find-bare-attribute.mdl` and `1002-filter-bad-iterator.fail.mdl`.", "insight": "Rewrite a bare name that PROVABLY resolves to a member of the list's element entity into `$currentObject/`; refuse one that does not resolve (it used to surface as CE0117 at build time); pass through untouched when the element entity cannot be determined, which proves nothing either way. An association takes its module qualifier in the path, an attribute does not. The predicate is a frozen `SourceExpr` (`buildSourceExpression` in the visitor), so the rewrite has to patch the source TEXT, skipping single-quoted literals — `filter($L, Qty > 0 and Status != 'Amount')` must not rewrite the `'Amount'` inside the literal. Do not blame the Mendix version without running both: the issue reported \"reproduces on 11.13.0, not 11.11.0\" and the identical 7-error probe on mxbuild 11.11.0 and 11.13.0 disproves it — the fork is in mxcli, and nothing in it depends on the version. `mx check -j ` is the attribution tool: the console output names only the activity (\"List operation activity 'Filter by expression'\"), so two errors on one line read as one microflow; the JSON carries `document-name` per location. MDL-LISTOP01 keys on scope, not on the name — `$item` is valid in a predicate when it is the enclosing loop's iterator, which is exactly how CLAUDE.md's O(N) `find` idiom is written, so flagging the name would break the documented pattern. Control for the whole fix: stub `qualifyIteratorAttributes` to return the condition unchanged and the repro goes 0 → 7 x CE0117 with the two `=` cases staying green, which is also the #343 regression guard.", "refs": ["#1002", "#343"], "ce": ["CE0117", "CE0109"], "rules": ["MDL-LISTOP01"]} {"area": "mdl/executor", "date": "2026-08-31", "symptom": "A copied Atlas layout (or any page mxcli authors) fails `mx check` with **CE0463** on its Image widget, and a full field-level diff against Studio Pro's own widget differs in exactly **one line of 1480** — `maxHeight`, mxcli's `0` against the installed package's declared default", "cause": "Same class as the §69 width/height fix, which did not cover it — and why is the point. The reset was applied in a loop over the definition's **property mappings**, and a mapping is what gives a property an MDL keyword: `width`/`height` have one, `maxHeight` has none, so it was never visited and the widget TEMPLATE's captured value stood. Two further halves, each independently load-bearing (stub either and the test fails): a rule whose CONDITION is an unmapped property (`maxHeight` is hidden when `maxHeightUnit` = \"none\") was always indeterminable so never fired — the declared default is the right fallback there, because nothing can have moved an unnamable property off it; and `def.PropertyVisibility` is **empty** for every widget whose rules are lifted live from the `.mpk`, which is most of them, so a lookup keyed on that field silently finds nothing (both consumers now share `visibilityRules()`).", "file": "`mdl/executor/widget_engine.go` (`unmappedHiddenResets`, `visibilityRules`, the condition fallback in `hiddenUnnamedProperties`)", "insight": "The general rule: **the set of properties that must be default-valued is the widget's editorConfig to decide, not mxcli's** — making it a subset of what MDL has words for was the mistake. The third trap was found only end-to-end; the unit test passed while the command still wrote the wrong value. Control, end-to-end on a real 11.13 project with the Image `.mpk` patched to declare `maxHeight` 250 (the 1.6.0 value the finding measured, since that version was not obtainable): pre-fix binary writes 0 and all four mxcli-authored Images fail CE0463; fixed binary writes 250 and all four are clean, while `minHeight` stays at its own declared 0 — per-property from the package, not a blanket value. The project's ~60 Studio Pro-authored Images stay stale under both, which is the package change itself and not mxcli: the Step 0 discrimination diagnose-ce0463.md asks for. Reported as mxcli-ledger FINDINGS §142."} +{"area": "mdl/executor", "date": "2026-09-01", "symptom": "A hand-placed microflow/nanoflow/rule parameter is silently moved onto a grid at 200;53, 300;53, \u2026 by any rewrite \u2014 including a describe \u2192 exec of mxcli's OWN output. Reported as a feature request for `@position` on a parameter; the missing feature and the silent loss are the same defect seen from two sides.", "cause": "Microflows$MicroflowParameter is a stored node with RelativeMiddlePoint + Size, but the semantic type had no position field, so NEITHER reader carried one and BOTH writers could only recompute `200+idx*100;53` inline. The grammar had no slot for the annotation either, so there was no way to state a position and no way to preserve one.", "file": "sdk/microflows/microflows.go (Position + DerivedParameterPosition/AuthoredParameterPosition), mdl/grammar/domains/MDLMicroflow.g4 (annotation* on microflowParameter), mdl/visitor/visitor_microflow.go, mdl/backend/modelsdk/microflow.go + microflow_write.go, sdk/mpr/parser_microflow.go + writer_microflow.go, mdl/executor/cmd_microflows_parameter_position.go", "insight": "The design was already litigated one node family over and should be COPIED, not re-derived: @start's authoredStartPosition (#884 + #951) settles that a node at the layout's own derived spot carries no intent and must be re-derived, while one anywhere else was placed by a person and must survive. Carrying stored coordinates over unconditionally is the trap \u2014 inserting a parameter would strand the existing ones on the old grid while the new one lands on top. Put the arbitration in the READER so a non-nil Position means intent everywhere downstream; Position must be a POINTER because 0;0 is a coordinate a person can choose (two flows in the reference project use it). Measurement that framed the work: 20 of 28 parameters in a real 1971-unit project sit off the derived grid, so nearly every rewrite moved one. Trap when measuring: `Unchanged` on a SECOND exec proves the round trip reaches a fixed point, NOT that the first write changed only the thing you are looking at \u2014 this document also loses ExportLevel and a DestinationControlVector, unrelated and still open. Diff the raw unit BSON sorted by path, since the rebuild reorders ObjectCollection.Objects and a line-diff is then all noise.", "refs": ["ako/mxcli#993", "#951", "#884"], "rules": ["MDL059"]} +{"area": "mdl/executor", "date": "2026-08-29", "symptom": "A mapping sourced from an imported web service (SOAP) loses its binding on `create or replace|modify`: ImportedWebService and RootElementName are removed, ServiceName and OperationName blanked, and mxbuild reports CE6896 \"A mapping must have exactly one schema source\" + CE0270 \"No root element could be found in the schema\". `describe` emits the mapping with NO source clause, so describe -> exec — how a document is copied — is what destroys it.", "cause": "A mapping has FOUR source kinds, not three: JSON structure, XML schema, message definition, and ImportedWebService (stored key ImportedWebService, SDK name wsdlFile) with ServiceName/OperationName/RootElementName qualifying it — plus ParameterName/IsHeader on export mappings. model.ImportMapping carried three and a comment saying \"Schema source (at most one is set)\", so neither engine read the fourth and every rebuild dropped it. modelsdk/gen already exposed all the accessors; nothing called them.", "file": "model/types.go (WebServiceMappingSource), sdk/mpr/parser_import_mapping.go + parser_export_mapping.go (parseWebServiceSource), mdl/backend/modelsdk/mapping_read.go, mdl/executor/validate_webservice_mapping.go", "insight": "**Guard-don't-drop, and REFUSE rather than preserve** (ADR-0005, same class as the queued-call guard). Carrying the binding through a rebuild would imply the rest of the document survives too, and mxcli cannot check that: a SOAP mapping's elements resolve against the WSDL's schema entries, which live INLINE on WebServices$ImportedWebService.WsdlDescription.SchemaEntries and are never standalone XmlSchemas$XmlSchema documents. That inline detail also means improving `with xml schema` support does nothing for SOAP — a natural but wrong assumption. **The corpus cannot tell you SOAP is rare**: 0 web services across the 9 demo apps, but they are modern AI/factory demos, so that is evidence about the sample, not the population; legacy estates are exactly where this bites. To get a reference document, PLANT one (set the four keys on a real unit via UpdateRawUnit) — and replace keys IN PLACE, because appending duplicates makes the two engines disagree (bson.Raw.LookupErr takes the FIRST occurrence, bson.Unmarshal into a map takes the LAST), which looks exactly like an engine bug and is not. `describe` marks the source as NOT REPRESENTABLE in a comment rather than emitting nothing: the silent output parses, and re-executing it is the deletion.", "refs": ["ako/mxcli#365"], "ce": ["CE6896", "CE0270"]} +{"area": "mdl/executor", "date": "2026-09-02", "symptom": "A `create or replace` / `create or modify` that says nothing about documentation **deletes the object's `/** … */` doc comment**. The run prints success, `mx check` is clean (a document with no documentation is valid), and the loss is visible only by diffing the stored unit or noticing the text gone in Studio Pro", "cause": "The handlers set `Documentation: s.Documentation` unconditionally, so a statement with no doc comment wrote the zero value **over** the stored one. Not a drop — an empty overwrite. `ALTER ENTITY … ADD ATTRIBUTE` was unaffected, which is what localised it to the rewrite paths", "file": "`mdl/executor/cmd_microflows_create.go` (the carry block at the `existing*` reads), `mdl/executor/cmd_entities.go`, `mdl/executor/cmd_enumerations.go`, `mdl/visitor/visitor_entity.go` (`findDocComment`)", "insight": "**Absent and empty are different facts and the parser is the only layer that knows which.** `findDocCommentText` returned `\"\"` for both, so no downstream code could preserve-on-absent and clear-on-empty; the fix is a second return value (`findDocComment`) and a `DocumentationSet` bit on the statement. There is no `ALTER MICROFLOW … SET DOCUMENTATION`, so an explicitly empty `/** */` is the *only* clearing spelling available — preserve-always would have made a microflow's documentation permanent. **The mechanism already existed**: the same handler carries folder, allowed module roles, `Excluded` and the toolbox `MicroflowActionInfo` off the stored document; `Documentation` was simply never added to that set, so the fix is one more member of a list of four, not new machinery. **Scope is 26 statement types, not the 2 reported** — every type with a doc comment and a rewrite flag — so a coverage test enumerates them from `mdl/ast/*.go` and fails on any that is in neither the done nor the pending list, making the remaining 23 a number rather than a forgotten task. Two traps while building it: the first survival test chained `&& echo SURVIVED` to `head -1`, which exits 0 on empty input and reported success regardless of what grep found; and the first control deleted the carry block, which failed to COMPILE (unused variables) rather than failing the test — stub the condition to `if false` instead, or the control proves nothing. The bystander control (a second object, untouched) is what separates \"rewrites drop documentation\" from \"writes drop documentation\". mendixlabs/mxcli#1018 **Completing the pass across all 29 doctypes taught three things the first two did not.** (1) **An untested carry is worth nothing**: `CreateODataServiceStmt` has TWO update paths, the first fix patched one, and the type sat in the done list passing every build while the defect was fully intact — so \"done\" was redefined as carried AND covered by a fixture, and seven doctypes that cannot be fixtured here (agent editor needs AgentEditorCommons and 11.9+; OData/REST clients need a reachable $metadata or an OpenAPI spec) stay PENDING with their blocker recorded as the list's value rather than being quietly promoted. (2) **A delete+create rewrite must capture the stored value BEFORE the delete** — REST clients, pages and snippets are rebuilt that way, and their existing-object scan already captured roles and container for exactly this reason. (3) **The guard's own regex hid two doctypes**: it required a name ending in `Stmt`, so every `...StmtV3` was invisible to a list whose entire job is completeness, and image collections were missed because they spell the field `Comment`. True scope was 29, not the 26 first counted. Also found while testing: a menu's doc comment was parsed and read by nobody, so it never reached the model at all — a WRITE gap surfaced only because the survival test failed at its precondition rather than its assertion.", "refs": ["mendixlabs/mxcli#1018", "mendixlabs/mxcli#1017"]} +{"area": "mdl/executor", "date": "2026-09-01", "symptom": "`mxcli diff` against the UNMODIFIED output of `mxcli describe` reports the script would delete activities the executor round-trips exactly \u2014 every `call java action`, `download file`, `show message`, and every @position/@start/@curve line renders as `-` with no `+`. Retrieve constraints re-render as a Go struct pointer (`where &{0x236139b542d0 index=0}`). An entity dump comes back 'modified' purely on `create PERSISTENT entity` vs `create persistent entity`. Users concluded describe\u2192exec was lossy and stopped trusting the gate.", "cause": "diff rendered its two sides with TWO DIFFERENT RENDERERS. The project side went through the real describer (renderMicroflowMDL, shared with diff-local); the script side went through microflowStmtToMDL in cmd_diff_mdl.go, a second AST\u2192MDL renderer. Its statement switch covered 18 of 43 microflow statement types and had NO default case, so an unhandled activity emitted zero lines silently; it never emitted canvas annotations at all; and diffExpressionToString ended in `default: fmt.Sprintf(\"%v\", expr)`, which prints `&{\u2026}` for the ast.SourceExpr / ast.IdentifierExpr cases it lacked. Separately, diffStatement's `default: return nil, nil // Skip unsupported statements` covered 6 of 158 top-level statement types.", "file": "mdl/executor/cmd_microflows_build.go (new: buildMicroflowFromStmt/buildNanoflowFromStmt split out of the create handlers), mdl/executor/cmd_diff_render.go (new: renderFlowFromModel), mdl/executor/cmd_diff.go, mdl/executor/cmd_diff_mdl.go (376 lines of duplicate renderer deleted)", "insight": "Fix the mechanism, not the cases: build the model from the AST the way exec does (minus the write) and render BOTH sides with the one describer, which makes the false-deletion class unrepresentable. Adding the 25 missing cases would have fixed the symptom and left the drift. Two traps in the build/write split: (1) the create handler's build phase MUTATES \u2014 findOrCreateModule and resolveFolder create documents, consumeDroppedMicroflow consumes session state \u2014 so the dry-run path needs an AllowCreate flag and a read-only lookupFolder, or a read-only `diff` starts writing; (2) exec-only refusals (guard-don't-drop on queued calls, the already-exists error, validateMicroflowRules) must be skipped in dry-run, or a refusal aborts the build and the user gets NO diff instead of a diff plus the warning exec will give them anyway. Silence is the worse half of this bug: a script of statements diff cannot compare summarised as `0 new, 0 modified, 0 unchanged` even when it would add a document \u2014 a wrong count is at least visible. Measuring note: the reporter's control (exec, then OS-diff the re-describe) is good, but `exec` reporting `Unchanged` is stronger \u2014 ADR-0008 elision proves the parsed script is semantically EQUAL to what is stored, i.e. a no-op, not merely lossless. Deleting a renderer orphans its unit tests: #913's split-indentation test drove the dead renderer and was retargeted at formatMicroflowActivities, where the rule now lives.", "refs": ["ako/mxcli#997", "#913"], "rules": []} +{"area": "mdl/executor", "cause": "Not the resolver -- the walk. iconRefsInStatement visited only CreatePageStmtV3.Widgets, AlterPageStmt operations and AlterNavigationStmt, so four icon-bearing shapes were never inspected at all, each holding its widgets in a field the walk did not visit: widgets in a `placeholder X { ... }` block (CreatePageStmtV3.Placeholders, held apart from the bare body in .Widgets), CreateSnippetStmtV3, CreateLayoutStmt, and CreateMenuStmt (whose NavMenuItemDef items carry icons exactly as a profile menu's do).", "ce": ["CE1613"], "date": "2026-09-01", "file": "mdl/executor/validate_icon_refs.go (iconRefsInStatement); cross-check mdl/ast/ast_page_v3.go for every widget-bearing field on a statement and mdl/ast/ast_navigation.go for item-bearing ones", "insight": "Enumerate the AST fields; do not trust the obvious one. `Widgets` is not a page's only widget slot. Grep the ast package for statements holding []*WidgetV3 or []NavMenuItemDef and confirm each appears in the switch. Test at the walker, which is where the omission lives -- a test of the resolver passes either way. Pair every refusal with a POSITIVE control: the same four shapes with a VALID icon must still pass --references, exec, and build to 0 errors, or an over-eager walk has traded a silent miss for a false refusal. Measured on 11.13: all four silent at exit 0 and each producing its own CE1613 before, all four caught after, valid variants 0 errors. Note the report was against 0.19.0 while the resolver shipped in 0.18.0 -- when a reported case reproduces as FIXED, probe the neighbouring shapes rather than closing it, because the reporter hit a sibling of what they described. Repro mdl-examples/bug-tests/icon-refs-1008-placeholder-snippet-layout-menu.mdl", "refs": ["mendixlabs/mxcli#1008"], "symptom": "`mxcli check --references` passes and `exec` succeeds on an icon reference naming an IMAGE collection (Images$ImageCollection) instead of an ICON collection, and the first sign is CE1613 \"The selected custom icon 'Mod.Icons_SVG.cmdFilter24' no longer exists.\" from MxBuild. The same bad reference IS caught when the button sits directly in a page body, which makes it read as a resolver bug rather than a coverage gap."} +{"area": "mdl/executor", "cause": "Nothing resolved the role. cmd_navigation.go turned the AST's ForRole into a string with QualifiedName.String() and all three navigation writers dropped it into the BSON UserRole key unexamined, so whatever was typed reached disk. The confusion the docs encoded is real: a Mendix USER role is project-level and its identifier is a BARE name, while a MODULE role is module-scoped -- and they share names, so a blank app has a user role Administrator plus module roles called Administrator in three modules. mxcli's own docs, skill and `mxcli syntax` output all recommended the module-qualified form.", "ce": ["CE1613", "StorageLoadException"], "date": "2026-09-01", "file": "mdl/executor/validate_navigation_roles.go (new); wired at mdl/executor/validate.go (--references pass) and mdl/executor/cmd_navigation.go (exec, before any write). Writers that pass the value through: sdk/mpr/writer_navigation.go, mdl/backend/modelsdk/navigation_write.go, modelsdk/mpr/nav_patch.go", "insight": "Grade the failure by SHAPE, not by whether it is caught -- the three values behave differently and only measurement separates them (blank 11.13 app): `for Administrator` 0 errors; `for Supervisor` CE1613, an ordinary build error; `for MyFirstModule.Administrator` a StorageLoadException at LOAD, before checking runs. The load failure is a tier worse and is the one the docs recommended. It also explains a reporting trap seen twice now (also mendixlabs/mxcli#1000): a load failure suppresses the \"The app contains: N errors\" line while still exiting 1, so people read `mx check` as having SUCCEEDED. Never conclude success from the absence of that line; read the exit code. Two controls are load-bearing: (a) exec must refuse AND leave the .mpr byte-identical (md5 before/after) -- a refusal after a partial write is not a fix; (b) a script that CREATEs a user role and then uses it must still pass, because collectDefinitions does not track user roles, so the validator has to scan the program for CreateUserRoleStmt itself or it refuses the ordinary way to write one. Role matching is case-sensitive in Mendix (measured: `for administrator` is CE1613), so report the declared casing rather than silently normalising. gofmt trap: Go 1.19+ doc-comment normalisation rewrites '' into a curly quote, corrupting a quoted Mendix message -- put verbatim error text in an indented (tab) block. Precedent to copy for any user-role reference: applyGuestAccess in mdl/executor/cmd_security_write.go. Repro mdl-examples/bug-tests/navigation-1001-home-page-for-user-role.mdl **The last lesson is the sharpest: a blocker nobody has tried is a guess with a citation.** Seven doctypes sat in the pending list with confident reasons — \"agent editor needs AgentEditorCommons and Mendix 11.9+\", \"needs a reachable $metadata\", \"needs an OpenAPI spec\" — each written by me into a test file where it read as a finding. Tested directly, ALL SEVEN author fine offline against an ordinary 11.13 project: an OData client is created with a warning when $metadata is unreachable, a REST client takes an inline operation with no spec, and the four agent-editor documents need no extra module. The corpus is for measurements; an untested reason belongs in it only when labelled as untested.", "refs": ["mendixlabs/mxcli#1001"], "symptom": "`create or replace navigation ... home page X for MyFirstModule.Administrator` passes `mxcli check --references`, execs cleanly, and produces a project Mendix CANNOT LOAD: \"StorageLoadException: Role based home page in has an invalid value '' for property UserRole. The text 'MyFirstModule.Administrator' is not a valid UserRoleIdentifier.\" `mx check` exits 1 but prints no error count, so it reads as success. The documented syntax was the module-qualified form."} +{"area": "mdl/executor", "date": "2026-09-03", "symptom": "describe -> exec of a mapping silently drops OriginalValue (the sample parsed from the JSON structure's snippet), and reformats the structure's snippet from one line to multi-line. No build error either way — pure diff churn against a Studio Pro original.", "cause": "OriginalValue was written empty on every element (import carried whatever the caller set, which was nothing; export hardcoded \"\"), on the strength of #882's measurement over TWO mappings a blank app ships. describe pretty-prints the snippet and exec stored the pretty form.", "file": "mdl/executor/mapping_original_value.go, mdl/executor/cmd_jsonstructures.go (sameJSONContent), model/types.go (ExportMappingElement.OriginalValue), mdl/backend/modelsdk/mapping_write.go, sdk/mpr/writer_export_mapping.go", "insight": "**Neither global default was right, and measuring the SPLIT is what showed it.** Across 3,042 value elements whose structure carries a sample, 2,322 (76%) store it and 720 do not — but the split is PER DOCUMENT: 145 mappings carry it on every element, 107 on none, 2 mixed. So which one a mapping gets is a property of how and when it was authored, not something derivable. Always-copy is wrong for 107 mappings; always-empty (the old behaviour) is wrong for 145. **A REWRITE does not have to choose** — it knows what was stored, so it carries it (guard-don't-drop, ADR-0005), matching stored to rebuilt by JsonPath because names and order can change while the schema binding cannot. That leaves #882's actual decision intact: a NEWLY authored mapping still writes empty, which is what that issue was about. The general lesson: when a measurement says 'always X' from a small sample and a wider one says 'sometimes X', check whether the split is per-document before picking a default — a per-document split usually means the answer is 'preserve', not 'choose'. The snippet half is the same shape: keep the stored formatting when the JSON is semantically equal (compare decoded values, not strings), so describe -> exec is a no-op instead of a reformat.", "refs": ["ako/mxcli#379", "ako/mxcli#882"]} +{"area": "mdl/executor", "date": "2026-09-03", "symptom": "Implementing a new document type from a corpus census alone produces a document that builds at 0 errors and still differs from Studio Pro's in five places — Path, the typed-array marker, an empty mandatory list, PrimitiveType, and a dropped authored field.", "cause": "The census was 36 marketplace-module collections. A module author and someone building an app by hand exercise different parts of a document, so a census over shipped modules misses whatever only hand-authoring sets, and averages away anything the modules happen not to use.", "file": "mdl/backend/modelsdk/messagedefinition_write.go, mdl/executor/cmd_messagedefinitions.go, testdata/TestApp.OrderMessageDefinitions.bson", "insight": "**One hand-authored reference document is worth more than a large census of marketplace modules.** ako/TestApp's OrderMessageDefinitions found five things a 36-collection / 4,686-element census had not: (1) Path is a chain of ORIGINAL names, not exposed ones, and an ASSOCIATION contributes TWO segments — `Order|OrderLine_Order|OrderLine|Amount` — confirmed afterwards at 4,707/4,707 once we knew to look; (2) typed-array marker is 2, the codec defaults to 3; (3) every element serializes Children even when empty (the bare [2], same MandatoryLists rule as a rule document's Flows); (4) PrimitiveType is MAPPED not passed through — Long->Integer, AutoNumber->Integer, Enumeration->String, 279 corpus elements a pass-through gets wrong; (5) Example is author-set — empty in 4,686/4,686 of the corpus, set in TestApp, so hardcoding it empty silently drops the one that exists. **The round-trip test is what finds these**: read a REAL stored document into the semantic model, re-encode, diff against the STORED BYTES. Do not diff against a re-encoding of the decoded original — a lazily-decoded element that was never marked dirty encodes as an empty document, so that baseline passes by comparing nothing to nothing. **A hand-authored document also tends to carry natural controls**: this one uses the same association in both directions, which is exactly the control the cardinality rule needed. **After a CREATE that resolves a folder, invalidate the hierarchy cache** — the cached hierarchy predates the new folder, so a later lookup by module fails and CREATE OR MODIFY writes a DUPLICATE (CE0122). The update branch gets this free from applyDocumentFolder; a create branch has to say it.", "refs": ["ako/mxcli#272"], "ce": ["CE0122", "CE1613"]} +{"area": "mdl/executor", "cause": "Two DataTypes$ sub-documents the writer never emitted, both on Microflows$CallExternalAction. (1) edmReturnTypeToKind mapped only EDM primitives and returned \"\" for anything else -- documented in-code as \"Complex / collection / entity-typed returns aren't yet mapped\" -- so an action returning an ENTITY got no VariableDataType at all. (2) ExternalActionParameterMapping.ParameterType was never written, though generated/metamodel declares it WITHOUT omitempty. Separately, mdl/catalog/builder_external.go catalogued only entities whose Source is Rest$ODataRemoteEntitySource, skipping every Rest$ODataEntityTypeSource -- the derived, abstract, contained and action parameter/return types that have no entity set.", "ce": ["CE7252", "CE7269", "CE0117", "CE7251"], "date": "2026-09-03", "file": "mdl/executor/cmd_microflows_builder_calls.go (resolveExternalActionReturnKind, resolveExternalActionParameterKinds, edmBareTypeName); sdk/mpr/writer_microflow_actions.go + mdl/backend/modelsdk/microflow_external_action_write.go (both writers); sdk/microflows/microflows_actions.go (ResultEntity, ParameterDataType/ParameterEntity); mdl/catalog/builder_external.go (isODataEntitySource); new validator mdl/executor/validate_external_action_calls.go", "insight": "**Read the CE code out of Mendix's own assemblies before theorising about it.** `strings Mendix.Modeler.Texts.dll | grep CE7252` gives the symbol, the English text AND the LOCATION comment -- here CallExternalAction.cs for both codes, which settles in one command that the entity import can never fix them and that the reporter was pulling the wrong lever. Same technique found CE7253 and the CE7251 constraint (Mendix's `call external action` takes OData ACTIONS only, not Functions, and an unbound action needs an in the EntityContainer or it is not callable at all). **A missing mandatory sub-document is the recurring shape here**: generated/metamodel omitting `omitempty` on a pointer property is the tell, and the same fix pattern applied twice in one bug. **The reporter's evidence was an artifact of OUR tool**: contract_entities.UsedByExternalEntity is an mxcli catalog column filled by joining external_entities on RemoteName, so while that table skipped type-sourced entities the column was structurally always empty for exactly the entities in question -- it read as 'not linked' whether or not the import had worked. When a report cites one of our own derived columns as evidence, verify the column can be non-empty for that case before believing it. **Verification without a fixture**: no $metadata in the repo declared an action, so the contract was served from `python3 -m http.server` on 127.0.0.1 and MetadataUrl pointed at it -- a local HTTP contract makes the whole consumed-OData path testable end to end. Controls: reverting the return resolver reproduces CE7269 verbatim; before the ParameterType fix, ANY parameter of ANY type produced CE7252 + one CE0117 per argument; after, 0 errors on all three shapes. Repro mdl-examples/bug-tests/odata-1020-external-action-types.mdl", "refs": ["mendixlabs/mxcli#1020"], "symptom": "CE7252 \"The parameters for remote action '' have changed\" and CE7269 \"The return type for remote action '' has changed\" persist after CREATE OR MODIFY EXTERNAL ENTITIES, which reports success and changes nothing. A SQL query over CATALOG.contract_entities shows UsedByExternalEntity empty for the action's parameter/response entities while entity-set entities populate it, which reads as a broken link between the imported entity and the contract."} diff --git a/.claude/skills/fix-issue/findings/mdl-other.jsonl b/.claude/skills/fix-issue/findings/mdl-other.jsonl index 7dc32ef07d..afdf7116b0 100644 --- a/.claude/skills/fix-issue/findings/mdl-other.jsonl +++ b/.claude/skills/fix-issue/findings/mdl-other.jsonl @@ -50,3 +50,5 @@ {"area": "mdl/offlinepaths", "date": "2026-08-27", "symptom": "Adding an offline navigation profile makes a build fail with **CE6206** (\"Attribute paths with multiple steps cannot be used on pages that are accessible through an offline-based navigation\") in pages the statement never mentioned", "cause": "An offline profile restricts every page it can REACH to at most one association hop. The pages were valid before; creating the profile is what invalidated them, and mxcli writes multi-step paths happily and said nothing", "file": "`mdl/offlinepaths/scan.go` (the stored-document scan), `mdl/executor/offline_profile_warning.go` (the report), `mdl/types/navigation_profile_kind.go` (`IsOfflineProfileKind`)", "insight": "**The threshold is TWO hops, not \"any indirect reference\"** — one hop is explicitly allowed, pinned by a control inside one page where mxbuild flags column 2 of 2 and accepts column 1. Scan for the stored `DomainModels$AttributeRef` whose `EntityRef.Steps` holds ≥2 steps, keying on `$Type` and not on a table of widget property names — a DataGrid2 column's binding is nested several levels inside a `CustomWidgets$WidgetValue`, so a property-name scan reports a clean project for the widget people actually use. Do NOT match every `IndirectEntityRef`: a data source that navigates an association is a different element and CE6206 does not reject it. Report as a **warning**, never a refusal — reachability needs the whole page graph (home pages, menu items, and every page those open), which mxcli does not walk, and the end-to-end control shows mxbuild at 0 errors while the flagged page is unreachable and CE6206 the moment the profile's home page points at it. ako/mxcli-maintenance", "ce": ["CE6206"]} {"area": "mdl/types", "date": "2026-08-28", "symptom": "`rename entity` / `rename module` reports success but prints no \"Updated N reference(s)\" line, and the next `mx check` fails **CE0174** \"Cannot resolve object name 'MyFirstModule.Period'\" on a view entity", "cause": "A view's OQL is the ONE place a qualified name is stored **embedded in a sentence** rather than as a property of its own. Both engines' rename walkers match a string that *equals* the old name or *begins with* it — correct for a BY_NAME property, and blind to `from MyFirstModule.Period as p` mid-query", "file": "`mdl/types/oql_rename.go` (`RewriteOQLQualifiedName`), wired at the `Oql` key in `mdl/backend/modelsdk/infrastructure_write.go` (`replaceQNInDocCounted`) and `sdk/mpr/writer_rename.go` (`replaceStringsInDoc`)", "insight": "**Ask where each reference is STORED, not just which documents reference it** — a scan built for whole-string properties silently covers 0 of the embedded ones, and reports 0, which reads like \"nothing referenced it\". Scope the rewrite to the `Oql` key: a blanket substring replace across every string reaches documentation and expressions, where a similar-looking name is not a reference. Three name-shaped ways it goes wrong, all covered by tests: a **longer name starting with the old one** (`M.PeriodDetail`) must not move, a **quoted** reference must stay quoted (bare is CE0174 — quoting is how a view names an entity called after an OQL reserved word), and a **module rename arrives as a prefix pair** (`\"Old.\" -> \"New.\"`) with no entity half to split, so it needs its own path or module rename stays broken while entity rename looks fixed. Where a query-local alias is spelled exactly like the module, the rewrite **declines** rather than guessing — 0 references reported is a visible non-event, a corrupted query is not. Both engines, one shared rewrite. ako/mxcli-captrack", "ce": ["CE0174"]} {"area": "mdl/translations", "date": "2026-08-30", "raw": "| `create or modify translations in for ` reports success (\"Set 212 nl_NL translation(s) across 20 document(s)\") and the app's pages switch language while the **menu does not** | `mdl/translations/outofscope.go` (new), `mdl/executor/cmd_translations.go`, `cmd/mxcli/syntax/features_misc.go` | The **navigation is a project-level document**, not a module one, so `in ` never reaches it. Measured on the reporting project: 151 strings scoped against 546 unscoped, and re-running the same file unscoped landed 65 more across 22 further documents. Nothing warned — the document count was the only tell, and only if you knew what number to expect. A scoped run now names **the file's own entries** it did not reach (`translations.OutOfScope`), not \"the project has other strings\", which is true of every scoped run and would warn forever — the per-module workflow is exactly what the scoping exists to support. Second, load-bearing half: those entries were previously swept into the **drift** warning, whose premise (\"no text has this as its source\") is *false* about them — they matched, out of scope. They are subtracted from it, so \"the text may have been deleted\" is only said where it is true. Controls: an unscoped run of the same file reports nothing new and lands the strings; a key matching nothing anywhere is still reported as drift. Reported as ledger #137 |", "refs": ["#137"]} +{"area": "mdl/catalog", "date": "2026-09-03", "symptom": "`SHOW LANGUAGES` omits a language the project really has (ar_DZ absent from a list of 8 where the project has 9), and `search ''` returns \"No matches found\" for a string `DESCRIBE TRANSLATIONS` lists. Nothing errors and the catalog builds clean.", "cause": "CATALOG.strings was filled by hand-written per-type extractors reaching five sites (page title, enum caption, three microflow message templates), so a text anywhere else — every widget caption, tooltip, validation message, client template — was never indexed.", "file": "mdl/catalog/builder_strings.go", "insight": "A language present only on an unindexed site is INVISIBLE, not undercounted, so it vanishes from SHOW LANGUAGES entirely and from lint rule QUAL005, which discovers its language set from the same table. The fix is not a sixth case — that is how five was ever the number. Index from the type-agnostic walk DESCRIBE TRANSLATIONS already uses (translations.SitesInUnit over ListRawUnitsByType(\"\")), leaving only non-Texts$Text strings in the typed path (URLs, log nodes, REST paths, documentation, and Microflows$StringTemplate, which holds a plain Text and cannot carry a translation). Derive ObjectType from the unit $Type mechanically rather than via a table. Measured before: 69 of 3265 texts, 8 of 9 languages, 66 en_US of 1045. After: 1496 rows, 9 languages, counts identical to an independent BSON walk. Atlas design templates are ~70% of the corpus and are indexed rather than excluded, because CREATE TRANSLATIONS writes them and a SHOW LANGUAGES that excluded them would reopen the same split. CONTROL: stub the walk and the run reports `strings: 3` with SHOW LANGUAGES reporting nothing at all.", "refs": ["#250"]} +{"area": "mdl/linter", "date": "2026-09-03", "symptom": "Lint rule QUAL005 reports no missing translation for an enumeration where only one value is translated (11 real gaps unreported), and likewise for a page's sibling action buttons.", "cause": "The rule grouped by (QualifiedName, StringContext) while ElementId sat unused in the strings table, so every sibling element of one type collapsed into one group and a single translated value made the set look complete.", "file": "mdl/linter/rules/missing_translations.go", "insight": "Add ElementId to the SELECT, the ORDER BY and the elementKey struct. No test caught it because the harness synthesized ElementId from QualifiedName+StringContext, giving every sibling the same value and reproducing the defect inside the fixture — a fixture that encodes the bug cannot detect it. CONTROL: with every sibling translated the run must stay at 0 violations, or the new violation is an artifact of splitting the group rather than the missing translation.", "refs": ["#250"]} diff --git a/.claude/skills/mendix/bootstrap-app/SKILL.md b/.claude/skills/mendix/bootstrap-app/SKILL.md index 8e40d71b4f..b1b85e07b2 100644 --- a/.claude/skills/mendix/bootstrap-app/SKILL.md +++ b/.claude/skills/mendix/bootstrap-app/SKILL.md @@ -56,6 +56,17 @@ it is building. `ledger` (light, dense, data-heavy), `console` (dark), or `none` for stock Atlas. Default `signal`. 7. **Mendix version.** Default `11.13.0`. +8. **Do you have requirements to work from?** A specification document, a + prototype, a wireframe, a long description — anything that is the source of + truth but is not in this repo. **Default: yes, record them.** If they say yes, + ask them to paste or point at it; if they say "just build it", say in one line + that you will record the slices you derive as you go, and carry on. + + This one is not cosmetic. Requirements that live in a Word document, a Figma + file or a chat window leave **no trace in git**: not in an issue, not in a + commit message. A session that resumes after an idle reap has no idea what it + was building towards, and neither does the next person. Recording them costs + a minute now and is unrecoverable later. If the user says "defaults" or ignores a question, choose something sensible for it, say what you chose in one line, and keep going — **do not block on them twice**. @@ -121,16 +132,46 @@ drop the `./` if it came pre-installed on `PATH`. applied, a `mxcli check` that passed but a real `mx check` later flagged. Note the Mendix + mxcli versions and how each finding was verified. This is durable context for the next session, and the most useful thing to share back to improve mxcli. -6. **COMMIT everything now** — `.mpr`, `.devcontainer/`, `.claude/` (the - SessionStart hook **and** `.claude/bootstrap-mxcli.sh`), `README.md` and - `FINDINGS.md`. This step is mandatory, not housekeeping: the seed prompt is a +6. **Record the plan in the brain** — unless the user opted out at Q8: + + ```bash + ./mxcli brain init -p .mpr + ./mxcli brain capture "" \ + --slice 01- -a @Module. \ + -p .mpr + ./mxcli brain staged -p .mpr # then promote each + ``` + + Split the requirements into **slices you could deliver one at a time**, named + with a numeric prefix so they sort into a roadmap: `01-accounts`, + `02-approvals`. Anchor each requirement at **what will implement it** — the + entity, microflow or page you are about to create. The anchor points forward, + so naming something that does not exist yet is correct here, and it is what + makes `./mxcli brain plan` a real progress report instead of a checklist. + + Read `.ai-context/skills/project-brain/SKILL.md` before doing this. + + Two things to keep straight, because they are easy to conflate: + + - `README.md` is the **brief** — what the app is, in the user's words, for a + human landing on the repo. One page, written once. + - `docs/brain/plan/` is the **scope** — the individual requirements, anchored + and countable, appended to as they emerge. + + **Never write a status or a tick-box next to a requirement.** Whether it is + built is computed by `./mxcli brain plan` from the model; a hand-maintained + status is wrong the moment anyone builds anything, and nothing will tell you. + +7. **COMMIT everything now** — `.mpr`, `.devcontainer/`, `.claude/` (the + SessionStart hook **and** `.claude/bootstrap-mxcli.sh`), `README.md`, + `FINDINGS.md` and `docs/brain/`. This step is mandatory, not housekeeping: the seed prompt is a *one-time* seed, and committing its output is what makes every later session bootstrap from files instead of from a re-paste. The `mxcli` binary itself stays git-ignored (~85 MB); the bootstrap script is what fetches it back into a fresh clone, so committing the script is what makes the hook survive a reap. -7. **Boot and verify:** `./mxcli run --local -p .mpr` in the background, then +8. **Boot and verify:** `./mxcli run --local -p .mpr` in the background, then confirm the app answers HTTP 200 at http://localhost:8080/ and report. -8. **(Optional) browser preview from a cloud session:** +9. **(Optional) browser preview from a cloud session:** `./mxcli run --hub https://hub.mxcli.org -p .mpr`, and report the preview URL it prints. Needs `MXCLI_HUB_KEY` on the environment; without it, continue as a normal local run. `--hub` ships in the **Linux** build only (a cloud session is a @@ -219,6 +260,9 @@ named after it. From the brief, propose in chat: only what the other app actually needs Show it as **MDL the user can read**, and wait for their go-ahead before executing it. +Name the elements the same way the plan's anchors do — if a requirement is anchored +`@Module.ACT_Approve`, propose that name — so `./mxcli brain plan` starts +counting the moment the work lands, without anyone editing the plan. If a design was handed to you, it is the source of truth for the model and the pages — see `migrate-design-prototype`. @@ -231,6 +275,18 @@ see `migrate-design-prototype`. ./mxcli exec change.mdl -p .mpr # edit the model; the loop hot-applies ``` +Keep the plan current as you go — it is the only record of scope that outlives the +conversation: + +```bash +./mxcli brain plan -p .mpr # what is outstanding, counted from the model +./mxcli brain capture "" --slice -a @Mod.Thing -p .mpr +``` + +Requirements arrive mid-build — the user says "and it should also…". Capture that +when it is said, not later. Do **not** tick anything off: `brain plan` derives what is +built from the model, so finishing the work is what moves the number. + In a solution, run one loop per app from its own folder, with the second app on the alternate ports, and start the producer first so the consumer's external entities resolve: diff --git a/.claude/skills/mendix/json-structures-and-mappings/SKILL.md b/.claude/skills/mendix/json-structures-and-mappings/SKILL.md index 24615c2d23..300767fe67 100644 --- a/.claude/skills/mendix/json-structures-and-mappings/SKILL.md +++ b/.claude/skills/mendix/json-structures-and-mappings/SKILL.md @@ -156,29 +156,29 @@ describe json structure Module.JSON_Pet; drop json structure Module.JSON_Pet; ``` -### The `with` clause is resolved, not written through +## Message Definitions -A mapping's schema source — `with json structure M.X` or `with xml schema M.Y` — -is checked against the project by both `mxcli check -p` and `exec`, and a name -that resolves to nothing is refused with the documents that would have worked. -mxbuild otherwise reports it as **CE1613** "… no longer exists" at the end of a -build (ako/mxcli#259). +A mapping's source can also be a **message definition** — 74 of the 327 mappings +in the demo corpus (22.6%), and the only non-JSON source MDL can create. It +holds nothing external: it is a **selection over the domain model**. -For JSON structures a typo used to be worse than a dangling reference: the schema -index is empty whenever the structure cannot be loaded **for any reason**, and an -empty index reads as "there is nothing to validate against" — so one typo in the -source name switched off every member check in the mapping. +```sql +create message definition collection Sales.MD_Order ( + definition OrderMessage for Sales.Order as 'Orders' ( + OrderId, + Sales.Order_Customer/Sales.Customer ( FirstName ) + ) +); +``` -Two things the check deliberately does not do: +A bare name is an attribute; `Assoc/Module.Entity` is an association. **Name the +target entity** — the stored cardinality follows the direction of traversal, so +the same association gives a single object one way and a list the other. -- A structure the **same script** creates counts as existing. Create the - structure, then map over it, is the normal shape. -- A project with **no** XML schemas disables the XML half rather than refusing - every mapping. There is no `create xml schema` in MDL — an XML schema is only - ever imported into the project by hand — so having none is ordinary, not - evidence of a typo. +The full vocabulary, the ALTER statements, inherited attributes and what mxcli +deliberately does not guess: +[reference/message-definitions.md](reference/message-definitions.md). ---- ## Import Mappings diff --git a/.claude/skills/mendix/json-structures-and-mappings/reference/message-definitions.md b/.claude/skills/mendix/json-structures-and-mappings/reference/message-definitions.md new file mode 100644 index 0000000000..99ddcc4f12 --- /dev/null +++ b/.claude/skills/mendix/json-structures-and-mappings/reference/message-definitions.md @@ -0,0 +1,117 @@ +# Message definitions + +Supporting reference for [json-structures-and-mappings](../SKILL.md). + +A mapping's source can also be a **message definition** — 74 of the 327 mappings +in the demo corpus (22.6%), and the only non-JSON source MDL can create. Unlike +an XML schema (an imported `.xsd`) or a web service (a WSDL), it holds nothing +external: it is a **selection over the domain model**. + +```sql +create message definition collection Sales.MD_Order + folder 'Messages' +( + definition OrderMessage for Sales.Order as 'Orders' ( + OrderId, + Total as 'GrandTotal', + Sales.OrderLine_Order/Sales.OrderLine as 'Lines' ( Sku, Quantity ), + Sales.Order_Customer/Sales.Customer ( FirstName, Address example 'Kerstraat 5' ) + ) +); +``` + +A **bare name is an attribute**; `Assoc/Module.Entity` is an **association** with +its own members — the same discriminator import and export mappings use. A +mapping then binds to `Module.Collection.Definition`, a three-part reference. + +**Name the association's target entity.** It is not decoration: the stored +cardinality tracks the **direction of traversal**, not the association's type. +Reaching `Customer` from `Order` follows the foreign key and gives a single +object; reaching `Order` from `Customer` is the reverse and gives a list — the +same association, both ways. An association that connects the two entities in +neither direction is **refused**, because a wrong cardinality builds cleanly and +would silently expose a list as a single object. + +**Inherited attributes are named like the entity's own.** mxcli resolves each to +the entity that declares it, which is what Mendix stores; qualifying one against +the entity that merely uses it is CE1613. + +### Editing one without restating it + +Real definitions nest deeply, so a whole-document rewrite is a poor tool for +"expose one more attribute". `ALTER` edits the stored document and leaves the +rest alone: + +```sql +alter message definition Sales.MD_Order.OrderMessage add member Total; +alter message definition Sales.MD_Order.OrderMessage add member LastName in Customer; +alter message definition Sales.MD_Order.OrderMessage set member Total as 'GrandTotal'; +alter message definition Sales.MD_Order.OrderMessage drop member Sku in Lines; + +alter message definition collection Sales.MD_Order add definition Line for Sales.Line ( Sku ); +alter message definition collection Sales.MD_Order rename definition Line to OrderLine; +alter message definition collection Sales.MD_Order drop definition if exists OrderLine; +``` + +`in ` reaches a nested member, written in **exposed names**. `SET` changes +only the exposed name — it is not a model rename, which is why the verb is not +`RENAME`. + +Dropping or renaming a definition a mapping still references is refused, naming +the mappings. + +### What mxcli does not guess + +Studio Pro **pluralises** a repeating element's exposed name (`Order` → +`Orders`). mxcli defaults to the entity's own name and lets `as 'Orders'` say +otherwise — reproducing English inflection needs `-y → -ies` and an +already-plural detector, and a name the author writes beats one a heuristic +guesses. Everything else is derived from the domain model. + +`show message definition collections [in Module]` lists them; `describe` emits +re-executable MDL. + +### SOAP-sourced mappings are refused, not rewritten + +A mapping can also be sourced from an **imported web service** — a WSDL binding +(which service, which operation, which root element). MDL cannot spell that, so +`create or replace|modify` over such a mapping is **refused** rather than +rebuilding it without the binding, which would leave it with no schema source at +all (**CE6896**, plus **CE0270**). `describe` marks the source instead of +emitting nothing, because the silent output parses and re-executing it is what +deletes the binding (ako/mxcli#365): + +```sql +create or modify import mapping Legacy.IMM_Order + -- SOURCE NOT REPRESENTABLE: imported web service Legacy.WS_Orders (service OrderService, operation GetOrder) + -- re-executing this statement would drop it (CE6896); mxcli refuses the rewrite +{ ... } +``` + +Edit such a mapping in Studio Pro. Note that a consumed SOAP service does **not** +create XML schema documents — the WSDL's XSDs are held inline on the web-service +document — so `with xml schema` is a different path and does not help here. + +### The `with` clause is resolved, not written through + +A mapping's schema source — `with json structure M.X` or `with xml schema M.Y` — +is checked against the project by both `mxcli check -p` and `exec`, and a name +that resolves to nothing is refused with the documents that would have worked. +mxbuild otherwise reports it as **CE1613** "… no longer exists" at the end of a +build (ako/mxcli#259). + +For JSON structures a typo used to be worse than a dangling reference: the schema +index is empty whenever the structure cannot be loaded **for any reason**, and an +empty index reads as "there is nothing to validate against" — so one typo in the +source name switched off every member check in the mapping. + +Two things the check deliberately does not do: + +- A structure the **same script** creates counts as existing. Create the + structure, then map over it, is the normal shape. +- A project with **no** XML schemas disables the XML half rather than refusing + every mapping. There is no `create xml schema` in MDL — an XML schema is only + ever imported into the project by hand — so having none is ordinary, not + evidence of a typo. + +--- diff --git a/.claude/skills/mendix/manage-navigation/SKILL.md b/.claude/skills/mendix/manage-navigation/SKILL.md index 198c611730..133217365d 100644 --- a/.claude/skills/mendix/manage-navigation/SKILL.md +++ b/.claude/skills/mendix/manage-navigation/SKILL.md @@ -68,16 +68,33 @@ create or replace navigation Responsive ### Role-Based Home Pages -Add `for Module.Role` to override the home page for specific user roles: +Add `for ` to override the home page for specific user roles. The +role is a **bare name** — user roles are project-level and have no module part: ```sql create or replace navigation Responsive home page MyModule.Home_Web - home page MyModule.AdminDashboard for Administration.Administrator - home page MyModule.CustomerPortal for MyModule.Customer + home page MyModule.AdminDashboard for Administrator + home page MyModule.CustomerPortal for Customer login page Administration.Login; ``` +**Never write a module-qualified role here.** `for Administration.Administrator` +is a *module role*, and Mendix cannot load a project containing one in this +position: + +``` +StorageLoadException: Role based home page has an invalid value '' for property +UserRole. The text 'Administration.Administrator' is not a valid UserRoleIdentifier. +``` + +That is worse than a build error — it happens before checking runs, so there is +no error code and no line number, and `mx check` exits non-zero *without* the +"The app contains: N errors" line. The two role kinds are easy to confuse +because they share names: a blank app has a user role `Administrator` and module +roles called `Administrator` in three modules. List the real ones with +`show user roles`; `mxcli check --references` refuses the wrong form. + ### Full Menu Tree The `menu (...)` block replaces the entire menu. Use `menu item` for leaf items and `menu 'caption' (...)` for sub-menus: diff --git a/.claude/skills/mendix/project-brain/SKILL.md b/.claude/skills/mendix/project-brain/SKILL.md new file mode 100644 index 0000000000..132bbeda91 --- /dev/null +++ b/.claude/skills/mendix/project-brain/SKILL.md @@ -0,0 +1,226 @@ +--- +name: project-brain +description: "Project-specific knowledge mxcli cannot compute — the requirements and slices being built from (a spec, a prototype, a conversation), why a pattern was chosen here, which marketplace version broke what. Use when starting from requirements that live outside git, before designing something that looks like it was decided before, and when an mxbuild error is resolved by something non-obvious." +--- + +# Project brain + +The brain holds what mxcli **cannot** compute about this project. Two halves: + +- **Decisions** — why a pattern was chosen here, which marketplace version broke + what, what a recurring mxbuild error means in *this* app. +- **The plan** — the requirements being built from and the slices they are + grouped into, when the source is a specification document, a prototype or a + conversation rather than GitHub issues. + +The plan half matters because hours of work can otherwise leave no trace: a +Word document and a chat transcript are not in git, so a session that resumes +later has no idea what it was building towards, and neither does the next +person. + +It lives in `docs/brain/`, is committed, and is reviewed in a pull request like +any other change. + +## The rule that makes it work + +**Anything mxcli can answer does not belong here.** Entities, microflows, pages, +bindings, references, callers, dead assets — all queryable. A note that +transcribes any of them is a note that will disagree with the project the moment +someone edits the model, and it will disagree silently. + +Before writing anything down, ask whether a command answers it: + +```bash +mxcli -p app.mpr -c "show entities" +mxcli -p app.mpr -c "show callers of MyModule.ACT_Thing" +mxcli -p app.mpr -c "describe microflow MyModule.ACT_Thing" +``` + +If one does, do not record it. Record only the **negative space** — the reason, +the constraint, the history that no query can reach. + +## Reading it + +``` +docs/brain/ + project.md cross-cutting decisions + modules/.md decisions anchored to one module + plan/.md requirements for one deliverable slice +``` + +**When building: read `project.md`, plus the shard for each module you are about +to touch.** That set is known before the work starts. + +**When planning, or when picking up work: read the plan.** `mxcli brain plan` +first — it says which slices are outstanding — then the shard for the slice you +are working on. + +**Never read the whole directory.** A large project has dozens of shards, and +reading them all reinstates exactly the context cost the split removed. If you +do not know which modules you are touching yet, read `project.md` and come back. + +## Writing to it + +An agent **captures**; a person **promotes**. Capturing is free and reversible; +promotion is the human's call about what is worth committing. + +```bash +mxcli brain capture "Orders are committed by Finance, not Sales" \ + -a @Sales.Order -a @Finance.ACT_Post -p app.mpr +``` + +The first line becomes the entry's title and the rest becomes its body, so a +one-argument capture can still carry an explanation: + +```bash +mxcli brain capture "Marketplace Administration 4.5.0 breaks the login flow +It changes Account's password-policy handling; we pinned 4.3.2 until the +custom login microflow is reworked." -a @Administration.Account -p app.mpr +``` + +Then leave it. `mxcli lint` reminds the developer that something is staged. + +## Recording requirements and slices + +When the source of truth is outside git — a specification document, a +prototype, a long conversation — record it as **requirements grouped into +slices** before building. Otherwise the work is invisible: not in an issue, not +in a commit message, and gone from the session that resumes tomorrow. + +```bash +mxcli brain capture "Orders must be approvable by a manager" \ + --slice 02-approvals -a @Sales.ACT_Order_Approve -p app.mpr +``` + +`--slice` is the only signal needed. It files the entry in `plan/02-approvals.md` +and makes it a requirement rather than a decision. + +**Slices are ordered by name**, so a numeric prefix is how a roadmap is +sequenced: `01-accounts`, `02-approvals`, `03-reporting`. That is your choice, +not something mxcli maintains. + +### A requirement's anchor points forward + +This is the difference that matters, and it is why requirements are not simply +more decisions: + +| | Anchor points | An anchor that does not resolve means | +|---|---|---| +| decision | backward, at what exists | the decision is **stale** — check fails | +| requirement | forward, at what is intended | **not built yet** — normal, check passes | + +So anchor a requirement at what you are *going to* build. `@Sales.ACT_Order_Approve` +before that microflow exists is correct, not a mistake. + +### Progress is derived, never written + +```bash +mxcli brain plan -p app.mpr +``` + +``` +SLICE BUILT PLANNED UNANCHORED +01-accounts 1 0 +02-approvals 0 1 1 + +1 of 3 requirements built, across 2 slice(s). +``` + +A requirement is **built** when its anchors resolve against the model. Nothing +in the file says "done" — building the thing is what moves the number. + +**Never write a status into a requirement**, and never keep a checklist beside +it. A hand-maintained status is wrong the moment someone builds something, and +nothing will tell you. + +A requirement with **no anchor** is counted separately as unanchored: it cannot +be measured. That is a prompt to anchor it once you know what will implement it, +not an error. + +### Slices have a generous cap, and that is the slicing discipline + +A slice holds source material, so its budget is much larger than a decision +shard's — and it is not loaded every session. But it is still a budget: **a +slice too long to read is a slice that should be split.** + +## Write the anchor, not the name + +`@Sales.Order.Status` is what makes an entry **routable** (its module decides +the file) and **checkable** (`mxcli brain check` verifies it still resolves). +The same fact written as prose — "the Status attribute on the Sales order +entity" — is neither. + +| Anchor | Names | +|---|---| +| `@Sales` | a module | +| `@Sales.Order` | a document: entity, microflow, page, workflow, … | +| `@Sales.Order.Status` | a member: an attribute | + +An entry's **first** anchor decides its shard. An entry with no anchor is +cross-cutting and goes to `project.md`. + +An entry may anchor into more than one module — "`Sales.Order` is committed by +`Finance.ACT_Post`" genuinely spans two — and that is fine as long as one anchor +belongs to the shard it is filed in. + +## Checking it + +```bash +mxcli brain check -p app.mpr # every shard +mxcli brain check --changed -p app.mpr # only shards this branch touched +``` + +Two independent things are checked, and only some outcomes are failures: + +| Outcome | Meaning | Fails? | +|---|---|---| +| resolved | the anchor names something that is there | no | +| **not found** | the anchor names nothing — the entry is stale | **yes** | +| not indexable | the target exists but its document type is not in the catalog's index | no | +| **misfiled** | no anchor belongs to the shard the entry sits in | **yes** | + +"Not indexable" is not a problem to fix. Treating it as missing would demand +edits to entries that are perfectly current. + +Misfiling is a **separate axis**, not a fourth anchor state: every anchor can +resolve and the entry still be in the wrong file. + +## Size + +Each shard has a line budget, and `promote` refuses rather than letting a shard +grow past it. `project.md` is the tightest — it is the only file loaded every +session. + +```bash +mxcli brain show -p app.mpr +``` + +Sizes are computed on every run and deliberately not written down anywhere. A +figure in prose is stale the next time anyone promotes. + +If a promotion is refused, the answer is to condense or drop, not to raise the +cap: the cap is what stops the store becoming a file nobody reads. + +## Commands + +| Command | Does | +|---|---| +| `mxcli brain init -p app.mpr` | Creates `docs/brain/`. Refuses a `docs/brain/` it did not write | +| `mxcli brain capture "" [-a @Anchor]…` | Queues an entry. Never commits | +| `mxcli brain staged` | Lists the queue with the shard each entry would land in | +| `mxcli brain promote [--to ]` | Writes it into its shard. The human step | +| `mxcli brain drop ` | Removes it from the queue or from its shard | +| `mxcli brain capture "" --slice [-a @Anchor]…` | Queues a **requirement** of that slice | +| `mxcli brain plan` | The roadmap: each slice's requirements counted against the model | +| `mxcli brain check [--changed]` | Anchors still resolve, entries in the right shard, plus slice progress | +| `mxcli brain show []` | Entries, lines and headroom per shard | + +## What not to record + +- Anything `show`, `describe` or the catalog answers — it will drift. +- Counts and sizes of anything, including the brain itself. +- **Status.** Whether a requirement is done is computed by `mxcli brain plan`. + A "✅" written beside one is wrong as soon as anyone builds anything. +- Sprint chatter and task assignment. Requirements and their slices, yes; who is + doing what this week, no — that belongs in an issue tracker. +- A restatement of Mendix documentation. Record what is true *here*. diff --git a/.claude/skills/mendix/write-microflows/reference/control-flow.md b/.claude/skills/mendix/write-microflows/reference/control-flow.md index 9175a0693f..72c39849f6 100644 --- a/.claude/skills/mendix/write-microflows/reference/control-flow.md +++ b/.claude/skills/mendix/write-microflows/reference/control-flow.md @@ -310,6 +310,14 @@ commit $Product; - `@position` always appears in DESCRIBE output; `@caption` only when custom; `@color` only when not Default - DESCRIBE MICROFLOW shows `@` annotations before their activities - `@start(x, y)` positions the **start event** and goes on the first statement, because the start has no statement of its own. Omit it and the start is derived — one spacing unit (160) left of the first activity, on its centre line — and a rewrite re-derives it so the start follows the activities when they move. A start that is not at the derived spot was placed by hand (in Studio Pro or with `@start`): it survives a rewrite that does not mention it, and DESCRIBE emits `@start` for it. An explicit `@start` overrides both (#951) +- `@position(x, y)` on a **parameter** goes inside the parameter list, ahead of the parameter it places — a parameter is a stored node with its own coordinates, and this is the only annotation it takes. Omit it and the parameters form a row along the top of the canvas (200;53, 300;53, …). The `@start` rule above applies unchanged: a parameter on that derived row is re-derived on a rewrite, one anywhere else was placed by hand, survives, and is emitted by DESCRIBE (#993). Before this, a hand-aligned parameter block was moved back onto the row by any rewrite — including a describe → exec of mxcli's own output: + + ``` + create or modify nanoflow MyModule.ACT_Clear ( + @position(-77, 0) + $Feedback: MyModule.Feedback + ) + ``` ## Error Handling MDL supports error handling for activities that may fail (microflow calls, commits, external service calls, etc.). diff --git a/.claude/skills/setup-devcontainer-ssh.md b/.claude/skills/setup-devcontainer-ssh.md new file mode 100644 index 0000000000..c1bceef3dc --- /dev/null +++ b/.claude/skills/setup-devcontainer-ssh.md @@ -0,0 +1,144 @@ +# Set Up SSH Into Your Dev Container + +A contributor workflow for getting an SSH key working so a **desktop app can open +a session in your dev container** — Claude Code Desktop's environment dropdown +has no "attach to container" option, so SSH is the only way in. + +The machinery is already in the repo (`.devcontainer/ssh/`, wired to +`postStartCommand`). What is *not* in the repo, and cannot be, is your key +material: it is gitignored, so **every developer generates their own**. This +skill is the procedure for doing that and proving it works. + +Reference for the setup itself — how the parts fit, and troubleshooting — +is [`.devcontainer/ssh/README.md`](../../.devcontainer/ssh/README.md). Do not +restate it here. + +## When to Use This Skill + +- First time you want to reach your dev container over SSH. +- `ssh` into the container fails and you need the elimination order. +- You are on a different host OS than the last person who touched this and the + paths do not match. + +Not needed for ordinary development — VS Code's own container attach is +unaffected by any of this. + +## The Short Version + +Inside the container: + +```bash +bash .devcontainer/ssh/generate-key.sh # once — your keypair, authorised +bash .devcontainer/ssh/start-sshd.sh # postStartCommand also runs this +``` + +Both scripts are idempotent. `generate-key.sh` refuses to overwrite an existing +key (that would lock out anything already trusting it), and `start-sshd.sh` +merges into `authorized_keys` rather than replacing it. + +Then, **on the host**, pre-trust the container and connect. Both values below are +per-developer — read them off your own run, do not copy them from a colleague or +from the README. + +```bash +ssh-keyscan -p 2222 -t ed25519 localhost >> ~/.ssh/known_hosts +chmod 600 /.devcontainer/ssh/id_devcontainer +ssh -p 2222 -i /.devcontainer/ssh/id_devcontainer vscode@localhost +``` + +## Work Out Your Own Environment First + +Three facts differ per machine. Establish them before debugging anything. + +| Question | Command | Why it matters | +|---|---|---| +| Is the account password-locked? | `passwd -S "$(id -un)"` | `L` means pubkey is the **only** route in — don't chase password auth. Most devcontainer images lock it. | +| Is there a usable agent key? | `ssh-add -l` | `SSH_AUTH_SOCK` is usually forwarded but the agent is usually **empty**, so there is typically nothing to pull from the host. | +| Where is the workspace on the host? | `findmnt -T "$PWD" -o TARGET,SOURCE` | If it's a bind mount, a key generated *inside* the container is **already on your host** — no copying. `generate-key.sh` prints the derived host path. | + +On Docker Desktop the `SOURCE` looks like +`/run/host_mark/Users[/you/GitHub/mxcli]`, which maps to `/Users/you/GitHub/mxcli`. +On a Linux host with a plain bind mount the derivation may not resolve — in that +case the host path is simply wherever you cloned the repo. + +## Verify It, Don't Assume It + +Run these in order. Each isolates one layer, and each has caught a real failure. + +```bash +# 1. Does sshd accept the key at all? (inside the container) +ssh -i .devcontainer/ssh/id_devcontainer -p 2222 \ + -o BatchMode=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/tmp/kh vscode@localhost 'echo OK' + +# 2. Does it survive a rebuild? ~/.ssh is on the container's writable layer. +rm -f ~/.ssh/authorized_keys +bash .devcontainer/ssh/start-sshd.sh +test -f ~/.ssh/authorized_keys && echo restored + +# 3. Does sshd serve the PINNED host key, not a fresh one? +diff <(ssh-keyscan -p 2222 -t ed25519 localhost 2>/dev/null | ssh-keygen -lf - | awk '{print $2}') \ + <(ssh-keygen -lf .devcontainer/ssh/hostkeys/ssh_host_ed25519_key.pub | awk '{print $2}') \ + && echo "pinned identity served" + +# 4. The one that matters: a GUI client's exact conditions — no TTY, strict. +ssh -o StrictHostKeyChecking=yes -o BatchMode=yes -o UserKnownHostsFile= \ + -i .devcontainer/ssh/id_devcontainer -p 2222 vscode@localhost 'echo OK' + +# 5. Is the hardening actually enforced? Probe the LIVE daemon. +ssh -p 2222 -o PreferredAuthentications=password -o PubkeyAuthentication=no \ + vscode@localhost 2>&1 | tail -1 +# want: Permission denied (publickey). +# bad: Permission denied (publickey,password). <- config not in effect +``` + +Check 4 is the meaningful one. A connection that works interactively can still +fail from a desktop app, because the app cannot answer *"Are you sure you want to +continue connecting?"*. + +Check 5 catches a trap worth knowing about: **`sshd -T` is not evidence about the +running daemon.** It reads the config off disk; sshd freezes its config when the +listener starts. The sshd feature starts a listener from its entrypoint about a +second into container start, well before `postStartCommand` — so on a fresh +container the daemon can be enforcing the *feature's* config while `sshd -T` +happily reports ours. (Host keys are different: those are re-read per connection, +which is why pinning still works.) `start-sshd.sh` replaces the listener for this +reason, killing only the pid in `PidFile` so established sessions survive. + +## Failure Modes + +| Symptom | Cause | Fix | +|---|---|---| +| `Host key verification failed` + *"No ED25519 host key is known"* | The client has **no** entry and cannot prompt you to add one. | Append the `ssh-keyscan` line to `~/.ssh/known_hosts` on the host. | +| `Host key verification failed` naming an offending line | **Stale** entry — a previous container's identity. | `ssh-keygen -R '[localhost]:2222'`, then re-add. | +| `Permission denied (publickey)` | Key not in `authorized_keys`, or `~/.ssh` perms are loose. sshd **fails closed and silently** on loose perms. | Re-run `start-sshd.sh` (it sets `700`/`600`), then read the log — see below. | +| Connection refused from the host, works inside | Port not published. `forwardPorts` is editor-managed and only exists while VS Code is attached. | `appPort` must be set; rebuild the container. | +| `openssh-server is not installed` | The sshd feature didn't apply. | Rebuild the container. | +| Host key changed after an image rebuild | Host keys generated at **image build** belong to the image, not the container. | Expected only if `hostkeys/` was deleted; otherwise pinning prevents it. | +| `sshd -T` disagrees with how the daemon behaves | sshd froze its config at listener start; the feature started one before `postStartCommand`. | Re-run `start-sshd.sh` — it replaces the listener. Trust the probe in check 5, not `sshd -T`. | + +## Reading sshd's Side of the Story + +There is no syslog daemon in a container, so `/var/log/auth.log` does not exist +and sshd's account of a failure goes **nowhere**. Run it in the foreground: + +```bash +sudo pkill -x sshd +sudo /usr/sbin/sshd -E /var/log/sshd.log -o LogLevel=VERBOSE +# reproduce, then: +sudo tail -50 /var/log/sshd.log +sudo pkill -x sshd && bash .devcontainer/ssh/start-sshd.sh # restore +``` + +## Do Not + +- **Do not commit key material.** Private user key, public key, and especially + `hostkeys/` are gitignored. A committed private *host* key would let anyone + with the repo impersonate a colleague's container to a client that trusts it. + Confirm with `git check-ignore -v ` before any `git add -A`. +- **Do not share a key or a `known_hosts` line between developers.** Each + container pins its own identity on first run; yours is not mine. +- **Do not swap `appPort` for `forwardPorts`** to "fix" a connection problem — + it will work while VS Code is attached and fail exactly when you need it. +- **Do not bind the port to `0.0.0.0`** unless you mean to expose it to your + LAN. The default is `127.0.0.1:2222:2222`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 46946ba38d..087a6c3dbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,41 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +- **`CATALOG.strings` indexes every translatable string, not five hand-picked kinds** — `SHOW LANGUAGES` listed 8 of a project's 9 languages and `search` could not find a widget caption that `DESCRIBE TRANSLATIONS` had just listed. The index was filled by per-type extractors reaching five sites (page title, enum caption, three microflow message templates), so a text anywhere else was never indexed: measured on a stock 11.13 app, **69 of 3265 texts and 8 of 9 languages**. A language present only on an unindexed site is *invisible* rather than undercounted, which also blinded lint rule QUAL005 — it discovers its language set from the same table. + + The rows now come from the type-agnostic `Texts$Text` walk that `DESCRIBE TRANSLATIONS` already uses, so the two subsystems cannot disagree about what the project contains; the typed path keeps only the strings that are *not* translatable (URLs, log node names, REST paths, documentation, and the `Microflows$StringTemplate` a workflow name is stored in). `StringContext` now names the site — `Forms$ActionButton.Caption` rather than `page_title` — and `ObjectType` is derived from the unit `$Type` mechanically, so a document type Mendix adds later is named correctly with nobody maintaining a list. Same project after: 1496 rows, 9 languages, counts identical to an independent BSON walk. Atlas design templates are ~70% of the corpus and are indexed rather than dropped, because `CREATE TRANSLATIONS` writes them and a `SHOW LANGUAGES` that excluded them would reopen the same split; `ObjectType` is how a consumer filters them. + +- **QUAL005 reports the sibling elements it used to fold together** — the rule grouped by `(QualifiedName, StringContext)` while `ElementId` sat unused in the table, so an enumeration's twelve values became one group and translating any single value made the whole set look complete. Grouping now includes `ElementId`. The existing test harness synthesized that column from `QualifiedName+StringContext`, which is why no test caught it. + +- **`call external action` now types its return value and its parameters** (mendixlabs/mxcli#1020) — a call against a consumed OData service was written with neither the result variable's type nor its parameters' types, so Mendix reported `CE7269` ("the return type for remote action … has changed") and `CE7252` ("the parameters … have changed"), and re-running `CREATE OR MODIFY EXTERNAL ENTITIES` never cleared them. + + It never could. Both codes are defined on `CallExternalAction.cs` — they are raised by the microflow **activity**, not by the entity — which is what made the reported remedy the wrong lever. Two omissions of the same shape, each a `DataTypes$` sub-document that was never written: + + - The return-type resolver mapped only EDM primitives, so an action returning an **entity** (or a collection of them) got no `VariableDataType` at all. It now resolves to `DataTypes$ObjectType` / `DataTypes$ListType` naming the external entity imported for that type — the same linkage the entity import writes. + - `ExternalActionParameterMapping.ParameterType` was never written, though `generated/metamodel` declares it **without `omitempty`**. Measured: a call with *any* parameter, of any type, produced CE7252 plus one `CE0117` "Error(s) in expression" per argument — an argument cannot be type-checked against an untyped parameter. + + Measured on 11.13 against a contract with three action shapes: before, a no-parameter entity return was CE7269, a one-string-parameter call was CE7252 + 1× CE0117, and a two-parameter call CE7252 + 2× CE0117; after, all three build at **0 errors**. Reverting the return resolver reproduces CE7269 verbatim. + + `mxcli check --references` now also resolves the call against the cached contract, so an unknown action, an argument the action does not declare, a declared parameter left unsupplied, or an entity return whose entity has not been imported are reported with the statement that fixes them instead of surfacing a build later. + +- **The catalog listed only half the external entities** — `external_entities` skipped every entity stored as `Rest$ODataEntityTypeSource`, which is what `CREATE EXTERNAL ENTITIES` writes for any type the contract gives no entity set: derived, abstract, contained, and an action's parameter and return types. They were absent from `CATALOG.external_entities` and from module entity counts. + + The consequence was worse than the under-count. `contract_entities.UsedByExternalEntity` is filled by joining that table on `RemoteName`, so for exactly those entities the column was **structurally always empty** — it read as "this contract entity is linked to nothing" whether or not the import had worked, which is the evidence #1020 was diagnosed from. Both sources are now catalogued, and the reporter's query resolves. + +- **`HOME PAGE … FOR` resolves the user role, and the docs no longer recommend the form that breaks the project** (mendixlabs/mxcli#1001) — the role was written straight through to BSON, and what it did depended on its shape. Measured on a blank 11.13 app: `for Administrator` builds at 0 errors; `for Supervisor` (bare, unknown) is an ordinary `CE1613`; and `for MyFirstModule.Administrator` produces a project Mendix **cannot load** — `StorageLoadException: … 'MyFirstModule.Administrator' is not a valid UserRoleIdentifier`, raised before any checking runs, so there is no error code and no location. + + The third was the form **mxcli's own documentation, skill and `mxcli syntax` output all recommended**, across ten places including one runnable example. `FOR` binds a *user* role, which is project-level and written bare; a *module* role is module-scoped and shares the name — a blank app has a user role `Administrator` and module roles called `Administrator` in three modules, so the wrong one reads as correct. + + Both `check --references` and `exec` now refuse it through the same function, naming the bare form to write. A bare unknown role and a case mismatch (Mendix matches exactly, so `for administrator` really is CE1613) get their own messages, since they need different fixes. Two controls: a refused `exec` leaves the `.mpr` byte-identical, and a script that creates a user role and then uses it still passes — the ordinary way to write one, which a project-only lookup would have refused (`mdl-examples/bug-tests/navigation-1001-home-page-for-user-role.mdl`). + + Worth knowing beyond this bug: a load failure suppresses the `The app contains: N errors` line while still exiting 1, so `mx check` reads as having succeeded. It has now been misread that way in two reports. + +- **`check --references` now resolves icon references in placeholders, snippets, layouts and menu documents** (mendixlabs/mxcli#1008) — an icon naming an **image** collection instead of an **icon** collection passed `check --references`, executed fine, and surfaced only at build as `CE1613 "The selected custom icon … no longer exists."`. The reported case (a button in a page body) was already caught, which is what made this look like a resolver bug; it is a **coverage** gap. `iconRefsInStatement` walked only `CreatePageStmtV3.Widgets`, `AlterPageStmt` operations and `AlterNavigationStmt`, so four icon-bearing shapes were never inspected: widgets inside a `placeholder X { … }` block (a page's `Placeholders` is held apart from the bare-body `Widgets`), `CREATE SNIPPET`, `CREATE LAYOUT`, and `CREATE MENU` — whose items carry icons exactly as a navigation profile's do. + + Layouts matter most of the four: a layout's topbar is shared, so one wrong icon there is an error on every page that uses it. + + Measured on 11.13, with the positive control that makes the refusal trustworthy: before, all four were silent at exit 0 and each produced its own CE1613; after, all four are caught at check time, and the same four shapes with a **valid** icon still pass `--references`, exec, and build to 0 errors. An over-eager walk would have traded a silent miss for a false refusal, which is why the valid variants are part of the fixture rather than a separate exercise (`mdl-examples/bug-tests/icon-refs-1008-placeholder-snippet-layout-menu.mdl`). + - **Navigation writers emit the typed-array markers Studio Pro writes** — every list in a profile mxcli authored carried the leading marker `1`: `HomeItems`, `Menu.Items`, each menu item's sub-`Items`, its caption's `Texts$Text.Items`, `ParameterMappings` and `PagesForSpecializations`. Studio Pro writes `2` or `3` for those fields, and mxcli's own menu-document codec path already wrote `3` for the same `Menus$` item collections, so the two paths disagreed with each other. The values are a census over 19,078 unit files in 54 projects rather than a reading of the metamodel, because the marker is a **per-field constant and not a function of the list's contents** — `Forms$FormSettings.ParameterMappings` is `2` in 816 empty *and* 306 non-empty occurrences. Five of the six fields are settled by the census outright. The sixth, `HomeItems`, is empty in all 51 stored profiles, so it took a project that actually uses the feature: ako/TestApp's Studio Pro-authored profile carries two `Navigation$RoleBasedHomePage` elements at marker **2**, which confirms the rewrite paths and corrects the create path, which had been writing `3`. @@ -16,6 +51,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The legacy engine read the same bytes correctly the whole time, which is both the diagnosis and the control: the two engines now print identical output for the same document. The two slots were wrong in opposite directions. For the login page a blank app's own navigation document and `generated/metamodel` agree with the writers, so **gen** is wrong. For the not-found page — Studio Pro's **"Fallback page"** — gen and `generated/metamodel` agree with each other and mxcli's three **writers** were the odd one out, storing `Navigation$HomePage` where Studio Pro stores `Navigation$NotFoundHomePage`; the writers are corrected and now reproduce a Studio Pro-authored fallback page exactly. mxbuild accepts either spelling, so only a reference document could separate them. The reader keeps accepting both, because every not-found page mxcli wrote before this carries the old spelling and has to keep round-tripping. + The legacy engine read the same bytes correctly the whole time, which is both the diagnosis and the control: the two engines now print identical output for the same document. The two slots were wrong in opposite directions. For the login page a blank app's own navigation document and `generated/metamodel` agree with the writers, so **gen** is wrong. For the not-found page — Studio Pro's **"Fallback page"** — gen and `generated/metamodel` agree with each other and mxcli's three **writers** were the odd one out, storing `Navigation$HomePage` where Studio Pro stores `Navigation$NotFoundHomePage`; the writers are corrected and now reproduce a Studio Pro-authored fallback page exactly. + + **Correction to an earlier claim in this entry:** it previously said mxbuild accepts either spelling, and that only a reference document could separate them. That is wrong, and it understated the bug. Measured on 11.13 against a build emitting the old spelling, both `mx check` and `mxbuild --target=deploy` exit 1 with `Object of type 'Mendix.Modeler.WebUI.Navigation.HomePage' cannot be converted to type 'Mendix.Modeler.WebUI.Navigation.NotFoundHomePage'` — the project cannot be **loaded**, so every check downstream is lost with it. Nothing caught it because nothing had ever *built* a project with a fallback page set: the automated `mx check` coverage runs `doctype-tests/` only, and no script there sets one — the first that does was added by this fix. The reader keeps accepting both spellings for a different reason than stated: a project written before this does not build at all, and mxcli parses the BSON directly, so reading the old spelling is what lets it open that project and repair it. + - **Fixed navigation page actions overriding page titles with an empty template** — `CREATE OR REPLACE NAVIGATION` wrote `FormSettings.TitleOverride` as an empty `Microflows$TextTemplate`. That is an explicit override to an empty string, not “no override”, so every newly authored page menu item added a CW0263 warning and could render without the page title. All three navigation writers now emit the Studio Pro-authored shape, an explicit null, matching the already-correct page-button and show-page writers. ## [0.20.0] - 2026-08-28 @@ -460,7 +499,6 @@ Headline: **A statement mxcli accepts is now a statement mxcli honours.** This r - An additive chain keeps its operators in the order they were written. - A building-block datasource override is rebound by widget type, not by one that happens to be present already. - ## [0.18.0] - 2026-08-14 Headline: **mxcli can now maintain a project it did not author.** Marketplace modules install and update headlessly — carrying the GUIDs the database keys on, the role grants that live outside the module, and the MPR v2 format `mx module-import` silently collapses — and `marketplace diff` reports which elements were edited locally before an update replaces them. Alongside that, a write that changes nothing no longer touches the file, five more document types become authorable (task queues, scheduled events, regular expressions, validation rules, menus), and a long tail of activities that could be written but not read back stop disappearing from the describe → edit → re-exec loop. Separately, the Windows and macOS binaries stop shipping the embedded tunnel — 13.5 MB smaller, and no longer carrying the tunnelling stack that had Defender and enterprise EDR blocking mxcli on managed corporate endpoints. diff --git a/CLAUDE.md b/CLAUDE.md index b9efce96af..f523e4965c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -715,6 +715,7 @@ go build -o bin/mxcli ./cmd/mxcli | **Marketplace drift** | `mxcli marketplace diff -p app.mpr [--to V] [--json]` | Which elements of an installed marketplace module have been edited locally, and what an upgrade would overwrite | | **Model repair** | `mxcli fix widgets`, `mxcli fix design-properties` | Runs `mx update-widgets` / `mx rename-design-properties` and **persists** the result without their MPR v2 → v1 collapse (harvest: let the tool convert, read the units back, restore v2, write the changed ones through mxcli's writer). Clears CE0463 / CE6087 after a headless install — measured 203 → 0 errors on a vanilla 11.12.1 app | | **Diagnostics** | `mxcli diag [--bundle]` | Session logs, version info, bug report bundles | +| **Project brain** | `mxcli brain init\|capture\|staged\|promote\|drop\|check\|show\|plan` | Opt-in store in `docs/brain/` for what mxcli **cannot** compute (why a pattern was chosen here, which marketplace version broke what). Sharded by module — an entry's first anchor names its file — so a session loads `project.md` plus the modules it is touching, not the whole store. Also holds the **plan**: requirements grouped into slices, whose anchors point *forward*, so `brain plan` reports progress **derived from the model** rather than from a status column. An agent captures to a git-ignored queue; a person promotes | | **New project** | `mxcli new --version X.Y.Z [--output-dir dir] [--theme none] [--layout none]` | Downloads mxbuild, creates blank project, applies default styling, scaffolds a project-owned layout, runs init, installs Linux mxcli for devcontainer | | **Default styling** | `mxcli theme list\|show\|apply\|remove` | Applies a theme (signal/ledger/console) — files under `theme/` only, the model is never touched | | **Project themes** | `mxcli theme create [--from ]` | Scaffolds a theme the project owns into `theme/mxcli-themes/`; `--from ` seeds the palette from `--mxt-*` declarations | @@ -770,6 +771,7 @@ Regenerate after modifying `MDLLexer.g4`, `MDLParser.g4`, or any `domains/*.g4` - `.claude/skills/design-mdl-syntax.md` - **READ before designing new MDL syntax** - Design principles, decision framework, anti-patterns, checklist - `.claude/skills/write-microflows.md` - Microflow syntax, common mistakes, validation checklist - `.claude/skills/write-nanoflows.md` - Nanoflow syntax, restrictions, disallowed activities, validation checklist +- `.claude/skills/mendix/project-brain/SKILL.md` - **Project brain** (`mxcli brain`): the opt-in store for what mxcli cannot compute; why anything derivable from the model must never be written there, how anchors route an entry to its shard, and which `check` outcomes are failures - `.claude/skills/mendix/write-rules.md` - **Rules** (CREATE/LIST/DESCRIBE/DROP/MOVE RULE): a rule returns Boolean or an enumeration and is callable only from a decision; what its body may not contain and the CE numbers behind each refusal; why there is no `grant execute on rule` - `.claude/skills/write-workflows.md` - **Workflow authoring** (CREATE/DROP/ALTER WORKFLOW): activities (user task, decision, parallel split, jump, wait, boundary events), header options, gotchas. Workflows are authorable, not read-only. - `.claude/skills/create-page.md` - Page/widget syntax reference @@ -810,6 +812,7 @@ Full syntax tables for all MDL statements (microflows, pages, security, navigati ## Current Implementation Status **Implemented:** +- Project brain (`mxcli brain init/capture/staged/promote/drop/check/show`): an **opt-in** store in `docs/brain/` for the project knowledge mxcli cannot compute. The governing rule is that anything derivable from the model is answered by a command and never written down — a note that transcribes the model disagrees with it silently. Records shard by **anchor scope**: an entry's first anchor names its file (`@Sales.Order` → `modules/Sales.md`), an anchorless entry is cross-cutting (`project.md`), and there is no index to maintain because the module prefix *is* the file name. That is what makes the cap per-shard rather than a project-wide budget, and lets a session load `project.md` plus the modules it is touching. `check` answers two independent questions: each anchor is **resolved / not found / not indexable** — only the middle one fails, and the third exists because the catalog's `objects` view covers the describable types only, so a scheduled event would otherwise read as *missing* (separated with `FindDocumentUnit`, which cannot miss a kind because it never asks what kind anything is). Misfiling is a **second axis, not a fourth state**: every anchor can resolve and the entry still be in the wrong file, and it is only decided when something resolved — judging it on an all-not-indexable entry reintroduced the same false staleness through the other axis (caught by a test, with the guard stubbed as the control). An agent `capture`s to a git-ignored queue and a person `promote`s; the queue is deliberately **not** sharded, because routing it would force the file decision before a human has looked at the entry. `mxcli lint` prints the unpromoted-queue count, because a report only `brain check` prints is a report nothing demands. Sizes are computed by `brain show` and never written into a committed file. A second record kind, **requirement**, lives in `plan/.md` and inverts the anchor's meaning: a decision's anchor points backward (not resolving = stale, fails), a requirement's points forward (not resolving = not built yet, passes). Measured: filed as an ordinary entry, one unbuilt requirement takes `brain check` to exit 1 — which is why it is a separate kind rather than more entries in the same files. That inversion is also what makes `brain plan` a real progress report: a requirement is *built* when its anchors resolve, so creating the microflow it names moves the count with the plan file untouched (measured 0/1 → 1/0). A status written beside a requirement is therefore refused by the skill, not just discouraged. Slices are ordered by name (`01-accounts`), span modules by design (so misfiling does not apply), and carry a generous cap that enforces the slicing discipline — a slice too long to read should be split. `bootstrap-app` asks for requirements at the interview and records them by default. Package: `cmd/mxcli/brain/`. See `docs-site/src/tools/project-brain.md` and `docs/11-proposals/PROPOSAL_project_brain.md` - Default styling + runtime theme switching (`mxcli theme list/show/create/apply/remove/switcher`, `mxcli new --theme`): three embedded themes (**signal** light-first, **ledger** light-first, **console** dark-first), each a palette in `theme/web/custom-variables.scss` + a shared Atlas wiring partial + a theme partial imported from `theme/web/main.scss` (which compiles last), plus vendored fonts. **No model changes**, so it hot-applies under `run --local --watch` and cannot affect a build. Generated regions are digest-fenced: a block carrying local edits is refused rather than overwritten. Applying a theme removes the previous one. `--variant auto` (default) ships both palettes — the app follows `prefers-color-scheme` before first paint and honours a `theme-light`/`theme-dark` class on ``; `light`/`dark` bakes one. `theme switcher install` is the only part that writes to the model (JS actions + a nanoflow for a toggle button). A project can add its own themes under `theme/mxcli-themes//` (committed, not compiled); `theme create [--from ]` scaffolds one from an existing theme, renaming the identifiers built from the name and optionally seeding the palette from `--mxt-*` declarations in a design artifact. A local theme shadows a built-in of the same name. Package: `cmd/mxcli/theme/`. See `docs/11-proposals/PROPOSAL_default_styling.md` - MPR v1/v2 reading and writing - Idempotent writes (ADR-0008): a unit whose new content is semantically equal to what is stored is **not written**, so re-running an MDL script against an in-sync project leaves the `.mpr` and `mprcontents/` byte-identical and Studio Pro shows no version-control changes. Comparison is on a canonical form (element `$ID`s normalised away — a rebuild mints them randomly, so byte comparison would skip nothing); `Microflows$Microflow.StableId` is carried from the stored document rather than re-minted, because the build derives every client-callable microflow's operation id from it. When a write **does** land, `canon.TransplantIDs` matches the rebuild against the stored document and reuses its element `$ID`s (rewriting every pointer in the same pass), so a changed document's diff is the change rather than a wholesale replacement — measured on #910's nanoflow: 1 of 37 identities survived an argument edit before, 37 of 37 after, and a change plus its revert returns to the original bytes. Inserting or deleting an activity mints IDs only for the genuinely new elements. One policy in `modelsdk/canon`, called from both engines' write choke points. `MXCLI_ALWAYS_WRITE=1` disables elision (not preservation) for bisecting — which means it no longer changes the resulting bytes, only the mtimes. The executor's output distinguishes the two: `Unchanged nanoflow: …` where the write was skipped. See `docs-site/src/internals/idempotent-writes.md` diff --git a/cmd/mxcli/brain/anchor.go b/cmd/mxcli/brain/anchor.go new file mode 100644 index 0000000000..4ebc4ca48e --- /dev/null +++ b/cmd/mxcli/brain/anchor.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 + +// anchor.go - references from a brain entry into the model. +// +// An anchor is what makes an entry checkable and routable. It is written the +// way a Mendix developer already names things — `@Sales.Order` — and its module +// prefix decides which shard the entry lives in, so routing is a string split +// rather than an index someone has to maintain (PROPOSAL_project_brain.md A8). +package brain + +import ( + "fmt" + "regexp" + "strings" +) + +// ProjectShard is the shard for entries that carry no anchor: facts about the +// project as a whole. It is the only file loaded unconditionally, which is why +// it carries the tightest cap. +const ProjectShard = "project" + +// PlanPrefix marks a shard that holds requirements rather than decisions. The +// prefix is part of the shard name so that one identifier addresses all three +// kinds of file — project.md, modules/.md and plan/.md — and no +// caller has to carry a second "is this a plan shard" flag alongside it. +const PlanPrefix = "plan/" + +// PlanShard names the shard holding a slice's requirements. +func PlanShard(slice string) string { return PlanPrefix + slice } + +// IsPlanShard reports whether a shard holds requirements. +func IsPlanShard(shard string) bool { return strings.HasPrefix(shard, PlanPrefix) } + +// SliceOf returns the slice a plan shard belongs to, or "". +func SliceOf(shard string) string { + if !IsPlanShard(shard) { + return "" + } + return strings.TrimPrefix(shard, PlanPrefix) +} + +// identifier is the Mendix name shape — a leading letter or underscore, then +// letters, digits and underscores. Anything else is rejected at parse time +// rather than becoming an anchor that can never resolve. +var identifier = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// Anchor is a parsed reference into the model at one of three granularities: +// a module (`@Sales`), a document (`@Sales.Order`), or a member +// (`@Sales.Order.Status`). +type Anchor struct { + Module string + Element string // empty for a module anchor + Member string // empty unless the anchor is member-scoped +} + +// ParseAnchor reads one anchor. The leading '@' is optional so that callers can +// pass either the written form or a bare qualified name. +func ParseAnchor(s string) (Anchor, error) { + raw := strings.TrimSpace(s) + raw = strings.TrimPrefix(raw, "@") + if raw == "" { + return Anchor{}, fmt.Errorf("empty anchor") + } + parts := strings.Split(raw, ".") + if len(parts) > 3 { + return Anchor{}, fmt.Errorf("anchor %q has %d parts; anchors go at most three deep (@Module.Entity.Attribute)", s, len(parts)) + } + for _, p := range parts { + if !identifier.MatchString(p) { + return Anchor{}, fmt.Errorf("anchor %q: %q is not a Mendix identifier", s, p) + } + } + a := Anchor{Module: parts[0]} + if len(parts) > 1 { + a.Element = parts[1] + } + if len(parts) > 2 { + a.Member = parts[2] + } + return a, nil +} + +// ParseAnchors reads a list, reporting the first failure rather than silently +// dropping an anchor that would then never be checked. +func ParseAnchors(ss []string) ([]Anchor, error) { + out := make([]Anchor, 0, len(ss)) + for _, s := range ss { + a, err := ParseAnchor(s) + if err != nil { + return nil, err + } + out = append(out, a) + } + return out, nil +} + +// String renders the anchor as it is written in a shard. +func (a Anchor) String() string { + sb := "@" + a.Module + if a.Element != "" { + sb += "." + a.Element + } + if a.Member != "" { + sb += "." + a.Member + } + return sb +} + +// QualifiedName is the name to look up in the catalog. For a member anchor that +// is the *owning document's* qualified name — the member itself is resolved +// separately, because attributes live outside the objects view (A2). +func (a Anchor) QualifiedName() string { + if a.Element == "" { + return a.Module + } + return a.Module + "." + a.Element +} + +// IsMember reports whether resolution needs the second, member-level query. +func (a Anchor) IsMember() bool { return a.Member != "" } + +// ShardFor derives an entry's destination: the module of its first anchor, or +// the project shard when it has none. Deriving rather than asking is what makes +// the routing checkable — see MisfiledIn. +func ShardFor(anchors []Anchor) string { + if len(anchors) == 0 { + return ProjectShard + } + return anchors[0].Module +} diff --git a/cmd/mxcli/brain/brain_test.go b/cmd/mxcli/brain/brain_test.go new file mode 100644 index 0000000000..02c54bda63 --- /dev/null +++ b/cmd/mxcli/brain/brain_test.go @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +var day = time.Date(2026, 9, 3, 0, 0, 0, 0, time.UTC) + +func mustEntry(t *testing.T, text string, anchors ...string) Entry { + t.Helper() + e, err := NewEntry(text, anchors, day) + if err != nil { + t.Fatalf("NewEntry(%q): %v", text, err) + } + return e +} + +func TestParseAnchor(t *testing.T) { + for _, tc := range []struct { + in string + ok bool + mod, elem, member string + }{ + {"@Sales", true, "Sales", "", ""}, + {"@Sales.Order", true, "Sales", "Order", ""}, + {"@Sales.Order.Status", true, "Sales", "Order", "Status"}, + {"Sales.Order", true, "Sales", "Order", ""}, // the '@' is optional + {"@Sales.Order.Status.Extra", false, "", "", ""}, + {"@9Sales", false, "", "", ""}, + {"@Sales..Order", false, "", "", ""}, + {"@", false, "", "", ""}, + } { + a, err := ParseAnchor(tc.in) + if tc.ok != (err == nil) { + t.Errorf("ParseAnchor(%q): ok=%v, err=%v", tc.in, tc.ok, err) + continue + } + if !tc.ok { + continue + } + if a.Module != tc.mod || a.Element != tc.elem || a.Member != tc.member { + t.Errorf("ParseAnchor(%q) = %+v", tc.in, a) + } + } +} + +func TestShardIsDerivedFromTheFirstAnchor(t *testing.T) { + e := mustEntry(t, "Orders are committed by Finance", "@Sales.Order", "@Finance.ACT_Post") + if got := e.Shard(); got != "Sales" { + t.Errorf("shard = %q, want Sales", got) + } + if got := mustEntry(t, "We deploy on Fridays").Shard(); got != ProjectShard { + t.Errorf("anchorless shard = %q, want %q", got, ProjectShard) + } +} + +// The whole file format rests on this: what promote writes, check must be able +// to read back. A body containing text that looks like a heading or a metadata +// line is included on purpose. +func TestShardRoundTrips(t *testing.T) { + in := []Entry{ + mustEntry(t, "Orders are committed by Finance\nNot by Sales, despite the entity living there.", "@Sales.Order", "@Finance.ACT_Post"), + mustEntry(t, "No anchors here"), + mustEntry(t, "Body with tricky text\nAnchors: not really a meta line\nand a - hyphen", "@Sales.Order"), + } + out, malformed, err := ParseShard("Sales", RenderShard("Sales", in)) + if err != nil { + t.Fatal(err) + } + if len(malformed) != 0 { + t.Fatalf("malformed blocks: %v", malformed) + } + if len(out) != len(in) { + t.Fatalf("got %d entries, want %d", len(out), len(in)) + } + for i := range in { + if out[i].ID != in[i].ID || out[i].Title != in[i].Title || + out[i].Body != in[i].Body || out[i].Date != in[i].Date || + strings.Join(out[i].Anchors, ",") != strings.Join(in[i].Anchors, ",") { + t.Errorf("entry %d round-trip:\n got %+v\nwant %+v", i, out[i], in[i]) + } + } +} + +func TestMalformedEntryIsReportedNotSkipped(t *testing.T) { + content := "# Sales\n\n" + shardMarker + "\n\n## A heading with no metadata line\n\nsome prose\n" + entries, malformed, err := ParseShard("Sales", content) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("got %d entries, want 0", len(entries)) + } + if len(malformed) != 1 { + t.Fatalf("malformed = %v, want 1 block reported", malformed) + } +} + +func TestPromoteRefusesPastTheCapAndAcceptsBelowIt(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + + // Control: a first entry fits and is written. + if err := s.Promote(mustEntry(t, "First"), ProjectShard); err != nil { + t.Fatalf("first promote should fit: %v", err) + } + + // Fill until the cap refuses, then assert nothing was written past it. + var capErr *ErrCapExceeded + for i := 0; i < 200; i++ { + err := s.Promote(mustEntry(t, "Filler entry number "+string(rune('a'+i%26))+strings.Repeat("x", i)), ProjectShard) + if err == nil { + continue + } + var ok bool + if capErr, ok = err.(*ErrCapExceeded); !ok { + t.Fatalf("unexpected error: %v", err) + } + break + } + if capErr == nil { + t.Fatal("cap never bit; the budget is not being enforced") + } + b, err := os.ReadFile(s.ShardPath(ProjectShard)) + if err != nil { + t.Fatal(err) + } + if got := CountLines(string(b)); got > ProjectShardCap { + t.Errorf("shard is %d lines, past its %d cap — the refusal did not prevent the write", got, ProjectShardCap) + } +} + +func TestModuleShardGetsMoreRoomThanProject(t *testing.T) { + if CapFor("Sales") <= CapFor(ProjectShard) { + t.Fatal("project.md must be the tightest: it is the only file loaded unconditionally") + } +} + +func TestDropDeletesAnEmptiedModuleShardButKeepsProject(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + mod := mustEntry(t, "Only entry", "@Sales.Order") + proj := mustEntry(t, "Only project entry") + if err := s.Promote(mod, "Sales"); err != nil { + t.Fatal(err) + } + if err := s.Promote(proj, ProjectShard); err != nil { + t.Fatal(err) + } + + if _, _, err := s.Drop(mod.ID); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(s.ShardPath("Sales")); !os.IsNotExist(err) { + t.Error("an emptied module shard must be removed, not left as a husk") + } + + // Control: emptying project.md leaves the file, because it is the store's + // permanent home for cross-cutting facts. + if _, _, err := s.Drop(proj.ID); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(s.ShardPath(ProjectShard)); err != nil { + t.Errorf("project.md must survive being emptied: %v", err) + } +} + +func TestInitRefusesAForeignStoreAndAdoptsItsOwn(t *testing.T) { + dir := t.TempDir() + foreign := filepath.Join(dir, "docs", "brain") + if err := os.MkdirAll(foreign, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(foreign, "README.md"), []byte("# Our team notes\n"), 0644); err != nil { + t.Fatal(err) + } + if _, err := NewStore(dir).Init(); err == nil { + t.Fatal("init must refuse a docs/brain/ it did not write") + } + + // Control: the same call on a store mxcli wrote is idempotent, not refused. + own := t.TempDir() + s := NewStore(own) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + if _, err := s.Init(); err != nil { + t.Fatalf("re-init of mxcli's own store must succeed: %v", err) + } +} + +func TestInitNeverClobbersEntries(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + e := mustEntry(t, "Load-bearing decision") + if err := s.Promote(e, ProjectShard); err != nil { + t.Fatal(err) + } + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + entries, _, err := s.LoadShard(ProjectShard) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("a second init discarded entries: %d left", len(entries)) + } +} + +func TestQueueRefusesADuplicateButNotADifferentFact(t *testing.T) { + dir := t.TempDir() + q := NewQueue(dir) + first := mustEntry(t, "Marketplace 4.5.0 broke the login flow", "@Administration.Account") + added, err := q.Append(first) + if err != nil || !added { + t.Fatalf("first append: added=%v err=%v", added, err) + } + again := mustEntry(t, "marketplace 4.5.0 broke the LOGIN flow", "@Administration.Account") + added, err = q.Append(again) + if err != nil { + t.Fatal(err) + } + if added { + t.Error("the same fact captured twice must not queue twice") + } + // Control: a genuinely different fact is queued. + added, err = q.Append(mustEntry(t, "Something else entirely")) + if err != nil || !added { + t.Fatalf("a different fact must queue: added=%v err=%v", added, err) + } + entries, err := q.Load() + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 { + t.Fatalf("queue has %d entries, want 2", len(entries)) + } +} + +// stubResolver answers from a fixed table so the awkward states can be tested; +// a fixture project cannot readily produce a NotIndexable. +type stubResolver map[string]Resolution + +func (s stubResolver) Resolve(a Anchor) (Resolution, error) { + if r, ok := s[a.String()]; ok { + return r, nil + } + return Resolution{State: NotFound}, nil +} + +func TestCheckFailsOnMissingButNotOnNotIndexable(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + notIndexable := mustEntry(t, "Uses a document type the catalog does not index", "@Sales.SomeExoticDoc") + if err := s.Promote(notIndexable, "Sales"); err != nil { + t.Fatal(err) + } + r := stubResolver{"@Sales.SomeExoticDoc": {State: NotIndexable, Kind: "exotic"}} + + // A NotIndexable anchor cannot fail the check: the entry is current, the + // index simply does not cover its target. + rep, err := Check(s, r, []string{"Sales"}) + if err != nil { + t.Fatal(err) + } + if rep.Failed() { + t.Errorf("NotIndexable must not fail the check: %+v", rep) + } + if len(rep.Findings) != 1 || rep.Findings[0].State != NotIndexable { + t.Errorf("NotIndexable must still be reported: %+v", rep.Findings) + } + + // Control, same shape in the other direction: an anchor to nothing fails. + gone := mustEntry(t, "Points at a deleted microflow", "@Sales.ACT_Gone") + if err := s.Promote(gone, "Sales"); err != nil { + t.Fatal(err) + } + rep, err = Check(s, r, []string{"Sales"}) + if err != nil { + t.Fatal(err) + } + if !rep.Failed() { + t.Error("an anchor that names nothing must fail the check") + } +} + +func TestMisfilingIsASeparateAxisFromResolution(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + // Every anchor resolves perfectly; the entry is simply in the wrong file. + e := mustEntry(t, "Filed under Sales but only mentions Finance", "@Finance.ACT_Post") + if err := s.Promote(e, "Sales"); err != nil { + t.Fatal(err) + } + r := stubResolver{"@Finance.ACT_Post": {State: Resolved, Module: "Finance", Kind: "microflow"}} + rep, err := Check(s, r, []string{"Sales"}) + if err != nil { + t.Fatal(err) + } + if len(rep.Findings) != 0 { + t.Errorf("no anchor problem expected: %+v", rep.Findings) + } + if len(rep.Misfiled) != 1 || rep.Misfiled[0].Belongs != "Finance" { + t.Fatalf("expected one misfiled entry belonging to Finance: %+v", rep.Misfiled) + } + if !rep.Failed() { + t.Error("a misfiled entry must fail the check") + } +} + +// The relaxation is deliberate: a two-module fact keeps its home shard as long +// as one anchor belongs there. +func TestCrossModuleEntryIsNotMisfiled(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + e := mustEntry(t, "Orders are committed by Finance", "@Sales.Order", "@Finance.ACT_Post") + if err := s.Promote(e, "Sales"); err != nil { + t.Fatal(err) + } + r := stubResolver{ + "@Sales.Order": {State: Resolved, Module: "Sales", Kind: "entity"}, + "@Finance.ACT_Post": {State: Resolved, Module: "Finance", Kind: "microflow"}, + } + rep, err := Check(s, r, []string{"Sales"}) + if err != nil { + t.Fatal(err) + } + if len(rep.Misfiled) != 0 { + t.Errorf("an anchor into another module must not misfile the entry: %+v", rep.Misfiled) + } + if rep.Failed() { + t.Errorf("report should be clean: %+v", rep) + } +} + +func TestProjectShardIsNeverMisfiled(t *testing.T) { + if MisfiledIn(ProjectShard, nil) { + t.Error("the catch-all shard cannot be misfiled") + } +} + +func TestUsageIsComputedFromTheFilesThemselves(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + if err := s.Promote(mustEntry(t, "One", "@Sales.Order"), "Sales"); err != nil { + t.Fatal(err) + } + usage, err := s.Usage() + if err != nil { + t.Fatal(err) + } + var seen bool + for _, u := range usage { + if u.Shard != "Sales" { + continue + } + seen = true + if u.Entries != 1 || u.Lines == 0 || u.Cap != ModuleShardCap { + t.Errorf("unexpected usage: %+v", u) + } + if u.Headroom() != u.Cap-u.Lines { + t.Errorf("headroom is not derived: %+v", u) + } + } + if !seen { + t.Fatalf("Sales shard missing from usage: %+v", usage) + } +} + +// The README must not carry a figure that promotion would falsify (A6). +func TestReadmeCarriesNoCountsOrSizes(t *testing.T) { + readme := readmeContent() + for _, bad := range []string{"entries)", "lines)", "currently", "so far"} { + if strings.Contains(strings.ToLower(readme), bad) { + t.Errorf("README states %q — computed figures must not be written down", bad) + } + } +} + +// Regression: an entry whose only anchor is NotIndexable was reported as +// misfiled, because nothing had resolved to compare the shard against. That +// reintroduced A1's false-staleness signal through the other axis — the entry +// is perfectly current and the index simply does not cover its target. +func TestNotIndexableAnchorDoesNotMakeAnEntryLookMisfiled(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + e := mustEntry(t, "Anchored at a type the objects view skips", "@Sales.SomeExoticDoc") + if err := s.Promote(e, "Sales"); err != nil { + t.Fatal(err) + } + rep, err := Check(s, stubResolver{"@Sales.SomeExoticDoc": {State: NotIndexable}}, []string{"Sales"}) + if err != nil { + t.Fatal(err) + } + if len(rep.Misfiled) != 0 { + t.Fatalf("misfiling is undecidable with nothing resolved: %+v", rep.Misfiled) + } + + // Control: once one anchor resolves, misfiling is decidable again and this + // same entry in the same shard IS reported. + e2 := mustEntry(t, "Filed in Sales, resolves only in Finance", "@Finance.ACT_Post") + if err := s.Promote(e2, "Sales"); err != nil { + t.Fatal(err) + } + rep, err = Check(s, stubResolver{ + "@Sales.SomeExoticDoc": {State: NotIndexable}, + "@Finance.ACT_Post": {State: Resolved, Module: "Finance"}, + }, []string{"Sales"}) + if err != nil { + t.Fatal(err) + } + if len(rep.Misfiled) != 1 || rep.Misfiled[0].EntryID != e2.ID { + t.Fatalf("the resolvable entry must still be caught: %+v", rep.Misfiled) + } +} + +func mustRequirement(t *testing.T, text, slice string, anchors ...string) Entry { + t.Helper() + e, err := NewRequirement(text, anchors, slice, day) + if err != nil { + t.Fatalf("NewRequirement(%q): %v", text, err) + } + return e +} + +// The central claim, with its control. A requirement's anchor points forward: +// not resolving means not built yet, which is the normal state. The identical +// entry recorded as a decision must still fail, or the distinction is doing +// nothing. +func TestUnbuiltRequirementDoesNotFailButTheSameDecisionDoes(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + r := stubResolver{} // resolves nothing + + req := mustRequirement(t, "Orders must be approvable", "02-approvals", "@Sales.ACT_Approve") + if err := s.Promote(req, req.Shard()); err != nil { + t.Fatal(err) + } + rep, err := Check(s, r, []string{req.Shard()}) + if err != nil { + t.Fatal(err) + } + if rep.Failed() { + t.Errorf("an unbuilt requirement must not fail the check: %+v", rep) + } + if len(rep.Slices) != 1 || rep.Slices[0].Planned != 1 || rep.Slices[0].Built != 0 { + t.Fatalf("expected one planned requirement: %+v", rep.Slices) + } + + // Control: the same sentence and anchor, recorded as a decision, fails. + dec := mustEntry(t, "Orders must be approvable", "@Sales.ACT_Approve") + if err := s.Promote(dec, "Sales"); err != nil { + t.Fatal(err) + } + rep, err = Check(s, r, []string{"Sales"}) + if err != nil { + t.Fatal(err) + } + if !rep.Failed() { + t.Error("a decision anchored at something missing must still fail — otherwise the kind distinction changes nothing") + } +} + +// Progress is derived from resolving anchors, so building the thing is what +// moves the number. Nothing in the file says "done". +func TestSliceProgressIsDerivedFromTheModel(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + built := mustRequirement(t, "Accounts can be created", "01-accounts", "@Admin.NewAccount") + planned := mustRequirement(t, "Accounts can be archived", "01-accounts", "@Admin.ArchiveAccount") + unanchored := mustRequirement(t, "It should feel fast", "01-accounts") + for _, e := range []Entry{built, planned, unanchored} { + if err := s.Promote(e, e.Shard()); err != nil { + t.Fatal(err) + } + } + + before := stubResolver{} + rep, err := Check(s, before, []string{PlanShard("01-accounts")}) + if err != nil { + t.Fatal(err) + } + got := rep.Slices[0] + if got.Built != 0 || got.Planned != 2 || got.Unanchored != 1 { + t.Fatalf("before: %+v", got) + } + + // Build one of them — only the resolver changes, not the store. + after := stubResolver{"@Admin.NewAccount": {State: Resolved, Module: "Admin", Kind: "microflow"}} + rep, err = Check(s, after, []string{PlanShard("01-accounts")}) + if err != nil { + t.Fatal(err) + } + got = rep.Slices[0] + if got.Built != 1 || got.Planned != 1 || got.Unanchored != 1 { + t.Fatalf("after: %+v — progress must follow the model, not the file", got) + } + if got.Total() != 3 { + t.Errorf("total = %d, want 3", got.Total()) + } +} + +// A requirement anchored at a document type the catalog does not index is +// BUILT: the thing exists. Counting it as planned would report finished work as +// outstanding — the same false signal as A1, in the progress report. +func TestNotIndexableCountsAsBuilt(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + e := mustRequirement(t, "A nightly job trims the audit log", "04-ops", "@Ops.NightlyTrim") + if err := s.Promote(e, e.Shard()); err != nil { + t.Fatal(err) + } + rep, err := Check(s, stubResolver{"@Ops.NightlyTrim": {State: NotIndexable, Kind: "scheduled event"}}, + []string{e.Shard()}) + if err != nil { + t.Fatal(err) + } + if rep.Slices[0].Built != 1 { + t.Fatalf("a not-indexable target exists and must count as built: %+v", rep.Slices[0]) + } +} + +// A slice spans modules on purpose, so the misfiling rule that keeps decisions +// honest must not apply to it. +func TestCrossModuleSliceIsNeverMisfiled(t *testing.T) { + dir := t.TempDir() + s := NewStore(dir) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + e := mustRequirement(t, "Approving an order posts it to the ledger", "02-approvals", + "@Sales.Order", "@Finance.ACT_Post") + if err := s.Promote(e, e.Shard()); err != nil { + t.Fatal(err) + } + rep, err := Check(s, stubResolver{ + "@Sales.Order": {State: Resolved, Module: "Sales"}, + "@Finance.ACT_Post": {State: Resolved, Module: "Finance"}, + }, []string{e.Shard()}) + if err != nil { + t.Fatal(err) + } + if len(rep.Misfiled) != 0 { + t.Fatalf("a slice spans modules by design: %+v", rep.Misfiled) + } +} + +// The same sentence can legitimately be a requirement of two slices; the id has +// to distinguish them or the second is refused as a duplicate. +func TestSameTextInTwoSlicesIsTwoRequirements(t *testing.T) { + a := mustRequirement(t, "The list must paginate", "01-accounts") + b := mustRequirement(t, "The list must paginate", "02-approvals") + if a.ID == b.ID { + t.Fatal("requirements in different slices must not collide") + } + // Control: the same text in the SAME slice is one requirement. + c := mustRequirement(t, "The list must paginate", "01-accounts") + if a.ID != c.ID { + t.Fatal("the same requirement in the same slice must be one entry") + } +} + +func TestRequirementRoundTripsThroughItsSlice(t *testing.T) { + in := []Entry{mustRequirement(t, "A title\nand a body", "02-approvals", "@Sales.Order")} + shard := PlanShard("02-approvals") + out, malformed, err := ParseShard(shard, RenderShard(shard, in)) + if err != nil || len(malformed) != 0 { + t.Fatalf("err=%v malformed=%v", err, malformed) + } + if len(out) != 1 { + t.Fatalf("got %d entries", len(out)) + } + // The kind comes from the file, not from a second copy inside the entry. + if out[0].EntryKind() != KindRequirement || out[0].Slice != "02-approvals" { + t.Errorf("kind/slice not recovered from the shard: %+v", out[0]) + } +} + +func TestSliceNamesAreValidated(t *testing.T) { + for _, bad := range []string{"", "has space", "../escape", "a/b"} { + if _, err := NewRequirement("x", nil, bad, day); err == nil { + t.Errorf("slice %q should be refused", bad) + } + } + for _, ok := range []string{"01-accounts", "approvals", "a_b", "2"} { + if _, err := NewRequirement("x", nil, ok, day); err != nil { + t.Errorf("slice %q should be accepted: %v", ok, err) + } + } +} + +func TestPlanSlicesGetMoreRoomThanDecisions(t *testing.T) { + if CapFor(PlanShard("01-x")) <= CapFor("Sales") { + t.Fatal("a slice holds source material and is not loaded every session; it needs more room than a decision shard") + } +} diff --git a/cmd/mxcli/brain/caps.go b/cmd/mxcli/brain/caps.go new file mode 100644 index 0000000000..f9521b7fc8 --- /dev/null +++ b/cmd/mxcli/brain/caps.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +// caps.go - the size limits, and where they bite. +// +// Caps are per shard rather than per store. A single file would make the cap a +// project-wide budget, so recording a Sales decision would compete with a +// Finance one and promotion would start refusing on exactly the projects that +// most need the store (PROPOSAL_project_brain.md §4.1). +package brain + +// Caps are measured in lines, because lines are what an agent pays for when a +// shard is loaded into context. +// +// project.md is the tightest: it is the only file loaded unconditionally, so +// every line in it is charged to every session on the project. A module shard +// is charged only to sessions touching that module, which is what buys it the +// larger allowance. +const ( + ProjectShardCap = 120 + ModuleShardCap = 240 + + // PlanSliceCap is generous because a slice holds source material — text + // lifted from a specification, not a distilled decision — and because a + // plan shard is read when planning rather than loaded every session. + // + // It is still a cap, and that is the point: a slice too long to read is a + // slice that should be split. Here the limit does not merely bound context + // cost, it enforces the slicing discipline the plan exists for. + PlanSliceCap = 600 +) + +// CapFor returns the line budget for a shard. +func CapFor(shard string) int { + switch { + case shard == ProjectShard: + return ProjectShardCap + case IsPlanShard(shard): + return PlanSliceCap + default: + return ModuleShardCap + } +} + +// Usage is what `brain show` reports. Every field is computed on the call — +// none of it is ever written into a committed file, because a figure in prose +// is stale the next time anyone promotes (A6). +type Usage struct { + Shard string + Entries int + Lines int + Cap int +} + +// Headroom is the number of lines still available. It goes negative for a shard +// that was edited past its cap by hand, which `show` reports rather than hides. +func (u Usage) Headroom() int { return u.Cap - u.Lines } + +// Over reports whether the shard is past its cap. +func (u Usage) Over() bool { return u.Lines > u.Cap } diff --git a/cmd/mxcli/brain/check.go b/cmd/mxcli/brain/check.go new file mode 100644 index 0000000000..17ca24410c --- /dev/null +++ b/cmd/mxcli/brain/check.go @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: Apache-2.0 + +// check.go - does the store still describe the project? +// +// Two independent questions, and conflating them is the mistake this file +// exists to avoid: +// +// - Does each anchor still resolve? Three states, and only one is a failure. +// An anchor to a document type the catalog does not index resolves as +// *missing* through the catalog alone, which is a false staleness signal — +// so a second, type-agnostic lookup separates "gone" from "not indexed". +// - Is the entry in the right shard? A separate axis, not a fourth state: an +// anchor can resolve perfectly and the entry still sit in the wrong file. +package brain + +import "sort" + +// AnchorState is the outcome of resolving one anchor. +type AnchorState int + +const ( + // Resolved: the anchor names something the catalog knows. + Resolved AnchorState = iota + // NotFound: the anchor names nothing in the project. The only failure. + NotFound + // NotIndexable: the target exists but is of a type the catalog's objects + // view does not cover. Reported, never failed — treating it as missing + // would make `check` demand edits to entries that are perfectly current. + NotIndexable +) + +func (s AnchorState) String() string { + switch s { + case Resolved: + return "resolved" + case NotFound: + return "not found" + default: + return "not indexable" + } +} + +// Resolution is what a Resolver reports about one anchor. +type Resolution struct { + State AnchorState + // Module is the module the target actually lives in, which is what the + // misfiling check compares against. Empty unless State is Resolved. + Module string + // Kind is the target's type, for the report ("entity", "microflow"). + Kind string +} + +// Resolver answers anchors against a project. It is an interface so the check +// logic is testable without a .mpr — the states that matter are the awkward +// ones, and a fixture project cannot readily produce a NotIndexable. +type Resolver interface { + Resolve(Anchor) (Resolution, error) +} + +// AnchorFinding is one anchor's outcome. +type AnchorFinding struct { + Shard string + EntryID string + Title string + Anchor string + State AnchorState + Kind string +} + +// MisfiledFinding is an entry sitting in a shard none of its anchors belong to. +type MisfiledFinding struct { + Shard string + EntryID string + Title string + Belongs string // the shard it should be in, from its first resolved anchor +} + +// SliceProgress is a slice's requirements counted against the model. Every +// figure is derived from resolving anchors, so nothing here is self-reported +// and no one has to maintain a status column that will go stale. +type SliceProgress struct { + Slice string + // Built is requirements whose anchors all resolve — the thing exists. + Built int + // Planned is requirements with at least one anchor that does not resolve + // yet. Not a failure: that is what a requirement is until it is built. + Planned int + // Unanchored is requirements with no anchor at all. They cannot be + // measured, and are counted apart rather than silently called planned. + Unanchored int +} + +// Total is every requirement in the slice. +func (p SliceProgress) Total() int { return p.Built + p.Planned + p.Unanchored } + +// Report is what `brain check` prints and exits on. +type Report struct { + Shards []string + Entries int + Anchors int + ResolvedN int + Findings []AnchorFinding // NotFound and NotIndexable only + Misfiled []MisfiledFinding + Malformed []string // entry blocks whose metadata line could not be read + Slices []SliceProgress +} + +// Failed reports whether the check should exit non-zero. +// +// Only two things fail: an anchor that names nothing at all, and an entry in +// the wrong shard. A NotIndexable anchor is information, not a defect — see the +// constant's comment. +func (r Report) Failed() bool { + if len(r.Misfiled) > 0 || len(r.Malformed) > 0 { + return true + } + for _, f := range r.Findings { + if f.State == NotFound { + return true + } + } + return false +} + +// Check validates the given shards. Passing a subset is how `--changed` avoids +// paying for shards a diff did not touch. +func Check(s *Store, r Resolver, shards []string) (Report, error) { + rep := Report{Shards: shards} + for _, shard := range shards { + entries, malformed, err := s.LoadShard(shard) + if err != nil { + return rep, err + } + for _, m := range malformed { + rep.Malformed = append(rep.Malformed, shard+": "+m) + } + if IsPlanShard(shard) { + progress, err := checkSlice(r, shard, entries) + if err != nil { + return rep, err + } + rep.Entries += len(entries) + rep.Anchors += progress.anchors + rep.ResolvedN += progress.resolved + rep.Slices = append(rep.Slices, progress.SliceProgress) + continue + } + for _, e := range entries { + rep.Entries++ + var resolvedModules []string + for _, a := range e.ParsedAnchors() { + rep.Anchors++ + res, err := r.Resolve(a) + if err != nil { + return rep, err + } + if res.State == Resolved { + rep.ResolvedN++ + resolvedModules = append(resolvedModules, res.Module) + continue + } + rep.Findings = append(rep.Findings, AnchorFinding{ + Shard: shard, EntryID: e.ID, Title: e.Title, + Anchor: a.String(), State: res.State, Kind: res.Kind, + }) + } + if MisfiledIn(shard, resolvedModules) { + rep.Misfiled = append(rep.Misfiled, MisfiledFinding{ + Shard: shard, EntryID: e.ID, Title: e.Title, + Belongs: belongsIn(resolvedModules), + }) + } + } + } + return rep, nil +} + +type sliceCounts struct { + SliceProgress + anchors, resolved int +} + +// checkSlice counts a slice's requirements against the model. It records no +// findings and no misfiling, and that is the point rather than an omission: +// +// - A requirement's anchor points FORWARD. Not resolving means not built, +// which is the normal state of a requirement and must never fail a check — +// measured: filed as an ordinary entry, one unbuilt requirement took +// `brain check` to exit 1. +// - A slice spans modules by design ("approvals" touches Sales and Finance), +// so the misfiling rule that keeps decisions honest does not apply. +func checkSlice(r Resolver, shard string, entries []Entry) (sliceCounts, error) { + out := sliceCounts{SliceProgress: SliceProgress{Slice: SliceOf(shard)}} + for _, e := range entries { + anchors := e.ParsedAnchors() + if len(anchors) == 0 { + out.Unanchored++ + continue + } + built := true + for _, a := range anchors { + out.anchors++ + res, err := r.Resolve(a) + if err != nil { + return out, err + } + // NotIndexable counts as built: the thing is there, the catalog + // simply does not index its type. Calling it planned would report + // finished work as outstanding. + if res.State == NotFound { + built = false + continue + } + out.resolved++ + } + if built { + out.Built++ + } else { + out.Planned++ + } + } + return out, nil +} + +// belongsIn names the shard an entry should have gone to. With no resolved +// anchor at all there is nothing to suggest, and the empty string says so +// rather than guessing. +func belongsIn(resolved []string) string { + if len(resolved) == 0 { + return "" + } + uniq := map[string]bool{} + for _, m := range resolved { + uniq[m] = true + } + keys := make([]string, 0, len(uniq)) + for k := range uniq { + keys = append(keys, k) + } + sort.Strings(keys) // deterministic output; maps iterate randomly + return keys[0] +} diff --git a/cmd/mxcli/brain/entry.go b/cmd/mxcli/brain/entry.go new file mode 100644 index 0000000000..320b89714f --- /dev/null +++ b/cmd/mxcli/brain/entry.go @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 + +// entry.go - the unit the brain stores. +// +// An entry is deliberately small: a title, an optional body, and the anchors +// that make it routable and checkable. Anything derivable from the model is not +// an entry — mxcli answers that with a query, and a store that transcribes it +// is a store that will disagree with the project. +package brain + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "regexp" + "strings" + "time" +) + +// Kind is what an entry records, and it decides what a failed anchor MEANS. +// +// This is the whole reason requirements are not simply more decisions. A +// decision's anchor points backward at something that exists, so an anchor that +// no longer resolves means the decision is stale. A requirement's anchor points +// forward at something intended, so an anchor that does not resolve means "not +// built yet" — the normal state of a requirement, and measured to fail the +// check outright when requirements were tried as ordinary entries. +type Kind string + +const ( + KindDecision Kind = "decision" + KindRequirement Kind = "requirement" +) + +// Entry is one recorded piece of project knowledge. +type Entry struct { + ID string `json:"id"` + // Kind is empty for a decision, so an entry written before requirements + // existed still reads correctly. Use EntryKind rather than this field. + Kind Kind `json:"kind,omitempty"` + // Slice is the deliverable a requirement belongs to. Empty for a decision. + Slice string `json:"slice,omitempty"` + Title string `json:"title"` + Body string `json:"body,omitempty"` + Anchors []string `json:"anchors,omitempty"` + Date string `json:"date"` +} + +// EntryKind normalises the zero value: an entry with no kind is a decision. +func (e Entry) EntryKind() Kind { + if e.Kind == KindRequirement { + return KindRequirement + } + return KindDecision +} + +// NewEntry builds an entry from captured text. The first line is the title and +// the remainder is the body, so `brain capture` can take one argument and still +// produce something that renders as a heading plus prose. +func NewEntry(text string, anchors []string, now time.Time) (Entry, error) { + title, body := splitTitle(text) + if title == "" { + return Entry{}, fmt.Errorf("an entry needs at least a title line") + } + if _, err := ParseAnchors(anchors); err != nil { + return Entry{}, err + } + e := Entry{ + Title: title, + Body: body, + Anchors: anchors, + Date: now.Format("2006-01-02"), + } + e.ID = e.computeID() + return e, nil +} + +// computeID derives the id from the content rather than from a random source. +// That is what makes the duplicate check of A5 free: capturing the same fact +// twice produces the same id, so the queue can refuse it without comparing +// prose. Date is deliberately excluded — the same fact captured on two days is +// the same fact. +func (e Entry) computeID() string { + h := sha256.New() + h.Write([]byte(strings.ToLower(strings.Join(strings.Fields(e.Title), " ")))) + if e.Slice != "" { + h.Write([]byte("\x01" + e.Slice)) + } + for _, a := range e.Anchors { + h.Write([]byte("\x00" + a)) + } + return hex.EncodeToString(h.Sum(nil))[:6] +} + +// ParsedAnchors returns the entry's anchors, ignoring any that do not parse. +// Callers that need to report a bad anchor use ParseAnchors directly; this is +// for the paths where a malformed anchor must not stop the rest of the work. +func (e Entry) ParsedAnchors() []Anchor { + out := make([]Anchor, 0, len(e.Anchors)) + for _, s := range e.Anchors { + if a, err := ParseAnchor(s); err == nil { + out = append(out, a) + } + } + return out +} + +// NewRequirement builds a requirement: a piece of intended scope belonging to a +// deliverable slice. Its anchors are forward references — they may name things +// that do not exist yet, and usually do. +func NewRequirement(text string, anchors []string, slice string, now time.Time) (Entry, error) { + e, err := NewEntry(text, anchors, now) + if err != nil { + return Entry{}, err + } + if !sliceName.MatchString(slice) { + return Entry{}, fmt.Errorf("slice %q: use letters, digits, '-' and '_' (a leading number orders it: 01-accounts)", slice) + } + e.Kind, e.Slice = KindRequirement, slice + // The id folds in the slice, so the same sentence can legitimately appear + // as a requirement of two slices without the second being refused as a + // duplicate. + e.ID = e.computeID() + return e, nil +} + +// sliceName is deliberately permissive about ordering: a slice is sorted by +// name, so a numeric prefix is how a roadmap gets its order, and that is the +// user's choice rather than a field mxcli maintains. +var sliceName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) + +// Shard is where this entry belongs. A requirement goes to its slice; a +// decision to the module of its first anchor. +func (e Entry) Shard() string { + if e.EntryKind() == KindRequirement { + return PlanShard(e.Slice) + } + return ShardFor(e.ParsedAnchors()) +} + +// MisfiledIn reports whether the entry sits in the wrong shard, given which of +// its anchors resolved and in which module. +// +// This is a second axis, not a fourth anchor state: an anchor can resolve +// perfectly and the entry still be in the wrong file. The rule is deliberately +// relaxed — *at least one* anchor must belong to the shard, and anchors into +// other modules are fine. A fact like "Sales.Order is committed by +// Finance.ACT_Post" is genuinely two-module, and forcing it into project.md +// would grow the one file that must stay small. +func MisfiledIn(shard string, resolvedModules []string) bool { + if shard == ProjectShard { + return false // the catch-all is never misfiled + } + if len(resolvedModules) == 0 { + // Nothing resolved, so there is no evidence about where the entry + // belongs — and an entry whose only anchor is NotIndexable would + // otherwise be reported as misfiled, reintroducing through this axis + // exactly the false staleness that A1's third state exists to prevent. + // An anchor that names nothing is already a failure on its own axis. + return false + } + for _, m := range resolvedModules { + if m == shard { + return false + } + } + return true +} + +func splitTitle(text string) (title, body string) { + text = strings.TrimSpace(text) + if text == "" { + return "", "" + } + if i := strings.IndexByte(text, '\n'); i >= 0 { + return strings.TrimSpace(text[:i]), strings.TrimSpace(text[i+1:]) + } + return text, "" +} diff --git a/cmd/mxcli/brain/shard.go b/cmd/mxcli/brain/shard.go new file mode 100644 index 0000000000..ca89fd2b17 --- /dev/null +++ b/cmd/mxcli/brain/shard.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: Apache-2.0 + +// shard.go - rendering a shard to Markdown and reading it back. +// +// The format is chosen for the reader, not the parser: a Mendix developer meets +// these files in a pull request diff, so the anchors are visible prose rather +// than metadata hidden in an HTML comment. There is exactly one copy of each +// fact in the file — a visible line that is also the parsed one — because a +// human-readable copy beside a machine-readable copy is two things that drift. +package brain + +import ( + "fmt" + "regexp" + "strings" +) + +// shardMarker identifies a file mxcli wrote. `brain init` refuses a docs/brain/ +// whose README does not carry it, so an existing folder of someone else's notes +// is never adopted by accident. +const shardMarker = "" + +// metaLine matches an entry's one metadata line. The separator is a middle dot +// so that a title or body containing a hyphen cannot be mistaken for it. +var metaLine = regexp.MustCompile("^Anchors: (.*?) · id `([0-9a-f]{6})` · (\\d{4}-\\d{2}-\\d{2})\\s*$") + +// anchorRef matches one backticked anchor inside the metadata line. +var anchorRef = regexp.MustCompile("`(@[A-Za-z_][A-Za-z0-9_.]*)`") + +// RenderShard writes a whole shard. Entries are emitted in the order given; +// promotion appends, so that is chronological, and a diff shows one added block. +func RenderShard(shard string, entries []Entry) string { + var b strings.Builder + fmt.Fprintf(&b, "# %s\n\n", shardTitle(shard)) + b.WriteString(shardMarker) + b.WriteString("\n\n") + b.WriteString(shardPreamble(shard)) + for _, e := range entries { + b.WriteString("\n") + b.WriteString(renderEntry(e)) + } + return b.String() +} + +func renderEntry(e Entry) string { + var b strings.Builder + fmt.Fprintf(&b, "## %s\n\n", e.Title) + fmt.Fprintf(&b, "Anchors: %s · id `%s` · %s\n", renderAnchors(e.Anchors), e.ID, e.Date) + if e.Body != "" { + fmt.Fprintf(&b, "\n%s\n", e.Body) + } + return b.String() +} + +func renderAnchors(anchors []string) string { + if len(anchors) == 0 { + return "none" + } + parts := make([]string, 0, len(anchors)) + for _, a := range anchors { + if !strings.HasPrefix(a, "@") { + a = "@" + a + } + parts = append(parts, "`"+a+"`") + } + return strings.Join(parts, ", ") +} + +// ParseShard reads entries back out of a rendered shard. A block that does not +// carry a well-formed metadata line is *reported*, not skipped: silently +// dropping it would make `check` claim a clean shard while an entry sits in it +// unchecked. +// +// The kind is taken from the shard, never from the entry's own text. A +// requirement is a requirement because it lives under plan/, so there is no +// second copy of that fact in the file to drift from the first. +func ParseShard(shard, content string) (entries []Entry, malformed []string, err error) { + for _, blk := range splitEntries(content) { + e, ok := parseEntry(blk) + if !ok { + malformed = append(malformed, firstLine(blk)) + continue + } + if IsPlanShard(shard) { + e.Kind, e.Slice = KindRequirement, SliceOf(shard) + } + entries = append(entries, e) + } + return entries, malformed, nil +} + +func splitEntries(content string) []string { + lines := strings.Split(content, "\n") + var blocks []string + var cur []string + in := false + for _, ln := range lines { + if strings.HasPrefix(ln, "## ") { + if in { + blocks = append(blocks, strings.Join(cur, "\n")) + } + in, cur = true, []string{ln} + continue + } + if in { + cur = append(cur, ln) + } + } + if in { + blocks = append(blocks, strings.Join(cur, "\n")) + } + return blocks +} + +func parseEntry(block string) (Entry, bool) { + lines := strings.Split(block, "\n") + if len(lines) == 0 || !strings.HasPrefix(lines[0], "## ") { + return Entry{}, false + } + e := Entry{Title: strings.TrimSpace(strings.TrimPrefix(lines[0], "## "))} + metaAt := -1 + for i := 1; i < len(lines); i++ { + if m := metaLine.FindStringSubmatch(strings.TrimSpace(lines[i])); m != nil { + for _, a := range anchorRef.FindAllStringSubmatch(m[1], -1) { + e.Anchors = append(e.Anchors, a[1]) + } + e.ID, e.Date = m[2], m[3] + metaAt = i + break + } + } + if metaAt < 0 { + return Entry{}, false + } + e.Body = strings.TrimSpace(strings.Join(lines[metaAt+1:], "\n")) + return e, true +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return strings.TrimSpace(s[:i]) + } + return strings.TrimSpace(s) +} + +func shardTitle(shard string) string { + switch { + case shard == ProjectShard: + return "Project" + case IsPlanShard(shard): + return "Slice: " + SliceOf(shard) + default: + return shard + } +} + +func shardPreamble(shard string) string { + if IsPlanShard(shard) { + return "Requirements for this slice. Anchors point FORWARD, at what the slice\n" + + "will build — an anchor that does not resolve yet means not built, not\n" + + "stale. `mxcli brain plan` counts them against the model.\n" + } + if shard == ProjectShard { + return "Decisions that are not about one module. This file is loaded every\n" + + "session, so it carries the tightest cap — see `mxcli brain show`.\n" + } + return fmt.Sprintf("Decisions anchored to the %s module. Loaded when %s is in play,\n"+ + "not otherwise.\n", shard, shard) +} + +// HasMarker reports whether a file was written by mxcli. +func HasMarker(content string) bool { return strings.Contains(content, shardMarker) } + +// CountLines is the size measure caps are expressed in, because lines are what +// an agent pays for when the shard is loaded into context. +func CountLines(content string) int { + content = strings.TrimRight(content, "\n") + if content == "" { + return 0 + } + return strings.Count(content, "\n") + 1 +} diff --git a/cmd/mxcli/brain/staged.go b/cmd/mxcli/brain/staged.go new file mode 100644 index 0000000000..9a47c73236 --- /dev/null +++ b/cmd/mxcli/brain/staged.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 + +// staged.go - the queue an agent writes to. +// +// The queue is deliberately NOT sharded. Staging is a queue, not a store, and +// routing it would force the shard decision before a human has looked at the +// entry — which is exactly the decision promotion exists to make. It lives +// under .mxcli/, which `mxcli init` git-ignores, so nothing reaches a pull +// request until someone promotes it. +package brain + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +// StagedPath is the queue's location relative to the project directory. +const StagedPath = ".mxcli/brain/staged.jsonl" + +// Queue is the staged half of the brain. +type Queue struct{ Path string } + +// NewQueue locates the queue for a project directory. +func NewQueue(projectDir string) *Queue { + return &Queue{Path: filepath.Join(projectDir, filepath.FromSlash(StagedPath))} +} + +// Load reads the queue. A missing file is an empty queue, not an error. +func (q *Queue) Load() ([]Entry, error) { + f, err := os.Open(q.Path) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + defer f.Close() + + var entries []Entry + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for line := 1; sc.Scan(); line++ { + text := strings.TrimSpace(sc.Text()) + if text == "" { + continue + } + var e Entry + if err := json.Unmarshal([]byte(text), &e); err != nil { + return nil, fmt.Errorf("%s line %d: %w", q.Path, line, err) + } + entries = append(entries, e) + } + return entries, sc.Err() +} + +// Append queues an entry, refusing one whose id is already present. +// +// The id is content-derived, so this duplicate check costs nothing and needs no +// prose comparison: capturing the same fact twice produces the same id. It is +// cheap insurance rather than a load-bearing guard — the duplicate flood that +// motivated it in mxcli's own findings store was a many-parallel-writers +// problem, and one developer on one project has little exposure to it (A5). +func (q *Queue) Append(e Entry) (added bool, err error) { + entries, err := q.Load() + if err != nil { + return false, err + } + for _, existing := range entries { + if existing.ID == e.ID { + return false, nil + } + } + if err := os.MkdirAll(filepath.Dir(q.Path), 0755); err != nil { + return false, err + } + f, err := os.OpenFile(q.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return false, err + } + defer f.Close() + b, err := json.Marshal(e) + if err != nil { + return false, err + } + if _, err := f.Write(append(b, '\n')); err != nil { + return false, err + } + return true, nil +} + +// Drop removes an entry from the queue by id. +func (q *Queue) Drop(id string) (bool, error) { + entries, err := q.Load() + if err != nil { + return false, err + } + kept := make([]Entry, 0, len(entries)) + found := false + for _, e := range entries { + if e.ID == id { + found = true + continue + } + kept = append(kept, e) + } + if !found { + return false, nil + } + return true, q.write(kept) +} + +func (q *Queue) write(entries []Entry) error { + if len(entries) == 0 { + err := os.Remove(q.Path) + if os.IsNotExist(err) { + return nil + } + return err + } + var b strings.Builder + for _, e := range entries { + line, err := json.Marshal(e) + if err != nil { + return err + } + b.Write(line) + b.WriteByte('\n') + } + if err := os.MkdirAll(filepath.Dir(q.Path), 0755); err != nil { + return err + } + return os.WriteFile(q.Path, []byte(b.String()), 0644) +} + +// Get returns the queued entry with the given id. +func (q *Queue) Get(id string) (Entry, bool, error) { + entries, err := q.Load() + if err != nil { + return Entry{}, false, err + } + for _, e := range entries { + if e.ID == id { + return e, true, nil + } + } + return Entry{}, false, nil +} diff --git a/cmd/mxcli/brain/store.go b/cmd/mxcli/brain/store.go new file mode 100644 index 0000000000..1637c2eff5 --- /dev/null +++ b/cmd/mxcli/brain/store.go @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: Apache-2.0 + +// store.go - the committed side of the brain: docs/brain/. +// +// The store lives under docs/ because it is reviewed in a pull request like any +// other change, and in its own subfolder because a Mendix project's docs/ may +// already be the customer's or Studio Pro's. A labelled subfolder can be added +// to someone else's docs tree; a decisions.md dropped into it cannot +// (PROPOSAL_project_brain.md §4.2). +package brain + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// StoreDir is the store's location relative to the project directory. +const StoreDir = "docs/brain" + +// Store is the committed half of the brain. The staged queue is deliberately +// not part of it — see staged.go. +type Store struct{ Root string } + +// NewStore locates the store for a project directory. +func NewStore(projectDir string) *Store { + return &Store{Root: filepath.Join(projectDir, filepath.FromSlash(StoreDir))} +} + +// ErrForeignStore is returned when docs/brain/ exists but mxcli did not write +// it. Adopting it would mean writing into someone else's notes. +var ErrForeignStore = errors.New("docs/brain/ exists but was not written by mxcli") + +// Exists reports whether the store has been initialised. +func (s *Store) Exists() bool { + _, err := os.Stat(filepath.Join(s.Root, "README.md")) + return err == nil +} + +// Init creates the store, returning the paths written. It refuses a docs/brain/ +// that exists without mxcli's marker, and is otherwise idempotent: an existing +// store is left alone rather than overwritten, so a second `init` cannot +// silently discard entries. +func (s *Store) Init() ([]string, error) { + if info, err := os.Stat(s.Root); err == nil && info.IsDir() { + readme, err := os.ReadFile(filepath.Join(s.Root, "README.md")) + if err != nil || !HasMarker(string(readme)) { + return nil, fmt.Errorf("%w: %s", ErrForeignStore, s.Root) + } + } + for _, sub := range []string{"modules", "plan"} { + if err := os.MkdirAll(filepath.Join(s.Root, sub), 0755); err != nil { + return nil, err + } + } + var written []string + for _, f := range []struct{ path, content string }{ + {filepath.Join(s.Root, "README.md"), readmeContent()}, + {filepath.Join(s.Root, "project.md"), RenderShard(ProjectShard, nil)}, + } { + if _, err := os.Stat(f.path); err == nil { + continue // never clobber an existing file + } + if err := os.WriteFile(f.path, []byte(f.content), 0644); err != nil { + return nil, err + } + written = append(written, f.path) + } + return written, nil +} + +// ShardPath is where a shard's Markdown lives. +func (s *Store) ShardPath(shard string) string { + switch { + case shard == ProjectShard: + return filepath.Join(s.Root, "project.md") + case IsPlanShard(shard): + return filepath.Join(s.Root, "plan", SliceOf(shard)+".md") + default: + return filepath.Join(s.Root, "modules", shard+".md") + } +} + +// LoadShard reads a shard. A shard that does not exist is empty, not an error — +// shards appear on promotion. +func (s *Store) LoadShard(shard string) ([]Entry, []string, error) { + b, err := os.ReadFile(s.ShardPath(shard)) + if os.IsNotExist(err) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + return ParseShard(shard, string(b)) +} + +// SaveShard writes a shard, deleting it when it has no entries left. Leaving an +// empty file behind would make the directory accumulate husks that read as +// "this module has decisions" when it has none. +func (s *Store) SaveShard(shard string, entries []Entry) error { + path := s.ShardPath(shard) + if len(entries) == 0 && shard != ProjectShard { + err := os.Remove(path) + if os.IsNotExist(err) { + return nil + } + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + return os.WriteFile(path, []byte(RenderShard(shard, entries)), 0644) +} + +// ListShards returns every shard that exists, project first and modules sorted. +func (s *Store) ListShards() ([]string, error) { + var shards []string + if _, err := os.Stat(s.ShardPath(ProjectShard)); err == nil { + shards = append(shards, ProjectShard) + } + mods, err := s.namesIn("modules", "") + if err != nil { + return nil, err + } + slices, err := s.namesIn("plan", PlanPrefix) + if err != nil { + return nil, err + } + shards = append(shards, mods...) + return append(shards, slices...), nil +} + +// ListSlices returns the plan shards, in name order — which is what gives a +// roadmap its order, since a slice is sorted by name and a numeric prefix is +// the user's way of sequencing it. +func (s *Store) ListSlices() ([]string, error) { return s.namesIn("plan", PlanPrefix) } + +func (s *Store) namesIn(sub, prefix string) ([]string, error) { + ents, err := os.ReadDir(filepath.Join(s.Root, sub)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []string + for _, e := range ents { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".md") { + continue + } + out = append(out, prefix+strings.TrimSuffix(e.Name(), ".md")) + } + sort.Strings(out) + return out, nil +} + +// ErrCapExceeded reports a promotion that would push a shard past its budget. +type ErrCapExceeded struct { + Shard string + Would, Cap int +} + +func (e *ErrCapExceeded) Error() string { + return fmt.Sprintf("promoting into %s would take it to %d lines, past its %d-line cap; "+ + "drop or condense an entry there first (mxcli brain show %s)", e.Shard, e.Would, e.Cap, e.Shard) +} + +// Promote appends an entry to a shard. This is the only writer of a committed +// file, and it is the point where the cap bites — refusing with the shard named +// and its occupancy, rather than letting the store grow past what a session can +// afford to load. +func (s *Store) Promote(e Entry, shard string) error { + entries, _, err := s.LoadShard(shard) + if err != nil { + return err + } + for _, existing := range entries { + if existing.ID == e.ID { + return fmt.Errorf("%s already carries entry %s (%q)", shard, e.ID, existing.Title) + } + } + next := append(entries, e) + if lines, limit := CountLines(RenderShard(shard, next)), CapFor(shard); lines > limit { + return &ErrCapExceeded{Shard: shard, Would: lines, Cap: limit} + } + return s.SaveShard(shard, next) +} + +// Drop removes an entry by id, reporting which shard it came from and whether +// that emptied the shard. +func (s *Store) Drop(id string) (shard string, deletedFile bool, err error) { + shards, err := s.ListShards() + if err != nil { + return "", false, err + } + for _, sh := range shards { + entries, _, err := s.LoadShard(sh) + if err != nil { + return "", false, err + } + kept := make([]Entry, 0, len(entries)) + found := false + for _, e := range entries { + if e.ID == id { + found = true + continue + } + kept = append(kept, e) + } + if !found { + continue + } + if err := s.SaveShard(sh, kept); err != nil { + return "", false, err + } + return sh, len(kept) == 0 && sh != ProjectShard, nil + } + return "", false, nil +} + +// Usage computes size and headroom for every shard. Nothing here is cached or +// written down (A6). +func (s *Store) Usage() ([]Usage, error) { + shards, err := s.ListShards() + if err != nil { + return nil, err + } + out := make([]Usage, 0, len(shards)) + for _, sh := range shards { + b, err := os.ReadFile(s.ShardPath(sh)) + if err != nil { + return nil, err + } + entries, _, err := ParseShard(sh, string(b)) + if err != nil { + return nil, err + } + out = append(out, Usage{Shard: sh, Entries: len(entries), Lines: CountLines(string(b)), Cap: CapFor(sh)}) + } + return out, nil +} + +func readmeContent() string { + return `# Project brain + +` + shardMarker + ` + +Project knowledge that mxcli cannot compute: why a pattern was chosen here, +which marketplace version broke what, what a recurring mxbuild error means in +this app. + +**Anything mxcli can answer does not belong here.** Entities, microflows, pages, +bindings and references are all queryable — a note that transcribes them is a +note that will disagree with the project. + +## Layout + + project.md cross-cutting decisions; loaded every session + modules/.md decisions anchored to one module; loaded when it is in play + +An entry's anchors decide its file: ` + "`@Sales.Order`" + ` puts it in +` + "`modules/Sales.md`" + `. An entry with no anchor is cross-cutting and lives in +` + "`project.md`" + `. + +## Working with it + + mxcli brain capture "" --anchor @Module.Element queue something + mxcli brain staged review the queue + mxcli brain promote write it into a shard + mxcli brain check anchors still resolve? + mxcli brain show size and headroom + +Promotion is a human step on purpose: an agent queues, a person decides what is +worth committing. + +Sizes and headroom are computed by ` + "`mxcli brain show`" + ` and are deliberately +not written down anywhere, including in this file — a figure in prose is stale +the next time anyone promotes. +` +} diff --git a/cmd/mxcli/cmd_brain.go b/cmd/mxcli/cmd_brain.go new file mode 100644 index 0000000000..d710aea52d --- /dev/null +++ b/cmd/mxcli/cmd_brain.go @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: Apache-2.0 + +// cmd_brain.go - `mxcli brain` : project knowledge mxcli cannot compute +package main + +import ( + "fmt" + "os" + osexec "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/mendixlabs/mxcli/cmd/mxcli/brain" + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/catalog" + "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/spf13/cobra" +) + +var brainCmd = &cobra.Command{ + Use: "brain", + Short: "Project knowledge mxcli cannot compute (docs/brain/)", + Long: `Record and check the project knowledge mxcli cannot compute. + +Two halves. DECISIONS: why a pattern was chosen here, which marketplace version +broke what, what a recurring mxbuild error means in this app. THE PLAN: the +requirements being built from, grouped into slices, when the source is a +specification, a prototype or a conversation rather than issues in a tracker. + +Anything mxcli CAN answer does not belong here — entities, microflows, pages, +bindings and references are all queryable, and a note that transcribes them is a +note that will disagree with the project. + +Entries live in docs/brain/, committed and reviewed like any other change. A +decision's anchors decide its file: @Sales.Order puts it in modules/Sales.md, and +one with no anchor is cross-cutting and lives in project.md. A requirement goes +to its slice, plan/.md. That is what lets a session load the shards for +the modules it is touching instead of the whole store. + +The two kinds differ in which way their anchors point, and it decides everything +else. A decision's anchor points BACKWARD at what exists, so one that stops +resolving means the decision is stale and 'check' fails. A requirement's points +FORWARD at what is intended, so one that does not resolve means not built yet — +which is why 'brain plan' can report progress derived from the model instead of +from a status column that goes stale the moment someone builds something. + +An agent captures; a person promotes. The queue is in .mxcli/ and is git-ignored, +so nothing reaches a pull request until someone has looked at it.`, + Example: ` mxcli brain init -p app.mpr + mxcli brain capture "Orders are committed by Finance, not Sales" -a @Sales.Order -a @Finance.ACT_Post -p app.mpr + mxcli brain staged -p app.mpr + mxcli brain promote a1b2c3 -p app.mpr + mxcli brain capture "Orders must be approvable by a manager" --slice 02-approvals -a @Sales.ACT_Order_Approve -p app.mpr + mxcli brain plan -p app.mpr + mxcli brain check -p app.mpr + mxcli brain show -p app.mpr`, +} + +var brainInitCmd = &cobra.Command{ + Use: "init", + Short: "Create docs/brain/ (refuses a docs/brain/ it did not write)", + Run: func(cmd *cobra.Command, args []string) { + dir := brainProjectDir(cmd) + written, err := brain.NewStore(dir).Init() + if err != nil { + brainFatal(err) + } + if len(written) == 0 { + fmt.Printf("Already initialised: %s\n", filepath.Join(dir, brain.StoreDir)) + return + } + for _, p := range written { + fmt.Printf("Created %s\n", p) + } + fmt.Println("\nCapture something with: mxcli brain capture \"\" -a @Module.Element") + }, +} + +var brainCaptureCmd = &cobra.Command{ + Use: "capture ", + Short: "Queue something worth remembering (does not commit it)", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + dir := brainProjectDir(cmd) + anchors, _ := cmd.Flags().GetStringSlice("anchor") + slice, _ := cmd.Flags().GetString("slice") + // --slice is the only signal needed: a requirement is a requirement + // because it belongs to a deliverable slice, so there is no second + // --kind flag to contradict it. + var ( + e brain.Entry + err error + ) + if slice != "" { + e, err = brain.NewRequirement(args[0], anchors, slice, time.Now()) + } else { + e, err = brain.NewEntry(args[0], anchors, time.Now()) + } + if err != nil { + brainFatal(err) + } + added, err := brain.NewQueue(dir).Append(e) + if err != nil { + brainFatal(err) + } + if !added { + fmt.Printf("Already queued as %s — not added again.\n", e.ID) + return + } + fmt.Printf("Queued %s -> would promote into %s\n", e.ID, shardLabel(e.Shard())) + fmt.Println("Review with 'mxcli brain staged'; commit it with 'mxcli brain promote " + e.ID + "'.") + }, +} + +var brainStagedCmd = &cobra.Command{ + Use: "staged", + Short: "List the queue, with the shard each entry would land in", + Run: func(cmd *cobra.Command, args []string) { + entries, err := brain.NewQueue(brainProjectDir(cmd)).Load() + if err != nil { + brainFatal(err) + } + if len(entries) == 0 { + fmt.Println("Nothing staged.") + return + } + for _, e := range entries { + fmt.Printf("%s %-18s %s\n", e.ID, shardLabel(e.Shard()), e.Title) + if len(e.Anchors) > 0 { + fmt.Printf(" %s\n", strings.Join(e.Anchors, " ")) + } + } + fmt.Printf("\n%d staged. Promote with 'mxcli brain promote '.\n", len(entries)) + }, +} + +var brainPromoteCmd = &cobra.Command{ + Use: "promote ", + Short: "Write a staged entry into its shard (the human step)", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + dir := brainProjectDir(cmd) + store, queue := brain.NewStore(dir), brain.NewQueue(dir) + if !store.Exists() { + brainFatal(fmt.Errorf("no store yet — run 'mxcli brain init' first")) + } + e, ok, err := queue.Get(args[0]) + if err != nil { + brainFatal(err) + } + if !ok { + brainFatal(fmt.Errorf("no staged entry with id %s", args[0])) + } + shard := e.Shard() + if to, _ := cmd.Flags().GetString("to"); to != "" { + shard = to + } + if err := store.Promote(e, shard); err != nil { + brainFatal(err) + } + if _, err := queue.Drop(e.ID); err != nil { + brainFatal(err) + } + fmt.Printf("Promoted %s into %s\n", e.ID, store.ShardPath(shard)) + }, +} + +var brainDropCmd = &cobra.Command{ + Use: "drop ", + Short: "Remove an entry from the queue or from its shard", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + dir := brainProjectDir(cmd) + if dropped, err := brain.NewQueue(dir).Drop(args[0]); err != nil { + brainFatal(err) + } else if dropped { + fmt.Printf("Dropped %s from the queue.\n", args[0]) + return + } + shard, deletedFile, err := brain.NewStore(dir).Drop(args[0]) + if err != nil { + brainFatal(err) + } + if shard == "" { + brainFatal(fmt.Errorf("no entry with id %s, staged or committed", args[0])) + } + fmt.Printf("Dropped %s from %s\n", args[0], shardLabel(shard)) + if deletedFile { + fmt.Printf("%s had no entries left and was removed.\n", shardLabel(shard)) + } + }, +} + +var brainShowCmd = &cobra.Command{ + Use: "show [shard]", + Short: "Size and headroom per shard (computed, never written down)", + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + store := brain.NewStore(brainProjectDir(cmd)) + if !store.Exists() { + fmt.Println("No store yet. Create one with 'mxcli brain init'.") + return + } + usage, err := store.Usage() + if err != nil { + brainFatal(err) + } + // Width is computed from the names actually present: a module shard is + // named after its module, and those run long. + width := len("SHARD") + for _, u := range usage { + if n := len(shardLabel(u.Shard)); n > width { + width = n + } + } + fmt.Printf("%-*s %8s %8s %12s\n", width, "SHARD", "ENTRIES", "LINES", "HEADROOM") + for _, u := range usage { + if len(args) == 1 && u.Shard != args[0] { + continue + } + note := "" + if u.Over() { + note = " OVER CAP" + } + fmt.Printf("%-*s %8d %8d %7d/%-4d%s\n", width, shardLabel(u.Shard), u.Entries, u.Lines, u.Headroom(), u.Cap, note) + } + }, +} + +var brainPlanCmd = &cobra.Command{ + Use: "plan", + Short: "The roadmap: each slice's requirements counted against the model", + Long: `Show the plan — the slices, and how much of each is built. + +Every figure is DERIVED. A requirement is "built" when its anchors resolve +against the model, so nothing is self-reported and there is no status column to +maintain or to go stale. Building the thing is what moves the number. + +Slices are listed in name order, so a numeric prefix is how a roadmap is +sequenced: 01-accounts, 02-approvals. That is the user's choice, not a field +mxcli maintains.`, + Run: func(cmd *cobra.Command, args []string) { + projectPath := brainProjectPath(cmd) + store := brain.NewStore(filepath.Dir(projectPath)) + slices, err := store.ListSlices() + if err != nil { + brainFatal(err) + } + if len(slices) == 0 { + fmt.Println("No slices yet. Record one with:") + fmt.Println(" mxcli brain capture \"\" --slice 01- -a @Module.Element") + return + } + resolver, closeFn, err := openBrainResolver(projectPath) + if err != nil { + brainFatal(err) + } + defer closeFn() + + rep, err := brain.Check(store, resolver, slices) + if err != nil { + brainFatal(err) + } + printBrainPlan(rep.Slices) + }, +} + +func printBrainPlan(slices []brain.SliceProgress) { + width := len("SLICE") + for _, sl := range slices { + if n := len(sl.Slice); n > width { + width = n + } + } + var built, total int + fmt.Printf("%-*s %8s %8s %12s\n", width, "SLICE", "BUILT", "PLANNED", "UNANCHORED") + for _, sl := range slices { + un := "" + if sl.Unanchored > 0 { + un = fmt.Sprint(sl.Unanchored) + } + fmt.Printf("%-*s %8d %8d %12s\n", width, sl.Slice, sl.Built, sl.Planned, un) + built += sl.Built + total += sl.Total() + } + fmt.Printf("\n%d of %d requirements built, across %d slice(s).\n", built, total, len(slices)) +} + +var brainCheckCmd = &cobra.Command{ + Use: "check", + Short: "Do the anchors still resolve, and is every entry in the right shard?", + Long: `Validate the store against the model. + +Two independent questions. Each anchor is resolved, and reported as one of +three states — only "not found" is a failure. An anchor whose target exists but +is of a document type the catalog does not index is reported as NOT INDEXABLE +and passes: treating it as missing would demand edits to entries that are +perfectly current. + +Separately, each entry is checked for being in the right shard. That is a second +axis, not a fourth state — an anchor can resolve perfectly and the entry still +sit in the wrong file. At least one anchor must belong to the entry's shard; +anchors into other modules are fine, because a fact can genuinely span two.`, + Run: func(cmd *cobra.Command, args []string) { + projectPath := brainProjectPath(cmd) + store := brain.NewStore(filepath.Dir(projectPath)) + if !store.Exists() { + fmt.Println("No store yet. Create one with 'mxcli brain init'.") + return + } + shards, err := store.ListShards() + if err != nil { + brainFatal(err) + } + if changed, _ := cmd.Flags().GetBool("changed"); changed { + shards, err = changedShards(filepath.Dir(projectPath), shards) + if err != nil { + brainFatal(err) + } + if len(shards) == 0 { + fmt.Println("No brain shards changed.") + return + } + } + + resolver, closeFn, err := openBrainResolver(projectPath) + if err != nil { + brainFatal(err) + } + defer closeFn() + + rep, err := brain.Check(store, resolver, shards) + if err != nil { + brainFatal(err) + } + if ci, _ := cmd.Flags().GetBool("ci"); ci { + printBrainReportCI(rep) + } else { + printBrainReport(rep) + } + if rep.Failed() { + os.Exit(1) + } + }, +} + +// printBrainReportCI emits one stable, greppable line per problem and nothing +// at all when the store is clean — what a CI log wants. Informational states +// (a not-indexable anchor) are omitted rather than printed, because a line in a +// CI log reads as something to fix. +func printBrainReportCI(rep brain.Report) { + for _, m := range rep.Malformed { + fmt.Printf("brain: malformed entry: %s\n", m) + } + for _, f := range rep.Findings { + if f.State != brain.NotFound { + continue + } + fmt.Printf("brain: %s: %s: anchor %s not found\n", shardLabel(f.Shard), f.EntryID, f.Anchor) + } + for _, m := range rep.Misfiled { + fmt.Printf("brain: %s: %s: misfiled, resolves in %s\n", shardLabel(m.Shard), m.EntryID, m.Belongs) + } +} + +func printBrainReport(rep brain.Report) { + for _, m := range rep.Malformed { + fmt.Printf("MALFORMED %s\n", m) + } + for _, f := range rep.Findings { + label := "NOT FOUND " + if f.State == brain.NotIndexable { + label = "not indexable" + } + fmt.Printf("%-13s %s %s (%s: %q)\n", label, f.Anchor, shardLabel(f.Shard), f.EntryID, f.Title) + } + for _, m := range rep.Misfiled { + belongs := m.Belongs + if belongs == "" { + belongs = "unknown" + } + fmt.Printf("MISFILED %s: %q is in %s but resolves in %s\n", + m.EntryID, m.Title, shardLabel(m.Shard), belongs) + } + if len(rep.Slices) > 0 { + fmt.Println() + printBrainPlan(rep.Slices) + } + fmt.Printf("\n%d entries, %d anchors, %d resolved, across %d shard(s).\n", + rep.Entries, rep.Anchors, rep.ResolvedN, len(rep.Shards)) + if !rep.Failed() { + fmt.Println("OK") + } +} + +// changedShards narrows a check to the shards a diff touches. That is the cheap +// half of the CI answer: the catalog still has to be built once, but nothing +// pays to re-check shards nobody edited. +func changedShards(projectDir string, all []string) ([]string, error) { + out, err := runGit(projectDir, "diff", "--name-only", "HEAD") + if err != nil { + return nil, fmt.Errorf("--changed needs a git repository: %w", err) + } + staged, err := runGit(projectDir, "diff", "--name-only", "--cached") + if err != nil { + return nil, err + } + touched := map[string]bool{} + for _, line := range strings.Split(out+"\n"+staged, "\n") { + line = strings.TrimSpace(line) + if line == "" || !strings.Contains(line, brain.StoreDir+"/") { + continue + } + touched[shardForPath(line)] = true + } + var shards []string + for _, s := range all { + if touched[s] { + shards = append(shards, s) + } + } + sort.Strings(shards) + return shards, nil +} + +// shardForPath maps a file under docs/brain/ back to its shard name. The +// basename alone is not enough: a plan slice's shard carries the plan/ prefix, +// so mapping docs/brain/plan/01-accounts.md to "01-accounts" made every edited +// slice invisible to --changed. Found by editing one; the unit test had only +// ever used module shards, which is exactly the blind spot. +func shardForPath(path string) string { + base := strings.TrimSuffix(filepath.Base(path), ".md") + if strings.Contains(filepath.ToSlash(filepath.Dir(path)), brain.StoreDir+"/plan") { + return brain.PlanShard(base) + } + return base +} + +// catalogResolver answers anchors from the catalog, falling back to a +// type-agnostic unit lookup for the types the catalog's objects view does not +// cover. Without that fallback an anchor to such a document reads as missing, +// which is a false staleness signal (A1). +type catalogResolver struct { + cat *catalog.Catalog + be backend.FullBackend +} + +func (r *catalogResolver) Resolve(a brain.Anchor) (brain.Resolution, error) { + if a.IsMember() { + rows, err := r.query(fmt.Sprintf( + "SELECT ModuleName FROM attributes_data WHERE EntityQualifiedName = '%s' AND Name = '%s' LIMIT 1", + sqlLiteral(a.QualifiedName()), sqlLiteral(a.Member))) + if err != nil { + return brain.Resolution{}, err + } + if len(rows) == 1 { + return brain.Resolution{State: brain.Resolved, Module: brainCell(rows[0][0]), Kind: "attribute"}, nil + } + // The member is gone. Whether its entity survives changes nothing: the + // anchor as written names something that is not there. + return brain.Resolution{State: brain.NotFound}, nil + } + + rows, err := r.query(fmt.Sprintf( + "SELECT ObjectType, ModuleName, Name FROM objects WHERE QualifiedName = '%s' LIMIT 1", + sqlLiteral(a.QualifiedName()))) + if err != nil { + return brain.Resolution{}, err + } + if len(rows) == 1 { + objectType, moduleName, name := brainCell(rows[0][0]), brainCell(rows[0][1]), brainCell(rows[0][2]) + // A MODULE row carries no ModuleName — it IS the module — so the + // misfiling comparison has to be given the module's own name. + if objectType == "MODULE" { + moduleName = name + } + return brain.Resolution{State: brain.Resolved, Module: moduleName, Kind: strings.ToLower(objectType)}, nil + } + + if a.Element == "" { + return brain.Resolution{State: brain.NotFound}, nil // a module is always indexed + } + unit, err := r.be.FindDocumentUnit(a.Module, a.Element) + if err != nil || unit == nil { + return brain.Resolution{State: brain.NotFound}, nil + } + return brain.Resolution{State: brain.NotIndexable, Module: a.Module, Kind: unit.Kind}, nil +} + +func (r *catalogResolver) query(sql string) ([][]any, error) { + res, err := r.cat.Query(sql) + if err != nil { + return nil, err + } + return res.Rows, nil +} + +// sqlLiteral escapes a value for a SQL string literal. Anchors are validated as +// Mendix identifiers before they get here, so nothing can reach this with a +// quote in it — the escape is belt and braces, not the guard. +func sqlLiteral(s string) string { return strings.ReplaceAll(s, "'", "''") } + +func brainCell(v any) string { + if v == nil { + return "" + } + if s, ok := v.(string); ok { + return s + } + return fmt.Sprint(v) +} + +func openBrainResolver(projectPath string) (brain.Resolver, func(), error) { + // Catalog progress goes to stderr so stdout carries only the report — the + // same split cmd_lint.go makes, and what lets `brain check` be piped. + exec, logger := newLoggedExecutorTo("subcommand", os.Stderr) + cleanup := func() { logger.Close(); exec.Close() } + + for _, src := range []string{ + fmt.Sprintf("CONNECT LOCAL '%s'", visitor.QuoteString(projectPath)), + "REFRESH CATALOG", + } { + prog, errs := visitor.Build(src) + if len(errs) > 0 { + cleanup() + return nil, nil, errs[0] + } + for _, stmt := range prog.Statements { + if err := exec.Execute(stmt); err != nil { + cleanup() + return nil, nil, fmt.Errorf("%s: %w", src, err) + } + } + } + return &catalogResolver{cat: exec.Catalog(), be: exec.Backend()}, cleanup, nil +} + +// runGit shells out in the project directory. Only used by --changed, where the +// question is which files the working tree has touched. +func runGit(dir string, args ...string) (string, error) { + cmd := osexec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.Output() + return string(out), err +} + +func shardLabel(shard string) string { + switch { + case shard == brain.ProjectShard: + return "project.md" + case brain.IsPlanShard(shard): + return "plan/" + brain.SliceOf(shard) + ".md" + default: + return shard + ".md" + } +} + +func brainProjectPath(cmd *cobra.Command) string { + p, _ := cmd.Flags().GetString("project") + if p == "" { + fmt.Fprintln(os.Stderr, "Error: --project (-p) is required") + os.Exit(1) + } + return p +} + +func brainProjectDir(cmd *cobra.Command) string { return filepath.Dir(brainProjectPath(cmd)) } + +func brainFatal(err error) { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) +} + +func init() { + for _, c := range []*cobra.Command{ + brainInitCmd, brainCaptureCmd, brainStagedCmd, brainPromoteCmd, + brainDropCmd, brainShowCmd, brainCheckCmd, + } { + c.Flags().StringP("project", "p", "", "Path to the .mpr file") + } + brainCaptureCmd.Flags().StringSliceP("anchor", "a", nil, + "Anchor into the model (@Module, @Module.Element, @Module.Entity.Attribute); repeatable") + brainCaptureCmd.Flags().StringP("slice", "s", "", + "Record this as a requirement of the named slice (plan/.md) instead of a decision") + brainPromoteCmd.Flags().String("to", "", + "Override the derived shard (use 'project' for a cross-cutting fact)") + brainCheckCmd.Flags().Bool("changed", false, "Only check shards touched by the working tree") + brainCheckCmd.Flags().Bool("ci", false, "Machine-friendly output for CI") + + brainPlanCmd.Flags().StringP("project", "p", "", "Path to the .mpr file") + brainCmd.AddCommand(brainInitCmd, brainCaptureCmd, brainStagedCmd, + brainPromoteCmd, brainDropCmd, brainShowCmd, brainCheckCmd, brainPlanCmd) + rootCmd.AddCommand(brainCmd) +} diff --git a/cmd/mxcli/cmd_brain_test.go b/cmd/mxcli/cmd_brain_test.go new file mode 100644 index 0000000000..a112929967 --- /dev/null +++ b/cmd/mxcli/cmd_brain_test.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/mendixlabs/mxcli/cmd/mxcli/brain" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/mdl/catalog" + "github.com/mendixlabs/mxcli/mdl/types" +) + +func newTestCatalog(t *testing.T, inserts ...string) *catalog.Catalog { + t.Helper() + cat, err := catalog.New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + for _, ins := range inserts { + if _, err := cat.Query(ins); err != nil { + t.Fatalf("%s: %v", ins, err) + } + } + return cat +} + +// A MODULE row in the objects view carries an empty ModuleName — it *is* the +// module. Without the special case the misfiling check would compare a module +// anchor's shard against "" and report every one of them as misfiled. +func TestResolverNamesTheModuleForAModuleAnchor(t *testing.T) { + cat := newTestCatalog(t, + `INSERT INTO modules_data (Id, Name, QualifiedName) VALUES ('m1', 'Sales', 'Sales')`) + r := &catalogResolver{cat: cat, be: &mock.MockBackend{}} + + a, err := brain.ParseAnchor("@Sales") + if err != nil { + t.Fatal(err) + } + res, err := r.Resolve(a) + if err != nil { + t.Fatal(err) + } + if res.State != brain.Resolved { + t.Fatalf("state = %v, want resolved", res.State) + } + if res.Module != "Sales" { + t.Errorf("module = %q, want Sales — a module anchor must report its own name", res.Module) + } + if brain.MisfiledIn("Sales", []string{res.Module}) { + t.Error("a module anchor must not misfile its own shard") + } +} + +// The fallback is what separates "gone" from "of a type the index does not +// cover". Both directions are asserted, because a fallback that always says +// NotIndexable would hide real staleness just as badly. +func TestResolverSeparatesNotIndexableFromNotFound(t *testing.T) { + cat := newTestCatalog(t, + `INSERT INTO modules_data (Id, Name, QualifiedName) VALUES ('m1', 'Sales', 'Sales')`) + + present := &mock.MockBackend{ + FindDocumentUnitFunc: func(moduleName, name string) (*types.DocumentUnit, error) { + return &types.DocumentUnit{Name: name, Kind: "scheduled event"}, nil + }, + } + absent := &mock.MockBackend{ + FindDocumentUnitFunc: func(moduleName, name string) (*types.DocumentUnit, error) { + return nil, nil + }, + } + a, err := brain.ParseAnchor("@Sales.NightlyRun") + if err != nil { + t.Fatal(err) + } + + res, err := (&catalogResolver{cat: cat, be: present}).Resolve(a) + if err != nil { + t.Fatal(err) + } + if res.State != brain.NotIndexable { + t.Errorf("a document the catalog does not index must be NotIndexable, got %v", res.State) + } + if res.Kind != "scheduled event" { + t.Errorf("kind = %q, want the unit's own kind", res.Kind) + } + + res, err = (&catalogResolver{cat: cat, be: absent}).Resolve(a) + if err != nil { + t.Fatal(err) + } + if res.State != brain.NotFound { + t.Errorf("a document that is not there at all must be NotFound, got %v", res.State) + } +} + +// A module anchor never takes the fallback: modules are always indexed, so a +// miss is a miss. Asserted because the fallback would otherwise report a +// deleted module as merely "not indexable" and never fail. +func TestMissingModuleIsNotFoundNotNotIndexable(t *testing.T) { + cat := newTestCatalog(t) + be := &mock.MockBackend{ + FindDocumentUnitFunc: func(moduleName, name string) (*types.DocumentUnit, error) { + t.Fatal("a module anchor must not reach the document fallback") + return nil, nil + }, + } + a, _ := brain.ParseAnchor("@Gone") + res, err := (&catalogResolver{cat: cat, be: be}).Resolve(a) + if err != nil { + t.Fatal(err) + } + if res.State != brain.NotFound { + t.Errorf("state = %v, want not found", res.State) + } +} + +func TestChangedShardsMapsPathsToShards(t *testing.T) { + dir := t.TempDir() + run := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + run("init", "-q") + + store := brain.NewStore(dir) + if _, err := store.Init(); err != nil { + t.Fatal(err) + } + for _, shard := range []string{"Sales", "Finance"} { + e, err := brain.NewEntry("An entry", []string{"@" + shard + ".Thing"}, day()) + if err != nil { + t.Fatal(err) + } + if err := store.Promote(e, shard); err != nil { + t.Fatal(err) + } + } + // A plan slice too: its shard carries the plan/ prefix, so mapping a path + // by basename alone made every edited slice invisible to --changed. The + // first version of this test used only module shards and missed it. + req, err := brain.NewRequirement("A requirement", []string{"@Sales.Thing"}, "01-accounts", day()) + if err != nil { + t.Fatal(err) + } + if err := store.Promote(req, req.Shard()); err != nil { + t.Fatal(err) + } + run("add", "-A") + run("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "seed") + + all := []string{brain.ProjectShard, "Finance", "Sales", brain.PlanShard("01-accounts")} + + // Control: a clean tree changes nothing, so nothing is checked. + got, err := changedShards(dir, all) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("clean tree selected %v", got) + } + + // Touch exactly one shard; only it should come back. + path := filepath.Join(dir, "docs", "brain", "modules", "Sales.md") + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, append(b, []byte("\nedited\n")...), 0644); err != nil { + t.Fatal(err) + } + got, err = changedShards(dir, all) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0] != "Sales" { + t.Fatalf("got %v, want [Sales]", got) + } + + // The same for a plan slice, which is the case the basename mapping broke. + planPath := filepath.Join(dir, "docs", "brain", "plan", "01-accounts.md") + pb, err := os.ReadFile(planPath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(planPath, append(pb, []byte("\nedited\n")...), 0644); err != nil { + t.Fatal(err) + } + got, err = changedShards(dir, all) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0] != "Sales" || got[1] != brain.PlanShard("01-accounts") { + t.Fatalf("got %v, want [Sales plan/01-accounts]", got) + } +} + +func day() time.Time { return time.Date(2026, 9, 3, 0, 0, 0, 0, time.UTC) } diff --git a/cmd/mxcli/cmd_lint.go b/cmd/mxcli/cmd_lint.go index 790fcf84d2..c39241791b 100644 --- a/cmd/mxcli/cmd_lint.go +++ b/cmd/mxcli/cmd_lint.go @@ -5,6 +5,7 @@ package main import ( "context" "fmt" + "github.com/mendixlabs/mxcli/cmd/mxcli/brain" "os" "path/filepath" "strings" @@ -306,6 +307,14 @@ Examples: os.Exit(1) } + // A7: the gap that motivates curation is printed by a command that + // already runs. A queue nobody promotes is the brain's version of the + // on-demand digest that went three months without a run — and a report + // only `brain check` prints is a report nothing demands. One file read, + // silent when there is no store or no queue, and on stderr so it cannot + // corrupt --format json/sarif. + reportBrainGap(projectDir) + // Exit with error if there are errors summary := linter.Summarize(violations) if summary.Errors > 0 { @@ -361,3 +370,23 @@ func pluralItThem(n int) string { } return "them" } + +// reportBrainGap prints one line when the project brain has staged entries that +// nobody has promoted. It never fails lint and never touches the model: the +// brain is opt-in, so a project without a store hears nothing at all. +func reportBrainGap(projectDir string) { + if !brain.NewStore(projectDir).Exists() { + return + } + staged, err := brain.NewQueue(projectDir).Load() + if err != nil || len(staged) == 0 { + return + } + noun := "entries" + if len(staged) == 1 { + noun = "entry" + } + fmt.Fprintf(os.Stderr, + "\nProject brain: %d staged %s not yet promoted — 'mxcli brain staged' to review.\n", + len(staged), noun) +} diff --git a/cmd/mxcli/help.go b/cmd/mxcli/help.go index 5241698cb5..cd297b77a7 100644 --- a/cmd/mxcli/help.go +++ b/cmd/mxcli/help.go @@ -23,6 +23,7 @@ Top-level topics: domain-model - Entities, associations, enumerations, constants, keywords, types microflow - Microflow/nanoflow creation and activities page - Pages, snippets, fragments, widgets + layout - Layouts: regions, navigation, placeholders, repointing pages security - Roles, access control, demo users workflow - Workflows, user tasks, decisions, parallel splits navigation - Navigation profiles, menus, home pages diff --git a/cmd/mxcli/init_claudemd.go b/cmd/mxcli/init_claudemd.go index a969e3a898..5cce90c5f8 100644 --- a/cmd/mxcli/init_claudemd.go +++ b/cmd/mxcli/init_claudemd.go @@ -139,6 +139,7 @@ func generateClaudeMD(projectName, mprFile string) string { w("| " + bt + ".ai-context/skills/check-syntax/SKILL.md" + bt + " | **Pre-flight** validation checklist |\n") w("| " + bt + ".ai-context/skills/demo-data/SKILL.md" + bt + " | **READ for any database/import work** - Mendix ID system, demo data |\n") w("| " + bt + ".ai-context/skills/test-microflows/SKILL.md" + bt + " | **READ for testing** - test annotations, file formats, Docker setup |\n") + w("| " + bt + ".ai-context/skills/project-brain/SKILL.md" + bt + " | **Why was it done this way here?** - the project's recorded decisions in " + bt + "docs/brain/" + bt + " |\n") w("\n") w("**Always validate before presenting to user:**\n\n") w(bt3 + "bash\n") diff --git a/cmd/mxcli/lsp_completions_gen.go b/cmd/mxcli/lsp_completions_gen.go index f5b2701595..92a1de628c 100644 --- a/cmd/mxcli/lsp_completions_gen.go +++ b/cmd/mxcli/lsp_completions_gen.go @@ -539,6 +539,8 @@ var mdlGeneratedKeywords = []protocol.CompletionItem{ {Label: "OVER", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "FOR", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "REPLACE", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, + {Label: "EXAMPLE", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, + {Label: "MEMBER", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "MEMBERS", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "ATTRIBUTENAME", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, {Label: "FORMAT", Kind: protocol.CompletionItemKindKeyword, Detail: "Utility keyword"}, diff --git a/cmd/mxcli/lsp_diagnostics.go b/cmd/mxcli/lsp_diagnostics.go index c67d00ef1f..e2b48882e8 100644 --- a/cmd/mxcli/lsp_diagnostics.go +++ b/cmd/mxcli/lsp_diagnostics.go @@ -278,6 +278,18 @@ func (s *mdlServer) runSemanticValidation(text string) []protocol.Diagnostic { } if mfStmt, ok := stmt.(*ast.CreateMicroflowStmt); ok { violations = append(violations, executor.ValidateMicroflow(mfStmt)...) + violations = append(violations, executor.ValidateFlowParameterAnnotations( + "microflow '"+mfStmt.Name.String()+"'", mfStmt.Parameters)...) + } + // The editor reports an unusable parameter annotation for the same + // reason `check` does — a typo of @position parses and does nothing. + if nfStmt, ok := stmt.(*ast.CreateNanoflowStmt); ok { + violations = append(violations, executor.ValidateFlowParameterAnnotations( + "nanoflow '"+nfStmt.Name.String()+"'", nfStmt.Parameters)...) + } + if ruleStmt, ok := stmt.(*ast.CreateRuleStmt); ok { + violations = append(violations, executor.ValidateFlowParameterAnnotations( + "rule '"+ruleStmt.Name.String()+"'", ruleStmt.Parameters)...) } if setStmt, ok := stmt.(*ast.AlterSettingsStmt); ok { violations = append(violations, executor.ValidateSettings(setStmt)...) diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 63ba670d1f..62f41c06ce 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -540,6 +540,41 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, // ── JSON Structures ─────────────────────────────────────────────── + Register(SyntaxFeature{ + Path: "message-definition", + Summary: "Message definition collections — a mapping source built from the domain model", + Keywords: []string{ + "message definition", "message definition collection", "create message definition", + "exposed entity", "exposed attribute", "exposed association", + }, + Syntax: "SHOW MESSAGE DEFINITION COLLECTIONS [IN Module];\nDESCRIBE MESSAGE DEFINITION COLLECTION Module.Name;\nCREATE [OR MODIFY] MESSAGE DEFINITION COLLECTION Module.Name [FOLDER 'path']\n(\n DEFINITION Name FOR Module.Entity [AS 'Exposed'] (\n AttributeName [AS 'Exposed'] [EXAMPLE 'text'],\n Module.Assoc/Module.TargetEntity [AS 'Exposed'] ( ... )\n )\n);\nDROP MESSAGE DEFINITION COLLECTION Module.Name;\n\n" + + "ALTER MESSAGE DEFINITION COLLECTION Module.Name\n" + + " ADD DEFINITION [IF NOT EXISTS] Name FOR Module.Entity [AS 'X'] ( ... )\n" + + " | DROP DEFINITION [IF EXISTS] Name\n" + + " | RENAME DEFINITION Old TO New;\n\n" + + "ALTER MESSAGE DEFINITION Module.Collection.Definition\n" + + " ADD MEMBER [IF NOT EXISTS] [IN path]\n" + + " | DROP MEMBER [IF EXISTS] Name [IN path]\n" + + " | SET MEMBER Name [IN path] AS 'Exposed';\n\n" + + "A message definition is a SELECTION OVER THE DOMAIN MODEL — every element\n" + + "names an entity, an attribute or an association — which is why it is\n" + + "authorable where an XML schema or a WSDL is not. It is the source for 74 of\n" + + "the 327 mappings in the demo corpus.\n\n" + + "A bare name is an ATTRIBUTE; Assoc/Module.Entity is an ASSOCIATION.\n" + + "Naming the association's TARGET is required: the stored cardinality follows\n" + + "the DIRECTION of traversal, so the same association gives a single object\n" + + "one way and a list the other. One that connects neither way is refused,\n" + + "because a wrong cardinality builds cleanly.\n\n" + + "Inherited attributes are named like the entity's own and resolve to the\n" + + "entity that DECLARES them. Everything else — occurrence bounds, element\n" + + "types, paths, item names, primitive types — is derived. Studio Pro\n" + + "pluralises a repeating element's exposed name; mxcli defaults to the\n" + + "entity's own name and lets AS say otherwise.\n\n" + + "IN reaches a nested member, in exposed names. SET changes only the\n" + + "exposed name — it is not a model rename. Authoring is modelsdk-only.", + Example: "CREATE MESSAGE DEFINITION COLLECTION Sales.MD_Order\n(\n DEFINITION OrderMessage FOR Sales.Order AS 'Orders' (\n OrderId,\n Sales.OrderLine_Order/Sales.OrderLine AS 'Lines' ( Sku, Quantity ),\n Sales.Order_Customer/Sales.Customer ( FirstName )\n )\n);\n\nALTER MESSAGE DEFINITION Sales.MD_Order.OrderMessage ADD MEMBER LastName IN Customer;\n\nCREATE IMPORT MAPPING Sales.IMM_Order\n WITH MESSAGE DEFINITION Sales.MD_Order.OrderMessage\n{ create Sales.Order { OrderId = OrderId } };", + }) + Register(SyntaxFeature{ Path: "json-structure", Summary: "JSON structures — schema snapshots used by import/export mappings", diff --git a/cmd/mxcli/syntax/features_microflow.go b/cmd/mxcli/syntax/features_microflow.go index efdb3e46db..e896ef91c2 100644 --- a/cmd/mxcli/syntax/features_microflow.go +++ b/cmd/mxcli/syntax/features_microflow.go @@ -272,6 +272,7 @@ func init() { "annotation", "caption", "color", "excluded", "bezier", }, Syntax: "@position(x, y) -- the activity's centre point\n" + + "@position(x, y) -- also on a PARAMETER, in the ( … ) list\n" + "@start(x, y) -- the start event, on the FIRST statement\n" + "@anchor(from: right, to: left) -- which SIDE each end of the outgoing flow attaches to\n" + "@curve(from: (40, -90), to: (-40, 90)) -- the flow's bezier control vectors\n" + @@ -288,8 +289,15 @@ func init() { "start is placed one spacing unit left of the first activity, on its centre\n" + "line — and a rewrite MOVES it to follow the activities. A start that is not\n" + "at that derived spot was put there by hand: it survives a rewrite, and\n" + - "DESCRIBE emits @start for it so the description round-trips exactly.", - Example: "create microflow MyModule.ACT_Flow ($In: String)\nreturns String as $Out\nbegin\n" + + "DESCRIBE emits @start for it so the description round-trips exactly.\n\n" + + "A PARAMETER is a stored node with its own coordinates, so it takes\n" + + "@position too — written inside the parameter list, ahead of the parameter\n" + + "it places. It is the only annotation a parameter takes. Omit it and the\n" + + "parameters form a row along the top of the canvas at 200;53, 300;53, … ;\n" + + "the same derived/authored rule as @start then applies, so a parameter on\n" + + "that row is re-derived and one anywhere else survives a rewrite and is\n" + + "emitted by DESCRIBE.", + Example: "create microflow MyModule.ACT_Flow (\n @position(145, 0)\n $In: String\n)\nreturns String as $Out\nbegin\n" + " @start(145, 100)\n @position(200, 100)\n @anchor(from: bottom, to: top)\n" + " @curve(from: (40, -90), to: (-40, 90))\n declare $Tmp String = $In;\n" + " @position(200, 300)\n declare $Out String = $Tmp;\n return $Out;\nend;", diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index 9d0c26a976..f89423b980 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -156,7 +156,7 @@ DISCONNECT;`, }, Syntax: `CREATE OR REPLACE NAVIGATION HOME PAGE Module.Page - [HOME PAGE Module.Page FOR Module.UserRole] + [HOME PAGE Module.Page FOR UserRole] [LOGIN PAGE Module.LoginPage] [NOT FOUND PAGE Module.Custom404] [MENU ( @@ -164,6 +164,14 @@ DISCONNECT;`, MENU 'Group' [ICON Module.IconCollection.Name] ( ... ); )]; +-- FOR takes a USER role, written BARE (FOR Administrator). User roles are +-- project-level and have no module part; a module role is a different thing +-- that often shares the name (a blank app has a user role Administrator and +-- module roles called Administrator in three modules). A qualified name here +-- gives a project Mendix cannot LOAD -- StorageLoadException "not a valid +-- UserRoleIdentifier", raised before checking runs, so there is no error code +-- and no line number. List the real ones with SHOW USER ROLES. +-- -- ICON is a qualified name into an ICON COLLECTION (Atlas_Core.Atlas, -- Atlas_Core.Atlas_Filled, Atlas_Core.Atlas_Styling, or your own) -- a model -- reference, not a string. Hyphenated Atlas names are double-quoted: @@ -183,7 +191,7 @@ DISCONNECT;`, -- documents in the project that already exceed that.`, Example: `CREATE OR REPLACE NAVIGATION Responsive HOME PAGE MyModule.Home_Web - HOME PAGE MyModule.AdminDashboard FOR Administration.Administrator + HOME PAGE MyModule.AdminDashboard FOR Administrator LOGIN PAGE Administration.Login MENU ( MENU ITEM 'Home' PAGE MyModule.Home_Web ICON Atlas_Core.Atlas.home; diff --git a/docs-site/src/SUMMARY.md b/docs-site/src/SUMMARY.md index 6cdb537bbe..9b4342d070 100644 --- a/docs-site/src/SUMMARY.md +++ b/docs-site/src/SUMMARY.md @@ -131,6 +131,7 @@ - [Available Tables](tools/catalog-tables.md) - [SQL Queries](tools/catalog-sql.md) - [Use Cases](tools/catalog-use-cases.md) +- [Project Brain](tools/project-brain.md) - [Linting and Reports](tools/linting.md) - [Built-in Rules](tools/builtin-rules.md) - [Starlark Rules](tools/starlark-rules.md) @@ -249,6 +250,7 @@ - [ALTER PAGE / ALTER SNIPPET](reference/page/alter-page.md) - [DROP PAGE / SNIPPET](reference/page/drop-page.md) - [CREATE LAYOUT](reference/page/create-layout.md) + - [ALTER LAYOUT](reference/page/alter-layout.md) - [Security Statements](reference/security/README.md) - [CREATE MODULE ROLE](reference/security/create-module-role.md) - [CREATE USER ROLE](reference/security/create-user-role.md) diff --git a/docs-site/src/appendixes/quick-reference.md b/docs-site/src/appendixes/quick-reference.md index 06e865c52e..5c8b1bcbab 100644 --- a/docs-site/src/appendixes/quick-reference.md +++ b/docs-site/src/appendixes/quick-reference.md @@ -283,7 +283,7 @@ END WORKFLOW; ```sql CREATE OR REPLACE NAVIGATION Responsive HOME PAGE MyModule.Home_Web - HOME PAGE MyModule.AdminHome FOR MyModule.Administrator + HOME PAGE MyModule.AdminHome FOR Administrator LOGIN PAGE Administration.Login NOT FOUND PAGE MyModule.Custom404 MENU ( diff --git a/docs-site/src/internals/catalog-schema.md b/docs-site/src/internals/catalog-schema.md index 49f48079e3..01e3177e26 100644 --- a/docs-site/src/internals/catalog-schema.md +++ b/docs-site/src/internals/catalog-schema.md @@ -186,12 +186,31 @@ CREATE TABLE PERMISSIONS ( ```sql CREATE VIRTUAL TABLE STRINGS USING fts5( - name, -- Document qualified name - kind, -- Document type - strings, -- All text content concatenated - tokenize='porter unicode61' -); -``` + QualifiedName, -- Document qualified name, e.g. MyModule.Home + ObjectType, -- Document type, derived from the unit $Type: PAGE, + -- PAGE_TEMPLATE, BUILDING_BLOCK, MICROFLOW, ENUMERATION, ... + StringValue, -- The string itself + StringContext, -- Where it lives. For translatable text this is + -- ., e.g. Forms$ActionButton.Caption. + -- Non-translatable strings keep a plain label: page_url, + -- log_node, documentation, rest_path, task_name, ... + Language, -- Language code, empty for non-translatable strings + ElementId, -- The owning element's $ID — what distinguishes an + -- enumeration's twelve values from each other + ModuleName +); +``` + +Every `Texts$Text` in the project is indexed, found by a type-agnostic walk +rather than per-document-type extraction, so a caption in a document type mxcli +cannot otherwise read is still searchable. That includes Atlas's design +templates (`PAGE_TEMPLATE`, `BUILDING_BLOCK`), which are roughly 70% of a stock +project's text and never render in a running app — filter them out with +`ObjectType` when you want only the app's own strings. + +An empty translation — a text that exists but is not translated yet — is **not** +a row, so a language's presence in this table means it is actually translated +somewhere. ### SOURCE (FTS5) diff --git a/docs-site/src/language/home-pages.md b/docs-site/src/language/home-pages.md index 68fc3b5b45..65ae2084ec 100644 --- a/docs-site/src/language/home-pages.md +++ b/docs-site/src/language/home-pages.md @@ -15,15 +15,25 @@ CREATE OR REPLACE NAVIGATION Responsive ### Role-Specific Home Pages -Use `HOME PAGE ... FOR` to direct users to different pages based on their module role: +Use `HOME PAGE ... FOR` to direct users to different pages based on their **user role**: ```sql CREATE OR REPLACE NAVIGATION Responsive HOME PAGE MyModule.Home_Web - HOME PAGE MyModule.AdminDashboard FOR MyModule.Administrator - HOME PAGE MyModule.ManagerDashboard FOR MyModule.Manager; + HOME PAGE MyModule.AdminDashboard FOR Administrator + HOME PAGE MyModule.ManagerDashboard FOR Manager; ``` +> **Write the role as a bare name.** `FOR` takes a *user role*, which is +> project-level and has no module part — `FOR Administrator`, not +> `FOR MyModule.Administrator`. A *module role* is a different thing that +> happens to share the name: a blank app has a user role `Administrator` and +> module roles called `Administrator` in three modules, so the wrong one looks +> right. A module-qualified name here produces a project Mendix **cannot load** +> (`StorageLoadException: … is not a valid UserRoleIdentifier`), which is worse +> than a build error because it happens before checking runs. `mxcli check +> --references` refuses it. + When a user logs in, the runtime checks their roles and redirects to the most specific matching home page. If no role-specific page matches, the default home page is used. ### Login Page diff --git a/docs-site/src/language/navigation-profiles.md b/docs-site/src/language/navigation-profiles.md index a98680ed05..078614c320 100644 --- a/docs-site/src/language/navigation-profiles.md +++ b/docs-site/src/language/navigation-profiles.md @@ -18,7 +18,7 @@ Replaces an entire navigation profile: ```sql CREATE OR REPLACE NAVIGATION HOME PAGE . - [HOME PAGE . FOR .] + [HOME PAGE . FOR ] [LOGIN PAGE .] [NOT FOUND PAGE .] [MENU ( @@ -31,7 +31,7 @@ CREATE OR REPLACE NAVIGATION ```sql CREATE OR REPLACE NAVIGATION Responsive HOME PAGE MyModule.Home_Web - HOME PAGE MyModule.AdminHome FOR MyModule.Administrator + HOME PAGE MyModule.AdminHome FOR Administrator LOGIN PAGE Administration.Login NOT FOUND PAGE MyModule.Custom404 MENU ( diff --git a/docs-site/src/language/translations.md b/docs-site/src/language/translations.md index 0d4751ae57..7b031b7749 100644 --- a/docs-site/src/language/translations.md +++ b/docs-site/src/language/translations.md @@ -61,7 +61,7 @@ modules ship translations in **nine**, so "other languages already have translations here" is true and misleading. > `SHOW LANGUAGES` lists languages that **have translations**, which is a -> different list — a stock app reports eight while one is enabled. The enabled +> different list — a stock app reports nine while one is enabled. The enabled > list is in `DESCRIBE SETTINGS`. ## Drift: a source string that was edited diff --git a/docs-site/src/reference/integration/README.md b/docs-site/src/reference/integration/README.md index c332edb5bc..e09d8a7fd5 100644 --- a/docs-site/src/reference/integration/README.md +++ b/docs-site/src/reference/integration/README.md @@ -9,6 +9,7 @@ These document types support moving data between Mendix entities and external da | Statement | Description | |-----------|-------------| | [CREATE JSON STRUCTURE](create-json-structure.md) | Define the shape of a JSON document from a sample snippet | +| [CREATE MESSAGE DEFINITION COLLECTION](create-message-definition-collection.md) | Define a mapping source as a selection over the domain model | | [CREATE IMPORT MAPPING](create-import-mapping.md) | Map incoming JSON/XML to Mendix entities | | [CREATE EXPORT MAPPING](create-export-mapping.md) | Map Mendix entities to outgoing JSON/XML | | [CREATE DATA TRANSFORMER](create-data-transformer.md) | Transform raw JSON or XML with JSLT/XSLT steps | diff --git a/docs-site/src/reference/integration/create-message-definition-collection.md b/docs-site/src/reference/integration/create-message-definition-collection.md new file mode 100644 index 0000000000..f548d1f218 --- /dev/null +++ b/docs-site/src/reference/integration/create-message-definition-collection.md @@ -0,0 +1,134 @@ +# CREATE MESSAGE DEFINITION COLLECTION + +## Synopsis + +```sql +CREATE [ OR MODIFY ] MESSAGE DEFINITION COLLECTION module.Name + [ FOLDER 'folder_path' ] +( + DEFINITION DefName FOR module.Entity [ AS 'ExposedName' ] ( + AttributeName [ AS 'ExposedName' ] [ EXAMPLE 'text' ], + module.Association/module.TargetEntity [ AS 'ExposedName' ] ( ... ), + ... + ), + ... +); +``` + +## Description + +Creates a message definition collection — one of the four sources an import or +export mapping can be bound to, and the only one besides a JSON structure that +MDL can create. + +Unlike an XML schema (which holds an imported `.xsd`) or an imported web service +(which holds a WSDL), a message definition holds nothing external. It is a +**selection over the domain model**: every element names an entity, an attribute +or an association. A mapping then binds to `module.Collection.Definition`. + +### Members + +A **bare name** is an attribute. An **`Association/TargetEntity`** pair is an +association with its own member list — the same discriminator import and export +mappings use. + +Naming the association's target entity is required, and not decoration. The +stored cardinality tracks the **direction of traversal**, not the association's +type: + +| traversal | cardinality | +|---|---| +| from the association's FROM entity (following the foreign key) | a single object | +| from its TO entity (the reverse) | a list | + +So the same association gives a single object one way and a list the other. An +association that connects the two entities in **neither** direction is refused — +a wrong cardinality builds cleanly and would silently expose a list as a single +object. + +**Inherited attributes** are named exactly like the entity's own; mxcli resolves +each to the entity that declares it, which is what Mendix stores. + +### What is derived + +Almost everything. A statement says only what a person chooses — the collection's +name, each definition's name, its root entity, the members, and the occasional +rename. Occurrence bounds, nillability, precision, element types, paths, item +names and the primitive type all come from the domain model. + +`EXAMPLE 'text'` sets an element's sample value, the one other authored field. + +### What is not guessed + +Studio Pro **pluralises** a repeating element's exposed name (`Order` → +`Orders`). mxcli defaults to the entity's own name and lets `AS 'Orders'` say +otherwise: reproducing English inflection needs `-y → -ies` and an +already-plural detector, and a name the author writes beats one a heuristic +guesses. + +## Examples + +```sql +CREATE MESSAGE DEFINITION COLLECTION Sales.MD_Order + FOLDER 'Messages' +( + DEFINITION OrderMessage FOR Sales.Order AS 'Orders' ( + OrderId, + TotalAmount AS 'Total', + Sales.OrderLine_Order/Sales.OrderLine AS 'Lines' ( Sku, Quantity ), + Sales.Order_Customer/Sales.Customer ( FirstName, LastName ) + ), + DEFINITION CustomerOrders FOR Sales.Customer AS 'Customers' ( + FirstName, + Sales.Order_Customer/Sales.Order AS 'Orders' ( OrderId ) + ) +); + +CREATE IMPORT MAPPING Sales.IMM_Order + WITH MESSAGE DEFINITION Sales.MD_Order.OrderMessage +{ create Sales.Order { OrderId = OrderId } }; +``` + +## Editing without restating + +Definitions nest deeply, so a whole-document rewrite is a poor tool for "expose +one more attribute". + +```sql +ALTER MESSAGE DEFINITION Sales.MD_Order.OrderMessage ADD MEMBER TotalAmount; +ALTER MESSAGE DEFINITION Sales.MD_Order.OrderMessage ADD MEMBER LastName IN Customer; +ALTER MESSAGE DEFINITION Sales.MD_Order.OrderMessage SET MEMBER TotalAmount AS 'GrandTotal'; +ALTER MESSAGE DEFINITION Sales.MD_Order.OrderMessage DROP MEMBER Sku IN Lines; + +ALTER MESSAGE DEFINITION COLLECTION Sales.MD_Order ADD DEFINITION Line FOR Sales.Line ( Sku ); +ALTER MESSAGE DEFINITION COLLECTION Sales.MD_Order RENAME DEFINITION Line TO OrderLine; +ALTER MESSAGE DEFINITION COLLECTION Sales.MD_Order DROP DEFINITION IF EXISTS OrderLine; +``` + +The definition is addressed as `module.Collection.Definition` — the same +three-part reference `WITH MESSAGE DEFINITION` takes. + +`IN ` reaches a nested member, written in **exposed names**. `SET` changes +only the exposed name; it is not a model rename, which is why the verb is not +`RENAME`. + +Dropping or renaming a definition a mapping still references is refused, naming +the mappings. + +## Other statements + +```sql +SHOW MESSAGE DEFINITION COLLECTIONS [ IN module ]; +DESCRIBE MESSAGE DEFINITION COLLECTION module.Name; +DROP MESSAGE DEFINITION COLLECTION module.Name; +``` + +`DESCRIBE` emits re-executable MDL. `OR MODIFY` preserves the document UUID, so +mappings bound to it keep resolving. + +Authoring requires the modelsdk engine; `MXCLI_ENGINE=legacy` refuses. Reading +works on both. + +## See Also + +[CREATE IMPORT MAPPING](create-import-mapping.md), [CREATE EXPORT MAPPING](create-export-mapping.md), [CREATE JSON STRUCTURE](create-json-structure.md) diff --git a/docs-site/src/reference/navigation/alter-navigation.md b/docs-site/src/reference/navigation/alter-navigation.md index 61443c3711..0188ca3779 100644 --- a/docs-site/src/reference/navigation/alter-navigation.md +++ b/docs-site/src/reference/navigation/alter-navigation.md @@ -5,7 +5,7 @@ ```sql CREATE OR REPLACE NAVIGATION profile HOME PAGE module.PageName - [ HOME PAGE module.PageName FOR module.UserRole ] + [ HOME PAGE module.PageName FOR UserRole ] [ LOGIN PAGE module.PageName ] [ NOT FOUND PAGE module.PageName ] [ MENU ( @@ -44,9 +44,11 @@ Each `MENU ITEM` specifies a label and a target page. Menu items are terminated `HOME PAGE module.PageName` : The default home page for the profile. Required. The page must already exist. -`HOME PAGE module.PageName FOR module.UserRole` +`HOME PAGE module.PageName FOR UserRole` : Optional role-specific home page. Users with this role see a different home page than the default. Multiple role-specific home pages can be specified. + The role is a **user role**, written as a bare name (`FOR Administrator`). User roles are project-level and have no module part. A module-qualified name here — a module role, which often shares the name — produces a project Mendix cannot load: `StorageLoadException: … is not a valid UserRoleIdentifier`, raised before checking runs. `mxcli check --references` refuses it. + `LOGIN PAGE module.PageName` : Optional custom login page. If omitted, the default system login page is used. @@ -76,7 +78,7 @@ Full navigation with role-specific homes and menus: ```sql CREATE OR REPLACE NAVIGATION Responsive HOME PAGE MyModule.Home_Web - HOME PAGE MyModule.AdminHome FOR MyModule.Administrator + HOME PAGE MyModule.AdminHome FOR Administrator LOGIN PAGE Administration.Login NOT FOUND PAGE MyModule.Custom404 MENU ( diff --git a/docs-site/src/reference/page/README.md b/docs-site/src/reference/page/README.md index 9cebb5f68d..f5c4dcf5b3 100644 --- a/docs-site/src/reference/page/README.md +++ b/docs-site/src/reference/page/README.md @@ -10,4 +10,5 @@ Pages define the user interface of a Mendix application. Each page has a layout, | [CREATE SNIPPET](create-snippet.md) | Create a reusable widget fragment | | [ALTER PAGE / ALTER SNIPPET](alter-page.md) | Modify an existing page or snippet in-place | | [DROP PAGE / SNIPPET](drop-page.md) | Remove a page or snippet | -| [CREATE LAYOUT](create-layout.md) | Create a page layout | +| [CREATE LAYOUT](create-layout.md) | Create a layout — the frame a page renders inside | +| [ALTER LAYOUT](alter-layout.md) | Edit a layout in place; move pages onto a different one | diff --git a/docs-site/src/reference/page/alter-layout.md b/docs-site/src/reference/page/alter-layout.md new file mode 100644 index 0000000000..0196f6edec --- /dev/null +++ b/docs-site/src/reference/page/alter-layout.md @@ -0,0 +1,174 @@ +# ALTER LAYOUT + +## Synopsis + +```sql +ALTER LAYOUT module.Name { + operations +} +``` + +Where each operation is one of: + +```sql +-- Insert widgets as the last children of a container +INSERT INTO containerName { widget_definitions }; + +-- Insert widgets into a scroll-container region (addressed by slot) +INSERT INTO scrollContainerName.top { widget_definitions }; +INSERT INTO scrollContainerName.left { widget_definitions }; + +-- Insert widgets before or after a target +INSERT BEFORE widgetName { widget_definitions }; +INSERT AFTER widgetName { widget_definitions }; + +-- Set a property on a widget +SET property = value ON widgetName; + +-- Remove widgets +DROP WIDGET widgetName1, widgetName2; + +-- Replace a widget with new widgets +REPLACE widgetName WITH { widget_definitions }; +``` + +Repointing pages onto a different layout: + +```sql +-- One page +ALTER PAGE module.Page { SET Layout = module.Layout [MAP (Old AS New, …)]; }; + +-- Many at once — the migration form +ALTER PAGES [IN module] SET LAYOUT = module.Layout + [MAP (Old AS New, …)] [WHERE LAYOUT = module.OldLayout]; +``` + +## Description + +Modifies an existing layout in place. `ALTER LAYOUT` takes the whole +[ALTER PAGE](alter-page.md) operation vocabulary, because a layout's widget +tree *is* a page's plus four element types. + +This is the tool for changing a layout you did not write. `CREATE OR REPLACE +LAYOUT` rebuilds the document from your MDL, which means it can only reproduce +what MDL can express; `ALTER LAYOUT` edits what is stored and leaves everything +it was not asked about untouched — including widgets MDL has no syntax for. +Measured on `Atlas_Core.Atlas_SideBar`, a describe → rename → run copy loses +both of its `Forms$SidebarToggleButton` widgets. An `ALTER LAYOUT` against it +does not. + +Like `CREATE LAYOUT`, it refuses a Marketplace target and names the +copy-then-repoint route in the error. + +### Addressing a region + +A scroll-container region has no name of its own — its slot *is* its identity — +so it is addressed as `.`, reusing the dotted widget +reference that also serves DataGrid 2 columns. Which one is meant is decided by +the named widget's type. + +Only `INSERT INTO` accepts a region. `BEFORE` and `AFTER` position a widget +among its siblings, so name a widget for those. + +An empty slot has no stored region to insert into. Add the region with +`CREATE OR REPLACE LAYOUT`. + +### SET LAYOUT and the placeholder guard + +A page binds to its layout's placeholders by qualified name, so repointing +rewrites both the layout reference and every placeholder binding. + +Both forms **refuse** a repoint that would leave a page bound to a placeholder +the target layout does not declare: + +``` +Error: layout MyModule.Sidebarish does not declare the placeholder Main that +this page binds to; it has Content. Add the placeholder to the layout, or remap +the binding: `set Layout = MyModule.Sidebarish map (Main as Content)` +``` + +Without the check the rewrite produces a dangling binding that mxbuild reports +much later as CE1613, naming the page rather than the statement. The check runs +**after** `MAP` is applied, since `MAP` is the remedy. + +In the bulk form, pages in Marketplace modules are **skipped and named** rather +than refused — a project-wide repoint that stopped dead on Administration's +pages would be unusable. A `WHERE LAYOUT` that names a layout which does not +exist is an **error**, not a "0 pages" success for a typo: + +``` +Error: layout not found: MyModule.NoSuchLayout +``` + +## Examples + +Put a brand caption in the topbar of a layout you own: + +```sql +ALTER LAYOUT MyModule.App_Default { + INSERT INTO layoutContainer.top { DYNAMICTEXT brand (Content: 'My App') }; +}; +``` + +Change it, then remove it: + +```sql +ALTER LAYOUT MyModule.App_Default { + SET Content = 'Renamed' ON brand; +}; + +ALTER LAYOUT MyModule.App_Default { + DROP WIDGET brand; +}; +``` + +Add a theme switcher snippet to the topbar: + +```sql +ALTER LAYOUT MyModule.App_Default { + INSERT INTO layoutContainer.top { + SNIPPETCALL themeBar (Snippet: MyModule.SNIPPET_ThemeBar) + }; +}; +``` + +Move one page onto a new layout: + +```sql +ALTER PAGE MyModule.Home { + SET Layout = MyModule.App_Default; +}; +``` + +Move every page that is still on the Atlas default — the migration: + +```sql +ALTER PAGES SET LAYOUT = MyModule.App_Default + WHERE LAYOUT = Atlas_Core.Atlas_Default; +``` + +Scoped to one module, whatever each page is on now: + +```sql +ALTER PAGES IN MyModule SET LAYOUT = MyModule.App_Default; +``` + +Repoint onto a layout whose placeholder has a different name: + +```sql +ALTER PAGE MyModule.Split { + SET Layout = MyModule.Minimal MAP (HeaderLeft AS Main); +}; +``` + +## Notes + +- Layout authoring requires the default `modelsdk` engine; `--engine legacy` + refuses it. +- `DESCRIBE LAYOUT` emits re-executable MDL, so it is the way to see what a + layout currently contains before altering it. + +## See Also + +[CREATE LAYOUT](create-layout.md), [ALTER PAGE / ALTER SNIPPET](alter-page.md), +[CREATE PAGE](create-page.md) diff --git a/docs-site/src/reference/page/alter-page.md b/docs-site/src/reference/page/alter-page.md index a7dcae7a00..2e5f86c2f1 100644 --- a/docs-site/src/reference/page/alter-page.md +++ b/docs-site/src/reference/page/alter-page.md @@ -102,6 +102,8 @@ REPLACE dgProducts.Description WITH { COLUMN Notes (Attribute: Notes) } Changes the page's layout without rebuilding the widget tree. Placeholder names are auto-mapped by default. If the new layout has different placeholder names, use `MAP` to specify the mapping. +A repoint that would leave the page bound to a placeholder the target layout does not declare is **refused**, and the error names `MAP` as the remedy. The check runs after `MAP` is applied, so mapping onto an existing placeholder is always accepted. To move many pages at once, see [ALTER LAYOUT](alter-layout.md), which documents the bulk `ALTER PAGES … SET LAYOUT` form. + Not supported for snippets (snippets don't have layouts). ### ADD Variables / DROP Variables diff --git a/docs-site/src/reference/page/create-layout.md b/docs-site/src/reference/page/create-layout.md index 296870b917..203060fcda 100644 --- a/docs-site/src/reference/page/create-layout.md +++ b/docs-site/src/reference/page/create-layout.md @@ -3,71 +3,199 @@ ## Synopsis ```sql -CREATE LAYOUT module.Name +CREATE [OR REPLACE] LAYOUT module.Name +( + layouttype: 'Responsive', + class: 'layout-atlas layout-atlas-responsive-topbar' +) { - widget_tree + SCROLLCONTAINER name { + REGION top | right | bottom | left | center ( region_properties ) { + widgets + } + } } ``` ## Description -Creates a page layout in the specified module. Layouts define the overall structure of pages -- they typically include a header, navigation, content placeholder, and footer. Pages reference a layout via the `Layout` property. +Creates a layout — the frame every page renders inside: the topbar, the +navigation sidebar, and the hole a page's own content drops into. A page names +one with its `Layout` property, and its widgets land in the layout's `Main` +placeholder. + +A layout's body uses the whole page widget vocabulary plus four element types +that only a layout has: `SCROLLCONTAINER`, `REGION`, `PLACEHOLDER` and +`NAVIGATIONTREE` (with `MENUBAR` for the topbar's horizontal menu). + +`OR REPLACE` rewrites an existing layout. To change one without rebuilding it, +use [ALTER LAYOUT](alter-layout.md) instead — it edits the stored document and +leaves alone anything it was not asked about, including widgets MDL cannot +express. -Layout creation in MDL has limited support. Most Mendix projects use layouts provided by the Atlas UI module (e.g., `Atlas_Core.Atlas_Default`, `Atlas_Core.PopupLayout`) rather than creating custom layouts through MDL. For advanced layout customization, use Mendix Studio Pro. +### Do not write into a Marketplace module -### Common Atlas Layouts +Mendix's own guidance is *"Do not change the supplied layouts. Either create a +separate module with the custom layouts … or create your own."* A Marketplace +update replaces the module wholesale and every local edit is gone, silently. -These layouts are available in most Mendix projects that include Atlas Core: +`CREATE LAYOUT` refuses a Marketplace target for that reason: + +``` +Error: layout Atlas_Core.T2: Atlas_Core is a marketplace module — a layout +written there is overwritten by the next module update. Create the layout in a +module of your own (Mendix's own guidance) and point pages at it with +ALTER PAGE … SET LAYOUT. +``` + +The usual starting point is a copy of an Atlas layout. `DESCRIBE LAYOUT` emits +re-executable MDL, so describe → rename → run *is* the copy operation: + +```bash +mxcli -p app.mpr -c "describe layout Atlas_Core.Atlas_Default" > mine.mdl +# change the qualified name to your own module, then: +mxcli exec mine.mdl -p app.mpr +``` -| Layout | Description | -|--------|-------------| -| `Atlas_Core.Atlas_Default` | Standard responsive page with sidebar navigation | -| `Atlas_Core.Atlas_TopBar` | Page with top navigation bar | -| `Atlas_Core.PopupLayout` | Modal popup dialog | -| `Atlas_Core.Atlas_Default_NativePhone` | Native mobile layout | +Read the describe output before running it. A widget MDL cannot express is +emitted as a comment ending `-- NOT re-executable`, naming exactly what a +re-run would drop (`Atlas_Core.Atlas_SideBar`, for instance, loses both of its +`Forms$SidebarToggleButton` widgets). ## Parameters `module.Name` -: The qualified name of the layout (`Module.LayoutName`). +: The qualified name of the layout (`Module.LayoutName`). Must be a module you + own. + +`layouttype` +: **Required.** Omitting it is an error, not a default: + `Error: layout needs a layouttype`. + + | Platform | Values | + |----------|--------| + | Web | `Responsive`, `Phone`, `Tablet`, `ModalPopup` | + | Native | `Default`, `Popup` | + + The two sets are disjoint, so the platform is **inferred** from the type — + there is no separate `native:` flag to contradict it. + +`class` +: The layout's own CSS class. Not decoration — Atlas scopes around two dozen + of its layout rules to `.layout-atlas` and its variants, and every Atlas + layout with chrome carries one. A layout written without it builds cleanly, + passes `mx check`, and renders with **no topbar bar and no sidebar rail**. + + | Shape | Class | + |-------|-------| + | Topbar navigation | `layout-atlas layout-atlas-responsive-topbar` | + | Sidebar navigation | `layout-atlas layout-atlas-responsive-default` | + | Popup | *(none — `PopupLayout` is bare)* | + +`style` +: Inline CSS on the layout element. + +`layouttype`, `class` and `style` are the **only** header properties. Anything +else is an error rather than an ignored key: + +``` +Error: layout MyModule.T1: unknown property "bogus" (a layout header takes +layouttype, class and style; which placeholder is "main" is set by naming one +Main, not by a property) +``` + +### Region properties + +| Property | Values | Notes | +|----------|--------|-------| +| `size` | integer | Unset is Studio Pro's `200` | +| `sizemode` | `Fixed`, `Pixels`, `Auto` | Unset is `Auto` | +| `class` | CSS class | e.g. `region-topbar`, `region-content` | + +## The four elements only a layout has + +| Element | MDL | Notes | +|---------|-----|-------| +| Scroll container | `SCROLLCONTAINER name { … }` | The layout's root. Its children are **regions**, never widgets | +| Region | `REGION top \| right \| bottom \| left \| center` | Five **named slots**, not a list. One region per slot | +| Placeholder | `PLACEHOLDER Main` | The hole a page's content goes into. No properties, no body | +| Navigation tree | `NAVIGATIONTREE name (profile: 'Responsive')` | The sidebar menu — vertical | +| Menu bar | `MENUBAR name (profile: 'Responsive')` | The topbar menu — horizontal | ## Examples -Reference an existing layout when creating a page: +A topbar layout, which is the shape `mxcli new` scaffolds: ```sql -CREATE PAGE MyModule.Dashboard +CREATE OR REPLACE LAYOUT MyModule.App_Default ( - Title: 'Dashboard', - Layout: Atlas_Core.Atlas_Default + layouttype: 'Responsive', + class: 'layout-atlas layout-atlas-responsive-topbar' ) { - CONTAINER cntMain { - DYNAMICTEXT txtWelcome (Attribute: WelcomeMessage) + SCROLLCONTAINER layoutContainer { + REGION top (size: 60, sizemode: 'Fixed', class: 'region-topbar') { + MENUBAR mainMenu (profile: 'Responsive') + } + REGION center (class: 'region-content') { + PLACEHOLDER Main + } } }; ``` -Reference a popup layout for a dialog page: +A sidebar layout, with a navigation tree in the left region: ```sql -CREATE PAGE MyModule.ConfirmDelete +CREATE LAYOUT MyModule.App_Sidebar ( - Params: { $Item: MyModule.Item }, - Title: 'Confirm Delete', - Layout: Atlas_Core.PopupLayout + layouttype: 'Responsive', + class: 'layout-atlas layout-atlas-responsive-default' ) { - DATAVIEW dvItem (DataSource: $Item) { - DYNAMICTEXT txtMessage (Attribute: Name) - FOOTER footer1 { - ACTIONBUTTON btnDelete (Caption: 'Delete', Action: DELETE, ButtonStyle: Danger) - ACTIONBUTTON btnCancel (Caption: 'Cancel', Action: CANCEL_CHANGES) + SCROLLCONTAINER layoutContainer { + REGION left (size: 232, sizemode: 'Pixels', class: 'region-sidebar') { + NAVIGATIONTREE navMenu (profile: 'Responsive') + } + REGION center (class: 'region-content') { + PLACEHOLDER Main } } }; ``` +A page then names the layout, and its widgets land in `Main`: + +```sql +CREATE PAGE MyModule.Dashboard +( + Title: 'Dashboard', + Layout: MyModule.App_Default +) +{ + CONTAINER cntMain { + DYNAMICTEXT txtWelcome (Content: 'Welcome') + } +}; +``` + +## Notes + +- **A layout must declare at least one placeholder.** Without one a page's + content has nowhere to go, and the statement is refused: + `layout "X" declares no placeholder`. +- **Name one placeholder `Main`.** `Forms$Layout` has no property recording + which placeholder is the main one — the convention is the mechanism, and all + 22 layouts Atlas ships follow it. There is deliberately no + `mainplaceholder:` property: writing the underlying key produces a layout + that builds cleanly and that Studio Pro cannot open. +- **A placeholder's name is API.** Pages bind to it as `Module.Layout.Name`. + Renaming one unbinds every page that used it — those pages still build, and + their content vanishes. +- Layout authoring requires the default `modelsdk` engine; `--engine legacy` + refuses it. + ## See Also -[CREATE PAGE](create-page.md), [SHOW PAGES](/reference/query/show-pages.md) +[ALTER LAYOUT](alter-layout.md), [CREATE PAGE](create-page.md), +[ALTER PAGE / ALTER SNIPPET](alter-page.md) diff --git a/docs-site/src/tools/bootstrap-prompt.md b/docs-site/src/tools/bootstrap-prompt.md index 2d6a85736a..ff52d6c0c3 100644 --- a/docs-site/src/tools/bootstrap-prompt.md +++ b/docs-site/src/tools/bootstrap-prompt.md @@ -55,9 +55,9 @@ a longer prompt. ## What the skill does once it takes over 1. **Interviews you** — one app or a solution, app name, what the app is for, what it - keeps track of, who logs in, theme, Mendix version. The app name comes first - because it becomes the `.mpr` file name, the Studio Pro app name and the path baked - into the SessionStart hook. + keeps track of, who logs in, theme, Mendix version, and **whether you have + requirements to work from**. The app name comes first because it becomes the `.mpr` + file name, the Studio Pro app name and the path baked into the SessionStart hook. 2. **Provisions** — `mxcli new` into a subfolder and moves it to the repo root (the root is where `.claude/` and `./mxcli` must live), `mxcli init --tool claude`, then `run --local --setup --ensure-db` to cache MxBuild + runtime and create the @@ -65,9 +65,17 @@ a longer prompt. 3. **Writes the brief** — `README.md` (what is being built, in your words) and `FINDINGS.md` (anything surprising or broken, appended as work proceeds). These are what an idle-reaped session reads to know what it is working on. -4. **Commits, then boots and verifies** — HTTP 200 at `http://localhost:8080/`, plus +4. **Records the plan** — the requirements, grouped into deliverable slices, in + [`docs/brain/plan/`](project-brain.md). On by default; say so at the interview if + you would rather skip it. This exists because a specification in a Word document, a + Figma file or a chat window leaves **no trace in git** — not an issue, not a commit + message — so hours of work can end up with nothing recording what it was for. + Requirements anchor at what will implement them, so `mxcli brain plan` reports + progress **derived from the model**: building the thing moves the number, and there + is no status column to maintain. +5. **Commits, then boots and verifies** — HTTP 200 at `http://localhost:8080/`, plus an optional `run --hub` preview URL. -5. **Proposes the model in MDL and waits** — module, entities, roles, pages — before +6. **Proposes the model in MDL and waits** — module, entities, roles, pages — before building anything. For a solution repo it also covers the parts that bite: per-app ports, a hostname per diff --git a/docs-site/src/tools/diff.md b/docs-site/src/tools/diff.md index 7dea1e15da..525e8ef5f8 100644 --- a/docs-site/src/tools/diff.md +++ b/docs-site/src/tools/diff.md @@ -19,6 +19,36 @@ This shows: Use `mxcli diff` to review changes before applying them, especially when working with AI-generated scripts. +### What diff does not compare + +`diff` compares entities, view entities, enumerations, associations, microflows +and nanoflows. Every other statement — `grant`, `create constant`, pages, +navigation, settings — is listed under **Not compared** after the summary: + +``` +Summary: 0 new, 0 modified, 1 unchanged + +Not compared (2 statement(s)) — diff has no comparison for these, +so they are absent from the summary above, not unchanged: + create constant x1 + grant microflow access x1 +``` + +Read that list. The counts describe only the statements diff understands, so a +script made entirely of the others summarises as all zeros — which means "not +examined", not "no change". Those statements were previously skipped without a +word, so the summary looked like a clean bill of health for a script that would +add documents (#997). + +### Both sides go through one renderer + +The project side and the script side are rendered by the same describer +`describe microflow` uses, so an unmodified `describe` dump diffs as +**unchanged**. Before this, the script side had a renderer of its own that +covered 18 of 43 activity types and silently emitted nothing for the rest, so a +java-action call, a `download file` or a canvas annotation appeared as a +deletion in a script that changed nothing at all. + ## mxcli diff-local Compares local changes against a git reference for MPR v2 projects. MPR v2 (Mendix >= 10.18) stores documents as individual files in an `mprcontents/` folder, making git diff feasible. diff --git a/docs-site/src/tools/project-brain.md b/docs-site/src/tools/project-brain.md new file mode 100644 index 0000000000..546fc67859 --- /dev/null +++ b/docs-site/src/tools/project-brain.md @@ -0,0 +1,217 @@ +# Project Brain + +`mxcli brain` records the project knowledge mxcli **cannot** compute. Two halves: + +- **Decisions** — why a pattern was chosen here, which marketplace version broke + what, what a recurring mxbuild error means in *this* app. +- **The plan** — the requirements being built from, grouped into slices, when the + source is a specification document, a prototype or a conversation. + +The plan half exists because that source leaves **no trace in git**. A Word +document, a Figma file and a chat window are not an issue and not a commit +message, so hours of work can end with nothing recording what it was for — and a +session resuming tomorrow has no idea what it was building towards. + +It is opt-in. A project without `docs/brain/` never hears about it. `mxcli`'s +bootstrap interview asks for requirements by default. + +## The rule + +**Anything mxcli can answer does not belong in the brain.** Entities, +microflows, pages, bindings, references and callers are all queryable. A note +that transcribes any of them will disagree with the project the moment someone +edits the model — and it will disagree silently, because nothing checks prose. + +The brain stores only the negative space: the reason, the constraint, the +history that no query can reach. + +## Layout + +``` +docs/brain/ + README.md what the folder is + project.md cross-cutting decisions + modules/Sales.md decisions anchored to the Sales module + modules/Finance.md … + plan/01-accounts.md requirements for one deliverable slice + plan/02-approvals.md … +``` + +Committed, and reviewed in a pull request like any other change. + +The split is not cosmetic. A single file would make the size cap a project-wide +budget — recording a `Sales` decision would compete with a `Finance` one — and +every session would load every module's decisions. With one file per module, a +session loads `project.md` plus the shards for the modules it is touching. + +## Anchors + +An entry's anchors are what make it routable and checkable. + +| Anchor | Names | +|---|---| +| `@Sales` | a module | +| `@Sales.Order` | a document: entity, microflow, page, workflow, … | +| `@Sales.Order.Status` | a member: an attribute | + +The **first** anchor decides the file: `@Sales.Order` puts the entry in +`modules/Sales.md`. An entry with no anchor is cross-cutting and goes to +`project.md`. + +There is no index to maintain — the module prefix *is* the file name. + +## Workflow + +An agent captures; a person promotes. + +```bash +mxcli brain init -p app.mpr + +mxcli brain capture "Orders are committed by Finance, not Sales" \ + -a @Sales.Order -a @Finance.ACT_Post -p app.mpr + +mxcli brain staged -p app.mpr # review the queue +mxcli brain promote a1b2c3 -p app.mpr +``` + +Captures go to `.mxcli/brain/staged.jsonl`, which `mxcli init` git-ignores — so +nothing reaches a pull request until someone has looked at it. `mxcli lint` +prints a one-line reminder when the queue is not empty. + +The queue is deliberately not sharded: routing it would force the file decision +before a human has looked at the entry, which is the decision promotion exists +to make. + +## Checking + +```bash +mxcli brain check -p app.mpr # every shard +mxcli brain check --changed -p app.mpr # only shards this branch touched +``` + +Two independent things are checked. + +**Does each anchor still resolve?** Three outcomes, one of which is a failure: + +| Outcome | Meaning | Fails | +|---|---|---| +| resolved | the anchor names something that is there | no | +| **not found** | the anchor names nothing — the entry is stale | **yes** | +| not indexable | the target exists, but its document type is not in the catalog's index | no | + +The third state matters. The catalog's `objects` view covers the describable +document types, not all of them. Without this distinction an anchor to, say, a +scheduled event would read as *missing*, and `check` would demand edits to +entries that are perfectly current. mxcli separates the two with a +type-agnostic unit lookup that cannot miss a kind, because it never asks what +kind anything is. + +**Is each entry in the right shard?** A separate axis, not a fourth anchor +state: every anchor can resolve and the entry still be in the wrong file. At +least one anchor must belong to the shard the entry sits in; anchors into other +modules are reported but do not fail, because a fact like "`Sales.Order` is +committed by `Finance.ACT_Post`" genuinely spans two modules. + +Misfiling is only decided when something resolved. An entry whose anchors are +all *not indexable* is left alone — there is no evidence about where it belongs. + +## Requirements and slices + +When the source of truth lives outside git, record it before building: + +```bash +mxcli brain capture "Orders must be approvable by a manager" \ + --slice 02-approvals -a @Sales.ACT_Order_Approve -p app.mpr +``` + +`--slice` is the only signal needed: it files the entry in `plan/02-approvals.md` +and makes it a requirement. Slices sort by name, so a numeric prefix is how a +roadmap is sequenced — that is your choice, not a field mxcli maintains. + +### A requirement's anchor points forward + +| | Anchor points at | An anchor that does not resolve means | +|---|---|---| +| decision | what exists | the decision is **stale** — `check` fails | +| requirement | what is intended | **not built yet** — normal, `check` passes | + +Same syntax, opposite meaning. Anchoring a requirement at a microflow that does +not exist yet is correct: it is the forward reference that later becomes the +progress signal. + +This is not a theoretical distinction. Recorded as an ordinary entry, a single +unbuilt requirement takes `mxcli brain check` to exit 1 — which is what made +requirements a separate kind rather than more entries in the same files. + +### Progress is derived + +```bash +mxcli brain plan -p app.mpr +``` + +``` +SLICE BUILT PLANNED UNANCHORED +01-accounts 1 0 +02-approvals 0 1 1 + +1 of 3 requirements built, across 2 slice(s). +``` + +A requirement is **built** when its anchors resolve against the model. Nothing in +the file says "done": create the microflow a requirement points at and the count +moves on the next run, with the plan file untouched. + +That is the reason this belongs in mxcli rather than in a hand-kept markdown +checklist. A status column is wrong the moment someone builds something, and +nothing tells you; a derived one cannot be. + +A requirement with **no anchor** is counted apart as *unanchored* rather than +silently called planned — it cannot be measured until you say what will +implement it. + +Misfiling is not checked for slices: a slice spans modules by design. + +## Size + +```bash +mxcli brain show -p app.mpr +``` + +``` +SHARD ENTRIES LINES HEADROOM +project.md 3 28 92/120 +Administration.md 2 16 224/240 +``` + +Each shard has a line budget and `promote` refuses rather than exceeding it, +naming the shard and its occupancy. `project.md` is the tightest: it is the only +file loaded every session, so every line in it is charged to every session. + +A plan slice gets far more room — it holds source material and is read when +planning rather than loaded every session. It is still a budget, and that is the +point: **a slice too long to read is a slice that should be split.** Here the cap +does not merely bound context cost, it enforces the slicing discipline. + +When a promotion is refused the answer is to condense or drop, not to raise the +cap. The cap is what stops the store becoming a file nobody reads. + +Sizes are computed on every run and are deliberately not written into any +committed file, including the store's own `README.md` — a figure in prose is +stale the next time anyone promotes. + +## Commands + +| Command | Does | +|---|---| +| `brain init` | Creates `docs/brain/`. Refuses a `docs/brain/` it did not write | +| `brain capture "" [-a @Anchor]…` | Queues an entry. Never commits | +| `brain staged` | Lists the queue with the shard each entry would land in | +| `brain promote [--to ]` | Writes it into its shard | +| `brain drop ` | Removes it from the queue or from its shard | +| `brain capture "" --slice [-a @Anchor]…` | Queues a **requirement** of that slice | +| `brain plan` | Each slice's requirements counted against the model | +| `brain check [--changed]` | Anchors resolve, entries filed correctly, plus slice progress | +| `brain show []` | Entries, lines and headroom per shard | + +Dropping the last entry from a module shard deletes the file, so the directory +does not accumulate husks that read as "this module has decisions". diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index 8fcdef36ae..9ffb5dff36 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -536,6 +536,7 @@ it is for pages. | Validation | `validation feedback $entity/attribute message 'message';` | Requires attribute path + MESSAGE | | Log | `log info\|warning\|error [node 'name'] 'message';` | | | Position | `@position(x, y)` | Canvas position (before activity) | +| Parameter position | `@position(x, y)` before a parameter, **inside** the `( … )` list | The only annotation a parameter takes. Omit it and parameters form a row at 200;53, 300;53, …; a parameter off that row is treated as hand-placed, survives a rewrite, and is emitted by DESCRIBE (#993) | | Start event | `@start(x, y)` | Canvas position of the start, on the **first** statement. Omit it and the start is placed one spacing unit left of the first activity and MOVES with it on a rewrite; a start that is not at that derived spot is treated as hand-placed, survives a rewrite, and is emitted by DESCRIBE (#951) | | Caption | `@caption 'text'` | Custom caption (before activity) | | Color | `@color Green` | Background color (before activity) | @@ -763,7 +764,7 @@ alter workflow Module.OrderApproval ```sql create or replace navigation Responsive home page MyModule.Home_Web - home page MyModule.AdminHome for MyModule.Administrator + home page MyModule.AdminHome for Administrator login page Administration.Login not found page MyModule.Custom404 menu ( @@ -1084,6 +1085,11 @@ source json '{"latitude": 51.9, "current": {"temp": 12.8}}' | Create or modify | `create or modify json structure Module.Name snippet '...';` | Preserves UUID — preferred for AI agents | | Create with name map | `create json structure Module.Name snippet '...' CUSTOM NAME map ('jsonKey' as 'CustomName', ...);` | Override auto-generated ExposedNames | | Name an array's item | `CUSTOM NAME map (item of 'lines' as 'OrderLine')` | An item has no JSON key; `item of 'Root'` for a root array | +| Message definition collection | `create [or modify] message definition collection M.Name [folder '...'] ( definition D for M.Entity [as 'X'] ( members ) );` | A selection over the domain model — the one non-JSON mapping source MDL can create | +| Message definition member | attribute: `OrderId [as 'X'] [example '...']`; association: `M.Assoc/M.Entity [as 'X'] ( ... )` | Naming the target sets the traversal direction, which decides the cardinality | +| Alter a definition's members | `alter message definition M.Coll.Def add\|drop\|set member X [in path] [as 'Y']` | Addressed as Module.Collection.Definition; `set` changes only the exposed name | +| Alter a collection | `alter message definition collection M.Coll add\|drop\|rename definition ...` | | +| Browse | `show message definition collections [in M]`, `describe message definition collection M.Name` | | | Drop structure | `drop json structure Module.Name;` | | ## Import Mappings diff --git a/docs/05-mdl-specification/01-language-reference.md b/docs/05-mdl-specification/01-language-reference.md index 5cbfb97f98..fc788f49a3 100644 --- a/docs/05-mdl-specification/01-language-reference.md +++ b/docs/05-mdl-specification/01-language-reference.md @@ -1218,7 +1218,7 @@ describe navigation [] -- Full MDL output (round-trippable) ```sql create or replace navigation home page Module.HomePage - [home page Module.AdminHome for Module.AdminRole] + [home page Module.AdminHome for AdminUserRole] [login page Module.LoginPage] [not found page Module.Custom404] [menu ( @@ -1233,7 +1233,7 @@ create or replace navigation ```sql create or replace navigation Responsive home page MyModule.Home_Web - home page MyModule.AdminHome for MyModule.Administrator + home page MyModule.AdminHome for Administrator login page Administration.Login menu ( menu item 'Home' page MyModule.Home_Web; diff --git a/docs/11-proposals/PROPOSAL_authorable_message_definitions.md b/docs/11-proposals/PROPOSAL_authorable_message_definitions.md new file mode 100644 index 0000000000..d692bbb961 --- /dev/null +++ b/docs/11-proposals/PROPOSAL_authorable_message_definitions.md @@ -0,0 +1,402 @@ +--- +title: Authorable message definitions — the last mapping source MDL cannot write +status: implemented +date: 2026-09-01 +related: + - PROPOSAL_mapping_coverage.md + - docs/13-decisions/0003-mdl-is-sql-shaped.md + - docs/13-decisions/0005-semantic-model-interface-currency.md + - .claude/skills/mendix/json-structures-and-mappings/SKILL.md +--- + +# Authorable message definitions — the last mapping source MDL cannot write + +## Problem Statement + +A mapping is bound to one of four schema sources. MDL can create exactly one of +them. + +| source | mappings in the corpus | authorable in MDL | +|--------|------------------------|-------------------| +| JSON structure | 250 (76.5%) | yes | +| **message definition** | **74 (22.6%)** | **no — read-only** | +| XML schema | 3 (0.9%) | no | +| imported web service (SOAP) | 0 in the corpus | no — and now [refused rather than dropped](https://github.com/ako/mxcli/issues/365) | + +Measured across all 327 import/export mappings in the nine demo apps. + +So a project built entirely through MDL can express three-quarters of the +mappings a real app has. The remaining quarter needs a document mxcli can read +and map over, but only a human in Studio Pro can create. `CREATE IMPORT MAPPING +… WITH MESSAGE DEFINITION` already works; there is simply no way to bring the +definition into existence. + +This is the last of the four that is both **worth doing** and **doable**: + +- **XML schema** holds an imported `.xsd` (`FilePath='C:\Users\…\Response.xsd'`). + Authoring it means parsing XSD — imports, includes, complexTypes, substitution + groups — for 0.9% of mappings. +- **Imported web service** holds a WSDL, with its schema entries inline. Same + problem, larger. +- **Message definition** holds *nothing external*. It is a selection over the + domain model. Every element names an entity, an attribute or an association. + +## What the document actually is + +Measured over all **36 collections / 56 definitions / 4,686 elements** in the +corpus, read through the unit table (not by grepping — see the note at the end). + +Four element types, and every property present on every instance. There are no +optional keys to guess at: + +``` +MessageDefinitions$MessageDefinitionCollection 36 Name, Documentation, Excluded, ExportLevel, MessageDefinitions + MessageDefinitions$EntityMessageDefinition 56 Name, Documentation, ExposedEntity + MessageDefinitions$ExposedEntity 56 Entity, ExposedName, ExposedItemName, OriginalName, Path, + ElementType, MinOccurs, MaxOccurs, Nillable, PrimitiveType, + MaxLength, FractionDigits, TotalDigits, IsDefaultType, + Example, ErrorMessage, WarningMessage, Documentation, Children + MessageDefinitions$ExposedAttribute 3697 (same, plus Attribute) + MessageDefinitions$ExposedAssociation 933 (same, plus Association + Entity) +``` + +Collections are small in one dimension and not at all in the other. 28 of 36 +hold a single definition, 4 hold two, 3 hold four, 1 holds eight — but the +definitions themselves are **deep**: + +| nesting depth | elements | | +|---|---|---| +| 0 (the definition root) | 56 | | +| 1 | 356 | | +| 2 | 265 | | +| 3 | 142 | | +| 4 | 208 | | +| 5 | 477 | | +| 6 | 974 | | +| **7** | **2,208** | the maximum observed | + +Object elements carry a mean of 4.7 members, and the tail is long — 93 have 15 +members, one has 22. + +That shape decides more of this proposal than anything else. A definition is not +a flat field list you would happily restate; a whole-document `CREATE OR MODIFY` +is a poor tool for "expose one more attribute", and any addressing scheme for +targeted edits has to reach seven levels down. + +### Almost every property is a constant or is derived + +Across all 4,686 elements, with no exceptions: + +| property | value | | +|---|---|---| +| `MinOccurs` | `0` | constant | +| `Nillable` | `true` | constant | +| `IsDefaultType` | `false` | constant | +| `MaxLength`, `FractionDigits`, `TotalDigits` | `-1` | constant | +| `Example`, `ErrorMessage`, `WarningMessage`, `Documentation` | `""` | constant | +| `ElementType` | `Object` / `Value` | derived from the element kind | +| `PrimitiveType` | `Unknown` for objects, the attribute's type for values | derived | +| `OriginalName` | the entity / attribute / association's own name | derived | +| `Path` | the position in the tree (`Reference\|Content`) | derived | +| `MaxOccurs` | `1` or `-1` | derived — **see below** | +| `ExposedItemName` | `OriginalName` when `MaxOccurs = -1`, `""` otherwise | derived, 461/461 | + +That leaves exactly four things an author chooses: the collection's name, each +definition's name, which entity it roots at, and which members to expose — plus +an optional rename per element. + +### The one derivation that is not obvious + +`MaxOccurs` on an exposed association is **not** a function of the association's +type. All 927 resolvable associations in the corpus are `Reference`, yet 526 +carry `MaxOccurs = 1` and 401 carry `-1`. + +It tracks the **direction of traversal**, with zero counter-examples: + +| the element's holder is | traversal | `MaxOccurs` | count | +|---|---|---|---| +| the association's **FROM** entity | child → parent, following the FK | `1` | 496 | +| the association's **TO** entity | parent → children, in reverse | `-1` | 401 | + +(30 further elements are cross-module and were not resolved by the census +script; all carry `MaxOccurs = 1`, consistent with FROM.) + +This is the `ParentPointer` / `ChildPointer` inversion from CLAUDE.md wearing a +different hat, and it is the one place a plausible implementation goes silently +wrong: get it backwards and the definition exposes a list as a single object, or +a single object as a list. A build error is not guaranteed — the mapping over it +simply carries the wrong cardinality. + +**Design consequence: the statement names the target entity**, so direction is +explicit in the source text rather than inferred: + +```sql +Sales.Order_Line/Sales.Line as 'Lines' ( ... ) +``` + +This is also the shape import and export mappings already use for a nested +object (`Assoc/Module.Child`), so it is not a new idea to learn. + +## Proposed syntax + +```sql +create message definition collection Sales.MD_Order ( + definition Order for Sales.Order as 'Orders' ( + OrderId, + OrderDate, + Total, + Sales.Order_Line/Sales.Line as 'Lines' ( + Sku, + Quantity, + Price + ), + Sales.Order_Customer/Sales.Customer ( + Name, + Email + ) + ) +); +``` + +Following [ADR-0003](../13-decisions/0003-mdl-is-sql-shaped.md) and +`design-mdl-syntax`: + +- `create` / `drop` / `describe` / `show`, not a custom verb. +- Qualified names everywhere; no implicit module. +- `as 'Name'` for a name-to-name mapping, matching `CUSTOM NAME MAP` and + `ALTER ENTITY … RENAME … AS …`. (`:` is for property values; this is a rename.) +- A bare identifier is an **attribute**; a qualified `Assoc/Module.Entity` is an + **association**. Same discriminator import and export mappings use. +- One member per line, trailing comma allowed, so adding a field is a one-line + diff. + +### Statement set + +**Whole document:** + +| statement | notes | +|---|---| +| `CREATE [OR MODIFY] MESSAGE DEFINITION COLLECTION M.Name ( ... )` | `OR MODIFY` preserves the UUID — mappings reference the collection by qualified name, and a fresh document would break every `WITH MESSAGE DEFINITION` | +| `DROP MESSAGE DEFINITION COLLECTION M.Name` | refuse when a mapping still references it, naming the mappings | +| `DESCRIBE MESSAGE DEFINITION COLLECTION M.Name` | re-executable output; this is what makes describe → rename → exec the copy operation | +| `SHOW MESSAGE DEFINITION COLLECTIONS [IN Module]` | there is no listing today at all | + +`FOLDER 'path'` on create, as every other document type takes. + +**Targeted edits.** `CREATE OR MODIFY` alone is not enough, for the reason the +depth table shows: adding one attribute to a definition seven levels deep would +mean restating the whole document, which is precisely the diff-unfriendliness +[ADR-0003](../13-decisions/0003-mdl-is-sql-shaped.md) argues against, and is why +`ALTER ENTITY ADD ATTRIBUTE` exists rather than only `CREATE OR MODIFY ENTITY`. + +Definitions within a collection — this is the 8 collections that hold more than +one: + +```sql +alter message definition collection Sales.MD_Order + add definition Line for Sales.Line as 'Lines' ( Sku, Quantity ); + +alter message definition collection Sales.MD_Order drop definition Line; +alter message definition collection Sales.MD_Order rename definition Line to OrderLine; +``` + +Members within a definition — this applies to all 36: + +```sql +alter message definition Sales.MD_Order.Order add member Total; +alter message definition Sales.MD_Order.Order + add member Sales.Order_Line/Sales.Line as 'Lines' ( Sku, Quantity ); + +alter message definition Sales.MD_Order.Order drop member Total; +alter message definition Sales.MD_Order.Order set member Total as 'GrandTotal'; +``` + +Three deliberate choices here: + +- **The definition is addressed as `Module.Collection.Definition`** — the same + three-part reference `WITH MESSAGE DEFINITION` already takes. Nothing new to + learn, and the two cannot drift apart. +- **`SET member … AS`, not `RENAME member … TO`.** `ALTER ENTITY RENAME + ATTRIBUTE` renames the attribute *in the model* and rewrites every reference to + it. This changes only the element's `ExposedName` and touches nothing else, so + borrowing `RENAME` would promise something far larger than it does. `SET` is + the established verb for changing a property, and `as` for a name-to-name + mapping. +- **`IF NOT EXISTS` / `IF EXISTS`** on add and drop, so a definition script + re-runs cleanly — the same treatment `ALTER ENTITY` gives attributes. + +**Reaching a nested member.** Members live up to seven levels down, so the +address needs a path. It is written in **exposed names**, the names the document +itself carries: + +```sql +alter message definition Sales.MD_Order.Order + add member Price in Lines/Prices; + +alter message definition Sales.MD_Order.Order + drop member Sku in Lines; +``` + +`in ` rather than a `/`-joined member name, because `/` already means +"association to entity" inside a member (`Sales.Order_Line/Sales.Line`) and +overloading it in the same clause would be ambiguous to a reader even where the +grammar could tell them apart. + +New members append. `describe` emits stored order, so the round trip is stable +either way; a positional form (`before` / `after`) is deliberately not proposed +until something needs it. + +## The name of a repeating element, and why not to guess it + +Studio Pro **pluralises** `ExposedName` for a repeating element while keeping +`ExposedItemName` at the singular. Across all 461 repeating elements: + +| pattern | count | example | +|---|---|---| +| `ExposedName == OriginalName + "s"` | 386 | `Reference` → `References` | +| an English plural | 71 | `Factory` → `Factories` | +| unchanged (the original is already plural) | 4 | `Parts` → `Parts` | +| `ExposedItemName == OriginalName` | **461 / 461** | | + +`ExposedItemName` is therefore free — it is always the original name. The plural +is not: reproducing it needs `-y → -ies`, an already-plural detector, and +whatever the next corpus turns up. + +**Recommendation: do not implement English inflection.** Default `ExposedName` to +`OriginalName` and make `as 'Orders'` the way to get the plural — the same +conclusion reached for array-item naming in +[#272](https://github.com/ako/mxcli/issues/272), and for the same reason: a rule +fitted to one corpus is wrong on the next, and a name the author writes is +better than a name a heuristic guesses. + +**The honest cost**, stated so nobody is surprised: an mxcli-written definition +that omits `as` will differ from a Studio-Pro-written one in `ExposedName` for +every repeating element. That is a describe-diff, not a build error — but it is +real, and it is the argument for making `as` prominent in the docs rather than a +footnote. + +An alternative worth considering in review: **require** `as` on a repeating +element. It removes the silent divergence at the cost of some friction, and it +makes the cardinality visible at the point the author writes it. + +## Scope + +**In:** the statements above — whole-document and targeted; attributes, +associations in both directions, nesting to the depth the corpus shows; +`OR MODIFY`; folder placement; the `describe` round trip; a check-time rule for +members that do not exist on the entity, and for an `in` path that reaches +nothing (the shape `MDL-JSON01` / `MDL-JSON02` established). + +**Out:** + +- ~~**Inherited attributes.**~~ **Measured and IN scope.** 398 of 3,697 exposed + attributes are inherited (10.8%) — e.g. `Email_Connector.Attachment` exposing + `System.FileDocument.FileID`. Far too common to refuse, and the stored + `Attribute` names the DECLARING entity, which `DeclaringMemberRef` already + resolves for mappings. +- **`Documentation` / `ErrorMessage` / `WarningMessage`.** Empty in 4,707 of + 4,707. Add them when a document needs them, not before. +- ~~**`Example`.**~~ **In scope after all.** The corpus said empty in 4,686 of + 4,686 — but that corpus is marketplace modules. ako/TestApp's hand-authored + definition sets one, so it is author-set and rare rather than unused, and + hardcoding it empty would silently drop the one that exists. `example '...'` + is now syntax. +- **Published message definitions** (the Business Events surface). A different + document. +- **A wildcard member** (`definition Order for Sales.Order ( * )` to expose every + attribute). Studio Pro's UI is a tree of checkboxes, so "tick everything" is a + natural thing to want, and with a mean of 4.7 members and a tail out to 22 it + would save real typing. But `*` appears nowhere else in MDL, and no measurement + here says anyone wants it — proposing it now would be designing for a shape + with no document behind it. Raised as an open question instead. + +## Open questions for review + +1. **Require `as` on a repeating element?** See above — it removes the silent + divergence from Studio Pro at the cost of friction. +2. **A wildcard member.** Worth it, or speculative? +3. **Inherited attributes** — refuse, or block the proposal until measured? +4. **Legacy engine** — author there too, or modelsdk-only like rules, menus and + layouts? + +## What implementation changed + +Five things the census had not shown, all found by round-tripping ako/TestApp's +hand-authored collection against its stored bytes: + +1. **`Path` is a chain of ORIGINAL names**, not exposed ones, and an + **association contributes two segments** — its own name, then the target + entity's: `Order|OrderLine_Order|OrderLine|Amount`. Confirmed afterwards at + 4,707 of 4,707 elements once we knew to look for it. +2. **The typed-array marker is 2**, where the codec defaults to 3. +3. **Every element serializes `Children` even when empty** — a leaf attribute + stores the bare `[2]`, the same `MandatoryLists` rule as a rule document's + `Flows`. +4. **`PrimitiveType` is mapped, not passed through**: `Long → Integer`, + `AutoNumber → Integer`, `Enumeration → String`, everything else identity. + A pass-through gets 279 corpus elements wrong. +5. **`Example` is author-set** — see the scope note above. + +The lesson is worth more than the list: **one hand-authored reference document +is worth more than a large census of marketplace modules.** A module author and +someone building an app by hand exercise different parts of the same document. + +## Implementation outline + +Full-stack, per the CLAUDE.md checklist: + +1. **Grammar** — `createMessageDefinitionCollectionStatement`, plus `drop` / + `describe` / `show`, and `alterMessageDefinitionCollectionStatement` / + `alterMessageDefinitionStatement`. New keyword: none required; `MESSAGE` and + `DEFINITION` already exist (`WITH MESSAGE DEFINITION`), `COLLECTION` is used + by image and icon collections, and `IN` / `AS` / `SET` / `ADD` / `DROP` / + `RENAME` are all in the lexer. +2. **AST** — `CreateMessageDefinitionCollectionStmt`, with a member tree that + distinguishes attribute from association by the presence of a qualified name. +3. **Visitor** — bridge, as usual. +4. **Backend** — `CreateMessageDefinitionCollection` / + `UpdateMessageDefinitionCollection` / `DeleteMessageDefinitionCollection` on + `MappingBackend`, beside the existing `ListMessageDefinitionCollections`; + implemented in both engines and stubbed in the mock. +5. **Executor** — thin handlers. The element tree is built from the domain model, + which is where `MaxOccurs` and `PrimitiveType` are resolved. The ALTER path + edits the stored document rather than rebuilding it, so an untouched + definition is never round-tripped through the describer — the argument that + made `ALTER LAYOUT` a capability rather than a convenience. +6. **`describe`** — re-executable, deterministic order. + +The document is written through the codec on the modelsdk engine. Whether the +legacy engine should author it at all is an open question — recent document +types (rules, menus, layouts) are modelsdk-only and legacy refuses. + +## Verification plan + +The corpus is the oracle, and it is unusually good here: 36 real documents, all +uniform. + +1. **Round-trip fixtures.** Pin three real collections in + `mdl/executor/testdata/mapping-fixtures/` — one single-definition, one + multi-definition, one with a reverse-direction association — and assert + `describe` → re-exec → `canon.Equal`, the methodology used for scheduled + events and rules. +2. **The cardinality control.** A definition exposing the *same* association in + both directions, asserting `MaxOccurs` 1 and -1 respectively. Getting this + backwards is the failure mode with no build error behind it. +3. **mxbuild.** 0 errors on a project holding an mxcli-written collection *and* + an import mapping bound to it, against a base-project control. +4. **The constants.** One test asserting `MinOccurs`/`Nillable`/`MaxLength`/… + against the measured values, so a future change that starts varying one of + them fails loudly rather than drifting. +5. **ALTER leaves the rest alone.** Add one member to a pinned collection and + assert every *other* element is byte-identical — the control that separates a + targeted edit from a rebuild that happened to produce the same thing. + +## A note on measuring this document type + +The census figures above were read through `mprbson.units()` / the reader, not +by grepping the extracted project tree. Grep gives false negatives here: it can +only see units that are *files*, which is MPR v2 only. An MPR v1 project keeps +its units in the SQLite `Unit.Contents` blob, where the type string is not +greppable — which is exactly how an earlier count of XML schemas came back as +zero when the answer was three +([#259](https://github.com/ako/mxcli/issues/259) follow-up). diff --git a/docs/11-proposals/PROPOSAL_project_brain.md b/docs/11-proposals/PROPOSAL_project_brain.md index 00f96f6887..14399f51b1 100644 --- a/docs/11-proposals/PROPOSAL_project_brain.md +++ b/docs/11-proposals/PROPOSAL_project_brain.md @@ -1,232 +1,448 @@ --- -title: Project Brain — Persistent Knowledge and Session Scaffolding for Long-Term AI Collaboration +title: Project brain — an opt-in store, in a user's Mendix project, for what mxcli cannot compute status: draft +date: 2026-09-01 +related: + - .claude/skills/fix-issue.md + - .claude/skills/maintain-wiki.md + - docs/13-decisions/0003-mdl-is-sql-shaped.md + - PROPOSAL_ai_capability_dataset.md --- -# Proposal: Project Brain — Persistent Knowledge and Session Scaffolding for Long-Term AI Collaboration +# Project brain — an opt-in store, in a user's Mendix project, for what mxcli cannot compute -## Problem Statement +> Written in response to mendixlabs/mxcli#1017. The issue asks that the brief's +> assumptions be verified against the codebase and that conflicts be flagged +> rather than followed. §2 does that; four of the four assumptions needed +> amending, and one of them is wrong outright. -mxcli is developed with significant AI involvement, yet the project's knowledge infrastructure is built for static human reading rather than persistent, connected memory. Three related friction points compound over time: +## 1. Problem -1. **Documentation goes stale without detection** — Proposals describe features as future work after they have already shipped (e.g., the registry implementation predating its own proposal by weeks). There is no mechanism to detect or prevent this drift. +**Audience: users of mxcli building Mendix projects** — not mxcli's own +development. The store lives in the user's Mendix project, is maintained by that +developer and their agent, and is read by people who may never see mxcli's +source. This matters throughout: it sets the scale (tens of lines, not +hundreds of entries), the number of writers (one developer, not many parallel +sessions), and the review audience (a Mendix developer reading a pull request). -2. **No orientation layer for new contributors or agent sessions** — Skills files cover specific tasks well, but there is no single place that answers: *what is the current state of the project, what is actively being worked on, and what is the path for adding something new end-to-end?* A new human contributor or an AI agent dropped into a fresh session must reconstruct this from scattered files, CLAUDE.md, and git history. +An agent working on a Mendix project accumulates knowledge it loses each +session: why a pattern was chosen, which marketplace version broke what, which +mxbuild error means what *here*. The usual answers — a hand-maintained +`CLAUDE.md`, a memory-bank tool — rot, and duplicate what mxcli can already +answer. -3. **Knowledge is not connected** — The internals documentation in `docs-site/src/internals/` covers the pipeline and key subsystems, but the pages are isolated documents rather than a traversable network. There is no way to navigate from "executor" to "backend interface" to "MPR backend" as a linked path, and no graph view for humans to explore topics. +The governing principle from the brief is right and is what makes this +Mendix-specific rather than another memory tool: **store only the negative +space.** Anything derivable from the model must be answered by a command and +never written down. mxcli can already query entities, microflows, pages, +bindings and references; a store that transcribes any of that is a store that +will disagree with the project. -These are not separate problems. They share the same root: the project has grown to a point where implicit conventions and scattered documents work for a small team in a single session but break down for parallel development, onboarding, and long-running AI collaboration across many sessions. +## 2. Assumptions verified -## Current State +### 2.1 Can MDL read and write `Documentation`? — Yes, via doc comments, and it is the supported spelling -The project already has several knowledge layers that are good individually but unconnected: - -| Layer | Location | Audience | Character | -|---|---|---|---| -| Kitchen-sink AI context | `CLAUDE.md` | AI agents | Procedural, comprehensive, monolithic | -| Maintainer task skills | `.claude/skills/*.md` | AI agents (mxcli-dev) | Procedural ("how to do X") | -| User task skills | `.claude/skills/mendix/*.md` | AI agents (shipped to users) | Procedural ("how to do X") | -| Internals docs | `docs-site/src/internals/` | Curious users + contributors | Explanatory, isolated pages | -| Proposals | `docs/11-proposals/` | Contributors | Forward-looking, no lifecycle | -| Architecture docs | `docs/01-project/`, `docs/03-development/` | Contributors | Structural, sparse | - -The gaps are: links between topics (graph), decision lifecycle (why things are the way they are), extension guides (how to add new things end-to-end), a raw material inbox, and a live "what's next" layer connected to GitHub data. - -## Proposed System: Four Connected Layers - -### Layer 1: Raw Inbox (`docs/raw/`) - -An immutable drop zone for unprocessed source material: GitHub issue exports, research notes, discussion transcripts, external references. Files land here and are never edited after landing. The `/brain-ingest` command processes them into the wiki and marks each file as processed via frontmatter. The raw material remains as provenance. - -This separates input from synthesised knowledge cleanly. An agent or contributor drops a file here; the brain processes it on the next ingest run. - -### Layer 2: The Wiki (`wiki/`) - -A contributor-facing wiki at the repo root. The character is **explanatory** — how things work and why — distinct from skills (procedural) and proposals (decisions in flight). One concept per file, explicit links between topics, with a Mermaid index in `README.md` that renders as a graph. - -Both humans and agents write to the wiki. The agent drafts and updates via commands; humans edit directly. The wiki is not agent-owned — it is collaboratively maintained, with the agent doing the bulk of routine updates. - -The `docs-site/src/internals/` pages are good raw material; overlapping topics link to the wiki or migrate into it. The distinction is audience: docs-site explains internals to curious users, the wiki explains them to contributors who need to change things. - -**Structure:** +A `/** … */` comment before a statement becomes the object's `Documentation` and +is written into the model. Measured on Mendix 11.13, executing against a real +project and then searching the **stored bytes**, not a read-back: ``` -wiki/ - README.md # visual index: Mermaid graph of clusters + entry points - log.md # chronological record of all wiki operations - MAP.md # source path → wiki topic manifest (drives freshness hooks) - - pipeline/ # the journey from MDL text to MPR write - overview.md # full pipeline diagram, links to each step - grammar.md # ANTLR4, domain split, make grammar - ast.md # statement node hierarchy, how kinds map to types - visitor.md # ANTLR listener → AST construction - executor.md # registry, dispatch, ExecContext, register_stubs.go - backend.md # hexagonal boundary, interface design, why it exists - mpr.md # BSON, reader/writer, storage names, v1 vs v2 - - design/ # why things are the way they are (ADRs) - mdl-syntax-principles.md # design guidelines, anti-patterns, decision framework - backend-boundary.md # no sdk/mpr in executor, why enforced by checklist - registry-explicit-wiring.md # why init() was rejected, emptyRegistry() testing - grammar-domain-split.md # how the domain split reduces conflict surface - - subsystems/ # key components in isolation - catalog.md # SQLite catalog, what it indexes, query interface - lsp.md # LSP capabilities, how diagnostics flow, wiring - widget-engine.md # widget registry, .def.json, template loading - version-awareness.md # feature registry, checkFeature(), version gates - repl.md # REPL architecture, session state - - extending/ # end-to-end contributor guides - new-command.md # full path: grammar → AST → visitor → executor → backend → MPR - new-document-type.md # adding a new Mendix document type - new-backend-method.md # interface → MPR implementation → mock stub - new-lint-rule.md # rule registration, Starlark vs Go rules - new-widget.md # .def.json, widget registry, template extraction +create microflow … with a /** … */ header + -> text present in mprcontents/d4/a4/….mxunit +create entity … with a /** … */ header + -> text present in mprcontents/84/77/….mxunit +describe microflow … + -> the comment comes back verbatim, above `create or modify microflow` ``` -**ADR format for `design/`:** Each decision file uses: -- **Context** — what problem prompted this decision -- **Decision** — what was chosen and what was explicitly rejected -- **Status** — `accepted` | `superseded` | `under review` -- **Consequences** — what the decision implies for contributors and agents +`create … comment 'text'` was removed **because this exists**, not because the +capability was missing — two spellings, one of which wrote nothing, is worse +than one. Reading the removal as evidence of a gap is the wrong inference, and +the earlier draft of this proposal made it. + +The doc comment is wired at **28 sites** across `mdl/visitor/`, covering entity, +microflow, page, association, enumeration, workflow, scheduled event, queue, +regular expression, JSON structure, image collection, OData, REST, business +events and the agent-editor documents. -**Links to skills:** Each `extending/` guide links to the relevant maintainer skill for the step-by-step checklist. The wiki provides narrative and rationale; the skill provides the checklist and gotchas. +**But a rewrite destroys it.** Writing works; *surviving* does not. Measured on +the same project, checking the stored bytes after each step, with the untouched +object as the control: -**`MAP.md` — the source-to-topic manifest:** -Maps source paths to wiki topics, enabling automated freshness checking: ``` -mdl/executor/registry.go → wiki/pipeline/executor.md -mdl/grammar/domains/ → wiki/pipeline/grammar.md -mdl/backend/ → wiki/pipeline/backend.md -sdk/mpr/ → wiki/pipeline/mpr.md + microflow doc entity doc +after create PRESENT PRESENT +after `create or replace` the microflow + ABSENT PRESENT <- control holds +after `create or modify` the entity + ABSENT ABSENT ``` -### Layer 3: Live Project State (Connected to GitHub) - -The wiki provides static knowledge. Current project state — what is in progress, ready to pick up, blocked — lives in GitHub (issues, milestones, project board) and must not be duplicated in the wiki, where it would go stale immediately. - -A scheduled Claude Code agent generates `wiki/CURRENT.md` periodically — not raw GitHub data, but a synthesised briefing: given open issues, active PRs, and recent decisions, what does the project need next and why. This gives contributor personal second-brain systems a pull-able summary, and gives AI agents dropping into a fresh session an orientation point without querying the GitHub API themselves. - -### Layer 4: CDC Feed (`feed/brain.xml`) +Each rewrite destroys **its own** object's documentation and leaves the other +alone. So a statement that says nothing about documentation — adding an +attribute, changing a flow — silently deletes whatever was promoted there. +`mx check` is clean throughout: a document with no documentation is valid. + +This is the guard-don't-drop class, and it is **fatal to tier 1 as the brief +describes it**. "Knowledge attached to the object travels with the object and is +deleted with it" is true, and the unstated half is that it is also deleted by an +ordinary edit that has nothing to do with the knowledge. An agent that promotes a +decision into a microflow's documentation and later adds a parameter has thrown +the decision away, with every signal reporting success. + +**Consequence for the design — since fixed.** Tier 1 is the strongest idea in the +brief and was **blocked** until rewrites preserved documentation the statement +does not restate. That was a fix in mxcli's writers, not in the brain. It has +landed: every rewrite path now carries the stored value when the statement is +silent, the way it already carried folder, allowed module roles and element +identity, and a fixture asserts survival for **all 29 rewrite-capable document +types** (`documentation_preserved_test.go`, with an untouched-object control and +an empty-comment-clears case). A source-scanning guard fails when a new +rewrite-capable type appears without one. Tier 1 is therefore a phase-3 task +rather than a precondition of it. + +Two mistakes made on the way there are worth carrying into the brain's own +tests. A control that does not *compile* is not a control — deleting the carry +block failed on unused variables instead of failing the test, so the condition +had to be stubbed to `if false` instead. And a type counted as done because the +carry was written, while the statement had a second update path the fixture never +exercised; "done" had to be redefined as **carried and covered**. + +A caveat on this measurement: it was made with a corrected test. The first +version chained `&& echo SURVIVED` to `head -1`, which exits 0 on empty input, so +it reported success regardless of what grep found — the same shape as the test +failures catalogued in §3. + +### 2.2 What does the catalog give us, and how fast? — Everything needed, and it is free + +The `objects` view (`mdl/catalog/tables.go:1028`) unions **43 document types** +with a `QualifiedName` column — module, entity, association, microflow, +nanoflow, rule, page, snippet, layout, workflow, and so on. Resolving +`Sales.ACT_Order_Approve` is one equality query. + +Measured against a real project's catalog (`/home/vscode/ord`, 382 objects, +1.6 MB SQLite): -An Atom feed that publishes insight events whenever the wiki gains new or significantly revised knowledge. This is the machine-readable layer that other brains — personal (meowary) or project — subscribe to and integrate. - -**Feed schema** uses a `brain:` namespace for knowledge-specific metadata: - -```xml - - - mxcli Project Brain - CDC feed — insight events from the mxcli knowledge base - - [PAGES_URL]/feed/brain.xml - [ISO8601] - mxcli Brain Agent - - +``` +resolve one anchor: 0.038 ms (mean of 1000, no index on QualifiedName) ``` -Each entry includes: -- `brain:event_type` — `concept-created` | `concept-revised` | `adr-added` | `adr-superseded` | `divergence-detected` | `synthesis-updated` -- `brain:confidence` — 0.0–1.0 (agent confidence in the insight) -- `brain:trigger` — `empirical` | `discussion` | `research` | `contradiction` | `synthesis` -- `brain:supersedes` — URI of the previous entry if this revises an earlier one - -**Feed entries are append-only.** Never delete or modify existing entries. The feed is the immutable audit log of the project brain's evolution. - -A `feed/.feedmeta` file holds feed configuration and the list of external brain feed URLs to sync from (e.g., meowary feeds from contributors). - -**Emission rules** — a feed entry must be emitted when: -- A new page is created in `wiki/design/` (ADR) or `wiki/pipeline/` -- An existing ADR is superseded or deprecated -- A `wiki/extending/` guide is substantially revised -- A DIVERGENCE is detected during `/brain-sync` -- `wiki/CURRENT.md` is regenerated with a materially different project state - -## Agent Commands - -### `/brain-ingest` -Processes new files in `docs/raw/`: -1. Read each unprocessed file -2. Extract key concepts, decisions, insights -3. Create or update relevant wiki pages with cross-links to existing pages -4. Mark source file as processed (add `processed: true` to frontmatter) -5. Update `wiki/log.md` -6. Evaluate each changed wiki page against feed emission rules; append entries to `feed/brain.xml` - -### `/brain-sync` -Consumes external brain feeds listed in `feed/.feedmeta`: -1. For each new feed item since last sync, determine if it extends, confirms, or contradicts current wiki knowledge -2. If extends: draft wiki update (propose, do not auto-commit) -3. If contradicts: create a DIVERGENCE entry in `wiki/synthesis/open-questions.md` -4. If confirms: note corroboration on the relevant wiki page -5. Emit a feed event summarising what was integrated - -This is the primary mechanism for meowary (retran's personal second brain) to contribute knowledge back to the project brain, and for the project brain to receive updates from contributor systems. - -### `/mxcli-dev:update-wiki` -End-of-feature wiki update command. Reads the current diff, identifies affected topics via `MAP.md`, and drafts updates to relevant wiki pages. Invoked intentionally at the end of a feature as part of the definition of done. - -### `/brain-lint` -Health-checks the wiki: -- Verify all `[[wikilinks]]` resolve -- Check `wiki/README.md` lists all pages -- Flag concepts mentioned across multiple pages but without their own topic page -- Flag ADRs that may be invalidated by recent wiki changes -- Flag MAP.md entries whose source path no longer exists -- Report `wiki/design/` entries whose status has not been reviewed in 90 days - -## Freshness Enforcement +A hundred anchors is under 4 ms. **Speed is a non-issue and should not shape the +design.** + +Two caveats that do: + +- **The `objects` view indexes only describable types.** A recorded finding says + it in as many words — "do not measure coverage from the catalog… enumerate raw + unit `$Type`s instead". An anchor to a document type outside the view resolves + as *missing*, which is a false staleness signal. `check` must distinguish + "resolved", "not found", and **"cannot be resolved by this index"**, and only + the middle one is a failure. +- **Member-level anchors need a second query.** `@Sales.Order` resolves through + `objects`; `@Sales.Order.Status` does not — attributes live in + `attributes_data` (`EntityQualifiedName` + `Name`, 254 rows in the sample + project). Support both or document that anchors are document-scoped; do not + let an attribute anchor silently fail. + +**Staleness of the catalog itself** is an `.mpr` **mtime** comparison +(`cmd_catalog.go:296`). `brain check --ci` on a fresh clone gets fresh mtimes, so +CI rebuilds the catalog every run. That cost is unmeasured here and needs a +number before `--ci` is promised. + +### 2.3 Does the Starlark engine support generated rules? — The question does not arise + +Rules are **discovered from files**: `FindLintRulesDir` walks up for +`.claude/lint-rules/`, and every `*.star` in it is loaded. There is no compiled-in +registry to extend, so "a rule generated at runtime from a template" is just **a +generated file**. No engine work is needed. + +One hazard the brief does not mention: `mxcli init` writes the bundled rules into +that same directory with `os.WriteFile` per file (`init.go:351`). It does **not** +wipe the directory, so a generated rule survives an upgrade — *unless its +filename collides with a shipped one*. Generated rules therefore need a reserved +prefix (`brain_*.star`) and a rule-ID namespace outside the shipped `ARCH` / +`CONV` / `QUAL` / `SEC` / `MDL` sets. + +### 2.4 Release and skill mechanics — the brief's premise is wrong + +**There is no goreleaser.** No `.goreleaser.yml` exists. Releases run +`make release` from `.github/workflows/release.yml`. + +Skills ship by embedding: `//go:embed all:skills` over `cmd/mxcli/skills/` +(`skills_content.go:26`), which `make sync-skills` mirrors from +`.claude/skills/mendix/` with **`rsync --delete`**. So shipping a skill means +adding `.claude/skills/mendix//SKILL.md` and nothing else — and editing the +embed directory directly is always wrong, because the next sync deletes it. The +`all:` prefix is load-bearing (a plain `go:embed` skips `_`-prefixed files). + +`.mxcli/` **is** gitignored by `mxcli init` (`constant_gitignore.go`), so the +brief's split between committed docs and tool-owned state holds as written. + +## 3. Evidence from an analogous store — with the differences stated + +mxcli maintains a store of a similar shape: the bug findings under +`.claude/skills/fix-issue/findings/`, digested into `docs-wiki/bug-patterns/`. + +**It is not this feature and the audience is different.** That store is for +developing *mxcli itself* — a Go repository, many parallel agent sessions, +hundreds of entries accumulated over months, read by people working on the tool. +The brain proposed here is for a *user's Mendix project*: one developer and their +agent, a few dozen lines, read by someone who may never see mxcli's source. The +scale differs by two orders of magnitude and the number of concurrent writers by +more. + +So this is an analogy, not a precedent. Three of its failures transfer, one does +not, and saying which is the point of including it. + +**Transfers — unbounded growth destroys the artifact.** The findings began as a +table inside a skill file and reached 1.05 MB across 630 rows: past a context +window, past what GitHub's web editor will open. The instruction to read it +before diagnosing was unfollowable for months and nobody noticed, because an +unread file has no failure mode. This transfers *more* strongly here, not less: +a project brain is loaded into an agent's context every session, so its size is a +recurring tax rather than an occasional one. **The brief's caps are the single +most important thing in it**, and `promote` refusing when a cap would be exceeded +is the right enforcement point. + +**Transfers — a curation step with no trigger stops.** Three digest pages were +written on one day and none was re-synced for three months while the corpus grew. +The step was on-demand and nothing demanded it. The brain has the same shape: +`staged.jsonl` fills automatically and `promote` is manual. What eventually +worked in the analogous case was printing the gap from a command that already +runs, rather than adding a report someone must remember to invoke. + +**Transfers — self-reported claims go stale.** A README and a PR body both said a +check ran in CI when it did not; coverage figures written into prose were stale +within days. Anything the brain says about itself — its size, its staleness — +should be computed by `brain show` / `brain check`, never written into a +committed file. + +**Does not transfer — silent duplicates from union merges.** `merge=union` let +256 duplicate findings reach the shared corpus unnoticed. That is a +many-parallel-writers problem; a single developer on one project has little +exposure to it. It justifies a cheap duplicate check on `staged.jsonl` and +nothing more, and the earlier draft over-weighted it. + +## 4. Design + +Adopt the brief as written, with the following amendments, each traceable to §2 +or §3. + +| # | Amendment | Because | +|---|---|---| +| A1 | `check` reports three anchor states — resolved, **not found**, **not indexable** — and fails only on the middle one | §2.2: the `objects` view is not a complete inventory | +| A2 | Anchors resolve at document *and* member granularity (`objects` + `attributes_data`) | §2.2: `@Mod.Entity.Attr` is the natural thing to write | +| A3 | Generated lint rules use a `brain_` filename prefix and a `BRAIN###` ID namespace | §2.3: `mxcli init` writes into the same directory | +| A4 | ~~Tier 1 is blocked~~ **Resolved.** Rewrites now carry documentation the statement does not restate, on all 29 rewrite-capable document types | §2.1: was measured broken, now fixture-covered with a control | +| A5 | `staged.jsonl` gets a cheap duplicate check | §3: cheap insurance, but a many-writers problem that mostly does not apply here | +| A6 | `brain show`'s size figure and any coverage number are computed, never written into a committed file | §3: prose figures went stale within days | +| A7 | The gap that motivates curation is printed by a command that already runs, not only by `brain check` | §3: the on-demand digest went three months without a run | +| A8 | Records shard by **anchor scope** — `project.md` for cross-cutting, `modules/.md` for anchored ones — created on demand, with caps applying per shard | §4.1: one file makes a single project-wide budget, and every session pays for every module | +| A9 | The store lives at **`docs/brain/`**, with a `README.md` written by `init` | §4.2: a bare `docs/` gives no signal that the files are brain-managed, and §6 Q2 | -**Hook — flag on edit:** -A `PostToolUse` hook on writes to mapped source paths looks up `MAP.md` and surfaces which wiki topics may be affected. It flags the connection; it does not generate content. +Everything else — the three tiers, the promote-only-through-a-human rule, the +non-goals — stands as written. The non-goals in particular should be treated as +load-bearing. -**Review integration:** -The existing `/mxcli-dev:review` command checks wiki freshness: for each source file changed in the PR, verify the corresponding wiki topic has been touched or explicitly noted as unaffected. +The storage layout and the CLI surface no longer stand as written: A8 and A9 +change the first, and the second has to follow it. §4.1 and §4.2 give the +layout; §4.3 restates the surface against it and §4.4 the skill. Both are +restated in full rather than by reference, because a reader of this proposal +cannot see the brief. -**CI check:** -A Makefile target compares modification dates of source files against their mapped wiki topics and warns when source is newer than documentation by more than a threshold. Staleness becomes visible at PR time. -## Skill Level Mapping +### 4.1 Storage layout at scale -| | Maintainer (mxcli-dev) | User (mendix/) | -|---|---|---| -| **Procedural skills** | `.claude/skills/*.md` | `.claude/skills/mendix/*.md` | -| **Explanatory wiki** | `wiki/` (this proposal) | `docs-site/src/` (exists) | -| **Live state** | `wiki/CURRENT.md` (generated) | — | -| **CDC feed** | `feed/brain.xml` | — | +The brief's single decisions file assumes a project whose decisions fit one +budget. Mendix projects routinely run to 100+ modules, and one file has two +failure modes there — both of which the caps make *worse* rather than better: -## Relationship to Meowary and Federated Brains +- **The cap becomes a project-wide budget.** Recording a `Sales` decision + competes with a `Finance` decision for the same allowance, so `promote` starts + refusing on exactly the projects that most need the store. +- **Every session pays for every module.** The store is loaded into context each + session; an agent working in one module carries the other ninety-nine. -Retran's meowary system (https://github.com/retran/meowary) is a personal second brain with GitHub CLI integration, PARA structure, and session-planning scaffolding. The project brain is the *supply side* that feeds meowary and equivalent systems: +Shard by **anchor scope**, which is derived rather than chosen: -- **Project brain publishes:** `feed/brain.xml` (knowledge events), `wiki/CURRENT.md` (project state) -- **Meowary subscribes:** adds `feed/brain.xml` to `feed/.feedmeta` subscribed_feeds; `/brain-sync` pulls new entries and integrates them into meowary's knowledge base -- **Meowary contributes back:** decisions and insights from retran's sessions can be published to meowary's own feed, which the project brain pulls via `/brain-sync` and proposes as wiki updates - -This is the federated model: each brain (project or personal) publishes a CDC feed; `/brain-sync` connects them bidirectionally. The `brain:` namespace makes feeds from different systems structurally compatible. +``` +docs/brain/ + README.md written by `brain init` — what the folder is, how it is checked + project.md cross-cutting, no anchor: always loaded, the tightest cap + modules/.md anchored to `Module.*`: loaded when that module is in play +``` -## What Is Not Changing +Three properties follow, and they are the reason to shard on this key rather +than by topic or by date: + +1. **There is no index to maintain.** An anchor is `@Sales.Order.Status`; its + module prefix *is* its file name. Routing is a string split — and A6 already + forbids a written index, which would be a self-reported claim that goes stale. +2. **Misfiling becomes a check rather than a style note.** The `objects` view + carries a `ModuleName` column (`mdl/catalog/tables.go:1028`), so `check` can + assert that every anchor in `modules/Sales.md` resolves with + `ModuleName = 'Sales'`. This check cannot exist for a single file: there is + nothing for an entry to be inconsistent *with*. +3. **`check` can scope to a diff.** `--changed` reads git's changed-file list and + validates only the affected shards — the cheap half of the answer to §6 Q1. + +**Shards are created on demand, and their universe is far smaller than the module +count.** Marketplace modules do not carry decisions: Mendix's own guidance is not +to edit them, and mxcli already refuses to write a layout into one. The catalog +distinguishes them (`modules.Source`), so the universe is computable rather than +guessed. Measured on the sample project: -- **`CLAUDE.md`** — remains the primary AI context file as-is. This proposal does not replace or restructure it. The wiki complements CLAUDE.md; it does not supersede it. -- **`.claude/skills/`** — remain the procedural task references. The wiki's `extending/` guides are the narrative counterpart, not a replacement. -- **`docs-site/src/`** — remains the user-facing published documentation. Overlapping internals topics link between the two rather than merging. -- **`docs/11-proposals/`** — remains for in-flight feature proposals. Accepted proposals that establish lasting architectural decisions migrate to `wiki/design/` as ADRs. +``` +9 modules, 7 of them Marketplace (Atlas_Core, Administration, NanoflowCommons, …) + -> 2 modules could ever own a shard +``` -## Summary and Priority +This is emphatically not the "one file per record" shape that turned the +analogous store into 600 files (§3). A shard holds many entries and is keyed by +something the project already has a name for; the file count tracks *modules the +team owns*, not decisions. + +**Phase 2 introduces a third key.** An mxbuild error → resolution record is +anchored to a CE number, not to a module, and belongs in its own `errors.md` +rather than being forced into one of the other two. Naming the axis now — +*anchored to a module, anchored to an error, anchored to nothing* — is cheaper +than discovering it once entries exist. + +### 4.2 Where the store lives + +`docs/brain/`, not `docs/`. + +The brief's argument for `docs/` is reviewability in a pull request. That is +right and is preserved unchanged. What a bare `docs/` does not give is any signal +that the files are brain-managed, and it walks straight into §6 Q2: a Mendix +project's `docs/` may already be the customer's, or Studio Pro's. + +A clearly-named subfolder answers both at once. Dropping a `decisions.md` into +someone else's docs tree is a collision; adding a labelled folder beside their +files is not. + +Two alternatives, and why not: + +- **`brain/` at the repository root.** More discoverable, but a Mendix project + root is already crowded (`mprcontents/`, `theme/`, `themesource/`, + `javascriptsource/`, `resources/`, `deployment/`, `.mxcli/`, `.claude/`, + `.ai-context/`), and discoverability is the skill's job — its `description` is + what decides whether the store is ever consulted at all (§2.4). +- **An `mxcli-` prefix, as in `theme/mxcli-themes/`.** That prefix exists where + mxcli *generates* files and must not clobber the user's. Brain entries are + written by the developer and promoted through a human, so the prefix would + signal tool ownership — the opposite of the intent. The `README.md` carries the + ownership statement instead; because it describes mechanism rather than state, + A6 still holds. + +### 4.3 CLI surface, restated against the sharded layout + +Seven verbs under `mxcli brain`. Sharding changes six of them and deliberately +leaves `capture` alone, so the surface is given in full rather than deferred to +the brief. + +| Command | Behaviour under A8/A9 | +|---|---| +| `brain init` | Creates `docs/brain/` with `README.md` and an empty `project.md`. `modules/` is created empty; shards appear on promotion. Adopts an existing `docs/` rather than claiming it, and **refuses** a `docs/brain/` it did not write (§6 Q2's residue) | +| `brain capture [@anchor…]` | Appends to `.mxcli/brain/staged.jsonl`. **Not sharded** — staging is a queue, not a store, and it is gitignored (§2.4). Sharding a queue buys nothing and costs a routing decision made before a human has looked at the entry | +| `brain staged` | Lists the queue with the shard each entry *would* land in, so the routing is visible before it happens | +| `brain promote [--to ]` | The only writer of a committed file, and still human-invoked. Destination is **derived**: the module of the entry's first anchor, or `project.md` when it has none. `--to project` is the escape hatch for a fact that is cross-cutting despite carrying an anchor | +| `brain drop ` | Removes an entry from the queue or from its shard, and **deletes a shard that becomes empty** — otherwise the directory accumulates husks that read as "this module has decisions" | +| `brain check [--changed] [--ci]` | Per-shard. Reports A1's three anchor states, and **separately** whether an entry is misfiled — a second axis, not a fourth state: an anchor can resolve perfectly and still sit in the wrong shard. `--changed` reads git's changed-file list and checks only the affected shards | +| `brain show []` | Per-shard size and cap headroom, **computed on every run** (A6). No shard's figure is ever written into a committed file, including the `README.md` | + +Two rules the table compresses: + +**The cap is per shard, and `promote` is where it bites.** A promotion that would +push its destination past the cap is refused, naming the shard and its current +occupancy. `project.md` carries the tightest cap of any shard, because it is the +only file loaded unconditionally. + +**Misfiling is a check, with one deliberate relaxation.** `check` requires that +**at least one** of an entry's anchors resolves with `ModuleName` equal to its +shard. Additional anchors into other modules are *reported, not failed* — a fact +like "`Sales.Order` is committed by `Finance.ACT_Post`" is genuinely two-module, +and forcing it into `project.md` would grow the one file that must stay small. +An entry with **zero** anchors in its own shard is misfiled, and that is the +failure the check exists for. + +**A7's host is `mxcli lint`, not `mxcli check`.** The staged-entry count and the +number of shards whose anchors no longer resolve have to surface from something +that already runs; `brain check` alone is a report nothing demands, which is how +the analogous store's curation step went three months without a run (§3). +`check` is the wrong host despite being the more frequently run of the two: it is +scoped to an **MDL script**, so a project-level staleness line has no business in +its output. `lint` already takes `-p app.mpr`, already runs in review, and is +already where the brain has a presence — A3's generated rules surface there. + +### 4.4 The skill + +Ship it at `.claude/skills/mendix/project-brain/SKILL.md`. The embed +(`//go:embed all:skills`) and `mxcli init` handle distribution, and the source of +truth is `.claude/skills/mendix/` — never the embed directory, which +`make sync-skills` rebuilds with `rsync --delete` (§2.4). + +The `description` is the routing mechanism, so it is phrased around symptoms +rather than around the feature: + +```yaml +--- +name: project-brain +description: "Project-specific knowledge mxcli cannot compute — why a pattern was + chosen here, which marketplace version broke what, what a recurring mxbuild error + means in this app. Use before designing something that looks like it was decided + before, and when an mxbuild error is resolved by something non-obvious." +--- +``` -| Component | Problem solved | Effort | -|---|---|---| -| `wiki/pipeline/` + `wiki/extending/` | Orients contributors; raw material exists in docs-site | Low | -| `wiki/design/` (ADRs) | Captures why; prevents decision re-litigation | Low per decision | -| `MAP.md` + hook | Freshness enforcement without manual discipline | Low | -| `docs/raw/` + `/brain-ingest` | Structured intake of source material | Low | -| `feed/brain.xml` + emission rules | CDC feed for federated brain subscriptions | Low | -| `/brain-sync` | Bidirectional meowary ↔ project brain connection | Medium | -| `wiki/CURRENT.md` (scheduled agent) | Live session briefing from GitHub data | Medium | -| `/brain-lint` | Automated health checks | Medium | - -**Recommended order:** `wiki/pipeline/` and `wiki/extending/` first — highest immediate value, raw material already exists. Add `MAP.md` and the hook immediately after. Then `docs/raw/`, `feed/brain.xml`, and `/brain-ingest` together as a batch — they form one coherent intake workflow. `/brain-sync`, CURRENT.md, and `/brain-lint` are independent and can follow in any order. +Sharding gives the skill four instructions it would not otherwise need, and the +first is the one that matters: + +1. **Read `project.md`, plus the shard for each module you are about to touch.** + That set is known before the work starts. **Never read the whole directory** — + the analogous store's failure was a file too large to read, and the sharded + equivalent is an agent that reads every shard and reinstates the cost the + sharding removed. +2. **Ask whether `mxcli` can answer it before writing anything down.** Entities, + microflows, pages, bindings and references are all queryable; a store that + transcribes them is a store that will disagree with the project (§1). +3. **`capture` during work; never `promote`.** Promotion is the human's step, and + the skill should say so rather than leaving an agent to infer it from the + command's absence in its instructions. +4. **Write the anchor, not the name.** `@Sales.Order.Status` is what makes an + entry checkable and routable; the same fact written as prose is neither. + +## 5. Phasing + +Unchanged from the brief, with A4 inserted: + +1. Storage, anchors, and the seven verbs of §4.3 — `init` / `capture` / + `staged` / `promote` / `drop` / `check` / `show` — plus the skill of §4.4. + Markdown destinations only. +2. The mxbuild error → resolution trigger. +3. **Documentation audit**, then promotion into model documentation and lint-rule + generation. No longer gated on A4. + +## 6. Open questions + +1. **What does `brain check --ci` cost on a cold clone?** It needs a catalog, and + catalog validity is an mtime comparison that a fresh checkout always fails. + Unmeasured. If a full build is expensive, `--ci` may need a cheaper anchor + index than the catalog. A8 narrows the question but does not close it: + `--changed` checks only the shards a diff touches, yet the *first* of those + still pays for the catalog build. +2. ~~**Is `docs/` the right home?**~~ **Answered — `docs/brain/` (A9, §4.2).** A + labelled subfolder is safe to add to a `docs/` that is Studio Pro's or a + customer's, which a bare `decisions.md` is not, and the folder name is what + tells a reviewer the files are brain-managed. `init`'s adoption step still has + to handle an existing `docs/brain/` that is *not* ours, which is now the only + collision left. +3. **THEORY.md does not exist.** The issue says to read it and to update it if the + working theory changes. There is no such file anywhere in the repository. Is it + expected to be created, or was another document meant? +4. ~~**Documentation preservation is filed as `mendixlabs/mxcli#1018`.**~~ + **Fixed** — all 29 rewrite-capable document types carry documentation the + statement does not restate, each with a fixture. The shape of the fix was the + one predicted here: carry the stored value the way the rewrite paths already + carry folder, allowed module roles and element identity. Upstream #1018 stays + open until the fork syncs. diff --git a/docs/11-proposals/PROPOSAL_translations.md b/docs/11-proposals/PROPOSAL_translations.md index 9d331e40fa..065b925a53 100644 --- a/docs/11-proposals/PROPOSAL_translations.md +++ b/docs/11-proposals/PROPOSAL_translations.md @@ -1,6 +1,6 @@ --- title: Translations — preserve, describe, author, and auto-translate -status: draft +status: partial date: 2026-08-23 related: - PROPOSAL_catalog_integration.md @@ -9,7 +9,7 @@ related: # Proposal: Translations — preserve, describe, author, and auto-translate -**Status:** Draft +**Status:** Partial — all four slices shipped; see Implementation Status. **Date:** 2026-08-23 A Mendix app ships its user-visible strings in every language it supports. @@ -307,6 +307,38 @@ makes deduplication and hand-editing work. This is the translation-memory problem, and TM tools answer it the same way: keep the dictionary source-keyed, flag "source changed", do not guess. +## Implementation Status + +All four slices below are shipped. What remains is one convenience flag and the +open questions at the end of this document; the feature itself is usable. + +| Slice | State | Landed in | +|-------|-------|-----------| +| 1 — Preservation | done | #245 | +| 2 — The `Texts$Text` overlay | done | #250 | +| 3 — `DESCRIBE TRANSLATIONS` | done | #250 | +| 4 — `CREATE TRANSLATIONS` + drift report | done, except `--untranslated` | #250 | + +Shipped beyond the original plan, because running the feature exposed the need: + +- **Enabled-language statements** — `ALTER SETTINGS ADD/REMOVE/ADD OR MODIFY + LANGUAGE`, which is what Open Question 2 turned into (#257 and follow-ups). + `create translations` now *warns* when the language is not enabled rather than + refusing or enabling it silently. +- **An out-of-scope report** (`mdl/translations/outofscope.go`) — a scoped run + names the entries its own scope kept it from reaching. Ledger #137: `in Ledger` + never reaches the project-level NAVIGATION, so the pages went Dutch and the + sidebar did not, under a message that read as success. +- **A removal form** — an `OR REPLACE` naming nothing takes a language's + translations back out, which is otherwise unexpressible. +- **A lint rule** (`mdl/linter/rules/missing_translations.go`), a skill + (`.claude/skills/mendix/translations/SKILL.md`), and a manual page + (`docs-site/src/language/translations.md`). + +Still unbuilt: **`--untranslated`** on `DESCRIBE`, to emit only the empty targets. +It is a cost optimisation for the LLM loop on an already-translated project, not +a correctness gap — the full describe is what the loop uses today. + ## Implementation Plan Four slices, each shippable alone. **Slice 1 is the bug and should go first.** @@ -419,23 +451,60 @@ observed uniformly (3299 of 3299) on 11.13.0 and is the same value ## Open Questions +Two of the five are settled by shipped work; the resolutions are recorded here +rather than deleted, since each was decided by a measurement. + +### Settled + +2. **The enabled-language list.** ~~Does a translation for an unenabled language + do anything?~~ **Measured**: a stock 11.13 app enables exactly one language + (`en_US`) while its documents carry translations in nine, all from marketplace + modules — so Studio Pro stores and keeps them, and `mx check` passes. But the + app does not *serve* an unenabled language, which makes translating 411 + strings into a language nobody can select the quiet failure here. Resolved by + doing neither of the two things this question offered: `CREATE TRANSLATIONS` + does not enable the language (that is a settings change, and not this + statement's business) and does not refuse (the translations are legitimately + stored) — it **warns**, and `ALTER SETTINGS ADD LANGUAGE` is the fix it names. + Note the trap the skill now documents: `show languages` lists languages that + have *translations*, not enabled ones, so it reports 8 where 1 is enabled. + + +3. **Catalog coverage.** ~~`CATALOG.strings` misses widget captions; worth + widening so `SHOW LANGUAGES` and `search` reflect reality.~~ **Done**, and + wider than the question assumed. Measured on a stock project the index held + **69 of 3265 texts and 8 of 9 languages** — a language present only on an + unindexed site was *invisible*, not undercounted, so `SHOW LANGUAGES` named 8 + and `search` returned nothing for a caption `DESCRIBE TRANSLATIONS` had just + listed. It also blinded the QUAL005 lint rule that shipped with slice 4, which + discovers its language set from the same table. + + The resolution was not to add cases. The five extractors were hand-written per + type, so a sixth site cost a sixth case; the index is now built from **this + proposal's own walk** (`translations.SitesInUnit`), which is what makes the two + subsystems structurally unable to disagree about what the project contains. + `StringContext` names the site (`Forms$ActionButton.Caption`) and `ObjectType` + is derived from the unit `$Type`, so a document type Mendix adds later is + handled with no list to maintain. Atlas design templates are ~70% of the corpus + and are indexed rather than excluded, because `CREATE TRANSLATIONS` writes them + — `ObjectType` is how a consumer filters them out. + +5. **Ordering inside `Items`.** ~~Does Studio Pro care, as it did for widget + `PropertyTypes`?~~ **No** — the patch preserves existing order and appends, + and projects patched this way open in Studio Pro and build clean. What *did* + bite, and was not this, is the **`$ID` form**: a new `Texts$Translation` must + carry a 16-byte `$ID` as its **first** property or the build fails with + `Expected '$ID' as the first property of a storage object`. See + `mdl-examples/bug-tests/translation-id-form.mdl`. + +### Still open + 1. **Homographs.** One source string needing different translations in different - contexts. Mendix's Excel export has the same limitation, so matching it is - defensible — and `in ` scoping covers most real cases. Worth deciding - explicitly rather than discovering. -2. **The enabled-language list.** `Settings$LanguageSettings.Languages` exists in - gen (`modelsdk/gen/settings/types.go:768`) and is surfaced nowhere — - `describe settings` shows only `DefaultLanguageCode`. Does adding a translation - for a language that is not enabled on the project do anything useful? Needs one - measurement in Studio Pro. If not, `CREATE TRANSLATIONS` should enable it or - refuse. -3. **Catalog coverage.** `CATALOG.strings` indexes 21 contexts and misses widget - captions — 39 texts in a page, 2 indexed. Worth widening so `SHOW LANGUAGES` - and `search` reflect reality, but it is a separate change and this proposal - does not depend on it. -4. **The 2 texts with translations but no default language.** Harmless to skip on - export, but the import should not silently create a default-language entry for - them. Confirm what Studio Pro does with such a text. -5. **Ordering inside `Items`.** The patch preserves existing order and appends. - Whether Studio Pro cares (it did for widget `PropertyTypes` — CE0463) is - unverified for texts; a Studio Pro open after an import settles it. + contexts. Unchanged, and now with usage behind it: `DESCRIBE` flags a + conflicting source (3 on a stock app), and `in ` resolves the common + case. Mendix's own Excel export has the same limitation. Worth deciding + explicitly rather than discovering, but nothing has forced the decision yet. + +4. **The 2 texts with translations but no default language.** Skipped on export, + as planned. What Studio Pro does with such a text is still unconfirmed, and + the import still does not invent a default-language entry for them. diff --git a/docs/11-proposals/README.md b/docs/11-proposals/README.md index a14affc701..5ad06d2367 100644 --- a/docs/11-proposals/README.md +++ b/docs/11-proposals/README.md @@ -26,7 +26,7 @@ for display in this README): -## Active Proposals (98) +## Active Proposals (103) ### In Progress (partial) (12) @@ -90,13 +90,15 @@ for display in this README): | [VS Code Search — Quick Pick + Workspace Symbol](PROPOSAL_vscode_search.md) | Proposed | Full-text search exists in mxcli (mxcli search) but is only accessible via the terminal. | | [Workflow Improvements: ALTER WORKFLOW + Cross-References](PROPOSAL_workflow_improvements.md) | Proposed | Workflow support in mxcli has full CREATE/DESCRIBE/DROP/SHOW coverage with 13 activity types and BSON round-trip fidelity. | -### Draft (42) +### Draft (45) | Proposal | Status | Summary | |----------|--------|---------| | [Agent Document Type Support in MDL](PROPOSAL_agent_document_support.md) | Draft | Mendix 11.9 introduces Agents as a first-class concept for building agentic AI applications. | | [Architecture Graph Visualization (communities, layers, god-nodes)](PROPOSAL_architecture_graph_visualization.md) | Draft | Issue: TBD (file before implementation) | | [Association Mapping in IMPORT](PROPOSAL_import_associations.md) | Draft | Parent: PROPOSAL_mxcli_sql.md (Phase 3 extension) | +| [Authorable layouts, page templates and building blocks — CREATE, not COPY](PROPOSAL_authorable_layouts.md) | Draft | An app built entirely through MDL cannot put anything in its own topbar. | +| [Authorable message definitions — the last mapping source MDL cannot write](PROPOSAL_authorable_message_definitions.md) | Draft | A mapping is bound to one of four schema sources. | | [Backend Strategy — adopt engalar's modelsdk base + multi-backend (MCP first)](PROPOSAL_backend_strategy.md) | Draft | - Adopt engalar's modelsdk foundation as the base rather than merging 1109 | | [Bulk Change Custom Widget Properties](PROPOSAL_bulk_widget_property_updates.md) | Draft | Custom widgets (pluggable widgets) in Mendix have complex nested property structures. | | [Bulk External Action Support from OData Contracts](PROPOSAL_external_actions_bulk_create.md) | Draft | Issue #143 requests importing all entities and actions from a consumed OData service. | @@ -106,6 +108,7 @@ for display in this README): | [Expression Type Checking for mxcli check](PROPOSAL_expression_type_checking.md) | Draft | mxcli check is currently a syntactic validator only. | | [Extend UPDATE WIDGETS to Built-in Widgets via Schema Registry](PROPOSAL_update_builtin_widget_properties.md) | Draft | update widgets currently only works for pluggable widgets (ComboBox, DataGrid2, etc.) — it cannot modify properties on built-in widgets like | | [GitHub Actions MDL Integration Tests](proposal-github-mdl-integration.md) | Draft | Add a GitHub Actions workflow that validates MDL example scripts against a real Mendix project after every merge to main. | +| [Import/export mapping coverage — what real mappings use, and the MDL to express it](PROPOSAL_mapping_coverage.md) | Draft | Measured against: 327 mapping documents in 8 demo/marketplace apps | | [Integration Pane — Unified View of External Service Assets](integration-pane-proposal.md) | Draft | Mendix Studio Pro has an Integration Pane that shows all connected services and lists the available assets from their contracts (OData $meta | | [MCP Backend — execute MDL against a live Studio Pro via its MCP server](PROPOSAL_mcp_backend.md) | Draft | Today mxcli writes model changes by editing the .mpr/mprcontents files | | [MCP BSON Benchmark — Correctness Oracle + Efficiency Measurement](PROPOSAL_mcp_bson_benchmark.md) | Draft | Use the Studio Pro MCP server as a dual-purpose tool: | @@ -127,12 +130,12 @@ for display in this README): | [mxcli Playground](mxcli-playground.md) | Draft | A public GitHub repository (mendixlabs/mxcli-playground) containing a ready-to-use Mendix project pre-configured with mxcli, Claude Code ski | | [Owning the modelsdk/gen codegen — why the vendored generator cannot be adopted as-is](PROPOSAL_codegen_ownership.md) | Draft | Constraint set by the maintainer (2026-08-13): the reflection data is an | | [Playwright Session Reuse and Lifecycle Control](PROPOSAL_playwright_session_reuse.md) | Draft | Builds on proposal-playwright-cli.md, which | -| [Project Brain — Persistent Knowledge and Session Scaffolding for Long-Term AI Collaboration](PROPOSAL_project_brain.md) | Draft | mxcli is developed with significant AI involvement, yet the project's knowledge infrastructure is built for static human reading rather than | +| [Project brain — an opt-in store, in a user's Mendix project, for what mxcli cannot compute](PROPOSAL_project_brain.md) | Draft | Audience: users of mxcli building Mendix projects — not mxcli's own | | [RENAME with Reference Refactoring](PROPOSAL_rename_refactoring.md) | Draft | Renaming entities, microflows, pages, and modules is one of the most common refactoring operations. | | [Replace Generated Playwright Tests with playwright-cli](proposal-playwright-cli.md) | Draft | The current approach (documented in proposal-playwright-testing.md) has Claude Code generate TypeScript test files (.spec.ts), then run them | | [Self-Describing Syntax Feature Registry](syntax-feature-registry.md) | Draft | Branch: research/recursive-help-discovery | | [Structured description of irreducible microflow graphs](PROPOSAL_structured_microflow_description.md) | Draft | DESCRIBE MICROFLOW renders a microflow's control flow as nested if/then/else. | -| [Translations — preserve, describe, author, and auto-translate](PROPOSAL_translations.md) | Draft | A Mendix app ships its user-visible strings in every language it supports. | +| [Translations — preserve, describe, author, and auto-translate](PROPOSAL_translations.md) | Partial | A Mendix app ships its user-visible strings in every language it supports. | | [Version-Aware Agent Support](PROPOSAL_version_aware_agent_support.md) | Draft | Three use cases require mxcli to be version-aware at the MDL level: | | [warm dev loop — Docker-free run and iPad split-screen preview](PROPOSAL_mxcli_dev_warm_loop.md) | Draft | Relates to: PROPOSAL_check_mxbuild_gap_heuristics.md (the static-check gate that | | [Workflow / Microflow Syntax Alignment](PROPOSAL_workflow_microflow_syntax_alignment.md) | Draft | MDL spells the same concept differently depending on which document type you are | diff --git a/go.mod b/go.mod index b6ad412aff..dfd3f2e60c 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( go.mongodb.org/mongo-driver/v2 v2.6.0 go.starlark.net v0.0.0-20260102030733-3fee463870c9 go.uber.org/zap v1.28.0 - golang.org/x/crypto v0.55.0 + golang.org/x/crypto v0.56.0 golang.org/x/net v0.57.0 golang.org/x/sync v0.22.0 golang.org/x/term v0.45.0 diff --git a/go.sum b/go.sum index e7cfa37c34..b1910872a8 100644 --- a/go.sum +++ b/go.sum @@ -192,8 +192,8 @@ go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= +golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= diff --git a/mdl-examples/bug-tests/catalog-strings-coverage.mdl b/mdl-examples/bug-tests/catalog-strings-coverage.mdl new file mode 100644 index 0000000000..3920c868e7 --- /dev/null +++ b/mdl-examples/bug-tests/catalog-strings-coverage.mdl @@ -0,0 +1,58 @@ +-- CATALOG.strings reached five hand-written sites, so most of a project's +-- translatable text was never indexed — and a language present only on an +-- unindexed site was INVISIBLE rather than undercounted. +-- +-- Measured on testdata/expr-checker/minimal.mpr (MPR v2), before the fix: +-- +-- indexed actually in the project +-- translatable texts ~69 3265 +-- languages 8 9 <- ar_DZ missing +-- en_US translations 66 1045 +-- extraction sites 5 17 +-- +-- The five were page titles, enum captions and three microflow message +-- templates, each hand-written against a typed reader. Everything else — every +-- widget caption, tooltip, validation message and client template — was +-- unreachable, and a sixth site cost another hand-written case. +-- +-- Two commands showed the split, on the same project: +-- +-- describe translations for nl_NL -> 'Save' as 'Opslaan' +-- search 'Opslaan' -> No matches found. +-- +-- The fix indexes from the type-agnostic Texts$Text walk that DESCRIBE +-- TRANSLATIONS already uses (translations.SitesInUnit), so the two subsystems +-- cannot disagree about what the project contains. +-- +-- After the fix, on the same project: 1496 rows, 9 languages, en_US 1045, +-- nl_NL 333, ar_DZ 4 — identical to an independent BSON walk of the units. +-- +-- CONTROL: stub buildTranslatableStrings and re-run. `strings: 3` (only the +-- non-translatable rows survive), and `show languages` reports nothing at all. +-- Without that line, "9 languages" is equally consistent with a build that +-- never had the fix. + +refresh catalog full; + +-- Every language in the project, not only the ones on a page title. +show languages; + +-- A widget caption is translatable text; before the fix this found nothing. +search 'Opslaan'; + +-- The context now names the site a text lives at, e.g. +-- Forms$ActionButton.Caption, rather than a hand-picked label like page_title. +select StringContext, count(*) as n +from CATALOG.strings +where Language != '' +group by StringContext +order by n desc; + +-- Atlas design templates are ~70% of the corpus and never render in the app. +-- They ARE indexed (DESCRIBE TRANSLATIONS reaches them, so SHOW LANGUAGES must +-- agree), and ObjectType is what lets a consumer filter them out. +select ObjectType, count(*) as n +from CATALOG.strings +where Language != '' +group by ObjectType +order by n desc; diff --git a/mdl-examples/bug-tests/diff-997-false-deletions.mdl b/mdl-examples/bug-tests/diff-997-false-deletions.mdl new file mode 100644 index 0000000000..27091a0234 --- /dev/null +++ b/mdl-examples/bug-tests/diff-997-false-deletions.mdl @@ -0,0 +1,49 @@ +-- ako/mxcli#997 — `mxcli diff` reported false deletions on an UNMODIFIED +-- `mxcli describe` dump, and leaked a Go struct pointer into retrieve +-- constraints. +-- +-- Root cause was one thing, not three: diff rendered its two sides with two +-- different renderers. The project side went through the real describer; the +-- script side went through a second AST-to-MDL renderer whose statement switch +-- covered 18 of 43 activity types and had NO default case — so every java +-- action call, `download file`, `show message` and canvas annotation emitted +-- zero lines and appeared in the diff as a deletion. +-- +-- Repro: run this, then +-- mxcli describe microflow MyFirstModule.ACT_Diff997 -p app.mpr > dump.mdl +-- mxcli diff dump.mdl -p app.mpr +-- Before: "0 new, 1 modified" with the java action, the annotations and the +-- constraint shown as deletions, the constraint re-rendered as `&{0x...}`. +-- After: "0 new, 0 modified, 1 unchanged". +-- +-- The control that matters is the pair: an unmodified dump must be UNCHANGED +-- and a real edit must still show. Change 'Ford' to 'Tesla' in the dump and +-- diff again — one line, and the java action stays as context. + +create or replace microflow MyFirstModule.ACT_Diff997 ( + $Email: String +) +returns Boolean as $Ok +begin + @position(200, 200) + retrieve $Cars from MyFirstModule.Car + where Brand = 'Ford' + limit 1; + @position(400, 200) + $Ok = call java action FeedbackModule.ValidateEmail(EmailAddress = $Email); + @position(600, 200) + return $Ok; +end; + +-- `show message` was also among the 25 unrendered types, and nanoflows went +-- through the same second renderer. +create or replace nanoflow MyFirstModule.NF_Diff997 ($In: String) +returns String as $Out +begin + @position(200, 200) + declare $Out String = $In; + @position(400, 200) + show message 'hi' type Information; + @position(600, 200) + return $Out; +end; diff --git a/mdl-examples/bug-tests/icon-refs-1008-placeholder-snippet-layout-menu.mdl b/mdl-examples/bug-tests/icon-refs-1008-placeholder-snippet-layout-menu.mdl new file mode 100644 index 0000000000..5489f56ed9 --- /dev/null +++ b/mdl-examples/bug-tests/icon-refs-1008-placeholder-snippet-layout-menu.mdl @@ -0,0 +1,97 @@ +-- ============================================================================ +-- mendixlabs/mxcli#1008: icon references that `check --references` never saw +-- ============================================================================ +-- +-- Report: an icon naming an IMAGE collection (Images$ImageCollection) instead of +-- an ICON collection (CustomIcons$CustomIconCollection) passed +-- `mxcli check --references`, executed fine, and only failed at build: +-- +-- [error] [CE1613] "The selected custom icon 'Mod.Icons_SVG.cmdFilter24' +-- no longer exists." at Action button 'abBad' +-- +-- The reported case — a button in a page body — was already caught (the +-- resolver shipped in 0.18.0). Four OTHER shapes were not, because +-- iconRefsInStatement walked only CreatePageStmtV3.Widgets, AlterPageStmt and +-- AlterNavigationStmt. Each of these holds its widgets in a field the walk never +-- visited, so the icon was never seen at all: +-- +-- 1. `placeholder X { … }` — CreatePageStmtV3.Placeholders, held apart from +-- the bare body in .Widgets +-- 2. CREATE SNIPPET — CreateSnippetStmtV3, not in the type switch +-- 3. CREATE LAYOUT — CreateLayoutStmt, not in the type switch +-- 4. CREATE MENU — CreateMenuStmt, whose items carry icons like a +-- profile menu's do +-- +-- All four were measured on Mendix 11.13: `check --references` exit 0 and +-- silent, then `mx check` reporting CE1613 for each one. +-- +-- This script is the POSITIVE control: the same four shapes with VALID icons. +-- It must exec cleanly and build to 0 errors, which is what proves the fix +-- rejects bad references without rejecting good ones — an over-eager walk would +-- break every one of these. +-- +-- To see the refusals, re-run any block with the icon replaced by an image +-- collection reference; `mxcli check --references` now names the collection and +-- lists the real ones, instead of deferring to MxBuild. +-- +-- Verify: +-- mxcli check mdl-examples/bug-tests/icon-refs-1008-*.mdl -p app.mpr --references +-- → Check passed! +-- mxcli exec mdl-examples/bug-tests/icon-refs-1008-*.mdl -p app.mpr +-- mx check app.mpr → 0 errors +-- ============================================================================ + +create module Icon1008; + +-- 1. A page whose button lives in a named layout placeholder, not the body. +create or replace page Icon1008.PlaceholderPage +( title: 'Placeholder', layout: Atlas_Core.Atlas_Default ) +{ + placeholder Main { + actionbutton abInPlaceholder ( + caption: 'Edit', + action: nothing, + icon: 'Atlas_Core.Atlas_Filled.pencil' + ) + } +} + +-- 2. A snippet. +create or replace snippet Icon1008.ButtonSnippet +{ + actionbutton abInSnippet ( + caption: 'Edit', + action: nothing, + icon: 'Atlas_Core.Atlas_Filled.pencil' + ) +} + +-- 3. A layout. The costliest of the four to get wrong: the topbar is shared, so +-- one bad icon here is an error on every page using the layout. +create or replace layout Icon1008.AppFrame +( layouttype: 'Responsive' ) +{ + scrollcontainer layoutContainer { + region top (class: 'region-topbar') { + actionbutton abInLayout ( + caption: 'Edit', + action: nothing, + icon: 'Atlas_Core.Atlas_Filled.pencil' + ) + } + region center (class: 'region-content') { + placeholder Main + } + } +} + +-- 4. A standalone menu document, sub-items included — the same NavMenuItemDef a +-- navigation profile's menu uses, which is why it needs the same recursion. +create or modify menu Icon1008.Main_Menu ( + menu item 'Home' page Icon1008.PlaceholderPage + icon Atlas_Core.Atlas_Filled.home; + menu 'More' icon Atlas_Core.Atlas_Filled.home ( + menu item 'Edit' page Icon1008.PlaceholderPage + icon Atlas_Core.Atlas_Filled.pencil; + ); +); diff --git a/mdl-examples/bug-tests/microflow-993-parameter-position.mdl b/mdl-examples/bug-tests/microflow-993-parameter-position.mdl new file mode 100644 index 0000000000..82e59ba4f1 --- /dev/null +++ b/mdl-examples/bug-tests/microflow-993-parameter-position.mdl @@ -0,0 +1,65 @@ +-- ako/mxcli#993 — @position on a flow parameter. +-- +-- A MicroflowParameter is a stored node with real geometry (RelativeMiddlePoint +-- + Size 30;30), and Studio Pro lets you drag it. Before this, no annotation +-- reached it: a generated flow's parameter block landed on mxcli's derived grid +-- and a hand-aligned one was moved back there by any rewrite. Measured on a real +-- project: a nanoflow parameter at -77;0 came back at 200;53 from a describe → +-- exec of mxcli's own output. +-- +-- Run, then `describe` each flow: NF_ParamPos round-trips its annotations, +-- NF_ParamDerived emits none because its parameters sit exactly where the layout +-- put them. + +create or replace nanoflow MyFirstModule.NF_ParamPos ( + @position(300, 100) + $A: Integer, + @position(200, 100) + $B: Integer +) +returns Integer as $R +begin + @position(300, 200) + declare $R Integer = $A + $B; + @position(500, 200) + return $R; +end; + +-- Control: no annotation, so both parameters go where the layout puts them — +-- 200;53 and 300;53. DESCRIBE must emit no @position for either, or every +-- rewritten flow would come back with its parameters pinned to the grid they +-- happened to be on (the #951 failure, one node family over). +create or replace nanoflow MyFirstModule.NF_ParamDerived ( + $A: Integer, + $B: Integer +) +returns Integer as $R +begin + @position(300, 200) + declare $R Integer = $A + $B; + @position(500, 200) + return $R; +end; + +-- Microflows and rules share the parameter grammar, so they take it too. +create or replace microflow MyFirstModule.MF_ParamPos ( + @position(140, -60) + $A: Integer +) +returns Integer as $R +begin + @position(300, 200) + declare $R Integer = $A + 1; + @position(500, 200) + return $R; +end; + +create or replace rule MyFirstModule.RL_ParamPos ( + @position(60, -40) + $A: Integer +) +returns Boolean +begin + @position(300, 200) + return $A > 0; +end; diff --git a/mdl-examples/bug-tests/navigation-1001-home-page-for-user-role.mdl b/mdl-examples/bug-tests/navigation-1001-home-page-for-user-role.mdl new file mode 100644 index 0000000000..b76411df38 --- /dev/null +++ b/mdl-examples/bug-tests/navigation-1001-home-page-for-user-role.mdl @@ -0,0 +1,82 @@ +-- ============================================================================ +-- mendixlabs/mxcli#1001: HOME PAGE ... FOR takes a USER role, not a module role +-- ============================================================================ +-- +-- `FOR` binds a Mendix **user role**. A user role is project-level, so its +-- identifier is a bare name (`Administrator`). A **module role** is +-- module-scoped (`Administration.Administrator`) and looks almost identical — +-- a blank app has a user role `Administrator` AND module roles of that name in +-- three modules, so the wrong one reads as correct. +-- +-- mxcli's own documentation recommended the module-qualified form, which is the +-- worst of the three possible values. Measured on a blank Mendix 11.13 app: +-- +-- for Administrator → mx check exit 0, 0 errors +-- for Supervisor (bare, unknown) → CE1613 "The selected user role +-- 'Supervisor' no longer exists" +-- for MyFirstModule.Administrator → the project will not LOAD: +-- +-- StorageLoadException: Role based home page in has an invalid value '' +-- for property UserRole. The text 'MyFirstModule.Administrator' is not a +-- valid UserRoleIdentifier. +-- +-- The third is a tier worse than a build error. It happens before checking +-- runs, so there is no error code and no location, and `mx check` exits 1 +-- WITHOUT the "The app contains: N errors" line that people read as the +-- verdict — which is why the report said check "reports success". +-- +-- Nothing resolved the role: it went to BSON verbatim from +-- cmd_navigation.go's `hp.ForRole.String()`. `mxcli check --references` passed +-- and `exec` wrote the unloadable project. Both now refuse it, through the same +-- function, naming the bare form to write instead. +-- +-- This script is the POSITIVE control: the forms that are correct. An +-- over-eager check would refuse these, and the failing direction would not +-- notice. In particular the last block — creating a user role and then using it +-- — is the ordinary way to write one, and must not be refused for naming a role +-- that is not in the project *yet*. +-- +-- To see the refusals, change a `for` value to a qualified name +-- (`for Nav1001.Supervisor`), an unknown name, or the wrong casing +-- (`for supervisor`). +-- +-- Verify: +-- mxcli check mdl-examples/bug-tests/navigation-1001-*.mdl -p app.mpr --references +-- → Check passed! +-- mxcli exec mdl-examples/bug-tests/navigation-1001-*.mdl -p app.mpr +-- mx check app.mpr → 0 errors +-- ============================================================================ + +create module Nav1001; + +create module role Nav1001.User; + +create page Nav1001.Home ( title: 'Home', layout: Atlas_Core.Atlas_Default ) +{ + container cHome { + dynamictext txtHome (content: 'Home') + } +} + +create page Nav1001.AdminHome ( title: 'Admin', layout: Atlas_Core.Atlas_Default ) +{ + container cAdmin { + dynamictext txtAdmin (content: 'Admin') + } +} + +-- 1. A bare user role: the correct form. `Administrator` is a project user +-- role in every Mendix app. +create or replace navigation Responsive + home page Nav1001.Home + home page Nav1001.AdminHome for Administrator; + +-- 2. A user role the script creates itself. The role does not exist in the +-- project when `check` runs, so the check has to read the script's own +-- CREATE USER ROLE statements as well as the project's — otherwise this +-- perfectly valid pairing is refused. +create user role Nav1001_Supervisor (System.User, Nav1001.User); + +create or replace navigation Responsive + home page Nav1001.Home + home page Nav1001.AdminHome for Nav1001_Supervisor; diff --git a/mdl-examples/bug-tests/odata-1020-external-action-types.mdl b/mdl-examples/bug-tests/odata-1020-external-action-types.mdl new file mode 100644 index 0000000000..fcb45f12af --- /dev/null +++ b/mdl-examples/bug-tests/odata-1020-external-action-types.mdl @@ -0,0 +1,89 @@ +-- ============================================================================ +-- mendixlabs/mxcli#1020: an external action call had no types on it +-- ============================================================================ +-- +-- Report: after importing external entities for an OData service whose actions +-- have complex parameter/return types, CE7252 and CE7269 persisted, and +-- re-running CREATE OR MODIFY EXTERNAL ENTITIES never cleared them. +-- +-- It never could. Both errors are raised on Microflows$CallExternalAction — the +-- microflow ACTIVITY — not on the entity. From Mendix 11.13's own +-- Mendix.Modeler.Texts.dll, both LOCATION: CallExternalAction.cs: +-- +-- CE7252 ACTION_PARAMETERS_UNALIGNED +-- "The parameters for remote action '{ACTION}' have changed." +-- CE7269 ACTION_RETURN_TYPE_UNALIGNED +-- "The return type for remote action '{ACTION}' has changed." +-- +-- Two separate omissions in the call, both of the same shape — a DataTypes$ +-- sub-document that was never written: +-- +-- CE7269 The return-type resolver mapped only EDM primitives, so an action +-- returning an ENTITY (or a collection of them) got NO +-- VariableDataType at all. Now resolved to DataTypes$ObjectType / +-- DataTypes$ListType naming the imported external entity. +-- +-- CE7252 ExternalActionParameterMapping.ParameterType was never written, +-- though generated/metamodel declares it WITHOUT omitempty. Measured: +-- a call with ANY parameter — of any type — produced CE7252 plus one +-- CE0117 "Error(s) in expression" per argument, because an argument +-- cannot be type-checked against an untyped parameter. +-- +-- Measured on Mendix 11.13 against a local $metadata with three action shapes: +-- +-- before GetAnyAirport (no params, entity return) -> CE7269 +-- FindAirport (one string param) -> CE7252 + 1x CE0117 +-- GetNearestAirport (two double params) -> CE7252 + 2x CE0117 +-- after all three -> 0 errors +-- +-- Also fixed alongside: the catalog listed only entity-set-backed external +-- entities, so an action's parameter/return-type entity was missing from +-- CATALOG.external_entities and contract_entities.UsedByExternalEntity was +-- STRUCTURALLY always empty for it — the observation the report was built on. +-- +-- This script cannot run standalone: it needs a consumed OData service whose +-- cached $metadata declares the actions. Point MetadataUrl at a service of your +-- own, or serve a contract locally, then run the blocks in order. The MDL below +-- is exactly what was executed for the measurement above. +-- +-- Verify: +-- mxcli exec -p app.mpr +-- mx check app.mpr -> 0 errors +-- ============================================================================ + +create module Ext; + +create constant Ext.SvcUrl type String default 'http://127.0.0.1:8899/'; + +-- The contract must declare the actions as with an in +-- the EntityContainer. Mendix's `call external action` takes OData ACTIONS +-- only, not Functions (CE7251), and an unbound action with no ActionImport is +-- not part of the service's callable surface. +create odata client Ext.TripPin ( + Version: '1.0', + ODataVersion: OData4, + MetadataUrl: 'http://127.0.0.1:8899/$metadata', + Timeout: 300, + ServiceUrl: @Ext.SvcUrl +); + +-- Imports BOTH kinds of entity: the entity-set-backed one (People) and the +-- action's return type (Airport), which has no entity set and is stored as +-- Rest$ODataEntityTypeSource. +create external entities from Ext.TripPin into Ext; + +-- The three shapes. Each was a separate error before the fix; all three build +-- to 0 errors now. +create microflow Ext.ACT_ExternalActions() returns boolean +begin + -- Entity return, no parameters. Was CE7269. + $A = call external action Ext.TripPin.GetAnyAirport(); + + -- One string parameter. Was CE7252 + CE0117. + $B = call external action Ext.TripPin.FindAirport(code = 'EHAM'); + + -- Two numeric parameters: one CE0117 per argument before the fix. + $C = call external action Ext.TripPin.GetNearestAirport(lat = 52.0, lon = 4.0); + + return true; +end; diff --git a/mdl-examples/doctype-tests/03-page-examples.mdl b/mdl-examples/doctype-tests/03-page-examples.mdl index 42c1cd627d..a2549d50e4 100644 --- a/mdl-examples/doctype-tests/03-page-examples.mdl +++ b/mdl-examples/doctype-tests/03-page-examples.mdl @@ -1953,13 +1953,62 @@ create page PgTest.StyledPage -- MARK: Image Widgets -- ============================================================================ --- Image widgets (Forms$StaticImageViewer, Forms$ImageViewer) are deprecated --- in Mendix 11.x React/optimized client (CE0582). No pluggable replacement --- exists. Image examples are therefore not included in this test file. --- --- If Mendix adds a pluggable image widget in the future, examples can be --- re-added using the same pattern as DATAGRID (pluggable widget template). +-- Image Widget Examples -- ============================================================================ +-- +-- The BUILT-IN image widgets (`staticimage` -> Forms$StaticImageViewer, +-- `dynamicimage` -> Forms$ImageViewer) are deprecated in the Mendix 11.x +-- React/optimized client (CE0582), so they are deliberately not exercised here. +-- +-- `image` is NOT one of them: it routes to the pluggable +-- com.mendix.widget.web.image.Image, which is the current widget and is +-- covered below. This file previously said no pluggable replacement existed +-- and skipped images entirely — so this gate built ZERO image widgets and ran +-- green through two separate image defects: a CE0463 on every image widget +-- mxcli wrote, and an image reference that could not be authored at all. Both +-- are fixed (597db1b2), and both were fixed with nothing here to catch a +-- regression — bug-tests/ documents them but this gate is what CI runs. +-- +-- `Image:` names an image-collection asset by its three-part qualified name. +-- Bare and quoted both parse (DESCRIBE emits the quoted form); the bare form +-- is used below to match every other qualified-name property on a page. +-- Atlas_Core.Layout.logo ships with Atlas_Core in every Mendix app. +-- +-- The property is effectively REQUIRED even though the widget package declares +-- it optional: an image widget with no source fails the build with +-- [Error] "No image selected.", which is what makes this example a real check +-- rather than a smoke test — omit the `Image:` line and mx check goes red. + +create page PgTest.P042_Image +( + title: 'Image Widget', + layout: Atlas_Core.Atlas_Default +) +{ + container cImages { + -- Sized in pixels, so `width` is a VISIBLE property and takes the authored + -- value by the ordinary route. + image imgLogoFixed ( + Image: Atlas_Core.Layout.logo, + WidthUnit: pixels, Width: 36, + AlternativeText: 'Company logo' + ) + + -- Default units, and the case that actually regressed: under `auto` the + -- width and height properties are HIDDEN, and a hidden property must be + -- written with its DECLARED default rather than skipped. Skipping leaves + -- whatever value the widget template happened to capture — image.json was + -- extracted from a 48px logo — and 48 against a declared 100 is CE0463 on + -- every image widget. Nothing in the authored MDL below distinguishes the + -- two behaviours, which is exactly why it needs a gate rather than a test + -- that asserts what the script says. + image imgLogoAuto (Image: Atlas_Core.Layout.logo) + + -- The URL mode, whose source lives in a text template rather than on the + -- Image field — the two modes reach the picture by different routes. + image imgFromUrl (ImageType: imageUrl, ImageUrl: 'img/logo.svg') + } +} -- MARK: Custom Container diff --git a/mdl-examples/doctype-tests/11-navigation-examples.mdl b/mdl-examples/doctype-tests/11-navigation-examples.mdl index a7e673cd30..7c880f554f 100644 --- a/mdl-examples/doctype-tests/11-navigation-examples.mdl +++ b/mdl-examples/doctype-tests/11-navigation-examples.mdl @@ -71,10 +71,13 @@ create or replace navigation Responsive home page NavTest.NavTest_Home login page Administration.login; --- Set home page with a role-based override +-- Set home page with a role-based override. +-- `for` takes a USER role, written bare. A module-qualified name here +-- (`Administration.Administrator` is a MODULE role) gives a project Mendix +-- cannot load at all -- see bug-tests/navigation-1001-home-page-for-user-role.mdl. create or replace navigation Responsive home page NavTest.NavTest_Home - home page NavTest.NavTest_Home for Administration.Administrator + home page NavTest.NavTest_Home for Administrator login page Administration.login; -- Replace the entire menu tree diff --git a/mdl-examples/doctype-tests/40-message-definition-examples.mdl b/mdl-examples/doctype-tests/40-message-definition-examples.mdl new file mode 100644 index 0000000000..8e22aef60a --- /dev/null +++ b/mdl-examples/doctype-tests/40-message-definition-examples.mdl @@ -0,0 +1,141 @@ +-- ############################################################################ +-- MESSAGE DEFINITION COLLECTIONS +-- ############################################################################ +-- +-- A message definition is one of the four sources a mapping can be bound to, +-- and it was the only non-JSON one a script could create (ako/mxcli#272). +-- Measured across the nine demo apps: 74 of 327 mappings (22.6%) are sourced +-- from one. +-- +-- Unlike an XML schema (which holds an imported .xsd) or an imported web +-- service (which holds a WSDL), a message definition holds nothing external. +-- It is a SELECTION OVER THE DOMAIN MODEL: every element names an entity, an +-- attribute or an association. That is what makes it authorable at all. +-- +-- Almost every stored property is derived, so a statement says only what a +-- person chooses: the collection's name, each definition's name, its root +-- entity, the members, and the occasional rename. + +create module MsgTest; + +create persistent entity MsgTest.Customer ( + FirstName: String(100), + LastName: String(100), + Address: String(200) +); +/ + +create persistent entity MsgTest.Order ( + OrderId: Long, + OrderDate: DateTime, + TotalAmount: Decimal +); +/ + +create persistent entity MsgTest.OrderLine ( + Sku: String(50), + Quantity: Integer +); +/ + +create association MsgTest.Order_Customer from MsgTest.Order to MsgTest.Customer; +create association MsgTest.OrderLine_Order from MsgTest.OrderLine to MsgTest.Order; + +-- ============================================================================ +-- Level 1: a collection with one definition +-- ============================================================================ +-- +-- An ATTRIBUTE is a bare name; an ASSOCIATION is `Assoc/Module.Entity` with its +-- own member list — the same discriminator import and export mappings use. +-- +-- The association's TARGET entity is spelled out, and that is load-bearing. +-- The stored MaxOccurs tracks the DIRECTION of traversal, not the association's +-- type: reaching Customer FROM Order follows the foreign key and is a single +-- object, while reaching Order FROM Customer is the reverse and is a list. +-- Naming the target makes the direction explicit rather than something a reader +-- has to work out. + +create message definition collection MsgTest.MD_Order + folder 'Messages' +( + definition OrderMessage for MsgTest.Order as 'Orders' ( + OrderId, + OrderDate, + TotalAmount as 'Total', + MsgTest.OrderLine_Order/MsgTest.OrderLine as 'Lines' ( + Sku, + Quantity + ), + MsgTest.Order_Customer/MsgTest.Customer ( + FirstName, + LastName, + Address example 'Kerstraat 5, Leiden, Netherlands' + ) + ) +); + +-- ============================================================================ +-- Level 2: several definitions, and the same association both ways +-- ============================================================================ +-- +-- OrderMessage above reaches Customer from Order and gets a single object. +-- CustomerOrders below reaches Order from Customer over the SAME association +-- and gets a list. One association, two directions, two cardinalities. + +create or modify message definition collection MsgTest.MD_Order + folder 'Messages' +( + definition OrderMessage for MsgTest.Order as 'Orders' ( + OrderId, + MsgTest.Order_Customer/MsgTest.Customer ( FirstName ) + ), + definition CustomerOrders for MsgTest.Customer as 'Customers' ( + FirstName, + MsgTest.Order_Customer/MsgTest.Order as 'Orders' ( OrderId, OrderDate ) + ) +); + +-- ============================================================================ +-- Level 3: targeted edits +-- ============================================================================ +-- +-- A whole-document CREATE OR MODIFY is a poor tool for "expose one more +-- attribute": real definitions nest to depth 7, so restating one to add a leaf +-- is the diff-unfriendliness ADR-0003 argues against. ALTER edits the stored +-- document instead, leaving the definitions it does not mention alone. +-- +-- The definition is addressed as Module.Collection.Definition — the same +-- three-part reference WITH MESSAGE DEFINITION uses. + +alter message definition MsgTest.MD_Order.OrderMessage add member TotalAmount; +alter message definition MsgTest.MD_Order.OrderMessage add member LastName in Customer; + +-- SET changes only the element's ExposedName. It is NOT a rename: the attribute +-- itself is untouched, which is why the verb is SET and not RENAME. +alter message definition MsgTest.MD_Order.OrderMessage set member TotalAmount as 'GrandTotal'; + +alter message definition MsgTest.MD_Order.OrderMessage drop member LastName in Customer; + +-- Definitions within the collection. +alter message definition collection MsgTest.MD_Order + add definition LineMessage for MsgTest.OrderLine ( Sku, Quantity ); +alter message definition collection MsgTest.MD_Order rename definition LineMessage to Lines; +alter message definition collection MsgTest.MD_Order drop definition if exists Lines; + +-- ============================================================================ +-- A mapping over a definition +-- ============================================================================ +-- +-- The reference is three parts: the collection, then the definition inside it. + +create import mapping MsgTest.IMM_Order + with message definition MsgTest.MD_Order.OrderMessage +{ + create MsgTest.Order { OrderId = OrderId } +}; + +-- MARK: Browse + +show message definition collections; +show message definition collections in MsgTest; +describe message definition collection MsgTest.MD_Order; diff --git a/mdl/ast/ast_agenteditor.go b/mdl/ast/ast_agenteditor.go index a744350d9f..4eb2281709 100644 --- a/mdl/ast/ast_agenteditor.go +++ b/mdl/ast/ast_agenteditor.go @@ -16,18 +16,19 @@ package ast // [, DeepLinkURL: '...'] // ); type CreateModelStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - Documentation string - Provider string // "MxCloudGenAI" by default - Key *QualifiedName // qualified name of the String constant holding the Portal key - DisplayName string // optional Portal-populated metadata - KeyName string // optional Portal-populated metadata - KeyID string // optional Portal-populated metadata - Environment string // optional Portal-populated metadata - ResourceName string // optional Portal-populated metadata - DeepLinkURL string // optional Portal-populated metadata - CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Provider string // "MxCloudGenAI" by default + Key *QualifiedName // qualified name of the String constant holding the Portal key + DisplayName string // optional Portal-populated metadata + KeyName string // optional Portal-populated metadata + KeyID string // optional Portal-populated metadata + Environment string // optional Portal-populated metadata + ResourceName string // optional Portal-populated metadata + DeepLinkURL string // optional Portal-populated metadata + CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE } func (s *CreateModelStmt) isStatement() {} @@ -59,6 +60,7 @@ type CreateConsumedMCPServiceStmt struct { Folder string // Folder path within module (empty = leave placement alone) Name QualifiedName OuterDocumentation string // /** ... */ doc comment + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears ProtocolVersion string Version string ConnectionTimeoutSeconds int @@ -94,6 +96,7 @@ type CreateKnowledgeBaseStmt struct { Folder string // Folder path within module (empty = leave placement alone) Name QualifiedName Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears Provider string Key *QualifiedName ModelDisplayName string @@ -125,23 +128,24 @@ func (s *AlterKnowledgeBaseStmt) isStatement() {} // CreateAgentStmt represents CREATE AGENT Module.Name (...) [{ body }]. type CreateAgentStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - Documentation string - UsageType string // "Task" or "Conversational" - Description string - Model *QualifiedName // reference to a Model document - Entity *QualifiedName // reference to a domain entity - MaxTokens *int - ToolChoice string - Temperature *float64 - TopP *float64 - SystemPrompt string - UserPrompt string - Variables []AgentVarDef - Tools []AgentToolDef - KBTools []AgentKBToolDef - CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + UsageType string // "Task" or "Conversational" + Description string + Model *QualifiedName // reference to a Model document + Entity *QualifiedName // reference to a domain entity + MaxTokens *int + ToolChoice string + Temperature *float64 + TopP *float64 + SystemPrompt string + UserPrompt string + Variables []AgentVarDef + Tools []AgentToolDef + KBTools []AgentKBToolDef + CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE } func (s *CreateAgentStmt) isStatement() {} diff --git a/mdl/ast/ast_association.go b/mdl/ast/ast_association.go index 0068da89a8..9cd6468f25 100644 --- a/mdl/ast/ast_association.go +++ b/mdl/ast/ast_association.go @@ -99,16 +99,17 @@ func (s StorageType) String() string { // CreateAssociationStmt represents: CREATE ASSOCIATION Module.Name FROM ... TO ... TYPE ... type CreateAssociationStmt struct { - Name QualifiedName - Parent QualifiedName - Child QualifiedName - Type AssociationType - Owner OwnerType - Storage StorageType - DeleteBehavior DeleteBehavior - Documentation string - Comment string - CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE + Name QualifiedName + Parent QualifiedName + Child QualifiedName + Type AssociationType + Owner OwnerType + Storage StorageType + DeleteBehavior DeleteBehavior + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Comment string + CreateOrModify bool // true for CREATE OR MODIFY / CREATE OR REPLACE // IfNotExists is CREATE ASSOCIATION IF NOT EXISTS: skip when it already // exists, leaving the stored definition untouched. IfNotExists bool diff --git a/mdl/ast/ast_businessevents.go b/mdl/ast/ast_businessevents.go index ad13625b90..4ab0071ade 100644 --- a/mdl/ast/ast_businessevents.go +++ b/mdl/ast/ast_businessevents.go @@ -4,13 +4,14 @@ package ast // CreateBusinessEventServiceStmt represents CREATE BUSINESS EVENT SERVICE. type CreateBusinessEventServiceStmt struct { - Name QualifiedName - ServiceName string - EventNamePrefix string - Messages []*BusinessEventMessageDef - CreateOrModify bool - Folder string - Documentation string + Name QualifiedName + ServiceName string + EventNamePrefix string + Messages []*BusinessEventMessageDef + CreateOrModify bool + Folder string + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears } func (s *CreateBusinessEventServiceStmt) isStatement() {} diff --git a/mdl/ast/ast_entity.go b/mdl/ast/ast_entity.go index a7dbc9d96a..321125eed5 100644 --- a/mdl/ast/ast_entity.go +++ b/mdl/ast/ast_entity.go @@ -41,7 +41,12 @@ type CreateEntityStmt struct { EventHandlers []EventHandlerDef // ON BEFORE/AFTER CREATE/COMMIT/DELETE/ROLLBACK CALL ... Position *Position Documentation string - CreateOrModify bool // true for CREATE OR MODIFY + // DocumentationSet records whether the statement carried a `/** … */` + // comment at all, as opposed to carrying an empty one. A rewrite that did + // not mention documentation preserves the stored value; an explicitly empty + // comment clears it (mendixlabs/mxcli#1018). + DocumentationSet bool + CreateOrModify bool // true for CREATE OR MODIFY // IfNotExists is CREATE ENTITY IF NOT EXISTS: skip entirely when the entity // is already there. Unlike CreateOrModify it never touches an existing // definition, so it is the safe way to make a domain script re-runnable. @@ -141,13 +146,14 @@ type OQLQuery struct { // CreateViewEntityStmt represents: CREATE [OR MODIFY|REPLACE] VIEW ENTITY Module.Name (attrs) AS SELECT ... type CreateViewEntityStmt struct { - Name QualifiedName - Attributes []ViewAttribute - Query OQLQuery - Position *Position - Documentation string - CreateOrModify bool - CreateOrReplace bool + Name QualifiedName + Attributes []ViewAttribute + Query OQLQuery + Position *Position + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + CreateOrModify bool + CreateOrReplace bool } func (s *CreateViewEntityStmt) isStatement() {} diff --git a/mdl/ast/ast_enumeration.go b/mdl/ast/ast_enumeration.go index ebc38a9153..0445342ad8 100644 --- a/mdl/ast/ast_enumeration.go +++ b/mdl/ast/ast_enumeration.go @@ -46,11 +46,16 @@ func (s *MoveFolderStmt) isStatement() {} // CreateEnumerationStmt represents: CREATE ENUMERATION Module.Name (values) COMMENT '...' type CreateEnumerationStmt struct { - Name QualifiedName - Values []EnumValue - Documentation string - Folder string // Module folder to place the enumeration in (Bug 12b) - CreateOrModify bool // True if CREATE OR MODIFY was used + Name QualifiedName + Values []EnumValue + Documentation string + // DocumentationSet records whether the statement carried a `/** … */` + // comment at all, as opposed to carrying an empty one. A rewrite that did + // not mention documentation preserves the stored value; an explicitly empty + // comment clears it (mendixlabs/mxcli#1018). + DocumentationSet bool + Folder string // Module folder to place the enumeration in (Bug 12b) + CreateOrModify bool // True if CREATE OR MODIFY was used } func (s *CreateEnumerationStmt) isStatement() {} @@ -89,14 +94,15 @@ func (s *DropEnumerationStmt) isStatement() {} // CreateConstantStmt represents: CREATE CONSTANT Module.Name TYPE type DEFAULT value [COMMENT '...'] type CreateConstantStmt struct { - Name QualifiedName - DataType DataType - DefaultValue any // The default value (can be string, number, boolean, etc.) - Documentation string - Comment string - Folder string // Folder path within module (e.g., "Resources/Constants") - ExposedToClient bool - CreateOrModify bool // True if CREATE OR MODIFY was used + Name QualifiedName + DataType DataType + DefaultValue any // The default value (can be string, number, boolean, etc.) + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Comment string + Folder string // Folder path within module (e.g., "Resources/Constants") + ExposedToClient bool + CreateOrModify bool // True if CREATE OR MODIFY was used } func (s *CreateConstantStmt) isStatement() {} diff --git a/mdl/ast/ast_imagecollection.go b/mdl/ast/ast_imagecollection.go index 926d07ad36..41451cfcb2 100644 --- a/mdl/ast/ast_imagecollection.go +++ b/mdl/ast/ast_imagecollection.go @@ -12,12 +12,13 @@ type ImageItem struct { // // CREATE IMAGE COLLECTION Module.Name [EXPORT LEVEL 'Public'] [COMMENT '...'] [(IMAGE "name" FROM FILE 'path', ...)] type CreateImageCollectionStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - CreateOrModify bool - ExportLevel string // "Hidden" (default) or "Public" - Comment string - Images []ImageItem + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + CreateOrModify bool + ExportLevel string // "Hidden" (default) or "Public" + Comment string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Images []ImageItem } func (s *CreateImageCollectionStmt) isStatement() {} diff --git a/mdl/ast/ast_javaaction.go b/mdl/ast/ast_javaaction.go index c8afea6f22..16677a3b58 100644 --- a/mdl/ast/ast_javaaction.go +++ b/mdl/ast/ast_javaaction.go @@ -22,17 +22,18 @@ type JavaActionParam struct { // EXPOSED AS 'caption' IN 'category' // AS $$ ... $$; type CreateJavaActionStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName // Qualified name (Module.ActionName) - Parameters []JavaActionParam // Input parameters - ReturnType DataType // Return type (can be nil for void) - JavaCode string // The executeAction() body - ExtraCode string // Optional extra code section - Imports []string // Optional additional imports - Documentation string // Optional documentation comment - TypeParameters []string // Type parameter names (e.g., ["pEntity"]) - ExposedCaption string // EXPOSED AS 'caption' - ExposedCategory string // IN 'category' + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName // Qualified name (Module.ActionName) + Parameters []JavaActionParam // Input parameters + ReturnType DataType // Return type (can be nil for void) + JavaCode string // The executeAction() body + ExtraCode string // Optional extra code section + Imports []string // Optional additional imports + Documentation string // Optional documentation comment + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + TypeParameters []string // Type parameter names (e.g., ["pEntity"]) + ExposedCaption string // EXPOSED AS 'caption' + ExposedCategory string // IN 'category' // NotExposed is NOT EXPOSED: remove the toolbox entry. Distinct from an // absent clause, which preserves whatever is stored. NotExposed bool @@ -62,15 +63,16 @@ func (s *DropJavaActionStmt) isStatement() {} // It mirrors CreateJavaActionStmt with an added Platform (Web/Native/Hybrid/All, // default Web). The inline source is JavaScript rather than Java. type CreateJavaScriptActionStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName // Qualified name (Module.ActionName) - Parameters []JavaActionParam // Input parameters - ReturnType DataType // Return type (can be nil for void) - JavaScriptCode string // The exported function body (user code) - Documentation string // Optional documentation comment - TypeParameters []string // Type parameter names (e.g., ["pEntity"]) - ExposedCaption string // EXPOSED AS 'caption' - ExposedCategory string // IN 'category' + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName // Qualified name (Module.ActionName) + Parameters []JavaActionParam // Input parameters + ReturnType DataType // Return type (can be nil for void) + JavaScriptCode string // The exported function body (user code) + Documentation string // Optional documentation comment + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + TypeParameters []string // Type parameter names (e.g., ["pEntity"]) + ExposedCaption string // EXPOSED AS 'caption' + ExposedCategory string // IN 'category' // NotExposed is NOT EXPOSED: remove the toolbox entry. Distinct from an // absent clause, which preserves whatever is stored. NotExposed bool diff --git a/mdl/ast/ast_jsonstructure.go b/mdl/ast/ast_jsonstructure.go index 46d96a200d..68ee75a2c7 100644 --- a/mdl/ast/ast_jsonstructure.go +++ b/mdl/ast/ast_jsonstructure.go @@ -6,12 +6,13 @@ package ast // // CREATE [OR REPLACE] JSON STRUCTURE Module.Name [COMMENT 'doc'] SNIPPET '...json...' [CUSTOM NAME MAP (...)]; type CreateJsonStructureStmt struct { - Name QualifiedName - JsonSnippet string // Raw JSON snippet - Documentation string // Optional documentation comment - Folder string // Optional folder path within module - CreateOrModify bool // true for CREATE OR MODIFY (or OR REPLACE, treated identically) - CustomNameMap map[string]string // Optional: JSON key → custom ExposedName + Name QualifiedName + JsonSnippet string // Raw JSON snippet + Documentation string // Optional documentation comment + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Folder string // Optional folder path within module + CreateOrModify bool // true for CREATE OR MODIFY (or OR REPLACE, treated identically) + CustomNameMap map[string]string // Optional: JSON key → custom ExposedName // CustomItemNameMap names the ITEM element of an array, keyed by the array's // JSON key ("Root" for a root-level array). An item has no key of its own, // so CustomNameMap cannot reach it and its name was derived and unspellable diff --git a/mdl/ast/ast_messagedefinition.go b/mdl/ast/ast_messagedefinition.go new file mode 100644 index 0000000000..dde6cd3d06 --- /dev/null +++ b/mdl/ast/ast_messagedefinition.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package ast + +// A message definition collection is a selection over the domain model: every +// element names an entity, an attribute or an association. That is what makes it +// authorable where a mapping's other two non-JSON sources are not — an XML +// schema holds an imported .xsd and an imported web service holds a WSDL. +// +// It is the source for 74 of the 327 mappings in the demo corpus (22.6%), and +// was the only one of the four a script could not create. + +// CreateMessageDefinitionCollectionStmt represents: +// +// CREATE [OR MODIFY] MESSAGE DEFINITION COLLECTION Module.Name +// [FOLDER 'path'] +// ( definition Name for Module.Entity [as 'Exposed'] ( members ) , ... ); +type CreateMessageDefinitionCollectionStmt struct { + Name QualifiedName + Folder string + CreateOrModify bool + Definitions []*MessageDefinitionDef +} + +func (s *CreateMessageDefinitionCollectionStmt) isStatement() {} + +// MessageDefinitionDef is one `definition for ` inside a +// collection. +// +// Name and ExposedName are independent: measured, 19 of 56 real definitions are +// named something other than their entity, and 52 of 56 root elements carry an +// ExposedName that differs from the entity's name. +type MessageDefinitionDef struct { + Name string + Entity QualifiedName + ExposedName string // "" means: use the entity's own name + Members []*MessageMemberDef +} + +// MessageMemberDef is an exposed attribute or an exposed association. +// +// Association is empty for an attribute. When it is set, Entity names the +// association's TARGET — spelled out rather than inferred, because the stored +// MaxOccurs tracks the DIRECTION of traversal and not the association's type +// (measured: all 927 resolvable associations in the corpus are `Reference`, yet +// 526 store MaxOccurs 1 and 401 store -1). +type MessageMemberDef struct { + // Attribute is the member's name on the holding entity, for a value member. + // An INHERITED attribute is named exactly like an own one — 398 of 3,697 + // real exposed attributes are inherited — and the executor resolves it to + // the entity that DECLARES it, which is what Mendix stores. + Attribute string + + Association QualifiedName // empty for an attribute + Entity QualifiedName // the association's target + + ExposedName string // "" means: use the member's own name + // Example is author-set sample text. Rare (1 of 4,707 elements measured) + // but real, and describe emitting nothing for it is what would make + // describe -> exec lossy. + Example string + Members []*MessageMemberDef +} + +// IsAssociation reports whether the member exposes an association rather than an +// attribute. +func (m *MessageMemberDef) IsAssociation() bool { return m.Association.Name != "" } + +// DropMessageDefinitionCollectionStmt represents: +// DROP MESSAGE DEFINITION COLLECTION Module.Name +type DropMessageDefinitionCollectionStmt struct { + Name QualifiedName +} + +func (s *DropMessageDefinitionCollectionStmt) isStatement() {} + +// AlterMessageDefinitionCollectionStmt adds, drops or renames a DEFINITION +// within a collection. +type AlterMessageDefinitionCollectionStmt struct { + Name QualifiedName + // Op is "ADD", "DROP" or "RENAME". + Op string + // Definition is the definition being added (ADD) or its name (DROP/RENAME). + Definition *MessageDefinitionDef + Target string // DROP / RENAME: the definition's name + NewName string // RENAME only + IfExists bool + IfNotExist bool +} + +func (s *AlterMessageDefinitionCollectionStmt) isStatement() {} + +// AlterMessageDefinitionStmt adds, drops or renames a MEMBER within one +// definition, addressed as Module.Collection.Definition — the same three-part +// reference `WITH MESSAGE DEFINITION` takes. +type AlterMessageDefinitionStmt struct { + // Collection and Definition come from the three-part name. + Collection QualifiedName + Definition string + // Op is "ADD", "DROP" or "SET". + Op string + // Member is the member being added (ADD). + Member *MessageMemberDef + // Target is the member's name for DROP and SET. + Target string + // ExposedName is the new exposed name for SET. SET changes only this — it is + // not a model rename, which is why the keyword is SET and not RENAME. + ExposedName string + // Path addresses a nested member, in exposed names. Members nest to depth 7 + // in the corpus, so this is not an edge case. + Path []string + IfExists bool + IfNotExist bool +} + +func (s *AlterMessageDefinitionStmt) isStatement() {} diff --git a/mdl/ast/ast_microflow.go b/mdl/ast/ast_microflow.go index db6fde1a81..dedcf2aa50 100644 --- a/mdl/ast/ast_microflow.go +++ b/mdl/ast/ast_microflow.go @@ -33,8 +33,15 @@ type ErrorHandlingClause struct { // MicroflowParam represents a microflow parameter. type MicroflowParam struct { - Name string // Parameter name (without $ prefix) - Type DataType // Parameter type + Name string // Parameter name (without $ prefix) + Type DataType // Parameter type + Position *Position // @position(x, y) on the parameter; nil to let the layout place it + // UnknownAnnotations holds annotation names written on the parameter that + // mxcli does not implement there. Collected rather than dropped so MDL059 + // can refuse them: an annotation that parses and does nothing loses whatever + // it was meant to express, silently (#884, the same reasoning one node + // family over). + UnknownAnnotations []string } // MicroflowReturnType represents a microflow return type. @@ -45,14 +52,19 @@ type MicroflowReturnType struct { // CreateMicroflowStmt represents: CREATE MICROFLOW Module.Name (params) RETURNS type BEGIN body END type CreateMicroflowStmt struct { - Name QualifiedName - Parameters []MicroflowParam - ReturnType *MicroflowReturnType - Body []MicroflowStatement - Documentation string - Folder string // Folder path within module (e.g., "Resources/Images") - CreateOrModify bool - Excluded bool // @excluded — document excluded from project + Name QualifiedName + Parameters []MicroflowParam + ReturnType *MicroflowReturnType + Body []MicroflowStatement + Documentation string + // DocumentationSet records whether the statement carried a `/** … */` + // comment at all, as opposed to carrying an empty one. A rewrite that did + // not mention documentation preserves the stored value; an explicitly empty + // comment clears it (mendixlabs/mxcli#1018). + DocumentationSet bool + Folder string // Folder path within module (e.g., "Resources/Images") + CreateOrModify bool + Excluded bool // @excluded — document excluded from project // Expose holds the EXPOSED AS … ACTION clauses. A microflow has two toolbox // entries — one for the microflow editor, one for the workflow editor — so // there can be one of each. @@ -98,14 +110,15 @@ func (s *DropMicroflowStmt) isStatement() {} // CreateNanoflowStmt represents: CREATE NANOFLOW Module.Name (params) RETURNS type BEGIN body END type CreateNanoflowStmt struct { - Name QualifiedName - Parameters []MicroflowParam - ReturnType *MicroflowReturnType - Body []MicroflowStatement - Documentation string - Folder string // Folder path within module - CreateOrModify bool - Excluded bool // @excluded — document excluded from project + Name QualifiedName + Parameters []MicroflowParam + ReturnType *MicroflowReturnType + Body []MicroflowStatement + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Folder string // Folder path within module + CreateOrModify bool + Excluded bool // @excluded — document excluded from project // Expose is parsed but refused: only Microflows$Microflow carries the toolbox // properties. Accepting it in the grammar and explaining the refusal beats a // parse error that says only "no viable alternative". @@ -120,14 +133,15 @@ func (s *CreateNanoflowStmt) isStatement() {} // are the same minus the ones a rule document has no property for (a rule stores // no AllowedModuleRoles, so there is nothing to grant). type CreateRuleStmt struct { - Name QualifiedName - Parameters []MicroflowParam - ReturnType *MicroflowReturnType - Body []MicroflowStatement - Documentation string - Folder string // Folder path within module - CreateOrModify bool - Excluded bool // @excluded — document excluded from project + Name QualifiedName + Parameters []MicroflowParam + ReturnType *MicroflowReturnType + Body []MicroflowStatement + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Folder string // Folder path within module + CreateOrModify bool + Excluded bool // @excluded — document excluded from project // Expose is parsed but refused — see CreateNanoflowStmt.Expose. Expose []ExposeActionClause } diff --git a/mdl/ast/ast_navigation.go b/mdl/ast/ast_navigation.go index dde4b3255c..8dd26a300a 100644 --- a/mdl/ast/ast_navigation.go +++ b/mdl/ast/ast_navigation.go @@ -39,11 +39,12 @@ type NavMenuItemDef struct { // Like CREATE NAVIGATION, this is a full replacement: the item list given is the // document's complete contents, so an omitted item is a removed item. type CreateMenuStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - Items []NavMenuItemDef - CreateOrModify bool // CREATE OR MODIFY / OR REPLACE - Documentation string + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + Items []NavMenuItemDef + CreateOrModify bool // CREATE OR MODIFY / OR REPLACE + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears } func (s *CreateMenuStmt) isStatement() {} diff --git a/mdl/ast/ast_odata.go b/mdl/ast/ast_odata.go index 5d97b9cd5a..9123dbd00f 100644 --- a/mdl/ast/ast_odata.go +++ b/mdl/ast/ast_odata.go @@ -16,6 +16,7 @@ type CreateODataClientStmt struct { ProxyType string Description string Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears Folder string // Folder path within module (e.g., "Integration/APIs") CreateOrModify bool // True if CREATE OR MODIFY was used @@ -84,16 +85,17 @@ func (s *DropODataClientStmt) isStatement() {} // CreateODataServiceStmt represents: CREATE ODATA SERVICE Module.Name (...) AUTHENTICATION ... { ... } type CreateODataServiceStmt struct { - Name QualifiedName - Path string - Version string - ODataVersion string - Namespace string - ServiceName string - Summary string - Description string - Documentation string - Folder string // Folder path within module (e.g., "Integration/APIs") + Name QualifiedName + Path string + Version string + ODataVersion string + Namespace string + ServiceName string + Summary string + Description string + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Folder string // Folder path within module (e.g., "Integration/APIs") // PublishAssociations selects how associations appear in the metadata: // true = as links, false = as an associated object id. The executor // defaults an unspecified value to true, so PublishAssociationsSet records @@ -217,6 +219,7 @@ type CreateExternalEntityStmt struct { AllowCreateChangeLocally *bool // "Allow creating and changing locally" flag Attributes []Attribute // reuse from ast_entity.go Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears CreateOrModify bool // UnknownProperties: see CreateODataServiceStmt.UnknownProperties. diff --git a/mdl/ast/ast_page_v3.go b/mdl/ast/ast_page_v3.go index f8494426c4..5ee5ff6129 100644 --- a/mdl/ast/ast_page_v3.go +++ b/mdl/ast/ast_page_v3.go @@ -37,11 +37,12 @@ type CreatePageStmtV3 struct { // `placeholder { … }` blocks (issue #532). `Widgets` above is the // bare-body content, which binds to the Main placeholder. A `placeholder // Main { … }` block merges into Main. - Placeholders []*PagePlaceholderV3 - Documentation string - IsReplace bool // CREATE OR REPLACE - IsModify bool // CREATE OR MODIFY - Excluded bool // @excluded — document excluded from project + Placeholders []*PagePlaceholderV3 + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + IsReplace bool // CREATE OR REPLACE + IsModify bool // CREATE OR MODIFY + Excluded bool // @excluded — document excluded from project // Pop-up dimensions (issue #661). nil means "not specified" — the executor // applies the Mendix defaults (600 / 600 / false). @@ -62,14 +63,15 @@ type PagePlaceholderV3 struct { // CreateSnippetStmtV3 represents a V3 snippet creation statement. type CreateSnippetStmtV3 struct { - Name QualifiedName - Parameters []PageParameter // From Params: { } block - Variables []PageVariable // From Variables: { } block - Folder string - Widgets []*WidgetV3 - Documentation string - IsReplace bool - IsModify bool + Name QualifiedName + Parameters []PageParameter // From Params: { } block + Variables []PageVariable // From Variables: { } block + Folder string + Widgets []*WidgetV3 + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + IsReplace bool + IsModify bool } func (s *CreateSnippetStmtV3) isStatement() {} @@ -80,12 +82,13 @@ func (s *CreateSnippetStmtV3) isStatement() {} // on the content wrapper rather than on the layout element — and which // placeholder a page's content goes into. type CreateLayoutStmt struct { - Name QualifiedName - Properties map[string]any - Widgets []*WidgetV3 - Documentation string - IsReplace bool - IsModify bool + Name QualifiedName + Properties map[string]any + Widgets []*WidgetV3 + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + IsReplace bool + IsModify bool } func (s *CreateLayoutStmt) isStatement() {} diff --git a/mdl/ast/ast_query.go b/mdl/ast/ast_query.go index 4d55ed5a6e..82cd695249 100644 --- a/mdl/ast/ast_query.go +++ b/mdl/ast/ast_query.go @@ -75,35 +75,36 @@ const ( ShowNavigationMenu // SHOW NAVIGATION MENU [profile] ShowNavigationHomes // SHOW NAVIGATION HOMES - ShowStructure // SHOW STRUCTURE [DEPTH n] [IN module] [ALL] - ShowWorkflows // SHOW WORKFLOWS [IN module] - ShowBusinessEventServices // SHOW BUSINESS EVENT SERVICES [IN module] - ShowBusinessEventClients // SHOW BUSINESS EVENT CLIENTS [IN module] - ShowBusinessEvents // SHOW BUSINESS EVENTS [IN module] (individual messages) - ShowSettings // SHOW SETTINGS - ShowFragments // SHOW FRAGMENTS - ShowDatabaseConnections // SHOW DATABASE CONNECTIONS [IN module] - ShowImageCollections // SHOW IMAGE COLLECTIONS [IN module] - ShowIconCollections // SHOW ICON COLLECTIONS [IN module] - ShowRestClients // SHOW REST CLIENTS [IN module] - ShowPublishedRestServices // SHOW PUBLISHED REST SERVICES [IN module] - ShowDataTransformers // LIST DATA TRANSFORMERS [IN module] - ShowConstantValues // SHOW CONSTANT VALUES [IN module] - ShowContractEntities // SHOW CONTRACT ENTITIES FROM Module.Service - ShowContractActions // SHOW CONTRACT ACTIONS FROM Module.Service - ShowContractChannels // SHOW CONTRACT CHANNELS FROM Module.Service (AsyncAPI) - ShowContractMessages // SHOW CONTRACT MESSAGES FROM Module.Service (AsyncAPI) - ShowLanguages // SHOW LANGUAGES - ShowJsonStructures // SHOW JSON STRUCTURES [IN module] - ShowImportMappings // SHOW IMPORT MAPPINGS [IN module] - ShowExportMappings // SHOW EXPORT MAPPINGS [IN module] - ShowModels // SHOW MODELS [IN module] (agent-editor Model documents) - ShowAgents // SHOW AGENTS [IN module] (agent-editor Agent documents) - ShowKnowledgeBases // SHOW KNOWLEDGE BASES [IN module] (agent-editor KB documents) - ShowConsumedMCPServices // SHOW CONSUMED MCP SERVICES [IN module] (agent-editor MCP documents) - ShowJarDependencies // LIST JAR DEPENDENCIES [IN module] - ShowBuildingBlocks // SHOW BUILDING BLOCKS [IN module] - ShowConnections // SHOW CONNECTIONS (open external SQL connections in this session) + ShowStructure // SHOW STRUCTURE [DEPTH n] [IN module] [ALL] + ShowWorkflows // SHOW WORKFLOWS [IN module] + ShowBusinessEventServices // SHOW BUSINESS EVENT SERVICES [IN module] + ShowBusinessEventClients // SHOW BUSINESS EVENT CLIENTS [IN module] + ShowBusinessEvents // SHOW BUSINESS EVENTS [IN module] (individual messages) + ShowSettings // SHOW SETTINGS + ShowFragments // SHOW FRAGMENTS + ShowDatabaseConnections // SHOW DATABASE CONNECTIONS [IN module] + ShowImageCollections // SHOW IMAGE COLLECTIONS [IN module] + ShowIconCollections // SHOW ICON COLLECTIONS [IN module] + ShowRestClients // SHOW REST CLIENTS [IN module] + ShowPublishedRestServices // SHOW PUBLISHED REST SERVICES [IN module] + ShowDataTransformers // LIST DATA TRANSFORMERS [IN module] + ShowConstantValues // SHOW CONSTANT VALUES [IN module] + ShowContractEntities // SHOW CONTRACT ENTITIES FROM Module.Service + ShowContractActions // SHOW CONTRACT ACTIONS FROM Module.Service + ShowContractChannels // SHOW CONTRACT CHANNELS FROM Module.Service (AsyncAPI) + ShowContractMessages // SHOW CONTRACT MESSAGES FROM Module.Service (AsyncAPI) + ShowLanguages // SHOW LANGUAGES + ShowJsonStructures // SHOW JSON STRUCTURES [IN module] + ShowMessageDefinitionCollections // SHOW MESSAGE DEFINITION COLLECTIONS [IN module] + ShowImportMappings // SHOW IMPORT MAPPINGS [IN module] + ShowExportMappings // SHOW EXPORT MAPPINGS [IN module] + ShowModels // SHOW MODELS [IN module] (agent-editor Model documents) + ShowAgents // SHOW AGENTS [IN module] (agent-editor Agent documents) + ShowKnowledgeBases // SHOW KNOWLEDGE BASES [IN module] (agent-editor KB documents) + ShowConsumedMCPServices // SHOW CONSUMED MCP SERVICES [IN module] (agent-editor MCP documents) + ShowJarDependencies // LIST JAR DEPENDENCIES [IN module] + ShowBuildingBlocks // SHOW BUILDING BLOCKS [IN module] + ShowConnections // SHOW CONNECTIONS (open external SQL connections in this session) // ShowAnnotations lists a domain model's canvas notes. ShowAnnotations ) @@ -237,6 +238,8 @@ func (t ShowObjectType) String() string { return "LANGUAGES" case ShowJsonStructures: return "JSON STRUCTURES" + case ShowMessageDefinitionCollections: + return "MESSAGE DEFINITION COLLECTIONS" case ShowImportMappings: return "IMPORT MAPPINGS" case ShowExportMappings: @@ -309,43 +312,44 @@ const ( DescribeLayout DescribeConstant DescribeJavaAction - DescribeJavaScriptAction // DESCRIBE JAVASCRIPT ACTION Module.Name - DescribeModuleRole // DESCRIBE MODULE ROLE Module.RoleName - DescribeUserRole // DESCRIBE USER ROLE Name - DescribeDemoUser // DESCRIBE DEMO USER 'name' - DescribeODataClient // DESCRIBE ODATA CLIENT Module.ServiceName - DescribeODataService // DESCRIBE ODATA SERVICE Module.ServiceName - DescribeExternalEntity // DESCRIBE EXTERNAL ENTITY Module.EntityName - DescribeNavigation // DESCRIBE NAVIGATION [profile] - DescribeWorkflow // DESCRIBE WORKFLOW Module.Name - DescribeBusinessEventService // DESCRIBE BUSINESS EVENT SERVICE Module.Name - DescribeDatabaseConnection // DESCRIBE DATABASE CONNECTION Module.Name - DescribeSettings // DESCRIBE SETTINGS - DescribeFragment // DESCRIBE FRAGMENT Name - DescribeImageCollection // DESCRIBE IMAGE COLLECTION Module.Name - DescribeIconCollection // DESCRIBE ICON COLLECTION Module.Name - DescribeRestClient // DESCRIBE REST CLIENT Module.Name - DescribePublishedRestService // DESCRIBE PUBLISHED REST SERVICE Module.Name - DescribeDataTransformer // DESCRIBE DATA TRANSFORMER Module.Name - DescribeContractEntity // DESCRIBE CONTRACT ENTITY Service.EntityName [FORMAT mdl] - DescribeContractAction // DESCRIBE CONTRACT ACTION Service.ActionName [FORMAT mdl] - DescribeContractMessage // DESCRIBE CONTRACT MESSAGE Service.MessageName - DescribeJsonStructure // DESCRIBE JSON STRUCTURE Module.Name - DescribeNanoflow // DESCRIBE NANOFLOW Module.Name - DescribeRule // DESCRIBE RULE Module.Name - DescribeImportMapping // DESCRIBE IMPORT MAPPING Module.Name - DescribeExportMapping // DESCRIBE EXPORT MAPPING Module.Name - DescribeModel // DESCRIBE MODEL Module.Name (agent-editor Model document) - DescribeAgent // DESCRIBE AGENT Module.Name (agent-editor Agent document) - DescribeKnowledgeBase // DESCRIBE KNOWLEDGE BASE Module.Name (agent-editor KB document) - DescribeConsumedMCPService // DESCRIBE CONSUMED MCP SERVICE Module.Name (agent-editor MCP document) - DescribeJarDependency // DESCRIBE JAR DEPENDENCY ModuleName 'group:artifact' - DescribeBuildingBlock // DESCRIBE BUILDING BLOCK Module.Name - DescribeMenu // DESCRIBE MENU Module.Name (standalone Menus$MenuDocument) - DescribeQueue // DESCRIBE QUEUE Module.Name - DescribeScheduledEvent // DESCRIBE SCHEDULED EVENT Module.Name - DescribeRegularExpression // DESCRIBE REGULAR EXPRESSION Module.Name - DescribeAuto // DESCRIBE Module.Name — type auto-detected at execution time + DescribeJavaScriptAction // DESCRIBE JAVASCRIPT ACTION Module.Name + DescribeModuleRole // DESCRIBE MODULE ROLE Module.RoleName + DescribeUserRole // DESCRIBE USER ROLE Name + DescribeDemoUser // DESCRIBE DEMO USER 'name' + DescribeODataClient // DESCRIBE ODATA CLIENT Module.ServiceName + DescribeODataService // DESCRIBE ODATA SERVICE Module.ServiceName + DescribeExternalEntity // DESCRIBE EXTERNAL ENTITY Module.EntityName + DescribeNavigation // DESCRIBE NAVIGATION [profile] + DescribeWorkflow // DESCRIBE WORKFLOW Module.Name + DescribeBusinessEventService // DESCRIBE BUSINESS EVENT SERVICE Module.Name + DescribeDatabaseConnection // DESCRIBE DATABASE CONNECTION Module.Name + DescribeSettings // DESCRIBE SETTINGS + DescribeFragment // DESCRIBE FRAGMENT Name + DescribeImageCollection // DESCRIBE IMAGE COLLECTION Module.Name + DescribeIconCollection // DESCRIBE ICON COLLECTION Module.Name + DescribeRestClient // DESCRIBE REST CLIENT Module.Name + DescribePublishedRestService // DESCRIBE PUBLISHED REST SERVICE Module.Name + DescribeDataTransformer // DESCRIBE DATA TRANSFORMER Module.Name + DescribeContractEntity // DESCRIBE CONTRACT ENTITY Service.EntityName [FORMAT mdl] + DescribeContractAction // DESCRIBE CONTRACT ACTION Service.ActionName [FORMAT mdl] + DescribeContractMessage // DESCRIBE CONTRACT MESSAGE Service.MessageName + DescribeJsonStructure // DESCRIBE JSON STRUCTURE Module.Name + DescribeMessageDefinitionCollection // DESCRIBE MESSAGE DEFINITION COLLECTION Module.Name + DescribeNanoflow // DESCRIBE NANOFLOW Module.Name + DescribeRule // DESCRIBE RULE Module.Name + DescribeImportMapping // DESCRIBE IMPORT MAPPING Module.Name + DescribeExportMapping // DESCRIBE EXPORT MAPPING Module.Name + DescribeModel // DESCRIBE MODEL Module.Name (agent-editor Model document) + DescribeAgent // DESCRIBE AGENT Module.Name (agent-editor Agent document) + DescribeKnowledgeBase // DESCRIBE KNOWLEDGE BASE Module.Name (agent-editor KB document) + DescribeConsumedMCPService // DESCRIBE CONSUMED MCP SERVICE Module.Name (agent-editor MCP document) + DescribeJarDependency // DESCRIBE JAR DEPENDENCY ModuleName 'group:artifact' + DescribeBuildingBlock // DESCRIBE BUILDING BLOCK Module.Name + DescribeMenu // DESCRIBE MENU Module.Name (standalone Menus$MenuDocument) + DescribeQueue // DESCRIBE QUEUE Module.Name + DescribeScheduledEvent // DESCRIBE SCHEDULED EVENT Module.Name + DescribeRegularExpression // DESCRIBE REGULAR EXPRESSION Module.Name + DescribeAuto // DESCRIBE Module.Name — type auto-detected at execution time ) // String returns the human-readable name of the describe object type. @@ -413,6 +417,8 @@ func (t DescribeObjectType) String() string { return "CONTRACT ACTION" case DescribeContractMessage: return "CONTRACT MESSAGE" + case DescribeMessageDefinitionCollection: + return "MESSAGE DEFINITION COLLECTION" case DescribeJsonStructure: return "JSON STRUCTURE" case DescribeNanoflow: diff --git a/mdl/ast/ast_queue.go b/mdl/ast/ast_queue.go index 51b767d678..97bf44be20 100644 --- a/mdl/ast/ast_queue.go +++ b/mdl/ast/ast_queue.go @@ -6,9 +6,10 @@ package ast // // CREATE [OR REPLACE|MODIFY] QUEUE Module.Name ( Parallelism: 3, ClusterWide: true ); type CreateQueueStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - Documentation string + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears // Parallelism is kept as written. Mendix stores it as an expression string // (Queues$BasicQueueConfig.ParallelismExpression), so `3` and `'3'` are the // same thing and an arbitrary expression is legal. diff --git a/mdl/ast/ast_regularexpression.go b/mdl/ast/ast_regularexpression.go index 94c1935428..2327efe138 100644 --- a/mdl/ast/ast_regularexpression.go +++ b/mdl/ast/ast_regularexpression.go @@ -8,9 +8,10 @@ package ast // Expression: '^[a-z]+$' // ); type CreateRegularExpressionStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - Documentation string + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears // Expression is the pattern, as written (unquoted). Expression string ExportLevel string diff --git a/mdl/ast/ast_rest.go b/mdl/ast/ast_rest.go index c1c9fc02a9..27dfc146e8 100644 --- a/mdl/ast/ast_rest.go +++ b/mdl/ast/ast_rest.go @@ -8,14 +8,15 @@ package ast // CreateRestClientStmt represents: CREATE REST CLIENT Module.Name BASE URL '...' AUTHENTICATION ... BEGIN ... END type CreateRestClientStmt struct { - Name QualifiedName - BaseUrl string - Authentication *RestAuthDef // nil = AUTHENTICATION NONE - Operations []*RestOperationDef - Documentation string - Folder string // Folder path within module - CreateOrModify bool // True if CREATE OR MODIFY was used - OpenApiPath string // Non-empty = spec-driven; operations come from spec not OPERATION blocks + Name QualifiedName + BaseUrl string + Authentication *RestAuthDef // nil = AUTHENTICATION NONE + Operations []*RestOperationDef + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears + Folder string // Folder path within module + CreateOrModify bool // True if CREATE OR MODIFY was used + OpenApiPath string // Non-empty = spec-driven; operations come from spec not OPERATION blocks } func (s *CreateRestClientStmt) isStatement() {} diff --git a/mdl/ast/ast_scheduledevent.go b/mdl/ast/ast_scheduledevent.go index 38d35e4eca..6b5a853a8b 100644 --- a/mdl/ast/ast_scheduledevent.go +++ b/mdl/ast/ast_scheduledevent.go @@ -13,9 +13,10 @@ package ast // "not mentioned" stays distinguishable from "mentioned as 0" — 0 is a real // hour, minute and month offset. type CreateScheduledEventStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - Documentation string + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears // Microflow is the qualified name of the microflow to run. Microflow string // Repeat names the schedule variant: Minutely, Hourly, Daily, Weekly, diff --git a/mdl/ast/ast_workflow.go b/mdl/ast/ast_workflow.go index 123f481ef5..0f417f67a2 100644 --- a/mdl/ast/ast_workflow.go +++ b/mdl/ast/ast_workflow.go @@ -4,10 +4,11 @@ package ast // CreateWorkflowStmt represents: CREATE WORKFLOW Module.Name ... type CreateWorkflowStmt struct { - Folder string // Folder path within module (empty = leave placement alone) - Name QualifiedName - CreateOrModify bool - Documentation string + Folder string // Folder path within module (empty = leave placement alone) + Name QualifiedName + CreateOrModify bool + Documentation string + DocumentationSet bool // see mendixlabs/mxcli#1018: absent preserves, empty clears // Context parameter entity ParameterVar string // e.g. "$WorkflowContext" diff --git a/mdl/backend/mapping.go b/mdl/backend/mapping.go index 4c57e327f5..14b915908f 100644 --- a/mdl/backend/mapping.go +++ b/mdl/backend/mapping.go @@ -24,9 +24,14 @@ type MappingBackend interface { MoveExportMapping(em *model.ExportMapping) error // ListMessageDefinitionCollections reads the message-definition documents. - // Read-only: a mapping can be authored OVER a definition (#263), but the - // definitions themselves are not authorable from MDL. ListMessageDefinitionCollections() ([]*model.MessageDefinitionCollection, error) + // CreateMessageDefinitionCollection / Update / Delete author the document + // (ako/mxcli#272). A message definition is a selection over the domain + // model — unlike an XML schema or a WSDL it holds nothing external — which + // is what makes it the one non-JSON mapping source a script can create. + CreateMessageDefinitionCollection(c *model.MessageDefinitionCollection) error + UpdateMessageDefinitionCollection(c *model.MessageDefinitionCollection) error + DeleteMessageDefinitionCollection(id string) error // ListXmlSchemas reads the project's XML schema documents. Read-only — // there is no CREATE for one — and used to resolve a mapping's diff --git a/mdl/backend/mcp/unsupported_gen.go b/mdl/backend/mcp/unsupported_gen.go index 7a31e76a03..9f940925c7 100644 --- a/mdl/backend/mcp/unsupported_gen.go +++ b/mdl/backend/mcp/unsupported_gen.go @@ -194,6 +194,11 @@ func (unsupportedBackend) CreateMenuDocument(_ *types.MenuDocument) (err0 error) return } +func (unsupportedBackend) CreateMessageDefinitionCollection(_ *model.MessageDefinitionCollection) (err0 error) { + err0 = errUnsupported("CreateMessageDefinitionCollection") + return +} + func (unsupportedBackend) CreateMicroflow(_ *microflows.Microflow) (err0 error) { err0 = errUnsupported("CreateMicroflow") return @@ -389,6 +394,11 @@ func (unsupportedBackend) DeleteMenuDocument(_ model.ID) (err0 error) { return } +func (unsupportedBackend) DeleteMessageDefinitionCollection(_ string) (err0 error) { + err0 = errUnsupported("DeleteMessageDefinitionCollection") + return +} + func (unsupportedBackend) DeleteMicroflow(_ model.ID) (err0 error) { err0 = errUnsupported("DeleteMicroflow") return @@ -1262,6 +1272,11 @@ func (unsupportedBackend) UpdateMenuDocument(_ *types.MenuDocument) (err0 error) return } +func (unsupportedBackend) UpdateMessageDefinitionCollection(_ *model.MessageDefinitionCollection) (err0 error) { + err0 = errUnsupported("UpdateMessageDefinitionCollection") + return +} + func (unsupportedBackend) UpdateMicroflow(_ *microflows.Microflow) (err0 error) { err0 = errUnsupported("UpdateMicroflow") return diff --git a/mdl/backend/mock/backend.go b/mdl/backend/mock/backend.go index b0f64e2733..99fdf10e36 100644 --- a/mdl/backend/mock/backend.go +++ b/mdl/backend/mock/backend.go @@ -209,25 +209,28 @@ type MockBackend struct { DeleteDataTransformerFunc func(id model.ID) error // MappingBackend - ListImportMappingsFunc func() ([]*model.ImportMapping, error) - GetImportMappingByQualifiedNameFunc func(moduleName, name string) (*model.ImportMapping, error) - CreateImportMappingFunc func(im *model.ImportMapping) error - UpdateImportMappingFunc func(im *model.ImportMapping) error - DeleteImportMappingFunc func(id model.ID) error - MoveImportMappingFunc func(im *model.ImportMapping) error - ListExportMappingsFunc func() ([]*model.ExportMapping, error) - GetExportMappingByQualifiedNameFunc func(moduleName, name string) (*model.ExportMapping, error) - CreateExportMappingFunc func(em *model.ExportMapping) error - UpdateExportMappingFunc func(em *model.ExportMapping) error - DeleteExportMappingFunc func(id model.ID) error - MoveExportMappingFunc func(em *model.ExportMapping) error - ListXmlSchemasFunc func() ([]*types.XmlSchema, error) - ListJsonStructuresFunc func() ([]*types.JsonStructure, error) - ListMessageDefinitionCollectionsFunc func() ([]*model.MessageDefinitionCollection, error) - GetJsonStructureByQualifiedNameFunc func(moduleName, name string) (*types.JsonStructure, error) - CreateJsonStructureFunc func(js *types.JsonStructure) error - UpdateJsonStructureFunc func(js *types.JsonStructure) error - DeleteJsonStructureFunc func(id string) error + ListImportMappingsFunc func() ([]*model.ImportMapping, error) + GetImportMappingByQualifiedNameFunc func(moduleName, name string) (*model.ImportMapping, error) + CreateImportMappingFunc func(im *model.ImportMapping) error + UpdateImportMappingFunc func(im *model.ImportMapping) error + DeleteImportMappingFunc func(id model.ID) error + MoveImportMappingFunc func(im *model.ImportMapping) error + ListExportMappingsFunc func() ([]*model.ExportMapping, error) + GetExportMappingByQualifiedNameFunc func(moduleName, name string) (*model.ExportMapping, error) + CreateExportMappingFunc func(em *model.ExportMapping) error + UpdateExportMappingFunc func(em *model.ExportMapping) error + DeleteExportMappingFunc func(id model.ID) error + MoveExportMappingFunc func(em *model.ExportMapping) error + ListXmlSchemasFunc func() ([]*types.XmlSchema, error) + ListJsonStructuresFunc func() ([]*types.JsonStructure, error) + ListMessageDefinitionCollectionsFunc func() ([]*model.MessageDefinitionCollection, error) + CreateMessageDefinitionCollectionFunc func(c *model.MessageDefinitionCollection) error + UpdateMessageDefinitionCollectionFunc func(c *model.MessageDefinitionCollection) error + DeleteMessageDefinitionCollectionFunc func(id string) error + GetJsonStructureByQualifiedNameFunc func(moduleName, name string) (*types.JsonStructure, error) + CreateJsonStructureFunc func(js *types.JsonStructure) error + UpdateJsonStructureFunc func(js *types.JsonStructure) error + DeleteJsonStructureFunc func(id string) error // JavaBackend ListJavaActionsFunc func() ([]*types.JavaAction, error) diff --git a/mdl/backend/mock/mock_mapping.go b/mdl/backend/mock/mock_mapping.go index b1441c38d8..cfae74c7d5 100644 --- a/mdl/backend/mock/mock_mapping.go +++ b/mdl/backend/mock/mock_mapping.go @@ -140,6 +140,27 @@ func (m *MockBackend) DeleteJsonStructure(id string) error { } // ListMessageDefinitionCollections returns the configured collections. +func (m *MockBackend) CreateMessageDefinitionCollection(c *model.MessageDefinitionCollection) error { + if m.CreateMessageDefinitionCollectionFunc != nil { + return m.CreateMessageDefinitionCollectionFunc(c) + } + return fmt.Errorf("MockBackend.CreateMessageDefinitionCollection not configured") +} + +func (m *MockBackend) UpdateMessageDefinitionCollection(c *model.MessageDefinitionCollection) error { + if m.UpdateMessageDefinitionCollectionFunc != nil { + return m.UpdateMessageDefinitionCollectionFunc(c) + } + return fmt.Errorf("MockBackend.UpdateMessageDefinitionCollection not configured") +} + +func (m *MockBackend) DeleteMessageDefinitionCollection(id string) error { + if m.DeleteMessageDefinitionCollectionFunc != nil { + return m.DeleteMessageDefinitionCollectionFunc(id) + } + return fmt.Errorf("MockBackend.DeleteMessageDefinitionCollection not configured") +} + func (m *MockBackend) ListMessageDefinitionCollections() ([]*model.MessageDefinitionCollection, error) { if m.ListMessageDefinitionCollectionsFunc != nil { return m.ListMessageDefinitionCollectionsFunc() diff --git a/mdl/backend/modelsdk/mapping_read.go b/mdl/backend/modelsdk/mapping_read.go index d31ea00644..3250620166 100644 --- a/mdl/backend/modelsdk/mapping_read.go +++ b/mdl/backend/modelsdk/mapping_read.go @@ -42,6 +42,12 @@ func (b *Backend) ListImportMappings() ([]*model.ImportMapping, error) { MessageDefinition: g.MessageDefinitionQualifiedName(), ParameterEntity: parameterEntityFromRaw(g.Raw()), MessageDefinition2: messageDefinition2FromRaw(g.Raw()), + WebServiceSource: model.WebServiceMappingSource{ + ImportedWebService: g.ImportedWebServiceQualifiedName(), + ServiceName: g.ServiceName(), + OperationName: g.OperationName(), + RootElementName: g.RootElementName(), + }, } im.ID = model.ID(g.ID()) im.TypeName = "ImportMappings$ImportMapping" @@ -77,6 +83,16 @@ func (b *Backend) ListExportMappings() ([]*model.ExportMapping, error) { MessageDefinition: g.MessageDefinitionQualifiedName(), NullValueOption: g.NullValueOption(), MessageDefinition2: messageDefinition2FromRaw(g.Raw()), + // ParameterName and IsHeader are export-only: which SOAP message + // part the mapping produces, and whether it is a header. + WebServiceSource: model.WebServiceMappingSource{ + ImportedWebService: g.ImportedWebServiceQualifiedName(), + ServiceName: g.ServiceName(), + OperationName: g.OperationName(), + RootElementName: g.RootElementName(), + ParameterName: g.ParameterName(), + IsHeader: g.IsHeader(), + }, } em.ID = model.ID(g.ID()) em.TypeName = "ExportMappings$ExportMapping" @@ -324,6 +340,7 @@ func exportMappingElementFromGen(el element.Element) *model.ExportMappingElement e.ExposedName = o.ExposedName() e.JsonPath = o.JsonPath() e.XmlPath = o.XmlPath() + e.OriginalValue = o.OriginalValue() e.MinOccurs = int(o.MinOccurs()) e.MaxOccurs = int(o.MaxOccurs()) e.MaxLength = int(o.MaxLength()) diff --git a/mdl/backend/modelsdk/mapping_write.go b/mdl/backend/modelsdk/mapping_write.go index 8f9fc18ca6..df1fb81b49 100644 --- a/mdl/backend/modelsdk/mapping_write.go +++ b/mdl/backend/modelsdk/mapping_write.go @@ -436,7 +436,10 @@ func exportValueElementToGen(id string, elem *model.ExportMappingElement, parent addBool(g, "IsKey", elem.IsKey) addBool(g, "IsContent", false) addBool(g, "IsXmlAttribute", false) - addStr(g, "OriginalValue", "") + // Carried, not hardcoded: whether a mapping stores the structure's sample is + // a per-document property, so a rewrite preserves what was there rather than + // deleting it (ako/mxcli#379). A newly authored mapping still gets "". + addStr(g, "OriginalValue", elem.OriginalValue) addStr(g, "XmlPrimitiveType", xmlPrimitiveTypeName(elem.DataType)) return g } diff --git a/mdl/backend/modelsdk/messagedefinition_read.go b/mdl/backend/modelsdk/messagedefinition_read.go index 7bb0b14d4c..717c57d1f1 100644 --- a/mdl/backend/modelsdk/messagedefinition_read.go +++ b/mdl/backend/modelsdk/messagedefinition_read.go @@ -23,31 +23,42 @@ func (b *Backend) ListMessageDefinitionCollections() ([]*model.MessageDefinition } out := make([]*model.MessageDefinitionCollection, 0, len(units)) for _, u := range units { - g := u.Element - c := &model.MessageDefinitionCollection{ - ContainerID: model.ID(u.ContainerID), - Name: g.Name(), - } - c.ID = model.ID(g.ID()) - c.TypeName = "MessageDefinitions$MessageDefinitionCollection" - for _, md := range g.MessageDefinitionsItems() { - em, ok := md.(*genMsg.EntityMessageDefinition) - if !ok { - // A definition kind this reader does not model. Recorded by name - // so a mapping referencing it is refused with "not found" rather - // than silently resolving against nothing. - continue - } - c.Definitions = append(c.Definitions, &model.MessageDefinition{ - Name: em.Name(), - Root: exposedNodeFromGen(em.ExposedEntity()), - }) - } - out = append(out, c) + out = append(out, messageCollectionFromGen(u.Element, string(u.ContainerID))) } return out, nil } +// messageCollectionFromGen converts one stored collection to the semantic model. +// +// Extracted from the loop so the write path's round-trip test can drive the same +// conversion a real read does — a test that rebuilt from a hand-made struct +// would prove nothing about documents Studio Pro actually writes. +func messageCollectionFromGen(g *genMsg.MessageDefinitionCollection, containerID string) *model.MessageDefinitionCollection { + c := &model.MessageDefinitionCollection{ + ContainerID: model.ID(containerID), + Name: g.Name(), + Documentation: g.Documentation(), + Excluded: g.Excluded(), + ExportLevel: g.ExportLevel(), + } + c.ID = model.ID(g.ID()) + c.TypeName = "MessageDefinitions$MessageDefinitionCollection" + for _, md := range g.MessageDefinitionsItems() { + em, ok := md.(*genMsg.EntityMessageDefinition) + if !ok { + // A definition kind this reader does not model. Skipped by name so a + // mapping referencing it is refused with "not found" rather than + // silently resolving against nothing. + continue + } + c.Definitions = append(c.Definitions, &model.MessageDefinition{ + Name: em.Name(), + Root: exposedNodeFromGen(em.ExposedEntity()), + }) + } + return c +} + // exposedNodeFromGen converts one node of a definition's exposed tree. func exposedNodeFromGen(n any) *model.MessageDefinitionElement { switch v := n.(type) { @@ -62,6 +73,8 @@ func exposedNodeFromGen(n any) *model.MessageDefinitionElement { Entity: v.EntityQualifiedName(), ExposedName: v.ExposedName(), ExposedItemName: v.ExposedItemName(), + OriginalName: v.OriginalName(), + Example: v.Example(), Path: v.Path(), MinOccurs: int(v.MinOccurs()), MaxOccurs: int(v.MaxOccurs()), @@ -81,8 +94,11 @@ func exposedNodeFromGen(n any) *model.MessageDefinitionElement { e := &model.MessageDefinitionElement{ Kind: "Entity", Association: v.AssociationQualifiedName(), + Entity: v.EntityQualifiedName(), ExposedName: v.ExposedName(), ExposedItemName: v.ExposedItemName(), + OriginalName: v.OriginalName(), + Example: v.Example(), Path: v.Path(), MinOccurs: int(v.MinOccurs()), MaxOccurs: int(v.MaxOccurs()), @@ -101,6 +117,8 @@ func exposedNodeFromGen(n any) *model.MessageDefinitionElement { Kind: "Attribute", Attribute: v.AttributeQualifiedName(), ExposedName: v.ExposedName(), + OriginalName: v.OriginalName(), + Example: v.Example(), Path: v.Path(), MinOccurs: int(v.MinOccurs()), MaxOccurs: int(v.MaxOccurs()), diff --git a/mdl/backend/modelsdk/messagedefinition_roundtrip_test.go b/mdl/backend/modelsdk/messagedefinition_roundtrip_test.go new file mode 100644 index 0000000000..3c66986c0c --- /dev/null +++ b/mdl/backend/modelsdk/messagedefinition_roundtrip_test.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "fmt" + "os" + "testing" + + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/codec" + genMsg "github.com/mendixlabs/mxcli/modelsdk/gen/messagedefinitions" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// The fixture is ako/TestApp's OrderMessageDefinitions — a collection authored +// by hand in Studio Pro rather than shipped in a marketplace module, which is +// what makes it the right oracle for a WRITE path. +// +// It also happens to contain the control the direction rule needs: the SAME +// association, Mappings.Order_Customer, appears in both definitions and stores +// a different MaxOccurs each time (1 from Order, -1 from Customer). Nothing +// synthetic would have made that as convincing. +const messageFixture = "testdata/TestApp.OrderMessageDefinitions.bson" + +func loadMessageFixture(t *testing.T) (*genMsg.MessageDefinitionCollection, []byte) { + t.Helper() + raw, err := os.ReadFile(messageFixture) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + g := genMsg.NewMessageDefinitionCollection() + g.InitFromRaw(bson.Raw(raw)) + return g, raw +} + +// TestMessageDefinitionCollectionRoundTrips reads a real document into the +// semantic model and writes it back, asserting the re-encoded document is +// semantically identical. +// +// This is the test that caught two derivations being wrong: Path is a chain of +// ORIGINAL names, not exposed ones, and an ASSOCIATION contributes TWO segments +// (its own name, then the target entity's). Both were confirmed afterwards at +// 4,707 of 4,707 elements across this document and the nine demo apps. +func TestMessageDefinitionCollectionRoundTrips(t *testing.T) { + orig, storedBytes := loadMessageFixture(t) + + // Read into the semantic model exactly as the backend does. + c := messageCollectionFromGen(orig, "") + + rebuilt, err := messageCollectionToGen(c) + if err != nil { + t.Fatalf("re-encode: %v", err) + } + after, err := (&codec.Encoder{}).Encode(rebuilt) + if err != nil { + t.Fatalf("encode rebuilt: %v", err) + } + + // Compare against the STORED BYTES, not against a re-encoding of the + // decoded original: a lazily-decoded element that was never marked dirty + // encodes as an empty document, so that baseline would pass by comparing + // nothing to nothing. + diffs := bsonDiff(t, storedBytes, after) + if len(diffs) > 0 { + for _, d := range diffs { + t.Errorf("DIFF %s", d) + } + } +} + +// TestFixtureCarriesTheDirectionControl pins the property that makes this +// fixture worth having: one association, two directions, two cardinalities. +// +// The rule is that MaxOccurs tracks the direction of traversal, not the +// association's type. Order_Customer is FROM Order TO Customer, so reaching +// Customer from Order is single (following the FK) and reaching Order from +// Customer is unbounded (the reverse). Getting this backwards produces a +// definition that exposes a list as a single object, with no build error behind +// it — which is why the control lives in a test rather than a comment. +func TestFixtureCarriesTheDirectionControl(t *testing.T) { + g, _ := loadMessageFixture(t) + c := messageCollectionFromGen(g, "") + + var found []string + var walk func(n *model.MessageDefinitionElement) + walk = func(n *model.MessageDefinitionElement) { + if n == nil { + return + } + if n.Association == "Mappings.Order_Customer" { + found = append(found, fmt.Sprintf("%s:%d", n.Entity, n.MaxOccurs)) + } + for _, c := range n.Children { + walk(c) + } + } + for _, def := range c.Definitions { + walk(def.Root) + } + if len(found) != 2 { + t.Fatalf("expected Order_Customer twice, got %v", found) + } + want := map[string]bool{"Mappings.Customer:1": true, "Mappings.Order:-1": true} + for _, f := range found { + if !want[f] { + t.Errorf("unexpected %s — the direction rule is: holder is FROM -> 1, holder is TO -> -1", f) + } + } +} + +// bsonDiff compares two encoded documents field by field and returns the paths +// that differ, so a failure names what changed rather than dumping two blobs. +func bsonDiff(t *testing.T, a, b []byte) []string { + t.Helper() + var da, db bson.M + if err := bson.Unmarshal(a, &da); err != nil { + t.Fatalf("unmarshal a: %v", err) + } + if err := bson.Unmarshal(b, &db); err != nil { + t.Fatalf("unmarshal b: %v", err) + } + var out []string + var cmp func(path string, x, y any) + cmp = func(path string, x, y any) { + mx, xok := asMap(x) + my, yok := asMap(y) + if xok && yok { + seen := map[string]bool{} + for k := range mx { + seen[k] = true + } + for k := range my { + seen[k] = true + } + for k := range seen { + if k == "$ID" { + continue // freshly minted on rebuild; not a semantic difference + } + cmp(path+"/"+k, mx[k], my[k]) + } + return + } + ax, xok := x.(bson.A) + ay, yok := y.(bson.A) + if xok && yok { + if len(ax) != len(ay) { + out = append(out, fmt.Sprintf("%s: len %d != %d", path, len(ax), len(ay))) + return + } + for i := range ax { + cmp(fmt.Sprintf("%s/%d", path, i), ax[i], ay[i]) + } + return + } + if fmt.Sprint(x) != fmt.Sprint(y) { + out = append(out, fmt.Sprintf("%s: %.90v -> %.90v", path, x, y)) + } + } + cmp("", da, db) + return out +} + +// asMap normalises the two shapes the driver decodes a sub-document into, so the +// walk recurses instead of falling through to a string compare — which is how a +// nested difference hides behind a matching prefix. +func asMap(v any) (bson.M, bool) { + switch t := v.(type) { + case bson.M: + return t, true + case bson.D: + m := make(bson.M, len(t)) + for _, e := range t { + m[e.Key] = e.Value + } + return m, true + default: + return nil, false + } +} diff --git a/mdl/backend/modelsdk/messagedefinition_write.go b/mdl/backend/modelsdk/messagedefinition_write.go new file mode 100644 index 0000000000..e1e7632d74 --- /dev/null +++ b/mdl/backend/modelsdk/messagedefinition_write.go @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/codec" + "github.com/mendixlabs/mxcli/modelsdk/element" + genMsg "github.com/mendixlabs/mxcli/modelsdk/gen/messagedefinitions" + mmpr "github.com/mendixlabs/mxcli/modelsdk/mpr" +) + +// Writing a message definition collection (ako/mxcli#272). +// +// Almost every stored property is a constant or is derived — measured across +// all 36 collections / 56 definitions / 4,686 elements in the demo corpus, with +// no exceptions: +// +// MinOccurs 0 always +// Nillable true always +// IsDefaultType false always +// MaxLength -1 always +// FractionDigits -1 always +// TotalDigits -1 always +// Documentation "" always (as are ErrorMessage/WarningMessage) +// +// Example is the exception and is CARRIED: it is author-set free text, rare (1 +// of 4,707 elements across the corpus and ako/TestApp) but real, and hardcoding +// it empty would silently drop the one that exists. +// ElementType Object/Value derived from the node's kind +// PrimitiveType Unknown for an object, the attribute's type for a value +// OriginalName the entity / attribute / target entity's own name +// Path the position in the tree — see messagePathSegment +// ExposedItemName OriginalName when MaxOccurs is -1, "" otherwise (461/461) +// +// So the caller supplies only what a person chooses, and this fills in the rest. + +func init() { + // Studio Pro writes typed-array marker 2 on these lists; the codec's default + // is 3. Measured across the demo corpus and ako/TestApp — 37 definition + // lists and 575 child lists, all marker 2 — and caught by the round-trip + // test against a real document, which is the only thing that would have. + codec.RegisterListMarker("MessageDefinitions$EntityMessageDefinition", 2) + codec.RegisterListMarker("MessageDefinitions$ExposedEntity", 2) + codec.RegisterListMarker("MessageDefinitions$ExposedAssociation", 2) + codec.RegisterListMarker("MessageDefinitions$ExposedAttribute", 2) + + // Every exposed element serializes Children even when it has none — a leaf + // attribute stores the bare [2]. Same MandatoryLists rule as a rule + // document's Flows and a custom handler's ParameterMappings; omitting it is + // a diff against every stored document, which is exactly what the + // round-trip against ako/TestApp reported before this was here. + for _, t := range []string{ + "MessageDefinitions$ExposedEntity", + "MessageDefinitions$ExposedAssociation", + "MessageDefinitions$ExposedAttribute", + } { + codec.RegisterTypeDefaults(t, codec.TypeDefaults{ + MandatoryListMarkers: map[string]int32{"Children": 2}, + }) + } +} + +// element constants, named rather than repeated so the measurement above has one +// place to be wrong. +const ( + mdMinOccurs = 0 + mdUnbounded = -1 + mdSingle = 1 + mdNillable = true + mdIsDefaultType = false + mdUnsetPrecision = -1 +) + +// CreateMessageDefinitionCollection writes a new collection. +func (b *Backend) CreateMessageDefinitionCollection(c *model.MessageDefinitionCollection) error { + if c == nil { + return fmt.Errorf("CreateMessageDefinitionCollection: nil collection") + } + if b.writer == nil { + return fmt.Errorf("CreateMessageDefinitionCollection: not connected for writing") + } + if c.ID == "" { + c.ID = model.ID(mmpr.GenerateID()) + } + contents, err := encodeMessageDefinitionCollection(c) + if err != nil { + return err + } + return b.writer.InsertUnit(string(c.ID), string(c.ContainerID), "Documents", + "MessageDefinitions$MessageDefinitionCollection", contents) +} + +// UpdateMessageDefinitionCollection rewrites a collection in place, preserving +// its ID — mappings reference the collection by qualified name, and a fresh +// document would break every `WITH MESSAGE DEFINITION`. +func (b *Backend) UpdateMessageDefinitionCollection(c *model.MessageDefinitionCollection) error { + if c == nil { + return fmt.Errorf("UpdateMessageDefinitionCollection: nil collection") + } + if b.writer == nil { + return fmt.Errorf("UpdateMessageDefinitionCollection: not connected for writing") + } + contents, err := encodeMessageDefinitionCollection(c) + if err != nil { + return err + } + return b.writer.UpdateRawUnit(string(c.ID), contents) +} + +// DeleteMessageDefinitionCollection removes a collection by ID. +func (b *Backend) DeleteMessageDefinitionCollection(id string) error { + if b.writer == nil { + return fmt.Errorf("DeleteMessageDefinitionCollection: not connected for writing") + } + return b.writer.DeleteUnit(id) +} + +func encodeMessageDefinitionCollection(c *model.MessageDefinitionCollection) ([]byte, error) { + g, err := messageCollectionToGen(c) + if err != nil { + return nil, err + } + contents, err := (&codec.Encoder{}).Encode(g) + if err != nil { + return nil, fmt.Errorf("message definition collection %s: encode: %w", c.Name, err) + } + return contents, nil +} + +func messageCollectionToGen(c *model.MessageDefinitionCollection) (*genMsg.MessageDefinitionCollection, error) { + g := genMsg.NewMessageDefinitionCollection() + g.SetID(element.ID(c.ID)) + g.SetName(c.Name) + g.SetDocumentation(c.Documentation) + g.SetExcluded(c.Excluded) + exportLevel := c.ExportLevel + if exportLevel == "" { + exportLevel = "Hidden" // what every collection in the corpus stores + } + g.SetExportLevel(exportLevel) + + for _, def := range c.Definitions { + if def == nil { + continue + } + d := genMsg.NewEntityMessageDefinition() + d.SetName(def.Name) + d.SetDocumentation("") + root, err := messageNodeToGen(def.Root, "") + if err != nil { + return nil, err + } + if root == nil { + return nil, fmt.Errorf("message definition %s has no root element", def.Name) + } + d.SetExposedEntity(root) + g.AddMessageDefinitions(d) + } + return g, nil +} + +// messageNodeToGen converts one node, filling in everything derived. +// +// parentPath is the enclosing node's Path. +func messageNodeToGen(n *model.MessageDefinitionElement, parentPath string) (element.Element, error) { + if n == nil { + return nil, nil + } + path := messagePath(parentPath, n) + + if n.Kind == "Attribute" { + a := genMsg.NewExposedAttribute() + a.SetAttributeQualifiedName(n.Attribute) + applyExposedCommon(a, n, path) + a.SetElementType("Value") + a.SetPrimitiveType(n.PrimitiveType) + a.SetMaxOccurs(mdSingle) // a value never repeats: 3697/3697 + a.SetExposedItemName("") + return a, nil + } + + // An object node is an ExposedEntity at the root of a definition and an + // ExposedAssociation everywhere else — the association is what reaches it. + if n.Association == "" { + e := genMsg.NewExposedEntity() + e.SetEntityQualifiedName(n.Entity) + applyExposedCommon(e, n, path) + e.SetElementType("Object") + e.SetPrimitiveType("Unknown") + e.SetMaxOccurs(mdUnbounded) // a definition root repeats: 56/56 + e.SetExposedItemName(n.ExposedItemName) + if err := addMessageChildren(e.AddChildren, n, path); err != nil { + return nil, err + } + return e, nil + } + + a := genMsg.NewExposedAssociation() + a.SetAssociationQualifiedName(n.Association) + a.SetEntityQualifiedName(n.Entity) + applyExposedCommon(a, n, path) + a.SetElementType("Object") + a.SetPrimitiveType("Unknown") + // MaxOccurs is the caller's, because it depends on the DIRECTION of + // traversal and not on the association: measured, all 927 resolvable + // associations in the corpus are `Reference`, yet 526 store 1 and 401 store + // -1. The executor resolves it against the domain model. + a.SetMaxOccurs(int32(n.MaxOccurs)) + a.SetExposedItemName(n.ExposedItemName) + if err := addMessageChildren(a.AddChildren, n, path); err != nil { + return nil, err + } + return a, nil +} + +func addMessageChildren(add func(element.Element), n *model.MessageDefinitionElement, path string) error { + for _, c := range n.Children { + child, err := messageNodeToGen(c, path) + if err != nil { + return err + } + if child != nil { + add(child) + } + } + return nil +} + +// messagePath builds a node's stored Path. +// +// Two things about it are easy to get wrong, and both were, until ako/TestApp's +// hand-authored collection showed the real shape (then confirmed at 4,707 of +// 4,707 elements across it and the demo corpus): +// +// - the chain is of ORIGINAL names, not exposed ones. A root exposed as +// "Orders" contributes "Order". +// +// - an ASSOCIATION contributes TWO segments — the association's own name and +// then the target entity's: +// +// Order|OrderLine_Order|OrderLine|Amount +// ^root ^association ^entity ^attribute +func messagePath(parentPath string, n *model.MessageDefinitionElement) string { + seg := messagePathSegment(n) + if parentPath == "" { + return seg + } + return parentPath + "|" + seg +} + +func messagePathSegment(n *model.MessageDefinitionElement) string { + if n.Association != "" { + return shortName(n.Association) + "|" + n.OriginalName + } + return n.OriginalName +} + +// shortName drops the module from a qualified name. +func shortName(qualified string) string { + if i := strings.LastIndex(qualified, "."); i >= 0 { + return qualified[i+1:] + } + return qualified +} + +// exposedCommon is the property set every node type shares. +type exposedCommon interface { + SetPath(string) + SetOriginalName(string) + SetExposedName(string) + SetMinOccurs(int32) + SetNillable(bool) + SetIsDefaultType(bool) + SetMaxLength(int32) + SetFractionDigits(int32) + SetTotalDigits(int32) + SetDocumentation(string) + SetExample(string) + SetErrorMessage(string) + SetWarningMessage(string) +} + +func applyExposedCommon(e exposedCommon, n *model.MessageDefinitionElement, path string) { + e.SetPath(path) + e.SetOriginalName(n.OriginalName) + e.SetExposedName(n.ExposedName) + e.SetMinOccurs(mdMinOccurs) + e.SetNillable(mdNillable) + e.SetIsDefaultType(mdIsDefaultType) + e.SetMaxLength(mdUnsetPrecision) + e.SetFractionDigits(mdUnsetPrecision) + e.SetTotalDigits(mdUnsetPrecision) + e.SetDocumentation("") + e.SetExample(n.Example) + e.SetErrorMessage("") + e.SetWarningMessage("") +} + +var _ backend.MappingBackend = (*Backend)(nil) diff --git a/mdl/backend/modelsdk/microflow.go b/mdl/backend/modelsdk/microflow.go index 6559a25792..f18d93ebfd 100644 --- a/mdl/backend/modelsdk/microflow.go +++ b/mdl/backend/modelsdk/microflow.go @@ -377,6 +377,13 @@ func splitFlowObjects(coll element.Element) ([]*microflows.MicroflowParameter, [ if po, ok := el.(*genMf.MicroflowParameter); ok { p := µflows.MicroflowParameter{Name: po.Name(), Type: dataTypeFromGen(po.ParameterType())} p.ID = model.ID(el.ID()) + // Carry the parameter's canvas position, but only when it is not the + // one the layout would derive for this index — a parameter sitting on + // the derived grid carries no intent, and pinning it would strand the + // others the moment a parameter is inserted (#993, and #951 before + // it). Without this the position was never read at all, so a rewrite + // moved every hand-placed parameter back onto the grid. + p.Position = microflows.AuthoredParameterPosition(pointFromGen(el), len(params)) params = append(params, p) continue } diff --git a/mdl/backend/modelsdk/microflow_external_action_returntype_test.go b/mdl/backend/modelsdk/microflow_external_action_returntype_test.go new file mode 100644 index 0000000000..145e74e530 --- /dev/null +++ b/mdl/backend/modelsdk/microflow_external_action_returntype_test.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/modelsdk/codec" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// TestExternalActionReturnTypeToGen covers the DataTypes$ element written into +// CallExternalAction.VariableDataType, asserted on the ENCODED document so the +// Entity is checked as a stored key rather than as a Go field. +// +// Object and List were unreachable before: the resolver mapped only EDM +// primitives, so an action returning an entity got no VariableDataType at all +// and Mendix reported CE7269 "The return type for remote action '' has +// changed" (mendixlabs/mxcli#1020). +func TestExternalActionReturnTypeToGen(t *testing.T) { + tests := []struct { + name string + kind string + entity string + wantType string + wantEntity string // "" = the key must be absent + }{ + {name: "object return", kind: "Object", entity: "Trippin.Airport", + wantType: "DataTypes$ObjectType", wantEntity: "Trippin.Airport"}, + {name: "list return", kind: "List", entity: "Trippin.Person", + wantType: "DataTypes$ListType", wantEntity: "Trippin.Person"}, + {name: "boolean", kind: "Boolean", wantType: "DataTypes$BooleanType"}, + {name: "string", kind: "String", wantType: "DataTypes$StringType"}, + {name: "integer", kind: "Integer", wantType: "DataTypes$IntegerType"}, + {name: "long is an integer", kind: "Long", wantType: "DataTypes$IntegerType"}, + {name: "decimal", kind: "Decimal", wantType: "DataTypes$DecimalType"}, + {name: "datetime", kind: "DateTime", wantType: "DataTypes$DateTimeType"}, + {name: "binary", kind: "Binary", wantType: "DataTypes$BinaryType"}, + {name: "void", kind: "Void", wantType: "DataTypes$VoidType"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := externalActionReturnTypeToGen(tt.kind, tt.entity) + if g == nil { + t.Fatal("externalActionReturnTypeToGen returned nil") + } + if g.TypeName() != tt.wantType { + t.Errorf("$Type = %q, want %q", g.TypeName(), tt.wantType) + } + + // Encoding is the real check: a mistyped property fails here rather + // than at Studio Pro. + raw, err := (&codec.Encoder{}).Encode(g) + if err != nil { + t.Fatalf("encode: %v", err) + } + doc := bson.Raw(raw) + + val, lookupErr := doc.LookupErr("Entity") + if tt.wantEntity == "" { + if lookupErr == nil { + t.Errorf("a primitive return must not carry an Entity (got %v)", val) + } + return + } + if lookupErr != nil { + t.Fatalf("Entity key missing — an ObjectType/ListType without one is as unaligned as no type at all") + } + if got, ok := val.StringValueOK(); !ok || got != tt.wantEntity { + t.Errorf("Entity = %v, want %q", val, tt.wantEntity) + } + }) + } +} + +// TestCallExternalActionParameterTypeIsWritten covers the CE7252 half. +// +// generated/metamodel declares ExternalActionParameterMapping.ParameterType +// WITHOUT omitempty, and mxcli never wrote it. Measured on Mendix 11.13: a call +// with any parameter produced CE7252 "The parameters for remote action '' +// have changed" plus one CE0117 "Error(s) in expression" per argument — an +// argument cannot be type-checked against a parameter that has no type. With +// ParameterType written, the same project builds at 0 errors. +func TestCallExternalActionParameterTypeIsWritten(t *testing.T) { + act := µflows.CallExternalAction{ + ConsumedODataService: "Ext.TripPin", + Name: "FindAirport", + ResultVariableName: "Airport", + ResultDataType: "Object", + ResultEntity: "Ext.Airport", + ParameterMappings: []*microflows.ExternalActionParameterMapping{ + {ParameterName: "code", Argument: "'EHAM'", ParameterDataType: "String"}, + {ParameterName: "near", Argument: "$A", ParameterDataType: "Object", ParameterEntity: "Ext.Airport"}, + }, + } + + g := callExternalActionToGen(act) + if g == nil { + t.Fatal("callExternalActionToGen returned nil") + } + raw, err := (&codec.Encoder{}).Encode(g) + if err != nil { + t.Fatalf("encode: %v", err) + } + doc := bson.Raw(raw) + + // The result variable's type still round-trips alongside the new parameter types. + vdt, err := doc.LookupErr("VariableDataType") + if err != nil { + t.Fatal("VariableDataType missing") + } + if got, _ := bson.Raw(vdt.Value).Lookup("$Type").StringValueOK(); got != "DataTypes$ObjectType" { + t.Errorf("VariableDataType $Type = %q, want DataTypes$ObjectType", got) + } + + arr, err := doc.LookupErr("ParameterMappings") + if err != nil { + t.Fatal("ParameterMappings missing") + } + vals, err := bson.Raw(arr.Value).Values() + if err != nil { + t.Fatalf("read ParameterMappings: %v", err) + } + // The typed-array marker leads the list; the mappings follow it. + var seen int + want := map[string][2]string{ + "code": {"DataTypes$StringType", ""}, + "near": {"DataTypes$ObjectType", "Ext.Airport"}, + } + for _, v := range vals { + md, ok := v.DocumentOK() + if !ok { + continue + } + name, _ := md.Lookup("ParameterName").StringValueOK() + exp, tracked := want[name] + if !tracked { + continue + } + seen++ + pt, err := md.LookupErr("ParameterType") + if err != nil { + t.Errorf("%s: ParameterType missing — this is CE7252", name) + continue + } + ptDoc := bson.Raw(pt.Value) + if got, _ := ptDoc.Lookup("$Type").StringValueOK(); got != exp[0] { + t.Errorf("%s: ParameterType $Type = %q, want %q", name, got, exp[0]) + } + if exp[1] != "" { + if got, _ := ptDoc.Lookup("Entity").StringValueOK(); got != exp[1] { + t.Errorf("%s: ParameterType Entity = %q, want %q", name, got, exp[1]) + } + } + } + if seen != len(want) { + t.Errorf("found %d parameter mappings, want %d", seen, len(want)) + } +} diff --git a/mdl/backend/modelsdk/microflow_external_action_write.go b/mdl/backend/modelsdk/microflow_external_action_write.go index 323389b5f4..30fdec21a8 100644 --- a/mdl/backend/modelsdk/microflow_external_action_write.go +++ b/mdl/backend/modelsdk/microflow_external_action_write.go @@ -31,7 +31,7 @@ func callExternalActionToGen(a *microflows.CallExternalAction) element.Element { // CE7269 ("return type has changed"); the executor resolves the kind from the // consumed service's cached $metadata. Omitted for void/unknown. if a.ResultDataType != "" { - addPart(g, "VariableDataType", externalActionReturnTypeToGen(a.ResultDataType)) + addPart(g, "VariableDataType", externalActionReturnTypeToGen(a.ResultDataType, a.ResultEntity)) } mappings := make([]element.Element, 0, len(a.ParameterMappings)) for _, pm := range a.ParameterMappings { @@ -39,6 +39,11 @@ func callExternalActionToGen(a *microflows.CallExternalAction) element.Element { addStr(m, "ParameterName", pm.ParameterName) addStr(m, "Argument", pm.Argument) addBool(m, "CanBeEmpty", pm.CanBeEmpty) + // generated/metamodel declares ParameterType without omitempty. Omitting + // it is CE7252 + a CE0117 per argument. + if pm.ParameterDataType != "" { + addPart(m, "ParameterType", externalActionReturnTypeToGen(pm.ParameterDataType, pm.ParameterEntity)) + } mappings = append(mappings, m) } if len(mappings) > 0 { @@ -50,7 +55,20 @@ func callExternalActionToGen(a *microflows.CallExternalAction) element.Element { // externalActionReturnTypeToGen maps a Mendix kind name to the DataTypes$ element // stored in CallExternalAction.VariableDataType. Mirrors // sdk/mpr.serializeExternalActionReturnType. -func externalActionReturnTypeToGen(kind string) element.Element { +// An Object or List return also carries the entity it is typed on: both +// DataTypes$ObjectType and DataTypes$ListType store an Entity by qualified +// name, and one without it is as unaligned as no type at all. +func externalActionReturnTypeToGen(kind, entity string) element.Element { + switch kind { + case "Object", "List": + t := "DataTypes$ObjectType" + if kind == "List" { + t = "DataTypes$ListType" + } + e := newElem(t, "") + addStr(e, "Entity", entity) + return e + } t := "DataTypes$VoidType" switch kind { case "Boolean": diff --git a/mdl/backend/modelsdk/microflow_parameter_position_test.go b/mdl/backend/modelsdk/microflow_parameter_position_test.go new file mode 100644 index 0000000000..e38901c87c --- /dev/null +++ b/mdl/backend/modelsdk/microflow_parameter_position_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "github.com/mendixlabs/mxcli/model" + genMf "github.com/mendixlabs/mxcli/modelsdk/gen/microflows" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// #993, codec engine side. The two engines derive the same grid, so a fix +// applied to one and not the other would make a parameter's placement depend on +// which engine happened to run. +func TestMicroflowParameterToGenKeepsAuthoredPosition(t *testing.T) { + authored := µflows.MicroflowParameter{ + Name: "Feedback", + Position: &model.Point{X: -77, Y: 0}, + } + g := microflowParameterToGen(authored, 0, 11).(*genMf.MicroflowParameter) + if got := g.RelativeMiddlePoint(); got != "-77;0" { + t.Errorf("authored position = %q, want -77;0", got) + } + + // Control: unannotated parameters still derive from the index. + derived := µflows.MicroflowParameter{Name: "Feedback"} + g = microflowParameterToGen(derived, 0, 11).(*genMf.MicroflowParameter) + if got := g.RelativeMiddlePoint(); got != "200;53" { + t.Errorf("derived position at index 0 = %q, want 200;53", got) + } + g = microflowParameterToGen(derived, 1, 11).(*genMf.MicroflowParameter) + if got := g.RelativeMiddlePoint(); got != "300;53" { + t.Errorf("derived position at index 1 = %q, want 300;53", got) + } +} diff --git a/mdl/backend/modelsdk/microflow_write.go b/mdl/backend/modelsdk/microflow_write.go index 20f639e0b6..826a152f81 100644 --- a/mdl/backend/modelsdk/microflow_write.go +++ b/mdl/backend/modelsdk/microflow_write.go @@ -1380,15 +1380,20 @@ func memberChangeToGen(m *microflows.MemberChange) element.Element { return g } -// microflowParameterToGen builds a gen MicroflowParameter (position derives from -// index, matching the legacy serializer). +// microflowParameterToGen builds a gen MicroflowParameter. An authored position +// is written as given; without one the position derives from the index, matching +// the legacy serializer. func microflowParameterToGen(p *microflows.MicroflowParameter, idx, major int) element.Element { g := genMf.NewMicroflowParameter() g.SetID(element.ID(p.ID)) g.SetDocumentation(p.Documentation) g.SetHasVariableNameBeenChanged(false) g.SetName(p.Name) - g.SetRelativeMiddlePoint(fmt.Sprintf("%d;53", 200+idx*100)) + pos := microflows.DerivedParameterPosition(idx) + if p.Position != nil { + pos = *p.Position + } + g.SetRelativeMiddlePoint(fmt.Sprintf("%d;%d", pos.X, pos.Y)) g.SetSize("30;30") if major >= 10 { g.SetDefaultValue("") diff --git a/mdl/backend/modelsdk/navigation_read.go b/mdl/backend/modelsdk/navigation_read.go index dbc4d250e6..916978a6e2 100644 --- a/mdl/backend/modelsdk/navigation_read.go +++ b/mdl/backend/modelsdk/navigation_read.go @@ -140,10 +140,19 @@ func navLoginPageOf(el element.Element) string { // Navigation$NotFoundHomePage, so gen and generated/metamodel were right and the // writers were wrong. They now emit that $Type. // -// Both are still accepted here, and must stay that way: every not-found page -// mxcli wrote before that fix carries the Navigation$HomePage spelling, and -// those documents have to keep round-tripping. mxbuild accepts either, which is -// why nothing caught it. +// Both are still accepted here, and must stay that way -- but not because the +// old spelling is benign. Mendix cannot LOAD a project carrying it: `mx check` +// and `mxbuild --target=deploy` both exit 1 with "Object of type +// '...Navigation.HomePage' cannot be converted to type +// '...Navigation.NotFoundHomePage'" (measured on 11.13). So a project mxcli +// wrote before the writer fix does not build at all, and reading the old +// spelling here is what lets mxcli open that project and repair it -- mxcli +// parses the BSON directly, so it is not bound by Mendix's load validation. +// Dropping this branch would strand exactly the projects that need fixing. +// +// Nothing caught the bug because nothing ever BUILT a project with a fallback +// page set: the automated mx-check coverage runs doctype-tests/ only, and no +// script there sets one. func navNotFoundPageOf(el element.Element) string { var page, microflow string switch nfp := el.(type) { diff --git a/mdl/backend/modelsdk/navigation_read_profile_pages_test.go b/mdl/backend/modelsdk/navigation_read_profile_pages_test.go index 179c630a5b..420d8bbb9f 100644 --- a/mdl/backend/modelsdk/navigation_read_profile_pages_test.go +++ b/mdl/backend/modelsdk/navigation_read_profile_pages_test.go @@ -68,8 +68,13 @@ func TestGetNavigation_ProfilePagesRoundTrip(t *testing.T) { // Studio Pro's "Fallback page" is a Navigation$NotFoundHomePage, not the // Navigation$HomePage the home-page slot takes -- measured on ako/TestApp. All -// three writers used to emit the latter; mxbuild accepts either, so only a -// reference document could tell them apart. +// three writers used to emit the latter, which gives a project Mendix cannot +// load: `mx check` and `mxbuild --target=deploy` both exit 1 with "Object of +// type '...Navigation.HomePage' cannot be converted to type +// '...Navigation.NotFoundHomePage'" (measured on 11.13). The build was never +// the safety net here -- it simply never ran against a project with a fallback +// page, since the automated mx-check coverage is doctype-tests/ and no script +// there sets one. This test is that missing coverage. func TestUpdateNavigationProfile_NotFoundPageType(t *testing.T) { proj := copyFixture(t) b := New() diff --git a/mdl/backend/modelsdk/navigation_write.go b/mdl/backend/modelsdk/navigation_write.go index 7543fea3ca..c200d97549 100644 --- a/mdl/backend/modelsdk/navigation_write.go +++ b/mdl/backend/modelsdk/navigation_write.go @@ -168,8 +168,16 @@ func navPatchWebProfile(doc bson.D, spec types.NavigationProfileSpec) bson.D { // Studio Pro's "Fallback page". The $Type is // Navigation$NotFoundHomePage, not the Navigation$HomePage the home // page slot takes -- measured on ako/TestApp, whose fallback page - // Studio Pro stored as Navigation$NotFoundHomePage/Page. mxbuild - // accepts either, so nothing caught this. + // Studio Pro stored as Navigation$NotFoundHomePage/Page. + // + // The wrong $Type here is not cosmetic: Mendix cannot LOAD the + // project. Both `mx check` and `mxbuild --target=deploy` exit 1 with + // "Object of type '...Navigation.HomePage' cannot be converted to + // type '...Navigation.NotFoundHomePage'" (measured on 11.13, against + // a build of this file emitting the old spelling). Nothing caught it + // because nothing ever BUILT a project with a fallback page set -- + // the automated mx-check coverage runs doctype-tests/ only, and no + // script there sets one. {Key: "$Type", Value: "Navigation$NotFoundHomePage"}, {Key: "Microflow", Value: ""}, {Key: "Page", Value: spec.NotFoundPage}, diff --git a/mdl/backend/modelsdk/testdata/TestApp.OrderMessageDefinitions.bson b/mdl/backend/modelsdk/testdata/TestApp.OrderMessageDefinitions.bson new file mode 100644 index 0000000000..d6db195619 Binary files /dev/null and b/mdl/backend/modelsdk/testdata/TestApp.OrderMessageDefinitions.bson differ diff --git a/mdl/backend/modelsdk/unimplemented_gen.go b/mdl/backend/modelsdk/unimplemented_gen.go index 138cfbaab5..5ee943e182 100644 --- a/mdl/backend/modelsdk/unimplemented_gen.go +++ b/mdl/backend/modelsdk/unimplemented_gen.go @@ -164,6 +164,10 @@ func (unimplemented) CreateMenuDocument(_ *types.MenuDocument) error { return errUnimplemented("CreateMenuDocument") } +func (unimplemented) CreateMessageDefinitionCollection(_ *model.MessageDefinitionCollection) error { + return errUnimplemented("CreateMessageDefinitionCollection") +} + func (unimplemented) CreateMicroflow(_ *microflows.Microflow) error { return errUnimplemented("CreateMicroflow") } @@ -321,6 +325,10 @@ func (unimplemented) DeleteMenuDocument(_ model.ID) error { return errUnimplemented("DeleteMenuDocument") } +func (unimplemented) DeleteMessageDefinitionCollection(_ string) error { + return errUnimplemented("DeleteMessageDefinitionCollection") +} + func (unimplemented) DeleteMicroflow(_ model.ID) error { return errUnimplemented("DeleteMicroflow") } @@ -1135,6 +1143,10 @@ func (unimplemented) UpdateMenuDocument(_ *types.MenuDocument) error { return errUnimplemented("UpdateMenuDocument") } +func (unimplemented) UpdateMessageDefinitionCollection(_ *model.MessageDefinitionCollection) error { + return errUnimplemented("UpdateMessageDefinitionCollection") +} + func (unimplemented) UpdateMicroflow(_ *microflows.Microflow) error { return errUnimplemented("UpdateMicroflow") } diff --git a/mdl/backend/mpr/backend.go b/mdl/backend/mpr/backend.go index 02753f5d3a..8b03cbadee 100644 --- a/mdl/backend/mpr/backend.go +++ b/mdl/backend/mpr/backend.go @@ -327,6 +327,23 @@ func (b *MprBackend) MoveRule(*microflows.Rule) error { return errors.New("moving a rule requires the modelsdk engine — rerun without MXCLI_ENGINE=legacy") } +// Message definition collections are authored through gen+codec only, as rules, +// menus and layouts are. The legacy writer has no serializer for the document +// and building one would duplicate a shape the codec already gets right +// (ako/mxcli#272). Reading works on both engines. + +func (b *MprBackend) CreateMessageDefinitionCollection(*model.MessageDefinitionCollection) error { + return errors.New("creating a message definition collection requires the modelsdk engine — rerun without MXCLI_ENGINE=legacy") +} + +func (b *MprBackend) UpdateMessageDefinitionCollection(*model.MessageDefinitionCollection) error { + return errors.New("modifying a message definition collection requires the modelsdk engine — rerun without MXCLI_ENGINE=legacy") +} + +func (b *MprBackend) DeleteMessageDefinitionCollection(string) error { + return errors.New("dropping a message definition collection requires the modelsdk engine — rerun without MXCLI_ENGINE=legacy") +} + // --------------------------------------------------------------------------- // PageBackend // --------------------------------------------------------------------------- diff --git a/mdl/catalog/builder_contract.go b/mdl/catalog/builder_contract.go index b334b455b3..9f0508e8ee 100644 --- a/mdl/catalog/builder_contract.go +++ b/mdl/catalog/builder_contract.go @@ -133,7 +133,17 @@ func (b *Builder) buildContractEntities() error { // consume them. Runs after both buildExternalEntities and buildContractEntities. // Targets the underlying _data table because contract_entities is a view. func (b *Builder) buildContractEntityUsage() error { - _, err := b.tx.Exec(` + _, err := b.tx.Exec(contractEntityUsageSQL) + return err +} + +// contractEntityUsageSQL fills contract_entities.UsedByExternalEntity by joining +// the imported external entities on their remote name. +// +// Shared with the test rather than copied into it: the column is only as +// meaningful as the rows external_entities holds, and it read as "not linked" +// for every entity that table was omitting (mendixlabs/mxcli#1020). +const contractEntityUsageSQL = ` UPDATE contract_entities_data SET UsedByExternalEntity = ( SELECT ee.QualifiedName @@ -144,9 +154,7 @@ func (b *Builder) buildContractEntityUsage() error { AND ee.RemoteName = contract_entities_data.EntityName LIMIT 1 ) - `) - return err -} + ` // buildContractMessages parses cached AsyncAPI documents from business event client // services and populates the contract_messages catalog table. diff --git a/mdl/catalog/builder_external.go b/mdl/catalog/builder_external.go index 3234f2360a..59628b6700 100644 --- a/mdl/catalog/builder_external.go +++ b/mdl/catalog/builder_external.go @@ -38,7 +38,18 @@ func (b *Builder) buildExternalEntities() error { moduleName := b.hierarchy.getModuleName(moduleID) for _, entity := range dm.Entities { - if entity.Source != "Rest$ODataRemoteEntitySource" { + // BOTH OData sources are external entities. Cataloguing only the + // entity-set-backed one hid every type-sourced entity — the derived, + // abstract, contained and action parameter/return types that + // CREATE EXTERNAL ENTITIES imports with no entity set of their own. + // + // That is not merely an under-count. contract_entities.UsedByExternalEntity + // is filled by joining this table on RemoteName, so for exactly those + // entities the column was STRUCTURALLY always empty, whether or not + // the import had worked. It read as "this contract entity is not + // linked to anything", which is what sent mendixlabs/mxcli#1020 + // looking for a linkage bug in the MPR that was never there. + if !isODataEntitySource(entity.Source) { continue } @@ -197,3 +208,20 @@ func (b *Builder) buildExternalActions() error { b.report("External Actions", len(actionMap)) return nil } + +// isODataEntitySource reports whether an entity's Source marks it as consumed +// from an OData service. +// +// Mendix stores two, and the difference is whether the contract gave the type +// an entity set: +// +// Rest$ODataRemoteEntitySource entity-set-backed, persistable +// Rest$ODataEntityTypeSource no entity set — derived, abstract, contained, +// or an action's parameter/return type +// +// Both are external entities and both are written by CREATE EXTERNAL ENTITIES +// (see applyExternalEntityFields), so anything asking "is this external?" has to +// accept both. The domain-model writer already does. +func isODataEntitySource(source string) bool { + return source == "Rest$ODataRemoteEntitySource" || source == "Rest$ODataEntityTypeSource" +} diff --git a/mdl/catalog/builder_external_source_test.go b/mdl/catalog/builder_external_source_test.go new file mode 100644 index 0000000000..20a8b92d44 --- /dev/null +++ b/mdl/catalog/builder_external_source_test.go @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import "testing" + +// TestIsODataEntitySource pins which entity Sources count as external. +// +// Mendix stores two, and only the entity-set-backed one was accepted. The other +// covers derived, abstract and contained types — and an OData action's +// parameter and return types, which have no entity set of their own. +func TestIsODataEntitySource(t *testing.T) { + tests := []struct { + source string + want bool + }{ + {"Rest$ODataRemoteEntitySource", true}, + // The one that was missing. CREATE EXTERNAL ENTITIES writes this for + // every type with no entity set (applyExternalEntityFields). + {"Rest$ODataEntityTypeSource", true}, + {"", false}, + {"Rest$ConsumedODataService", false}, + {"DomainModels$Entity", false}, + } + for _, tt := range tests { + if got := isODataEntitySource(tt.source); got != tt.want { + t.Errorf("isODataEntitySource(%q) = %v, want %v", tt.source, got, tt.want) + } + } +} + +// TestContractEntityUsageLinksTypeSourcedEntities is the reason the predicate +// matters, exercised through the real join. +// +// contract_entities.UsedByExternalEntity is filled by matching external_entities +// on RemoteName. While that table held only entity-set-backed entities, the +// column was structurally always empty for an action's parameter/return types — +// it read as "this contract entity is linked to nothing" whether or not the +// import had worked, which is what sent mendixlabs/mxcli#1020 hunting for an MPR +// linkage bug that did not exist. +// +// The two rows are the before and after of the catalog fix: Person is +// entity-set-backed and was always catalogued; Airport is an action's return +// type and is catalogued only now. +func TestContractEntityUsageLinksTypeSourcedEntities(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + defer cat.Close() + db := cat.CatalogDB() + + if _, err := db.Exec(` + INSERT INTO odata_clients_data (Id, Name, QualifiedName, ModuleName) + VALUES ('svc-1', 'TripPin', 'Ext.TripPin', 'Ext')`); err != nil { + t.Fatalf("seed odata_clients: %v", err) + } + + // Contract entities: one entity-set-backed, one that exists only as an + // action's return type. + for _, r := range []struct{ id, name, set string }{ + {"ce-person", "Person", "People"}, + {"ce-airport", "Airport", ""}, + } { + if _, err := db.Exec(` + INSERT INTO contract_entities_data (Id, ServiceId, ServiceQualifiedName, EntityName, EntitySetName) + VALUES (?, 'svc-1', 'Ext.TripPin', ?, ?)`, r.id, r.name, r.set); err != nil { + t.Fatalf("seed contract entity %s: %v", r.name, err) + } + } + + // External entities, as the fixed builder catalogues them: both sources. + for _, r := range []struct{ id, name, qn, set, remote string }{ + {"ee-person", "People", "Ext.People", "People", "Person"}, + {"ee-airport", "Airport", "Ext.Airport", "", "Airport"}, + } { + if _, err := db.Exec(` + INSERT INTO external_entities_data (Id, Name, QualifiedName, ModuleName, ServiceName, EntitySet, RemoteName) + VALUES (?, ?, ?, 'Ext', 'Ext.TripPin', ?, ?)`, + r.id, r.name, r.qn, r.set, r.remote); err != nil { + t.Fatalf("seed external entity %s: %v", r.name, err) + } + } + + if _, err := db.Exec(contractEntityUsageSQL); err != nil { + t.Fatalf("run usage join: %v", err) + } + + for _, tt := range []struct{ id, want string }{ + {"ce-person", "Ext.People"}, + // The row that was always empty before. + {"ce-airport", "Ext.Airport"}, + } { + var got *string + if err := db.QueryRow( + `SELECT UsedByExternalEntity FROM contract_entities WHERE Id = ?`, tt.id, + ).Scan(&got); err != nil { + t.Fatalf("read %s: %v", tt.id, err) + } + if got == nil { + t.Errorf("%s: UsedByExternalEntity is NULL, want %q", tt.id, tt.want) + continue + } + if *got != tt.want { + t.Errorf("%s: UsedByExternalEntity = %q, want %q", tt.id, *got, tt.want) + } + } +} + +// TestContractEntityUsageIsEmptyWhenNotImported is the control: the column is +// only meaningful because an absent import really does leave it empty. Without +// this, the test above would pass against a join that populated every row. +func TestContractEntityUsageIsEmptyWhenNotImported(t *testing.T) { + cat, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + defer cat.Close() + db := cat.CatalogDB() + + if _, err := db.Exec(` + INSERT INTO odata_clients_data (Id, Name, QualifiedName, ModuleName) + VALUES ('svc-1', 'TripPin', 'Ext.TripPin', 'Ext')`); err != nil { + t.Fatalf("seed odata_clients: %v", err) + } + if _, err := db.Exec(` + INSERT INTO contract_entities_data (Id, ServiceId, ServiceQualifiedName, EntityName) + VALUES ('ce-airport', 'svc-1', 'Ext.TripPin', 'Airport')`); err != nil { + t.Fatalf("seed contract entity: %v", err) + } + // No external entity imported for it. + + if _, err := db.Exec(contractEntityUsageSQL); err != nil { + t.Fatalf("run usage join: %v", err) + } + + var got *string + if err := db.QueryRow( + `SELECT UsedByExternalEntity FROM contract_entities WHERE Id = 'ce-airport'`, + ).Scan(&got); err != nil { + t.Fatalf("read: %v", err) + } + if got != nil && *got != "" { + t.Errorf("UsedByExternalEntity = %q, want empty — an unimported type must still read as unlinked", *got) + } +} diff --git a/mdl/catalog/builder_strings.go b/mdl/catalog/builder_strings.go index 3267792bd4..5f20eaf9f6 100644 --- a/mdl/catalog/builder_strings.go +++ b/mdl/catalog/builder_strings.go @@ -3,6 +3,13 @@ package catalog import ( + "sort" + "strings" + "unicode" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/translations" "github.com/mendixlabs/mxcli/sdk/microflows" "github.com/mendixlabs/mxcli/sdk/workflows" ) @@ -32,27 +39,22 @@ func (b *Builder) buildStrings() error { count++ } - // Extract from pages (title, URL) — using cached list + // Every TRANSLATABLE string comes from the walk below, not from here. What + // remains in the typed extractions is the strings that are not Texts$Text + // and so are invisible to it: URLs, log node names, REST paths, + // documentation, and the workflow templates (Microflows$StringTemplate, + // which holds a plain Text and cannot carry a translation). + + // Page URL (no language) pageList, err := b.cachedPages() if err == nil { for _, pg := range pageList { + if pg.URL == "" { + continue + } moduleID := b.hierarchy.findModuleID(pg.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) - qn := moduleName + "." + pg.Name - - pageID := string(pg.ID) - - // Page title translations (with language code) - if pg.Title != nil && pg.Title.Translations != nil { - for lang, t := range pg.Title.Translations { - insert(qn, "PAGE", t, "page_title", lang, pageID, moduleName) - } - } - - // Page URL (no language) - if pg.URL != "" { - insert(qn, "PAGE", pg.URL, "page_url", "", pageID, moduleName) - } + insert(moduleName+"."+pg.Name, "PAGE", pg.URL, "page_url", "", string(pg.ID), moduleName) } } @@ -66,36 +68,12 @@ func (b *Builder) buildStrings() error { mfID := string(mf.ID) - // Documentation (no language) + // Documentation (no language). The activities' message templates + // are Texts$Text and come from the walk. if mf.Documentation != "" { insert(qn, "MICROFLOW", mf.Documentation, "documentation", "", mfID, moduleName) } - - // Extract strings from activities - extractActivityStrings(mf.ObjectCollection, qn, "MICROFLOW", moduleName, insert) - } - } - - // Extract from enumerations (value captions) — using cached list - enums, err := b.cachedEnumerations() - if err == nil { - for _, enum := range enums { - moduleID := b.hierarchy.findModuleID(enum.ContainerID) - moduleName := b.hierarchy.getModuleName(moduleID) - qn := moduleName + "." + enum.Name - - enumID := string(enum.ID) - for _, val := range enum.Values { - if val.Caption != nil && val.Caption.Translations != nil { - valID := string(val.ID) - if valID == "" { - valID = enumID - } - for lang, t := range val.Caption.Translations { - insert(qn, "ENUMERATION", t, "enum_caption", lang, valID, moduleName) - } - } - } + extractLogNodeNames(mf.ObjectCollection, qn, "MICROFLOW", moduleName, insert) } } @@ -151,10 +129,131 @@ func (b *Builder) buildStrings() error { } } + b.buildTranslatableStrings(insert) + b.report("strings", count) return nil } +// buildTranslatableStrings indexes every Texts$Text in the project. +// +// It walks the RAW units rather than the typed readers, deliberately. The typed +// path reached five sites because each was hand-written, and a sixth cost +// another case; this reaches all of them — 17 distinct sites in a stock 11.13 +// app — with no per-type code, and covers document types mxcli cannot otherwise +// round-trip. Measured on that app, the typed path indexed ~69 of 3265 texts and +// saw 8 of 9 languages, so `ar_DZ` was invisible to SHOW LANGUAGES and to +// QUAL005 rather than merely undercounted. +// +// Atlas design templates (Forms$PageTemplate, Forms$BuildingBlock) are ~70% of +// the corpus and their captions never render in a running app. They are indexed +// anyway, with ObjectType naming the document type so a consumer can filter: +// DESCRIBE TRANSLATIONS reaches them and CREATE TRANSLATIONS writes them, so a +// SHOW LANGUAGES that excluded them would disagree with the statement that +// changes them — the same split this is closing. +// The caller's insert closure counts the rows it writes, so nothing is counted +// here — doing both reported twice the rows the table actually holds. +func (b *Builder) buildTranslatableStrings(insert func(string, string, string, string, string, string, string)) { + units, err := b.reader.ListRawUnitsByType("") + if err != nil { + return + } + + for _, u := range units { + if len(u.Contents) == 0 { + continue + } + var named struct { + Name string `bson:"Name"` + } + _ = bson.Unmarshal(u.Contents, &named) + + moduleName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(u.ContainerID)) + qn := named.Name + if moduleName != "" && qn != "" { + qn = moduleName + "." + qn + } + + for _, r := range translatableRows(u.Type, qn, moduleName, u.Contents) { + insert(r.QualifiedName, r.ObjectType, r.StringValue, r.StringContext, r.Language, r.ElementID, r.ModuleName) + } + } +} + +// stringRow is one row of the strings index. +type stringRow struct { + QualifiedName string + ObjectType string + StringValue string + StringContext string + Language string + ElementID string + ModuleName string +} + +// translatableRows turns one unit's stored bytes into index rows, one per +// (text, language). A language present with an empty string is a text that is +// not translated yet and is skipped: indexing it would make the language look +// present everywhere it is not, which is the opposite of what QUAL005 asks. +func translatableRows(unitType, qualifiedName, moduleName string, raw []byte) []stringRow { + sites, err := translations.SitesInUnit(raw) + if err != nil { + return nil + } + objType := catalogObjectType(unitType) + + var out []stringRow + for _, site := range sites { + ctx := site.OwnerType + "." + site.Property + for _, lang := range sortedLangs(site.Targets) { + if site.Targets[lang] == "" { + continue + } + out = append(out, stringRow{ + QualifiedName: qualifiedName, + ObjectType: objType, + StringValue: site.Targets[lang], + StringContext: ctx, + Language: lang, + ElementID: site.ElementID, + ModuleName: moduleName, + }) + } + } + return out +} + +// sortedLangs keeps row order deterministic — a map iteration here would make +// the catalog's bytes differ between two builds of an unchanged project. +func sortedLangs(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// catalogObjectType turns a unit's stored $Type into the catalog's object-type +// vocabulary — "Forms$PageTemplate" to "PAGE_TEMPLATE". Derived rather than +// looked up in a table, so a document type Mendix adds later is named correctly +// without anyone maintaining a list. It agrees with the hand-written values on +// every type they both cover (PAGE, MICROFLOW, ENUMERATION). +func catalogObjectType(unitType string) string { + name := unitType + if i := strings.LastIndex(name, "$"); i >= 0 { + name = name[i+1:] + } + var b strings.Builder + for i, r := range name { + if unicode.IsUpper(r) && i > 0 { + b.WriteByte('_') + } + b.WriteRune(unicode.ToUpper(r)) + } + return b.String() +} + // extractWorkflowFlowStrings extracts strings from workflow activities recursively. func extractWorkflowFlowStrings(flow *workflows.Flow, qn, moduleName string, insert func(string, string, string, string, string, string, string)) { for _, act := range flow.Activities { @@ -207,42 +306,21 @@ func extractWorkflowFlowStrings(flow *workflows.Flow, qn, moduleName string, ins } } -// extractActivityStrings extracts string literals from microflow/nanoflow activities. -func extractActivityStrings(oc *microflows.MicroflowObjectCollection, qn, objType, moduleName string, insert func(string, string, string, string, string, string, string)) { +// extractLogNodeNames indexes the one microflow-activity string that is NOT a +// Texts$Text. The message templates that used to be extracted here — log, show +// message, validation feedback — are Texts$Text and come from the walk in +// buildTranslatableStrings, which also reaches the ones this never listed. +func extractLogNodeNames(oc *microflows.MicroflowObjectCollection, qn, objType, moduleName string, insert func(string, string, string, string, string, string, string)) { if oc == nil { return } - for _, obj := range oc.Objects { act, ok := obj.(*microflows.ActionActivity) if !ok || act.Action == nil { continue } - - actID := string(act.ID) - - switch a := act.Action.(type) { - case *microflows.LogMessageAction: - if a.MessageTemplate != nil && a.MessageTemplate.Translations != nil { - for lang, t := range a.MessageTemplate.Translations { - insert(qn, objType, t, "log_message", lang, actID, moduleName) - } - } - if a.LogNodeName != "" { - insert(qn, objType, a.LogNodeName, "log_node", "", actID, moduleName) - } - case *microflows.ShowMessageAction: - if a.Template != nil && a.Template.Translations != nil { - for lang, t := range a.Template.Translations { - insert(qn, objType, t, "show_message", lang, actID, moduleName) - } - } - case *microflows.ValidationFeedbackAction: - if a.Template != nil && a.Template.Translations != nil { - for lang, t := range a.Template.Translations { - insert(qn, objType, t, "validation_message", lang, actID, moduleName) - } - } + if a, ok := act.Action.(*microflows.LogMessageAction); ok && a.LogNodeName != "" { + insert(qn, objType, a.LogNodeName, "log_node", "", string(act.ID), moduleName) } } } diff --git a/mdl/catalog/builder_strings_test.go b/mdl/catalog/builder_strings_test.go new file mode 100644 index 0000000000..305f74632c --- /dev/null +++ b/mdl/catalog/builder_strings_test.go @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/modelsdk/mpr" +) + +func txt(pairs ...string) bson.D { + items := bson.A{int32(3)} + for i := 0; i+1 < len(pairs); i += 2 { + items = append(items, bson.D{ + {Key: "$Type", Value: "Texts$Translation"}, + {Key: "LanguageCode", Value: pairs[i]}, + {Key: "Text", Value: pairs[i+1]}, + }) + } + return bson.D{{Key: "$Type", Value: "Texts$Text"}, {Key: "Items", Value: items}} +} + +func marshal(t *testing.T, d bson.D) []byte { + t.Helper() + raw, err := bson.Marshal(d) + if err != nil { + t.Fatal(err) + } + return raw +} + +// The gap this replaces: a widget caption is translatable text the hand-written +// extractor never reached, because it only ever read a page's Title. Measured on +// a stock project, the index held ~69 of 3265 texts. +func TestTranslatableRows_ReachesAWidgetCaptionNotJustThePageTitle(t *testing.T) { + raw := marshal(t, bson.D{ + {Key: "$ID", Value: mpr.IDToBsonBinary("11111111-1111-1111-1111-111111111111")}, + {Key: "$Type", Value: "Forms$Page"}, + {Key: "Name", Value: "Home"}, + {Key: "Title", Value: txt("en_US", "Home")}, + {Key: "Widgets", Value: bson.A{int32(3), + bson.D{ + {Key: "$ID", Value: mpr.IDToBsonBinary("22222222-2222-2222-2222-222222222222")}, + {Key: "$Type", Value: "Forms$ActionButton"}, + {Key: "Caption", Value: txt("en_US", "Save", "nl_NL", "Opslaan")}, + }, + }}, + }) + + rows := translatableRows("Forms$Page", "MyModule.Home", "MyModule", raw) + + var caption *stringRow + for i := range rows { + if rows[i].StringValue == "Opslaan" { + caption = &rows[i] + } + } + if caption == nil { + t.Fatalf("the button caption never reached the index; rows = %+v", rows) + } + if caption.Language != "nl_NL" { + t.Errorf("Language = %q, want nl_NL", caption.Language) + } + if caption.StringContext != "Forms$ActionButton.Caption" { + t.Errorf("StringContext = %q, want Forms$ActionButton.Caption", caption.StringContext) + } + if caption.ElementID != "22222222-2222-2222-2222-222222222222" { + t.Errorf("ElementID = %q, want the button's own", caption.ElementID) + } + if caption.ObjectType != "PAGE" { + t.Errorf("ObjectType = %q, want PAGE", caption.ObjectType) + } + + // The title must still be there — the walk replaces the typed extraction, + // it does not trade one site for another. + var sawTitle bool + for _, r := range rows { + if r.StringValue == "Home" && r.StringContext == "Forms$Page.Title" { + sawTitle = true + } + } + if !sawTitle { + t.Errorf("page title lost; rows = %+v", rows) + } +} + +// A language reaching the index at all is what decides whether SHOW LANGUAGES +// can list it and whether QUAL005 can reason about it. Measured: the old +// extractor saw 8 of the project's 9 languages, and ar_DZ was invisible rather +// than undercounted. +func TestTranslatableRows_ALanguageOnlyOnAWidgetStillReachesTheIndex(t *testing.T) { + raw := marshal(t, bson.D{ + {Key: "$Type", Value: "Forms$Page"}, + {Key: "Title", Value: txt("en_US", "Home")}, + {Key: "Widgets", Value: bson.A{int32(3), + bson.D{ + {Key: "$Type", Value: "Forms$Label"}, + {Key: "Caption", Value: txt("en_US", "Hello", "ar_DZ", "مرحبا")}, + }, + }}, + }) + + rows := translatableRows("Forms$Page", "MyModule.Home", "MyModule", raw) + + langs := map[string]bool{} + for _, r := range rows { + langs[r.Language] = true + } + if !langs["ar_DZ"] { + t.Fatalf("ar_DZ never reached the index, so SHOW LANGUAGES cannot list it; languages = %v", langs) + } +} + +// An empty translation is a text that exists but is not translated yet. Writing +// it as a row would make the language look present everywhere it is not, which +// is the opposite of what QUAL005 is for. +func TestTranslatableRows_AnEmptyTranslationIsNotARow(t *testing.T) { + raw := marshal(t, bson.D{ + {Key: "$Type", Value: "Forms$Page"}, + {Key: "Title", Value: txt("en_US", "Home", "nl_NL", "")}, + }) + + for _, r := range translatableRows("Forms$Page", "MyModule.Home", "MyModule", raw) { + if r.Language == "nl_NL" { + t.Fatalf("an untranslated nl_NL was indexed as though it were translated: %+v", r) + } + } +} + +// Atlas design templates are ~70% of a project's texts and their captions never +// render in the app, so a consumer has to be able to filter them out. They are +// indexed rather than dropped because DESCRIBE TRANSLATIONS reaches them, and a +// SHOW LANGUAGES that disagreed with it would be the same split being fixed here. +func TestCatalogObjectType_DerivesFromTheUnitTypeWithoutATable(t *testing.T) { + for unitType, want := range map[string]string{ + "Forms$Page": "PAGE", + "Forms$PageTemplate": "PAGE_TEMPLATE", + "Forms$BuildingBlock": "BUILDING_BLOCK", + "Microflows$Microflow": "MICROFLOW", + "Microflows$Nanoflow": "NANOFLOW", + "Enumerations$Enumeration": "ENUMERATION", + "Texts$SystemTextCollection": "SYSTEM_TEXT_COLLECTION", + "Navigation$NavigationDocument": "NAVIGATION_DOCUMENT", + "Forms$Layout": "LAYOUT", + } { + if got := catalogObjectType(unitType); got != want { + t.Errorf("catalogObjectType(%q) = %q, want %q", unitType, got, want) + } + } +} diff --git a/mdl/executor/cmd_agenteditor_models.go b/mdl/executor/cmd_agenteditor_models.go index a09f0faf1f..743937a0d3 100644 --- a/mdl/executor/cmd_agenteditor_models.go +++ b/mdl/executor/cmd_agenteditor_models.go @@ -192,6 +192,10 @@ func execCreateAgentEditorModel(ctx *ExecContext, s *ast.CreateModelStmt) error ResourceName: s.ResourceName, DeepLinkURL: s.DeepLinkURL, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + m.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existing.Documentation) + } if existing != nil { m.ID = existing.ID diff --git a/mdl/executor/cmd_agenteditor_write.go b/mdl/executor/cmd_agenteditor_write.go index 32bd18878c..d26aa64453 100644 --- a/mdl/executor/cmd_agenteditor_write.go +++ b/mdl/executor/cmd_agenteditor_write.go @@ -60,6 +60,10 @@ func execCreateConsumedMCPService(ctx *ExecContext, s *ast.CreateConsumedMCPServ InnerDocumentation: s.InnerDocumentation, ConnectionTimeoutSeconds: s.ConnectionTimeoutSeconds, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + c.Documentation = carriedDocumentation(s.DocumentationSet, s.OuterDocumentation, existing.Documentation) + } if existing != nil { c.ID = existing.ID @@ -163,6 +167,10 @@ func execCreateKnowledgeBase(ctx *ExecContext, s *ast.CreateKnowledgeBaseStmt) e Environment: s.Environment, DeepLinkURL: s.DeepLinkURL, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + k.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existing.Documentation) + } if existing != nil { k.ID = existing.ID @@ -253,6 +261,10 @@ func execCreateAgent(ctx *ExecContext, s *ast.CreateAgentStmt) error { Temperature: s.Temperature, TopP: s.TopP, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existingAgent != nil { + a.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingAgent.Documentation) + } // Resolve Model reference if s.Model != nil { diff --git a/mdl/executor/cmd_associations.go b/mdl/executor/cmd_associations.go index 15472b8528..c9189c445e 100644 --- a/mdl/executor/cmd_associations.go +++ b/mdl/executor/cmd_associations.go @@ -97,9 +97,8 @@ func execCreateAssociation(ctx *ExecContext, s *ast.CreateAssociationStmt) error assoc.Owner = owner assoc.StorageFormat = storageFormat assoc.ChildDeleteBehavior = &domainmodel.DeleteBehavior{Type: deleteBehavior} - if doc := associationDocumentation(s); doc != "" { - assoc.Documentation = doc - } + assoc.Documentation = carriedDocumentation( + associationDocumentationStated(s), associationDocumentation(s), assoc.Documentation) // Anchors are applied only when the statement names them — // silence preserves what is stored, so a `create or modify` // that is not about layout does not flatten a hand-tuned line. @@ -123,9 +122,8 @@ func execCreateAssociation(ctx *ExecContext, s *ast.CreateAssociationStmt) error ca.StorageFormat = storageFormat ca.ChildDeleteBehavior = &domainmodel.DeleteBehavior{Type: deleteBehavior} ca.ChildRef = childRef - if doc := associationDocumentation(s); doc != "" { - ca.Documentation = doc - } + ca.Documentation = carriedDocumentation( + associationDocumentationStated(s), associationDocumentation(s), ca.Documentation) if err := ctx.Backend.UpdateDomainModel(dm); err != nil { return mdlerrors.NewBackend("update cross-module association", err) } @@ -704,6 +702,15 @@ func associationExists(dm *domainmodel.DomainModel, name string) bool { // path already uses. `comment` survives here — and only here among the CREATE // statements — because it is an association's only inline spelling; everywhere // else the doc comment already worked, so the dead option was removed instead. +// associationDocumentationStated reports whether the statement said anything +// about documentation — a doc comment (even an empty one) or a COMMENT clause. +// The OR MODIFY path used `if doc != ""`, which preserved the stored value but +// also made it unclearable; #1018's rule is that an explicitly empty comment +// clears while an absent one preserves. +func associationDocumentationStated(s *ast.CreateAssociationStmt) bool { + return s.DocumentationSet || s.Comment != "" +} + func associationDocumentation(s *ast.CreateAssociationStmt) string { if s.Documentation != "" { return s.Documentation diff --git a/mdl/executor/cmd_businessevents.go b/mdl/executor/cmd_businessevents.go index 8f6145741a..1ff7d715a2 100644 --- a/mdl/executor/cmd_businessevents.go +++ b/mdl/executor/cmd_businessevents.go @@ -284,6 +284,8 @@ func createBusinessEventService(ctx *ExecContext, stmt *ast.CreateBusinessEventS // Placement is carried forward the same way when the statement is silent // about folders (#932). var existingContainerID model.ID + var existingDocumentation string + haveExistingSvc := false if existing, ok := pickLive(existingServices, func(svc *model.BusinessEventService) bool { return strings.EqualFold(h.GetModuleName(h.FindModuleID(svc.ContainerID)), moduleName) && @@ -297,6 +299,8 @@ func createBusinessEventService(ctx *ExecContext, stmt *ast.CreateBusinessEventS existingID = existing.ID existingExcluded = existing.Excluded existingContainerID = existing.ContainerID + existingDocumentation = existing.Documentation + haveExistingSvc = true } // Resolve folder if specified @@ -317,6 +321,10 @@ func createBusinessEventService(ctx *ExecContext, stmt *ast.CreateBusinessEventS ExportLevel: "Hidden", Excluded: existingExcluded, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExistingSvc { + svc.Documentation = carriedDocumentation(stmt.DocumentationSet, stmt.Documentation, existingDocumentation) + } if existingID != "" { svc.ID = existingID } diff --git a/mdl/executor/cmd_constants.go b/mdl/executor/cmd_constants.go index 6a45fb145b..06cdc1713e 100644 --- a/mdl/executor/cmd_constants.go +++ b/mdl/executor/cmd_constants.go @@ -287,11 +287,14 @@ func createConstant(ctx *ExecContext, stmt *ast.CreateConstantStmt) error { modName := h.GetModuleName(modID) if strings.EqualFold(modName, stmt.Name.Module) && strings.EqualFold(c.Name, stmt.Name.Name) { if stmt.CreateOrModify { - // Update existing constant — COMMENT takes precedence over doc-comment + // Update existing constant — COMMENT takes precedence over + // doc-comment, and a rewrite that mentioned NEITHER keeps + // what is stored (#1018). if stmt.Comment != "" { c.Documentation = stmt.Comment } else { - c.Documentation = stmt.Documentation + c.Documentation = carriedDocumentation( + stmt.DocumentationSet, stmt.Documentation, c.Documentation) } c.Type = constType c.DefaultValue = defaultValue diff --git a/mdl/executor/cmd_diff.go b/mdl/executor/cmd_diff.go index 086f57530b..816692dff6 100644 --- a/mdl/executor/cmd_diff.go +++ b/mdl/executor/cmd_diff.go @@ -4,10 +4,12 @@ package executor import ( - "bytes" "context" + "errors" "fmt" + "sort" "strings" + "unicode" "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" @@ -87,10 +89,20 @@ func diffProgram(ctx *ExecContext, prog *ast.Program, opts DiffOptions) error { processed := make(map[string]bool) // Process each statement + // Statements diff cannot compare, and statements whose comparison failed. + // Both are reported rather than dropped — see unsupportedDiffError. + skipped := map[string]int{} + var failures []string + for _, stmt := range prog.Statements { result, err := diffStatement(ctx, stmt) if err != nil { - // Skip statements that can't be diffed (e.g., connection statements) + var unsupported *unsupportedDiffError + if errors.As(err, &unsupported) { + skipped[unsupported.kind]++ + } else { + failures = append(failures, err.Error()) + } continue } if result != nil { @@ -135,10 +147,39 @@ func diffProgram(ctx *ExecContext, prog *ast.Program, opts DiffOptions) error { // Output summary fmt.Fprintf(ctx.Output, "\nSummary: %d new, %d modified, %d unchanged\n", newCount, modifiedCount, unchangedCount) + reportUndiffed(ctx, skipped, failures) return nil } +// reportUndiffed prints what the summary above does NOT account for. +// +// The counts only ever describe statements diff understands, so a script made +// entirely of statements it does not understand summarises as all zeros. That +// reads as "nothing would change" for a script that may add documents, which +// is exactly the wrong answer from a pre-apply safety gate. +func reportUndiffed(ctx *ExecContext, skipped map[string]int, failures []string) { + if len(skipped) > 0 { + kinds := make([]string, 0, len(skipped)) + for k := range skipped { + kinds = append(kinds, k) + } + sort.Strings(kinds) + total := 0 + for _, n := range skipped { + total += n + } + fmt.Fprintf(ctx.Output, "\nNot compared (%d statement(s)) — diff has no comparison for these,\n"+ + "so they are absent from the summary above, not unchanged:\n", total) + for _, k := range kinds { + fmt.Fprintf(ctx.Output, " %s x%d\n", k, skipped[k]) + } + } + for _, f := range failures { + fmt.Fprintf(ctx.Output, "\nCould not diff: %s\n", f) + } +} + // DiffProgram is a method wrapper for external callers. func (e *Executor) DiffProgram(prog *ast.Program, opts DiffOptions) error { return diffProgram(e.newExecContext(context.Background()), prog, opts) @@ -160,10 +201,36 @@ func diffStatement(ctx *ExecContext, stmt ast.Statement) (*DiffResult, error) { case *ast.CreateNanoflowStmt: return diffNanoflow(ctx, s) default: - return nil, nil // Skip unsupported statements + return nil, &unsupportedDiffError{kind: statementKindName(stmt)} } } +// unsupportedDiffError marks a statement diff has no comparison for. It is an +// error rather than a nil result so that diffProgram can SAY so: skipping +// silently made `diff` report "0 new, 0 modified, 0 unchanged" for a script +// that would genuinely add documents, which is worse than a wrong count +// because there is nothing on screen to disbelieve (#997). +type unsupportedDiffError struct{ kind string } + +func (e *unsupportedDiffError) Error() string { + return "diff does not compare " + e.kind + " statements" +} + +// statementKindName turns an AST statement type into something an MDL author +// recognises: *ast.GrantMicroflowAccessStmt → "grant microflow access". +func statementKindName(stmt ast.Statement) string { + name := strings.TrimPrefix(fmt.Sprintf("%T", stmt), "*ast.") + name = strings.TrimSuffix(name, "Stmt") + var out []rune + for i, r := range name { + if i > 0 && r >= 'A' && r <= 'Z' { + out = append(out, ' ') + } + out = append(out, unicode.ToLower(r)) + } + return string(out) +} + // diffEntity compares a CREATE ENTITY statement against the project func diffEntity(ctx *ExecContext, s *ast.CreateEntityStmt) (*DiffResult, error) { result := &DiffResult{ @@ -285,88 +352,71 @@ func diffAssociation(ctx *ExecContext, s *ast.CreateAssociationStmt) (*DiffResul // diffMicroflow compares a CREATE MICROFLOW statement against the project func diffMicroflow(ctx *ExecContext, s *ast.CreateMicroflowStmt) (*DiffResult, error) { - result := &DiffResult{ - ObjectType: "Microflow", - ObjectName: s.Name, - Proposed: microflowStmtToMDL(ctx, s), - } + result := &DiffResult{ObjectType: "Microflow", ObjectName: s.Name} - // Try to find existing microflow - h, err := getHierarchy(ctx) + // Build the flow the script describes, without writing anything, then + // render it through the SAME describer the stored side goes through. The + // second AST-to-MDL renderer this replaces dropped every activity type it + // did not know, which the diff then showed as a deletion (#997). + built, err := buildMicroflowFromStmt(ctx, s, buildFlowOpts{}) if err != nil { + return nil, err + } + proposed, err := renderFlowFromModel(ctx, "microflow", built.Microflow, s.Name) + if err != nil { + return nil, err + } + result.Proposed = proposed + + if built.ExistingID == "" { result.IsNew = true return result, nil } - mfs, err := ctx.Backend.ListMicroflows() - if err != nil { + stored, err := ctx.Backend.GetMicroflow(built.ExistingID) + if err != nil || stored == nil { result.IsNew = true return result, nil } - - for _, mf := range mfs { - modID := h.FindModuleID(mf.ContainerID) - modName := h.GetModuleName(modID) - if modName == s.Name.Module && mf.Name == s.Name.Name { - // Capture current MDL representation - var buf bytes.Buffer - oldOutput := ctx.Output - ctx.Output = &buf - describeMicroflow(ctx, s.Name) - ctx.Output = oldOutput - result.Current = strings.TrimSuffix(buf.String(), "\n") - result.Changes = compareMicroflows(ctx, result.Current, result.Proposed) - return result, nil - } + current, err := renderFlowFromModel(ctx, "microflow", stored, s.Name) + if err != nil { + return nil, err } - - result.IsNew = true + result.Current = current + result.Changes = compareMicroflows(ctx, result.Current, result.Proposed) return result, nil } // diffNanoflow compares a CREATE NANOFLOW statement against the project func diffNanoflow(ctx *ExecContext, s *ast.CreateNanoflowStmt) (*DiffResult, error) { - result := &DiffResult{ - ObjectType: "Nanoflow", - ObjectName: s.Name, - Proposed: nanoflowStmtToMDL(ctx, s), - } + result := &DiffResult{ObjectType: "Nanoflow", ObjectName: s.Name} - // Try to find existing nanoflow - // Errors treated as "new" to match diffMicroflow and other diff* functions - h, err := getHierarchy(ctx) + built, err := buildNanoflowFromStmt(ctx, s, buildFlowOpts{}) if err != nil { + return nil, err + } + proposed, err := renderFlowFromModel(ctx, "nanoflow", nanoflowAsMicroflow(built.Nanoflow), s.Name) + if err != nil { + return nil, err + } + result.Proposed = proposed + + if built.ExistingID == "" { result.IsNew = true return result, nil } - nfs, err := ctx.Backend.ListNanoflows() - if err != nil { + stored, err := ctx.Backend.GetNanoflow(built.ExistingID) + if err != nil || stored == nil { result.IsNew = true return result, nil } - - for _, nf := range nfs { - modID := h.FindModuleID(nf.ContainerID) - modName := h.GetModuleName(modID) - if modName == s.Name.Module && nf.Name == s.Name.Name { - // Capture current MDL representation - var buf bytes.Buffer - if err := func() error { - oldOutput := ctx.Output - ctx.Output = &buf - defer func() { ctx.Output = oldOutput }() - return describeNanoflow(ctx, s.Name) - }(); err != nil { - return nil, err - } - result.Current = strings.TrimSuffix(buf.String(), "\n") - result.Changes = compareMicroflows(ctx, result.Current, result.Proposed) - return result, nil - } + current, err := renderFlowFromModel(ctx, "nanoflow", nanoflowAsMicroflow(stored), s.Name) + if err != nil { + return nil, err } - - result.IsNew = true + result.Current = current + result.Changes = compareMicroflows(ctx, result.Current, result.Proposed) return result, nil } diff --git a/mdl/executor/cmd_diff_mdl.go b/mdl/executor/cmd_diff_mdl.go index 87a2b05502..c95acfc7b6 100644 --- a/mdl/executor/cmd_diff_mdl.go +++ b/mdl/executor/cmd_diff_mdl.go @@ -32,8 +32,11 @@ func entityStmtToMDL(ctx *ExecContext, s *ast.CreateEntityStmt) string { lines = append(lines, fmt.Sprintf("@Position(%d, %d)", s.Position.X, s.Position.Y)) } - // Entity type - entityType := s.Kind.String() + // Entity type. EntityKind.String() is upper case for error messages; the + // project side of the diff writes the keyword in lower case, and an + // unmodified describe dump was reported as modified purely on that casing + // (#997, the same two-renderer split as the flow body). + entityType := strings.ToLower(s.Kind.String()) lines = append(lines, fmt.Sprintf("create %s entity %s (", entityType, s.Name)) // Attributes @@ -194,337 +197,6 @@ func associationStmtToMDL(ctx *ExecContext, s *ast.CreateAssociationStmt) string return strings.Join(lines, "\n") } -// microflowStmtToMDL converts a CreateMicroflowStmt to MDL text -func microflowStmtToMDL(ctx *ExecContext, s *ast.CreateMicroflowStmt) string { - var lines []string - - // Annotations - if s.Excluded { - lines = append(lines, "@excluded") - } - - // Documentation - if s.Documentation != "" { - lines = append(lines, "/**") - for docLine := range strings.SplitSeq(s.Documentation, "\n") { - lines = append(lines, " * "+docLine) - } - lines = append(lines, " */") - } - - // CREATE [OR MODIFY] MICROFLOW header with parameters - header := "create" - if s.CreateOrModify { - header = "create or modify" - } - if len(s.Parameters) > 0 { - lines = append(lines, fmt.Sprintf("%s microflow %s (", header, s.Name)) - for i, param := range s.Parameters { - paramType := dataTypeToString(ctx, param.Type) - comma := "," - if i == len(s.Parameters)-1 { - comma = "" - } - lines = append(lines, fmt.Sprintf(" $%s: %s%s", param.Name, paramType, comma)) - } - lines = append(lines, ")") - } else { - lines = append(lines, fmt.Sprintf("%s microflow %s ()", header, s.Name)) - } - - // Folder - if s.Folder != "" { - lines = append(lines, fmt.Sprintf("folder '%s'", s.Folder)) - } - - // Return type - if s.ReturnType != nil { - returnType := dataTypeToString(ctx, s.ReturnType.Type) - if returnType != "Void" && returnType != "" { - returnLine := fmt.Sprintf("returns %s", returnType) - if s.ReturnType.Variable != "" { - returnLine += fmt.Sprintf(" as $%s", s.ReturnType.Variable) - } - lines = append(lines, returnLine) - } - } - - // BEGIN block - lines = append(lines, "begin") - - // Body statements - for _, stmt := range s.Body { - stmtLines := microflowStatementToMDL(ctx, stmt, 1) - lines = append(lines, stmtLines...) - } - - lines = append(lines, "end;") - lines = append(lines, "/") - - return strings.Join(lines, "\n") -} - -// nanoflowStmtToMDL converts a CreateNanoflowStmt to MDL text -func nanoflowStmtToMDL(ctx *ExecContext, s *ast.CreateNanoflowStmt) string { - var lines []string - - // Annotations - if s.Excluded { - lines = append(lines, "@excluded") - } - - // Documentation - if s.Documentation != "" { - lines = append(lines, "/**") - for docLine := range strings.SplitSeq(s.Documentation, "\n") { - lines = append(lines, " * "+docLine) - } - lines = append(lines, " */") - } - - // CREATE [OR MODIFY] NANOFLOW header with parameters - header := "create" - if s.CreateOrModify { - header = "create or modify" - } - if len(s.Parameters) > 0 { - lines = append(lines, fmt.Sprintf("%s nanoflow %s (", header, s.Name)) - for i, param := range s.Parameters { - paramType := dataTypeToString(ctx, param.Type) - comma := "," - if i == len(s.Parameters)-1 { - comma = "" - } - lines = append(lines, fmt.Sprintf(" $%s: %s%s", param.Name, paramType, comma)) - } - lines = append(lines, ")") - } else { - lines = append(lines, fmt.Sprintf("%s nanoflow %s ()", header, s.Name)) - } - - // Folder - if s.Folder != "" { - lines = append(lines, fmt.Sprintf("folder '%s'", s.Folder)) - } - - // Return type - if s.ReturnType != nil { - returnType := dataTypeToString(ctx, s.ReturnType.Type) - if returnType != "Void" && returnType != "" { - returnLine := fmt.Sprintf("returns %s", returnType) - if s.ReturnType.Variable != "" { - returnLine += fmt.Sprintf(" as $%s", s.ReturnType.Variable) - } - lines = append(lines, returnLine) - } - } - - // BEGIN block - lines = append(lines, "begin") - - // Body statements - for _, stmt := range s.Body { - stmtLines := microflowStatementToMDL(ctx, stmt, 1) - lines = append(lines, stmtLines...) - } - - lines = append(lines, "end;") - lines = append(lines, "/") - - return strings.Join(lines, "\n") -} - -// microflowStatementToMDL converts a microflow statement to MDL lines -func microflowStatementToMDL(ctx *ExecContext, stmt ast.MicroflowStatement, indent int) []string { - indentStr := strings.Repeat(" ", indent) - var lines []string - - switch s := stmt.(type) { - case *ast.DeclareStmt: - typeStr := dataTypeToString(ctx, s.Type) - initVal := "empty" - if s.InitialValue != nil { - initVal = diffExpressionToString(ctx, s.InitialValue) - } - lines = append(lines, fmt.Sprintf("%sdeclare $%s %s = %s;", indentStr, s.Variable, typeStr, initVal)) - - case *ast.MfSetStmt: - lines = append(lines, fmt.Sprintf("%sset $%s = %s;", indentStr, s.Target, diffExpressionToString(ctx, s.Value))) - - case *ast.ReturnStmt: - if s.Value != nil { - lines = append(lines, fmt.Sprintf("%sreturn %s;", indentStr, diffExpressionToString(ctx, s.Value))) - } else { - lines = append(lines, fmt.Sprintf("%sreturn;", indentStr)) - } - - case *ast.CreateObjectStmt: - mods := commitModifier(commitTypeOf(s.Commit)) + refreshModifier(s.RefreshInClient) - if len(s.Changes) > 0 { - var members []string - for _, c := range s.Changes { - members = append(members, fmt.Sprintf("%s = %s", c.Attribute, diffExpressionToString(ctx, c.Value))) - } - lines = append(lines, fmt.Sprintf("%s$%s = create %s (%s)%s;", indentStr, s.Variable, s.EntityType, strings.Join(members, ", "), mods)) - } else { - lines = append(lines, fmt.Sprintf("%s$%s = create %s%s;", indentStr, s.Variable, s.EntityType, mods)) - } - - case *ast.ChangeObjectStmt: - mods := commitModifier(commitTypeOf(s.Commit)) + refreshModifier(s.RefreshInClient) - if len(s.Changes) > 0 { - var members []string - for _, c := range s.Changes { - members = append(members, fmt.Sprintf("%s = %s", c.Attribute, diffExpressionToString(ctx, c.Value))) - } - lines = append(lines, fmt.Sprintf("%schange $%s (%s)%s;", indentStr, s.Variable, strings.Join(members, ", "), mods)) - } else { - lines = append(lines, fmt.Sprintf("%schange $%s%s;", indentStr, s.Variable, mods)) - } - - case *ast.MfCommitStmt: - // Same rendering rule as the describer: events ON is the default and - // stays unwritten, events OFF is spelled out. Dropping the modifiers - // here would make `mxcli diff` blind to a change in exactly the flags - // #895 was about. - suffix := "" - if s.WithoutEvents { - suffix += " without events" - } - suffix += refreshModifier(s.RefreshInClient) - lines = append(lines, fmt.Sprintf("%scommit $%s%s;", indentStr, s.Variable, suffix)) - - case *ast.DeleteObjectStmt: - lines = append(lines, fmt.Sprintf("%sdelete $%s%s;", indentStr, s.Variable, refreshModifier(s.RefreshInClient))) - - case *ast.RetrieveStmt: - var stmt string - if s.StartVariable != "" { - stmt = fmt.Sprintf("%sretrieve $%s from $%s/%s", indentStr, s.Variable, s.StartVariable, s.Source) - } else { - stmt = fmt.Sprintf("%sretrieve $%s from %s", indentStr, s.Variable, s.Source) - } - if s.Where != nil { - stmt += fmt.Sprintf("\n%s where %s", indentStr, diffExpressionToString(ctx, s.Where)) - } - if s.Limit != "" { - stmt += fmt.Sprintf("\n%s limit %s", indentStr, s.Limit) - } - lines = append(lines, stmt+";") - - case *ast.IfStmt: - lines = append(lines, fmt.Sprintf("%sif %s then", indentStr, diffExpressionToString(ctx, s.Condition))) - for _, thenStmt := range s.ThenBody { - lines = append(lines, microflowStatementToMDL(ctx, thenStmt, indent+1)...) - } - if s.HasElse || len(s.ElseBody) > 0 { - lines = append(lines, indentStr+"else") - for _, elseStmt := range s.ElseBody { - lines = append(lines, microflowStatementToMDL(ctx, elseStmt, indent+1)...) - } - } - lines = append(lines, indentStr+"end if;") - - // Branch bodies render at indent+2, one level in from their `when` at - // indent+1. Both splits rendered them at indent+1 — the same column as the - // branch keyword — which is unreadable once anything nests (#913). - case *ast.EnumSplitStmt: - lines = append(lines, fmt.Sprintf("%scase $%s", indentStr, s.Variable)) - for _, c := range s.Cases { - lines = append(lines, fmt.Sprintf("%s when %s then", indentStr, formatEnumSplitCaseValues(enumSplitCaseValues(c)))) - for _, caseStmt := range c.Body { - lines = append(lines, microflowStatementToMDL(ctx, caseStmt, indent+2)...) - } - } - if len(s.ElseBody) > 0 { - lines = append(lines, indentStr+" else") - for _, elseStmt := range s.ElseBody { - lines = append(lines, microflowStatementToMDL(ctx, elseStmt, indent+2)...) - } - } - lines = append(lines, indentStr+"end case;") - - case *ast.InheritanceSplitStmt: - lines = append(lines, fmt.Sprintf("%ssplit type $%s", indentStr, s.Variable)) - for _, c := range s.Cases { - lines = append(lines, fmt.Sprintf("%s when %s then", indentStr, c.Entity.String())) - for _, caseStmt := range c.Body { - lines = append(lines, microflowStatementToMDL(ctx, caseStmt, indent+2)...) - } - } - if len(s.ElseBody) > 0 { - // Mendix's `(empty)` flow (null object), not a default branch. - lines = append(lines, indentStr+" when (empty) then") - for _, elseStmt := range s.ElseBody { - lines = append(lines, microflowStatementToMDL(ctx, elseStmt, indent+2)...) - } - } - lines = append(lines, indentStr+"end split;") - - case *ast.CastObjectStmt: - if s.ObjectVariable == "" { - lines = append(lines, fmt.Sprintf("%scast $%s;", indentStr, s.OutputVariable)) - } else { - lines = append(lines, fmt.Sprintf("%s$%s = cast $%s;", indentStr, s.OutputVariable, s.ObjectVariable)) - } - - case *ast.LoopStmt: - lines = append(lines, fmt.Sprintf("%sloop $%s in $%s", indentStr, s.LoopVariable, s.ListVariable)) - for _, bodyStmt := range s.Body { - lines = append(lines, microflowStatementToMDL(ctx, bodyStmt, indent+1)...) - } - lines = append(lines, indentStr+"end loop;") - - case *ast.LogStmt: - nodeStr := defaultLogNodeExpression - if s.Node != nil { - nodeStr = diffExpressionToString(ctx, s.Node) - } - msgStr := diffExpressionToString(ctx, s.Message) - stmt := fmt.Sprintf("%slog %s node %s %s", indentStr, strings.ToLower(s.Level.String()), nodeStr, msgStr) - if len(s.Template) > 0 { - var params []string - for _, p := range s.Template { - params = append(params, fmt.Sprintf("{%d} = %s", p.Index, diffExpressionToString(ctx, p.Value))) - } - stmt += fmt.Sprintf(" with (%s)", strings.Join(params, ", ")) - } - lines = append(lines, stmt+";") - - case *ast.CallMicroflowStmt: - var params []string - for _, arg := range s.Arguments { - params = append(params, fmt.Sprintf("%s = %s", arg.Name, diffExpressionToString(ctx, arg.Value))) - } - paramStr := strings.Join(params, ", ") - if s.OutputVariable != "" { - lines = append(lines, fmt.Sprintf("%s$%s = call microflow %s(%s);", indentStr, s.OutputVariable, s.MicroflowName, paramStr)) - } else { - lines = append(lines, fmt.Sprintf("%scall microflow %s(%s);", indentStr, s.MicroflowName, paramStr)) - } - - case *ast.CallNanoflowStmt: - var params []string - for _, arg := range s.Arguments { - params = append(params, fmt.Sprintf("%s = %s", arg.Name, diffExpressionToString(ctx, arg.Value))) - } - paramStr := strings.Join(params, ", ") - if s.OutputVariable != "" { - lines = append(lines, fmt.Sprintf("%s$%s = call nanoflow %s(%s);", indentStr, s.OutputVariable, s.NanoflowName, paramStr)) - } else { - lines = append(lines, fmt.Sprintf("%scall nanoflow %s(%s);", indentStr, s.NanoflowName, paramStr)) - } - - case *ast.BreakStmt: - lines = append(lines, indentStr+"break;") - - case *ast.ContinueStmt: - lines = append(lines, indentStr+"continue;") - } - - return lines -} - // ============================================================================ // Project to MDL Converters // ============================================================================ @@ -807,48 +479,3 @@ func dataTypeToString(_ *ExecContext, dt ast.DataType) string { return "Unknown" } } - -// diffExpressionToString converts an expression to its string representation for diff output -func diffExpressionToString(ctx *ExecContext, expr ast.Expression) string { - if expr == nil { - return "empty" - } - - switch ex := expr.(type) { - case *ast.LiteralExpr: - if ex.Kind == ast.LiteralString { - return fmt.Sprintf("'%v'", ex.Value) - } - if ex.Kind == ast.LiteralEmpty { - return "empty" - } - if ex.Kind == ast.LiteralNull { - return "null" - } - return fmt.Sprintf("%v", ex.Value) - case *ast.VariableExpr: - return "$" + ex.Name - case *ast.AttributePathExpr: - return "$" + ex.Variable + "/" + strings.Join(ex.Path, "/") - case *ast.BinaryExpr: - return fmt.Sprintf("%s %s %s", diffExpressionToString(ctx, ex.Left), ex.Operator, diffExpressionToString(ctx, ex.Right)) - case *ast.UnaryExpr: - return fmt.Sprintf("%s%s", ex.Operator, diffExpressionToString(ctx, ex.Operand)) - case *ast.FunctionCallExpr: - var args []string - for _, arg := range ex.Arguments { - args = append(args, diffExpressionToString(ctx, arg)) - } - return fmt.Sprintf("%s(%s)", ex.Name, strings.Join(args, ", ")) - case *ast.TokenExpr: - return fmt.Sprintf("[%%%s%%]", ex.Token) - case *ast.ParenExpr: - return fmt.Sprintf("(%s)", diffExpressionToString(ctx, ex.Inner)) - case *ast.QualifiedNameExpr: - return ex.QualifiedName.String() - case *ast.ConstantRefExpr: - return "@" + ex.QualifiedName.String() - default: - return fmt.Sprintf("%v", expr) - } -} diff --git a/mdl/executor/cmd_diff_render.go b/mdl/executor/cmd_diff_render.go new file mode 100644 index 0000000000..94a21e4abf --- /dev/null +++ b/mdl/executor/cmd_diff_render.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package executor - rendering both sides of a diff through one describer. +package executor + +import ( + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// flowNameMaps builds the ID → qualified-name maps renderMicroflowMDL needs to +// print entity and flow references. Shared so that the two sides of a diff +// resolve names identically; a map built for one side only would show a +// reference as a name on one side and a stub on the other. +func flowNameMaps(ctx *ExecContext) (entityNames, microflowNames map[model.ID]string, err error) { + h, err := getHierarchy(ctx) + if err != nil { + return nil, nil, mdlerrors.NewBackend("build hierarchy", err) + } + + entityNames = make(map[model.ID]string) + domainModels, _ := ctx.Backend.ListDomainModels() + for _, dm := range domainModels { + modName := h.GetModuleName(dm.ContainerID) + for _, entity := range dm.Entities { + entityNames[entity.ID] = modName + "." + entity.Name + } + } + + microflowNames = make(map[model.ID]string) + allMicroflows, err := ctx.Backend.ListMicroflows() + if err != nil { + return nil, nil, mdlerrors.NewBackend("list microflows", err) + } + for _, mf := range allMicroflows { + microflowNames[mf.ID] = h.GetQualifiedName(mf.ContainerID, mf.Name) + } + allNanoflows, _ := ctx.Backend.ListNanoflows() + for _, nf := range allNanoflows { + microflowNames[nf.ID] = h.GetQualifiedName(nf.ContainerID, nf.Name) + } + return entityNames, microflowNames, nil +} + +// renderFlowFromModel renders an in-memory flow as MDL through the same +// describer DESCRIBE and diff-local use. +// +// This is the whole point of the #997 fix. diff used to render its script side +// with a second AST-to-MDL renderer, which covered 18 of 43 activity types and +// silently emitted nothing for the rest — so a java-action call, a `download +// file` or a canvas annotation appeared in the diff as a deletion, and mxcli +// confidently reported that a script would gut a microflow that exec proved +// was a no-op. One renderer for both sides makes that class of false report +// unrepresentable rather than fixed case by case. +func renderFlowFromModel(ctx *ExecContext, flowType string, mf *microflows.Microflow, name ast.QualifiedName) (string, error) { + entityNames, microflowNames, err := flowNameMaps(ctx) + if err != nil { + return "", err + } + return renderMicroflowMDL(ctx, flowType, mf, name, entityNames, microflowNames, nil), nil +} + +// nanoflowAsMicroflow wraps a Nanoflow so renderMicroflowMDL can print it. +// ContainerID is carried so the `folder` line matches the microflow path; +// without it one side of a nanoflow diff would print a folder and the other +// would not. +func nanoflowAsMicroflow(nf *microflows.Nanoflow) *microflows.Microflow { + if nf == nil { + return nil + } + return µflows.Microflow{ + BaseElement: nf.BaseElement, + ContainerID: nf.ContainerID, + Name: nf.Name, + Documentation: nf.Documentation, + Excluded: nf.Excluded, + Parameters: nf.Parameters, + ReturnType: nf.ReturnType, + ObjectCollection: nf.ObjectCollection, + AllowedModuleRoles: nf.AllowedModuleRoles, + } +} diff --git a/mdl/executor/cmd_diff_render_test.go b/mdl/executor/cmd_diff_render_test.go new file mode 100644 index 0000000000..1264101ffb --- /dev/null +++ b/mdl/executor/cmd_diff_render_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "errors" + "os" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// #997: diff rendered its script side with a second AST-to-MDL renderer whose +// statement switch had no default case, so an activity it did not know emitted +// zero lines and showed up in the diff as a deletion. mxcli reported that a +// script would gut a microflow that exec proved was a no-op. +// +// The fix is structural — one renderer for both sides — so the regression test +// that matters is that the dead renderer stays dead. A reviewer adding a case +// to a revived script-side renderer would re-create the drift; this fails +// first and says why. +func TestDiffHasNoSecondFlowRenderer(t *testing.T) { + for _, name := range []string{ + "microflowStmtToMDL", + "nanoflowStmtToMDL", + "microflowStatementToMDL", + "diffExpressionToString", + } { + if diffMDLSource(t, name) { + t.Errorf("%s is back in cmd_diff_mdl.go. diff must render its script side "+ + "through renderMicroflowMDL — the describer both `describe` and `diff-local` "+ + "use — so that an activity it cannot print is impossible rather than shown "+ + "as a deletion (#997).", name) + } + } +} + +// diffMDLSource reports whether cmd_diff_mdl.go still defines fn. +func diffMDLSource(t *testing.T, fn string) bool { + t.Helper() + b, err := os.ReadFile("cmd_diff_mdl.go") + if err != nil { + t.Fatalf("read cmd_diff_mdl.go: %v", err) + } + return strings.Contains(string(b), "func "+fn+"(") +} + +// nanoflowAsMicroflow must carry ContainerID: renderMicroflowMDL prints the +// `folder` line from it, so dropping it would make one side of a nanoflow diff +// print a folder and the other not — a false modification of exactly the kind +// this issue was about. +func TestNanoflowWrapperCarriesContainer(t *testing.T) { + nf := µflows.Nanoflow{Name: "NF", ContainerID: "folder-id"} + got := nanoflowAsMicroflow(nf) + if got.ContainerID != "folder-id" { + t.Errorf("ContainerID = %q, want folder-id — the folder line would differ between sides", got.ContainerID) + } + if nanoflowAsMicroflow(nil) != nil { + t.Error("nil nanoflow should wrap to nil") + } +} + +// A statement diff cannot compare must be reported, not dropped. Silently +// skipping made `diff` print "0 new, 0 modified, 0 unchanged" for a script that +// would genuinely add documents — worse than a wrong count, because there is +// nothing on screen to disbelieve. +func TestUnsupportedStatementIsAnError(t *testing.T) { + _, err := diffStatement(nil, &ast.CreateConstantStmt{}) + if err == nil { + t.Fatal("an unsupported statement returned no error — it would vanish from the summary") + } + var unsupported *unsupportedDiffError + if !errors.As(err, &unsupported) { + t.Fatalf("error = %T, want *unsupportedDiffError", err) + } + if !strings.Contains(unsupported.kind, "constant") { + t.Errorf("kind = %q, want it to name the statement", unsupported.kind) + } +} + +func TestStatementKindName(t *testing.T) { + if got := statementKindName(&ast.CreateConstantStmt{}); got != "create constant" { + t.Errorf("got %q, want \"create constant\"", got) + } +} diff --git a/mdl/executor/cmd_entities.go b/mdl/executor/cmd_entities.go index 641d8d6dda..50dbe5c38a 100644 --- a/mdl/executor/cmd_entities.go +++ b/mdl/executor/cmd_entities.go @@ -397,6 +397,12 @@ func execCreateEntity(ctx *ExecContext, s *ast.CreateEntityStmt) error { } // Carry forward (and prune) indexes so a dropped indexed attribute doesn't // leave an orphaned index that crashes `mx check` (finding #39). + // A rewrite that says nothing about documentation preserves what is + // stored; an explicitly empty `/** */` clears it. Same rule and same + // reason as the index carry below (mendixlabs/mxcli#1018). + if !s.DocumentationSet { + entity.Documentation = existingEntity.Documentation + } if droppedIdx := reconcileDroppedIndexes(entity, existingEntity); droppedIdx > 0 { fmt.Fprintf(ctx.Output, " Dropped %d index(es) that referenced removed attribute(s).\n", droppedIdx) @@ -664,6 +670,9 @@ func execCreateViewEntity(ctx *ExecContext, s *ast.CreateViewEntityStmt) error { // Update existing entity — preserve Source object ID to avoid CE-6770 entity.ID = existingEntity.ID entity.SourceObjectID = existingEntity.SourceObjectID + // A rewrite that carried no doc comment keeps the stored one (#1018). + entity.Documentation = carriedDocumentation( + s.DocumentationSet, s.Documentation, existingEntity.Documentation) if err := ctx.Backend.UpdateEntity(dm.ID, entity); err != nil { return mdlerrors.NewBackend("update view entity", err) } diff --git a/mdl/executor/cmd_enumerations.go b/mdl/executor/cmd_enumerations.go index 3c9078883c..b87d79e659 100644 --- a/mdl/executor/cmd_enumerations.go +++ b/mdl/executor/cmd_enumerations.go @@ -82,6 +82,10 @@ func execCreateEnumeration(ctx *ExecContext, s *ast.CreateEnumerationStmt) error // Excluded is model state, not script state: MDL cannot express it for // an enumeration, so the stored value is the one that survives (#914). enum.Excluded = existingEnum.Excluded + // Absent doc comment preserves, explicitly empty clears (#1018). + if !s.DocumentationSet { + enum.Documentation = existingEnum.Documentation + } // Placement is model state too when the statement is silent about it. if s.Folder == "" { enum.ContainerID = existingEnum.ContainerID diff --git a/mdl/executor/cmd_export_mappings.go b/mdl/executor/cmd_export_mappings.go index 1a3a4fa9e4..a1e172fa9c 100644 --- a/mdl/executor/cmd_export_mappings.go +++ b/mdl/executor/cmd_export_mappings.go @@ -122,6 +122,15 @@ func describeExportMapping(ctx *ExecContext, name ast.QualifiedName) error { // Dropped entirely before #263 — and the output still PARSED, so // re-executing a DESCRIBE rebuilt the mapping bound to nothing. fmt.Fprintf(ctx.Output, " with message definition %s\n", em.MessageDefinition) + } else if em.WebServiceSource.IsSet() { + // MDL has no `with web service` clause, so this cannot round-trip. + // Emitting NOTHING would be worse than saying so: the output parses, and + // re-executing it deletes the binding (ako/mxcli#365). The precedent is + // the range-bounded-by-attribute rule, which is marked rather than + // rendered wrong. + fmt.Fprintf(ctx.Output, " -- SOURCE NOT REPRESENTABLE: imported web service %s%s\n", + em.WebServiceSource.ImportedWebService, webServiceDetail(em.WebServiceSource)) + fmt.Fprintf(ctx.Output, " -- re-executing this statement would drop it (CE6896); mxcli refuses the rewrite\n") } if em.NullValueOption != "" && em.NullValueOption != "LeaveOutElement" { @@ -271,6 +280,13 @@ func execCreateExportMapping(ctx *ExecContext, s *ast.CreateExportMappingStmt) e if existing != nil && !s.CreateOrModify { return mdlerrors.NewAlreadyExists("export mapping", s.Name.String()) } + // A stored SOAP binding is not something the statement can restate, so a + // rewrite would delete it (ako/mxcli#365). + if existing != nil { + if err := checkNoWebServiceSource("export", s.Name.String(), existing.WebServiceSource); err != nil { + return err + } + } module, err := findModule(ctx, s.Name.Module) if err != nil { @@ -394,6 +410,9 @@ func finishExportMapping(ctx *ExecContext, s *ast.CreateExportMappingStmt, ) error { if existing != nil { em.ID = existing.ID + // A rewrite must not delete the samples the stored document carries + // (ako/mxcli#379). + carryExportOriginalValues(em, existing) if err := ctx.Backend.UpdateExportMapping(em); err != nil { return mdlerrors.NewBackend("update export mapping", err) } diff --git a/mdl/executor/cmd_imagecollections.go b/mdl/executor/cmd_imagecollections.go index e39fed927c..233b3d7cfb 100644 --- a/mdl/executor/cmd_imagecollections.go +++ b/mdl/executor/cmd_imagecollections.go @@ -49,6 +49,10 @@ func execCreateImageCollection(ctx *ExecContext, s *ast.CreateImageCollectionStm ExportLevel: s.ExportLevel, Documentation: s.Comment, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + ic.Documentation = carriedDocumentation(s.DocumentationSet, s.Comment, existing.Documentation) + } if existing != nil { ic.ID = existing.ID // Excluded is model state, not script state (#914). diff --git a/mdl/executor/cmd_import_mappings.go b/mdl/executor/cmd_import_mappings.go index ae14e46e51..5644c7ebf9 100644 --- a/mdl/executor/cmd_import_mappings.go +++ b/mdl/executor/cmd_import_mappings.go @@ -122,6 +122,15 @@ func describeImportMapping(ctx *ExecContext, name ast.QualifiedName) error { // Dropped entirely before #263 — and the output still PARSED, so // re-executing a DESCRIBE rebuilt the mapping bound to nothing. fmt.Fprintf(ctx.Output, " with message definition %s\n", im.MessageDefinition) + } else if im.WebServiceSource.IsSet() { + // MDL has no `with web service` clause, so this cannot round-trip. + // Emitting NOTHING would be worse than saying so: the output parses, and + // re-executing it deletes the binding (ako/mxcli#365). The precedent is + // the range-bounded-by-attribute rule, which is marked rather than + // rendered wrong. + fmt.Fprintf(ctx.Output, " -- SOURCE NOT REPRESENTABLE: imported web service %s%s\n", + im.WebServiceSource.ImportedWebService, webServiceDetail(im.WebServiceSource)) + fmt.Fprintf(ctx.Output, " -- re-executing this statement would drop it (CE6896); mxcli refuses the rewrite\n") } // The input object (#265). Printing it is what makes a `Param: parameter` // handler in the body re-executable at all. @@ -372,6 +381,13 @@ func execCreateImportMapping(ctx *ExecContext, s *ast.CreateImportMappingStmt) e if existing != nil && !s.CreateOrModify { return mdlerrors.NewAlreadyExists("import mapping", s.Name.String()) } + // A stored SOAP binding is not something the statement can restate, so a + // rewrite would delete it (ako/mxcli#365). + if existing != nil { + if err := checkNoWebServiceSource("import", s.Name.String(), existing.WebServiceSource); err != nil { + return err + } + } module, err := findModule(ctx, s.Name.Module) if err != nil { @@ -496,6 +512,9 @@ func finishImportMapping(ctx *ExecContext, s *ast.CreateImportMappingStmt, } if existing != nil { im.ID = existing.ID + // A rewrite must not delete the samples the stored document carries + // (ako/mxcli#379). + carryImportOriginalValues(im, existing) if err := ctx.Backend.UpdateImportMapping(im); err != nil { return mdlerrors.NewBackend("update import mapping", err) } @@ -593,14 +612,14 @@ func buildImportMappingElementModel(moduleName string, def *ast.ImportMappingEle elem.MinOccurs = jsElem.MinOccurs elem.MaxOccurs = jsElem.MaxOccurs elem.Nillable = jsElem.Nillable - // OriginalValue is deliberately NOT cloned. It is the sample value parsed - // out of the JSON structure's snippet ("42", "\"Widget\""), and it belongs - // to the STRUCTURE — Studio Pro leaves it empty on every mapping element. - // Measured across the two Studio-Pro-authored mappings a blank app ships - // (FeedbackModule's IMM_PostResponse and EMM_PostFeedback, ~15 value - // elements between them): all "", while their structures carry 17 non-empty - // samples. Copying the sample in makes an mxcli-written mapping differ from - // a Studio-Pro-written one over the same structure. (issue #882) + // OriginalValue is deliberately NOT cloned from the structure — a NEW + // mapping gets an empty one (#882). That decision stands, but its + // original measurement was too narrow: it read two mappings a blank app + // ships, and at corpus scale 2,322 of 3,042 value elements DO carry the + // sample. The split is per document (145 mappings all, 107 none, 2 + // mixed), so it is not derivable — which is why a REWRITE carries the + // stored value forward instead of choosing. See + // carryImportOriginalValues (ako/mxcli#379). elem.FractionDigits = jsElem.FractionDigits elem.TotalDigits = jsElem.TotalDigits elem.MaxLength = jsElem.MaxLength diff --git a/mdl/executor/cmd_javaactions.go b/mdl/executor/cmd_javaactions.go index 0cb447ca97..4d883a6eb3 100644 --- a/mdl/executor/cmd_javaactions.go +++ b/mdl/executor/cmd_javaactions.go @@ -321,6 +321,8 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { var existingContainer model.ID // Target the live action and carry its exclusion forward (#914). existingExcluded := false + var existingJADoc string + haveExistingJA := false var existingActionInfo *javaactions.MicroflowActionInfo if existing, ok := pickLive(jas, func(ja *types.JavaAction) bool { @@ -334,6 +336,8 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { existingJAID = existing.ID existingExcluded = existing.Excluded existingContainer = existing.ContainerID + existingJADoc = existing.Documentation + haveExistingJA = true // The toolbox entry holds four PNG bitmaps MDL cannot name, so the // stored one has to be read before the rewrite can carry them. if full, err := ctx.Backend.ReadJavaActionByName(s.Name.Module + "." + s.Name.Name); err == nil && full != nil { @@ -364,6 +368,10 @@ func execCreateJavaAction(ctx *ExecContext, s *ast.CreateJavaActionStmt) error { Documentation: s.Documentation, ExportLevel: "Public", } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExistingJA { + ja.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingJADoc) + } // Build type parameter definitions (with IDs for BY_ID references) typeParamNameToID := make(map[string]model.ID) diff --git a/mdl/executor/cmd_javascript_actions_write.go b/mdl/executor/cmd_javascript_actions_write.go index 10c066d01c..81098ca9f7 100644 --- a/mdl/executor/cmd_javascript_actions_write.go +++ b/mdl/executor/cmd_javascript_actions_write.go @@ -52,6 +52,8 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS // Target the live action and carry its exclusion forward (#914). existingExcluded := false var existingActionInfo *types.MicroflowActionInfo + var existingJSDoc string + haveExistingJS := false if ex, ok := pickLive(existing, func(a *types.JavaScriptAction) bool { return h.GetModuleName(h.FindModuleID(a.ContainerID)) == s.Name.Module && a.Name == s.Name.Name @@ -67,6 +69,8 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS // The toolbox entry's four PNG bitmaps are not expressible in MDL, so // they have to be carried rather than rebuilt. existingActionInfo = ex.MicroflowActionInfo + existingJSDoc = ex.Documentation + haveExistingJS = true } moduleID := containerID @@ -90,6 +94,10 @@ func execCreateJavaScriptAction(ctx *ExecContext, s *ast.CreateJavaScriptActionS ActionDefaultReturnName: "ReturnValueName", Platform: platformOrDefault(s.Platform), } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExistingJS { + jsa.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingJSDoc) + } // Type parameter definitions (with IDs for BY_ID references). typeParamNameToID := make(map[string]model.ID) diff --git a/mdl/executor/cmd_jsonstructures.go b/mdl/executor/cmd_jsonstructures.go index b55d0efad5..c2c0cd5e6f 100644 --- a/mdl/executor/cmd_jsonstructures.go +++ b/mdl/executor/cmd_jsonstructures.go @@ -4,7 +4,9 @@ package executor import ( + "encoding/json" "fmt" + "reflect" "sort" "strings" "unicode" @@ -295,6 +297,18 @@ func execCreateJsonStructure(ctx *ExecContext, s *ast.CreateJsonStructureStmt) e JsonSnippet: types.PrettyPrintJSON(s.JsonSnippet), Elements: elements, } + // Keep the stored snippet's FORMATTING when the content is the same. mxcli + // pretty-prints on describe, so describe -> exec — how a document is copied + // — otherwise rewrote a snippet Studio Pro had stored on one line into a + // multi-line one. Same JSON, different bytes, and a diff against the + // original for nothing (ako/mxcli#379). + if existing != nil && sameJSONContent(existing.JsonSnippet, js.JsonSnippet) { + js.JsonSnippet = existing.JsonSnippet + } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + js.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existing.Documentation) + } if existing != nil { // Excluded is model state, not script state (#914). js.Excluded = existing.Excluded @@ -361,3 +375,23 @@ func findJsonStructure(ctx *ExecContext, moduleName, structName string) *types.J } return nil } + +// sameJSONContent reports whether two snippets carry the same JSON, ignoring +// whitespace. Comparing the decoded values rather than the strings is the point: +// the question is whether a rewrite would change anything that matters. +// +// Anything that does not parse is treated as different, so a malformed snippet +// is replaced rather than silently kept. +func sameJSONContent(a, b string) bool { + if a == b { + return true + } + var va, vb any + if err := json.Unmarshal([]byte(a), &va); err != nil { + return false + } + if err := json.Unmarshal([]byte(b), &vb); err != nil { + return false + } + return reflect.DeepEqual(va, vb) +} diff --git a/mdl/executor/cmd_menus.go b/mdl/executor/cmd_menus.go index 2d66119b81..21fc9dcb02 100644 --- a/mdl/executor/cmd_menus.go +++ b/mdl/executor/cmd_menus.go @@ -85,6 +85,10 @@ func execCreateMenu(ctx *ExecContext, s *ast.CreateMenuStmt) error { Documentation: s.Documentation, Items: menuItemsFromAST(s.Items), } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + md.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existing.Documentation) + } if existing != nil { // Preserve the document's identity and the properties MDL does not diff --git a/mdl/executor/cmd_messagedefinitions.go b/mdl/executor/cmd_messagedefinitions.go new file mode 100644 index 0000000000..c9ff1d3537 --- /dev/null +++ b/mdl/executor/cmd_messagedefinitions.go @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// CREATE / DROP / DESCRIBE / SHOW MESSAGE DEFINITION COLLECTION, and the two +// ALTER families (ako/mxcli#272). +// +// The executor's job is resolution: turn what the author wrote into the +// properties Mendix stores. Almost all of them are derived — see the codec +// writer for the measured table — and the two that need the domain model are +// here: +// +// - an ATTRIBUTE resolves to the entity that DECLARES it, following the +// generalization chain. 398 of 3,697 exposed attributes in the corpus are +// inherited (10.8%), and qualifying one against the entity that merely uses +// it is CE1613. +// - an ASSOCIATION's MaxOccurs comes from the DIRECTION of traversal. See +// resolveAssociationCardinality. + +// execCreateMessageDefinitionCollection handles CREATE [OR MODIFY]. +func execCreateMessageDefinitionCollection(ctx *ExecContext, s *ast.CreateMessageDefinitionCollectionStmt) error { + if !ctx.ConnectedForWrite() { + return mdlerrors.NewNotConnectedWrite() + } + existing := findMessageCollection(ctx, s.Name.Module, s.Name.Name) + if existing != nil && !s.CreateOrModify { + return mdlerrors.NewAlreadyExists("message definition collection", s.Name.String()) + } + + module, err := findModule(ctx, s.Name.Module) + if err != nil { + return mdlerrors.NewNotFound("module", s.Name.Module) + } + containerID := module.ID + if s.Folder != "" { + folderID, ferr := resolveFolder(ctx, module.ID, s.Folder) + if ferr != nil { + return mdlerrors.NewBackend("resolve folder "+s.Folder, ferr) + } + containerID = folderID + } else if existing != nil { + // Without a folder clause an existing document stays where it is. + containerID = existing.ContainerID + } + + c := &model.MessageDefinitionCollection{ + ContainerID: containerID, + Name: s.Name.Name, + } + if existing != nil { + // Carry what the statement does not restate. + c.ID = existing.ID + c.Documentation = existing.Documentation + c.Excluded = existing.Excluded + c.ExportLevel = existing.ExportLevel + } + + for _, def := range s.Definitions { + built, berr := buildMessageDefinition(ctx, def, s.Name.String()) + if berr != nil { + return berr + } + c.Definitions = append(c.Definitions, built) + } + + if existing != nil { + if err := ctx.Backend.UpdateMessageDefinitionCollection(c); err != nil { + return mdlerrors.NewBackend("update message definition collection", err) + } + if _, err := applyDocumentFolder(ctx, c.ID, existing.ContainerID, containerID); err != nil { + return err + } + ctx.ReportMutation("Modified", "message definition collection: %s", s.Name.String()) + return nil + } + if err := ctx.Backend.CreateMessageDefinitionCollection(c); err != nil { + return mdlerrors.NewBackend("create message definition collection", err) + } + // The document — and any folder resolveFolder just created for it — is not + // in the cached hierarchy, so a later statement looking this collection up + // by module would not find it and would create a DUPLICATE (CE0122). The + // update branch gets this for free from applyDocumentFolder; the create + // branch has to say it. + invalidateHierarchy(ctx) + ctx.ReportMutation("Created", "message definition collection: %s", s.Name.String()) + return nil +} + +// findMessageCollection looks a collection up by module and name. +func findMessageCollection(ctx *ExecContext, moduleName, name string) *model.MessageDefinitionCollection { + all, err := ctx.Backend.ListMessageDefinitionCollections() + if err != nil { + return nil + } + h, herr := getHierarchy(ctx) + for _, c := range all { + if c == nil || c.Name != name { + continue + } + if herr != nil { + return c + } + if h.GetModuleName(h.FindModuleID(c.ContainerID)) == moduleName { + return c + } + } + return nil +} + +// lookupAssociation finds an association by qualified name, in the module the +// name gives — scanning every domain model would pick up a same-named +// association from elsewhere. +func lookupAssociation(ctx *ExecContext, assocQN string) (*domainmodel.Association, bool) { + parts := strings.SplitN(assocQN, ".", 2) + if len(parts) != 2 { + return nil, false + } + b, ok := ctx.Backend.(entityLookupBackend) + if !ok { + return nil, false + } + mod, err := b.GetModuleByName(parts[0]) + if err != nil || mod == nil { + return nil, false + } + dm, err := b.GetDomainModel(mod.ID) + if err != nil || dm == nil { + return nil, false + } + for _, a := range dm.Associations { + if a != nil && a.Name == parts[1] { + return a, true + } + } + // A cross-module association is stored on the module of its FROM entity, so + // it may not be in the module its name suggests. + for _, a := range dm.CrossAssociations { + if a != nil && a.Name == parts[1] { + return nil, false // shape differs; handled as unresolvable for now + } + } + return nil, false +} + +// entityQNByID resolves an entity ID to its qualified name. +func entityQNByID(ctx *ExecContext, id model.ID) string { + b, ok := ctx.Backend.(entityLookupBackend) + if !ok { + return "" + } + dms, err := b.ListDomainModels() + if err != nil { + return "" + } + h, herr := getHierarchy(ctx) + for _, dm := range dms { + for _, e := range dm.Entities { + if e != nil && e.ID == id { + if herr != nil { + return e.Name + } + return h.GetModuleName(h.FindModuleID(dm.ContainerID)) + "." + e.Name + } + } + } + return "" +} + +// buildMessageDefinition resolves one `definition for ` block. +func buildMessageDefinition(ctx *ExecContext, def *ast.MessageDefinitionDef, collection string) (*model.MessageDefinition, error) { + entityQN := def.Entity.String() + if _, ok := lookupEntity(ctx, entityQN); !ok { + return nil, mdlerrors.NewValidation(fmt.Sprintf( + "message definition collection %s: definition %s names the entity %s, which does not exist", + collection, def.Name, entityQN)) + } + + root := &model.MessageDefinitionElement{ + Kind: "Entity", + Entity: entityQN, + OriginalName: shortEntityName(entityQN), + // A definition's root always repeats — 56 of 56 in the corpus — so it + // always carries an item name, and that name is the entity's own. + MaxOccurs: -1, + ExposedItemName: shortEntityName(entityQN), + } + root.ExposedName = def.ExposedName + if root.ExposedName == "" { + // Studio Pro pluralises here. mxcli does not guess English: it defaults + // to the entity's own name and lets `as 'Orders'` say otherwise, the + // same conclusion reached for array-item naming in ako/mxcli#272. + root.ExposedName = root.OriginalName + } + + for _, m := range def.Members { + child, err := buildMessageMember(ctx, m, entityQN, collection, def.Name) + if err != nil { + return nil, err + } + root.Children = append(root.Children, child) + } + return &model.MessageDefinition{Name: def.Name, Root: root}, nil +} + +// buildMessageMember resolves one member against the entity that holds it. +func buildMessageMember(ctx *ExecContext, m *ast.MessageMemberDef, holderQN, collection, definition string) (*model.MessageDefinitionElement, error) { + where := fmt.Sprintf("message definition %s.%s", collection, definition) + + if !m.IsAssociation() { + ref, ok := declaringAttributeRef(ctx, holderQN, m.Attribute) + if !ok { + return nil, mdlerrors.NewValidation(fmt.Sprintf( + "%s: %s has no attribute %q%s", where, holderQN, m.Attribute, + availableAttributes(ctx, holderQN))) + } + e := &model.MessageDefinitionElement{ + Kind: "Attribute", + Attribute: ref, + OriginalName: m.Attribute, + ExposedName: m.ExposedName, + Example: m.Example, + MaxOccurs: 1, + PrimitiveType: resolveMessageMemberType(ctx, holderQN, m.Attribute), + } + if e.ExposedName == "" { + e.ExposedName = m.Attribute + } + return e, nil + } + + assocQN := m.Association.String() + targetQN := m.Entity.String() + maxOccurs, err := resolveAssociationCardinality(ctx, assocQN, holderQN, targetQN, where) + if err != nil { + return nil, err + } + + e := &model.MessageDefinitionElement{ + Kind: "Entity", + Association: assocQN, + Entity: targetQN, + OriginalName: shortEntityName(targetQN), + MaxOccurs: maxOccurs, + ExposedName: m.ExposedName, + } + if e.ExposedName == "" { + e.ExposedName = e.OriginalName + } + // ExposedItemName is set exactly when the element repeats — 461 of 461 — and + // its value is the target entity's own name. + if maxOccurs == -1 { + e.ExposedItemName = e.OriginalName + } + + for _, sub := range m.Members { + child, cerr := buildMessageMember(ctx, sub, targetQN, collection, definition) + if cerr != nil { + return nil, cerr + } + e.Children = append(e.Children, child) + } + return e, nil +} + +// resolveAssociationCardinality returns the MaxOccurs an exposed association +// stores, from the DIRECTION the definition traverses it in. +// +// This is the one derivation in the whole document that is not obvious, and +// getting it backwards has no build error behind it — the definition simply +// exposes a list as a single object, or the reverse. +// +// It is NOT a function of the association's type: measured across the demo +// corpus, all 927 resolvable associations are `Reference`, yet 526 store 1 and +// 401 store -1. It tracks direction, with zero counter-examples: +// +// holder is the FROM entity (child -> parent, following the FK) -> 1 (496) +// holder is the TO entity (parent -> children, in reverse) -> -1 (401) +// +// ako/TestApp confirms it in a single document: Mappings.Order_Customer appears +// in both of its definitions and stores 1 reaching Customer from Order and -1 +// reaching Order from Customer. +// +// An association that connects the two entities in NEITHER direction is refused +// rather than defaulted. A wrong cardinality is worse than a refusal: it builds. +func resolveAssociationCardinality(ctx *ExecContext, assocQN, holderQN, targetQN, where string) (int, error) { + assoc, ok := lookupAssociation(ctx, assocQN) + if !ok { + return 0, mdlerrors.NewValidation(fmt.Sprintf( + "%s: the association %s does not exist", where, assocQN)) + } + fromQN, toQN := associationEnds(ctx, assoc) + switch { + case fromQN == holderQN && toQN == targetQN: + // Following the foreign key: one target per holder. + return 1, nil + case toQN == holderQN && fromQN == targetQN: + // The reverse: many holders point at one target, so from the target's + // side the element repeats. + return -1, nil + } + return 0, mdlerrors.NewValidation(fmt.Sprintf( + "%s: the association %s does not connect %s to %s — it runs from %s to %s. "+ + "The direction decides whether the element is a single object or a list, "+ + "so mxcli refuses rather than guessing", + where, assocQN, holderQN, targetQN, fromQN, toQN)) +} + +// associationEnds returns the association's FROM and TO entity qualified names. +// +// ParentID is the FROM entity (the one that owns the foreign key) and ChildID +// the TO entity — the inversion documented in CLAUDE.md, and the reason this is +// a named function rather than two field reads at the call site. +func associationEnds(ctx *ExecContext, assoc *domainmodel.Association) (string, string) { + return entityQNByID(ctx, assoc.ParentID), entityQNByID(ctx, assoc.ChildID) +} + +func shortEntityName(qualified string) string { + if i := strings.LastIndex(qualified, "."); i >= 0 { + return qualified[i+1:] + } + return qualified +} + +// availableAttributes lists what the holder does have, so a typo says what would +// have worked — the shape #882 established for mapping members. +func availableAttributes(ctx *ExecContext, entityQN string) string { + b, ok := ctx.Backend.(entityLookupBackend) + if !ok { + return "" + } + var names []string + for _, mem := range EntityMembersFor(b, entityQN) { + names = append(names, mem.Name) + } + if len(names) == 0 { + return "" + } + sort.Strings(names) + return "; available: " + strings.Join(names, ", ") +} + +func declaringAttributeRef(ctx *ExecContext, entityQN, attr string) (string, bool) { + b, ok := ctx.Backend.(entityLookupBackend) + if !ok { + return "", false + } + return DeclaringMemberRef(b, entityQN, attr) +} + +// messagePrimitiveTypes maps a Mendix attribute type to the PrimitiveType a +// message definition stores. Measured across 3,372 exposed attributes in the +// demo corpus and ako/TestApp: +// +// Decimal -> Decimal 1104 +// Integer -> Integer 658 +// DateTime -> DateTime 567 +// String -> String 566 +// Enumeration -> String 263 +// Boolean -> Boolean 198 +// Long -> Integer 12 +// AutoNumber -> Integer 4 +// +// The three that are NOT identity are the point: passing the attribute's type +// straight through stores Long and AutoNumber where Mendix stores Integer, and +// Enumeration where it stores String — 279 elements, and the round trip against +// TestApp caught exactly that (ProductId is a Long). +var messagePrimitiveTypes = map[string]string{ + "Enumeration": "String", + "Long": "Integer", + "AutoNumber": "Integer", +} + +func resolveMessageMemberType(ctx *ExecContext, entityQN, attr string) string { + b, ok := ctx.Backend.(entityLookupBackend) + if !ok { + return "String" + } + t := ResolveMemberType(b, entityQN, attr) + if t == "" { + return "String" + } + if mapped, ok := messagePrimitiveTypes[t]; ok { + return mapped + } + return t +} + +func lookupEntity(ctx *ExecContext, entityQN string) (*domainmodel.Entity, bool) { + b, ok := ctx.Backend.(entityLookupBackend) + if !ok { + return nil, false + } + return findEntityByQN(b, entityQN) +} + +// execDropMessageDefinitionCollection deletes a collection. +// +// A mapping bound to it would be left dangling — mxbuild reports CE1613 — so a +// collection still referenced is refused, naming the mappings. That is the same +// courtesy `drop json structure` owes and the reason the refusal lists them +// rather than saying "in use". +func execDropMessageDefinitionCollection(ctx *ExecContext, s *ast.DropMessageDefinitionCollectionStmt) error { + if !ctx.ConnectedForWrite() { + return mdlerrors.NewNotConnectedWrite() + } + c := findMessageCollection(ctx, s.Name.Module, s.Name.Name) + if c == nil { + return mdlerrors.NewNotFound("message definition collection", s.Name.String()) + } + if users := mappingsUsingCollection(ctx, s.Name.String()); len(users) > 0 { + return mdlerrors.NewValidation(fmt.Sprintf( + "message definition collection %s is still used by %s — dropping it would leave "+ + "them bound to nothing (CE1613)", s.Name.String(), strings.Join(users, ", "))) + } + if err := ctx.Backend.DeleteMessageDefinitionCollection(string(c.ID)); err != nil { + return mdlerrors.NewBackend("drop message definition collection", err) + } + ctx.ReportMutation("Dropped", "message definition collection: %s", s.Name.String()) + return nil +} + +// mappingsUsingCollection returns the mappings whose source is a definition in +// this collection. A mapping's reference is three parts +// (Module.Collection.Definition), so the collection is its prefix. +func mappingsUsingCollection(ctx *ExecContext, collectionQN string) []string { + prefix := collectionQN + "." + var out []string + if ims, err := ctx.Backend.ListImportMappings(); err == nil { + for _, im := range ims { + if im != nil && strings.HasPrefix(im.MessageDefinition, prefix) { + out = append(out, "import mapping "+im.Name) + } + } + } + if ems, err := ctx.Backend.ListExportMappings(); err == nil { + for _, em := range ems { + if em != nil && strings.HasPrefix(em.MessageDefinition, prefix) { + out = append(out, "export mapping "+em.Name) + } + } + } + sort.Strings(out) + return out +} + +// execDescribeMessageDefinitionCollection prints re-executable MDL. +func execDescribeMessageDefinitionCollection(ctx *ExecContext, name ast.QualifiedName) error { + c := findMessageCollection(ctx, name.Module, name.Name) + if c == nil { + return mdlerrors.NewNotFound("message definition collection", name.String()) + } + fmt.Fprintf(ctx.Output, "create or modify message definition collection %s\n", name.String()) + if h, err := getHierarchy(ctx); err == nil { + if folder := h.BuildFolderPath(c.ContainerID); folder != "" { + fmt.Fprintf(ctx.Output, " folder '%s'\n", folder) + } + } + fmt.Fprintln(ctx.Output, "(") + for i, def := range c.Definitions { + sep := "," + if i == len(c.Definitions)-1 { + sep = "" + } + describeMessageDefinition(ctx, def, sep) + } + fmt.Fprintln(ctx.Output, ");") + return nil +} + +func describeMessageDefinition(ctx *ExecContext, def *model.MessageDefinition, sep string) { + if def == nil || def.Root == nil { + return + } + fmt.Fprintf(ctx.Output, " definition %s for %s%s (\n", + def.Name, def.Root.Entity, exposedClause(def.Root, shortEntityName(def.Root.Entity))) + describeMessageMembers(ctx, def.Root.Children, " ") + fmt.Fprintf(ctx.Output, " )%s\n", sep) +} + +func describeMessageMembers(ctx *ExecContext, members []*model.MessageDefinitionElement, indent string) { + for i, m := range members { + sep := "," + if i == len(members)-1 { + sep = "" + } + switch { + case m.Kind == "Attribute": + fmt.Fprintf(ctx.Output, "%s%s%s%s%s\n", indent, m.OriginalName, + exposedClause(m, m.OriginalName), exampleClause(m), sep) + case len(m.Children) == 0: + fmt.Fprintf(ctx.Output, "%s%s/%s%s ()%s\n", indent, m.Association, m.Entity, + exposedClause(m, shortEntityName(m.Entity)), sep) + default: + fmt.Fprintf(ctx.Output, "%s%s/%s%s (\n", indent, m.Association, m.Entity, + exposedClause(m, shortEntityName(m.Entity))) + describeMessageMembers(ctx, m.Children, indent+" ") + fmt.Fprintf(ctx.Output, "%s)%s\n", indent, sep) + } + } +} + +// exampleClause emits `example '...'`, the one authored field besides the name. +// Rare — 1 of 4,707 elements — but describe dropping it silently is what makes +// describe -> exec lossy, so it is emitted whenever set. +func exampleClause(e *model.MessageDefinitionElement) string { + if e.Example == "" { + return "" + } + return fmt.Sprintf(" example '%s'", strings.ReplaceAll(e.Example, "'", "''")) +} + +// exposedClause emits `as ''` only when the exposed name differs from what +// the executor would derive, so a describe does not restate every default. +func exposedClause(e *model.MessageDefinitionElement, derived string) string { + if e.ExposedName == "" || e.ExposedName == derived { + return "" + } + return fmt.Sprintf(" as '%s'", e.ExposedName) +} + +// execAlterMessageDefinitionCollection adds, drops or renames a DEFINITION. +// +// It edits the stored collection rather than rebuilding it from a statement, so +// the definitions the statement does not mention are never round-tripped through +// the describer — the argument that made ALTER LAYOUT a capability rather than a +// convenience. +func execAlterMessageDefinitionCollection(ctx *ExecContext, s *ast.AlterMessageDefinitionCollectionStmt) error { + if !ctx.ConnectedForWrite() { + return mdlerrors.NewNotConnectedWrite() + } + c := findMessageCollection(ctx, s.Name.Module, s.Name.Name) + if c == nil { + return mdlerrors.NewNotFound("message definition collection", s.Name.String()) + } + + switch s.Op { + case "ADD": + if idx := indexOfDefinition(c, s.Definition.Name); idx >= 0 { + if s.IfNotExist { + ctx.ReportMutation("Unchanged", "message definition collection: %s (definition %s already exists)", + s.Name.String(), s.Definition.Name) + return nil + } + return mdlerrors.NewAlreadyExists("message definition", s.Name.String()+"."+s.Definition.Name) + } + built, err := buildMessageDefinition(ctx, s.Definition, s.Name.String()) + if err != nil { + return err + } + c.Definitions = append(c.Definitions, built) + case "DROP": + idx := indexOfDefinition(c, s.Target) + if idx < 0 { + if s.IfExists { + ctx.ReportMutation("Unchanged", "message definition collection: %s (no definition %s)", + s.Name.String(), s.Target) + return nil + } + return mdlerrors.NewNotFound("message definition", s.Name.String()+"."+s.Target) + } + if users := mappingsUsingDefinition(ctx, s.Name.String()+"."+s.Target); len(users) > 0 { + return mdlerrors.NewValidation(fmt.Sprintf( + "message definition %s.%s is still used by %s — dropping it would leave them "+ + "bound to nothing (CE1613)", s.Name.String(), s.Target, strings.Join(users, ", "))) + } + c.Definitions = append(c.Definitions[:idx], c.Definitions[idx+1:]...) + case "RENAME": + idx := indexOfDefinition(c, s.Target) + if idx < 0 { + return mdlerrors.NewNotFound("message definition", s.Name.String()+"."+s.Target) + } + if indexOfDefinition(c, s.NewName) >= 0 { + return mdlerrors.NewAlreadyExists("message definition", s.Name.String()+"."+s.NewName) + } + // A mapping names the definition, so renaming one behind its back leaves + // the mapping dangling. Refuse rather than rename half the model. + if users := mappingsUsingDefinition(ctx, s.Name.String()+"."+s.Target); len(users) > 0 { + return mdlerrors.NewValidation(fmt.Sprintf( + "message definition %s.%s is used by %s, which names it — renaming it here would "+ + "leave them bound to nothing (CE1613)", + s.Name.String(), s.Target, strings.Join(users, ", "))) + } + c.Definitions[idx].Name = s.NewName + default: + return mdlerrors.NewValidation("unsupported ALTER MESSAGE DEFINITION COLLECTION operation") + } + + if err := ctx.Backend.UpdateMessageDefinitionCollection(c); err != nil { + return mdlerrors.NewBackend("update message definition collection", err) + } + ctx.ReportMutation("Modified", "message definition collection: %s", s.Name.String()) + return nil +} + +// execAlterMessageDefinition adds, drops or renames a MEMBER within one +// definition. +func execAlterMessageDefinition(ctx *ExecContext, s *ast.AlterMessageDefinitionStmt) error { + if !ctx.ConnectedForWrite() { + return mdlerrors.NewNotConnectedWrite() + } + c := findMessageCollection(ctx, s.Collection.Module, s.Collection.Name) + if c == nil { + return mdlerrors.NewNotFound("message definition collection", s.Collection.String()) + } + idx := indexOfDefinition(c, s.Definition) + if idx < 0 { + return mdlerrors.NewNotFound("message definition", s.Collection.String()+"."+s.Definition) + } + def := c.Definitions[idx] + + holder, err := resolveMemberPath(def.Root, s.Path, s.Collection.String()+"."+s.Definition) + if err != nil { + return err + } + + switch s.Op { + case "ADD": + name := memberName(s.Member) + if findChildByName(holder, name) != nil { + if s.IfNotExist { + ctx.ReportMutation("Unchanged", "message definition: %s.%s (member %s already exists)", + s.Collection.String(), s.Definition, name) + return nil + } + return mdlerrors.NewAlreadyExists("message definition member", name) + } + built, berr := buildMessageMember(ctx, s.Member, holder.Entity, s.Collection.String(), s.Definition) + if berr != nil { + return berr + } + holder.Children = append(holder.Children, built) + case "DROP": + child := findChildByName(holder, s.Target) + if child == nil { + if s.IfExists { + ctx.ReportMutation("Unchanged", "message definition: %s.%s (no member %s)", + s.Collection.String(), s.Definition, s.Target) + return nil + } + return mdlerrors.NewNotFound("message definition member", s.Target) + } + holder.Children = removeChild(holder.Children, child) + case "SET": + child := findChildByName(holder, s.Target) + if child == nil { + return mdlerrors.NewNotFound("message definition member", s.Target) + } + // SET changes the ExposedName and nothing else. It is not a model + // rename: the underlying attribute or association is untouched, which is + // why the keyword is SET rather than RENAME. + child.ExposedName = s.ExposedName + default: + return mdlerrors.NewValidation("unsupported ALTER MESSAGE DEFINITION operation") + } + + if err := ctx.Backend.UpdateMessageDefinitionCollection(c); err != nil { + return mdlerrors.NewBackend("update message definition collection", err) + } + ctx.ReportMutation("Modified", "message definition: %s.%s", s.Collection.String(), s.Definition) + return nil +} + +// resolveMemberPath walks `in a/b` down the definition's tree, in exposed names. +// +// Members nest to depth 7 in the corpus, so reaching one is ordinary rather than +// an edge case, and a path that reaches nothing is an error naming what is +// there — the shape MDL-JSON01 established. +func resolveMemberPath(root *model.MessageDefinitionElement, path []string, where string) (*model.MessageDefinitionElement, error) { + node := root + for _, seg := range path { + child := findChildByExposedName(node, seg) + if child == nil { + return nil, mdlerrors.NewValidation(fmt.Sprintf( + "%s: no member %q under %q%s", where, seg, node.ExposedName, availableChildren(node))) + } + if child.Kind != "Entity" { + return nil, mdlerrors.NewValidation(fmt.Sprintf( + "%s: %q is a value, so it has no members to reach into", where, seg)) + } + node = child + } + return node, nil +} + +func availableChildren(n *model.MessageDefinitionElement) string { + var names []string + for _, c := range n.Children { + names = append(names, c.ExposedName) + } + if len(names) == 0 { + return "" + } + sort.Strings(names) + return "; available: " + strings.Join(names, ", ") +} + +// findChildByName matches on the member's OWN name — the attribute's or the +// target entity's — which is what `add`/`drop`/`set member X` names. +func findChildByName(n *model.MessageDefinitionElement, name string) *model.MessageDefinitionElement { + for _, c := range n.Children { + if c.OriginalName == name { + return c + } + } + return nil +} + +// findChildByExposedName matches on the exposed name, which is what an `in` +// path segment names. +func findChildByExposedName(n *model.MessageDefinitionElement, name string) *model.MessageDefinitionElement { + for _, c := range n.Children { + if c.ExposedName == name { + return c + } + } + return nil +} + +func removeChild(children []*model.MessageDefinitionElement, target *model.MessageDefinitionElement) []*model.MessageDefinitionElement { + out := children[:0] + for _, c := range children { + if c != target { + out = append(out, c) + } + } + return out +} + +func memberName(m *ast.MessageMemberDef) string { + if m == nil { + return "" + } + if m.IsAssociation() { + return shortEntityName(m.Entity.String()) + } + return m.Attribute +} + +func indexOfDefinition(c *model.MessageDefinitionCollection, name string) int { + for i, d := range c.Definitions { + if d != nil && d.Name == name { + return i + } + } + return -1 +} + +// mappingsUsingDefinition returns the mappings bound to exactly this definition. +func mappingsUsingDefinition(ctx *ExecContext, definitionQN string) []string { + var out []string + if ims, err := ctx.Backend.ListImportMappings(); err == nil { + for _, im := range ims { + if im != nil && im.MessageDefinition == definitionQN { + out = append(out, "import mapping "+im.Name) + } + } + } + if ems, err := ctx.Backend.ListExportMappings(); err == nil { + for _, em := range ems { + if em != nil && em.MessageDefinition == definitionQN { + out = append(out, "export mapping "+em.Name) + } + } + } + sort.Strings(out) + return out +} + +// listMessageDefinitionCollections handles SHOW MESSAGE DEFINITION COLLECTIONS. +// +// There was no listing at all before this: a mapping could be authored over a +// definition, but nothing told you which definitions existed. +func listMessageDefinitionCollections(ctx *ExecContext, inModule string) error { + all, err := ctx.Backend.ListMessageDefinitionCollections() + if err != nil { + return mdlerrors.NewBackend("list message definition collections", err) + } + h, herr := getHierarchy(ctx) + + type row struct{ module, name, defs string } + var rows []row + for _, c := range all { + if c == nil { + continue + } + module := "" + if herr == nil { + module = h.GetModuleName(h.FindModuleID(c.ContainerID)) + } + if inModule != "" && !strings.EqualFold(module, inModule) { + continue + } + names := make([]string, 0, len(c.Definitions)) + for _, d := range c.Definitions { + if d != nil { + names = append(names, d.Name) + } + } + rows = append(rows, row{module, c.Name, strings.Join(names, ", ")}) + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].module != rows[j].module { + return rows[i].module < rows[j].module + } + return rows[i].name < rows[j].name + }) + + if len(rows) == 0 { + fmt.Fprintln(ctx.Output, "No message definition collections found") + return nil + } + fmt.Fprintln(ctx.Output, "| Module | Collection | Definitions |") + fmt.Fprintln(ctx.Output, "|--------|------------|-------------|") + for _, r := range rows { + fmt.Fprintf(ctx.Output, "| %s | %s | %s |\n", r.module, r.name, r.defs) + } + fmt.Fprintf(ctx.Output, "\n(%d collection(s))\n", len(rows)) + return nil +} diff --git a/mdl/executor/cmd_messagedefinitions_test.go b/mdl/executor/cmd_messagedefinitions_test.go new file mode 100644 index 0000000000..82d16bd873 --- /dev/null +++ b/mdl/executor/cmd_messagedefinitions_test.go @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// A message definition is a selection over the domain model, so the executor's +// whole job is resolution. Two resolutions need the model and are the ones that +// can go silently wrong (ako/mxcli#272). + +// mdFixture builds Order -> Customer (FROM Order, TO Customer), an OrderLine +// that inherits from a base, and the attributes each carries. +func mdFixture(t *testing.T) (*ExecContext, *model.MessageDefinitionCollection) { + t.Helper() + mod := &model.Module{Name: "Sales"} + mod.ID = nextID("mod") + + mkAttr := func(name string, typ domainmodel.AttributeType) *domainmodel.Attribute { + a := &domainmodel.Attribute{Name: name, Type: typ} + a.ID = nextID("attr" + name) + return a + } + base := &domainmodel.Entity{Name: "Base", Persistable: true} + base.ID = nextID("base") + base.Attributes = []*domainmodel.Attribute{mkAttr("Code", &domainmodel.StringAttributeType{})} + + order := &domainmodel.Entity{Name: "Order", Persistable: true, GeneralizationRef: "Sales.Base"} + order.ID = nextID("order") + order.Attributes = []*domainmodel.Attribute{ + mkAttr("OrderId", &domainmodel.LongAttributeType{}), + mkAttr("Status", &domainmodel.EnumerationAttributeType{}), + mkAttr("Total", &domainmodel.DecimalAttributeType{}), + } + customer := &domainmodel.Entity{Name: "Customer", Persistable: true} + customer.ID = nextID("cust") + customer.Attributes = []*domainmodel.Attribute{mkAttr("Name", &domainmodel.StringAttributeType{})} + + // ParentID is the FROM entity (the FK owner); ChildID the TO entity. + assoc := &domainmodel.Association{Name: "Order_Customer", ParentID: order.ID, ChildID: customer.ID} + assoc.ID = nextID("assoc") + + dm := &domainmodel.DomainModel{ContainerID: mod.ID, + Entities: []*domainmodel.Entity{base, order, customer}, + Associations: []*domainmodel.Association{assoc}, + } + dm.ID = nextID("dm") + h := mkHierarchy(mod) + withContainer(h, dm.ID, mod.ID) + + var written *model.MessageDefinitionCollection + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + GetModuleByNameFunc: func(string) (*model.Module, error) { return mod, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + ListMessageDefinitionCollectionsFunc: func() ([]*model.MessageDefinitionCollection, error) { return nil, nil }, + CreateMessageDefinitionCollectionFunc: func(c *model.MessageDefinitionCollection) error { + written = c + return nil + }, + } + ctx, _ := newMockCtx(t, withBackend(mb), withHierarchy(h)) + return ctx, written +} + +func mdCreate(defs ...*ast.MessageDefinitionDef) *ast.CreateMessageDefinitionCollectionStmt { + return &ast.CreateMessageDefinitionCollectionStmt{ + Name: ast.QualifiedName{Module: "Sales", Name: "MD"}, + Definitions: defs, + } +} + +func attrMember(name string) *ast.MessageMemberDef { return &ast.MessageMemberDef{Attribute: name} } + +func assocMember(assoc, entity string, kids ...*ast.MessageMemberDef) *ast.MessageMemberDef { + return &ast.MessageMemberDef{ + Association: ast.QualifiedName{Module: "Sales", Name: assoc}, + Entity: ast.QualifiedName{Module: "Sales", Name: entity}, + Members: kids, + } +} + +// runCreate executes a CREATE and returns the collection the backend received. +func runCreate(t *testing.T, stmt *ast.CreateMessageDefinitionCollectionStmt) (*model.MessageDefinitionCollection, error) { + t.Helper() + var written *model.MessageDefinitionCollection + ctx, _ := mdFixture(t) + ctx.Backend.(*mock.MockBackend).CreateMessageDefinitionCollectionFunc = + func(c *model.MessageDefinitionCollection) error { written = c; return nil } + err := execCreateMessageDefinitionCollection(ctx, stmt) + return written, err +} + +// TestAssociationCardinalityFollowsTheDirection is the important one. +// +// MaxOccurs is not a function of the association's type — measured, all 927 +// resolvable associations in the demo corpus are `Reference`, yet 526 store 1 +// and 401 store -1. It tracks the direction of traversal, and getting it +// backwards exposes a list as a single object with NO build error behind it. +func TestAssociationCardinalityFollowsTheDirection(t *testing.T) { + // Order is the FROM entity: reaching Customer follows the FK, so single. + c, err := runCreate(t, mdCreate(&ast.MessageDefinitionDef{ + Name: "OrderMsg", + Entity: ast.QualifiedName{Module: "Sales", Name: "Order"}, + Members: []*ast.MessageMemberDef{assocMember("Order_Customer", "Customer", attrMember("Name"))}, + })) + if err != nil { + t.Fatalf("create: %v", err) + } + if got := c.Definitions[0].Root.Children[0].MaxOccurs; got != 1 { + t.Errorf("Order -> Customer MaxOccurs = %d, want 1 (following the FK)", got) + } + + // Customer is the TO entity: reaching Order is the reverse, so unbounded. + c, err = runCreate(t, mdCreate(&ast.MessageDefinitionDef{ + Name: "CustMsg", + Entity: ast.QualifiedName{Module: "Sales", Name: "Customer"}, + Members: []*ast.MessageMemberDef{assocMember("Order_Customer", "Order", attrMember("OrderId"))}, + })) + if err != nil { + t.Fatalf("create: %v", err) + } + child := c.Definitions[0].Root.Children[0] + if child.MaxOccurs != -1 { + t.Errorf("Customer -> Order MaxOccurs = %d, want -1 (the reverse)", child.MaxOccurs) + } + // ExposedItemName is set exactly when the element repeats — 461 of 461. + if child.ExposedItemName != "Order" { + t.Errorf("ExposedItemName = %q, want Order", child.ExposedItemName) + } +} + +// TestAssociationThatConnectsNeitherWayIsRefused pins the refusal. Defaulting +// to 1 would build, and be wrong. +func TestAssociationThatConnectsNeitherWayIsRefused(t *testing.T) { + _, err := runCreate(t, mdCreate(&ast.MessageDefinitionDef{ + Name: "Bad", + Entity: ast.QualifiedName{Module: "Sales", Name: "Base"}, + Members: []*ast.MessageMemberDef{assocMember("Order_Customer", "Customer")}, + })) + if err == nil { + t.Fatal("accepted an association that connects neither entity — the cardinality would be a guess") + } + if !strings.Contains(err.Error(), "does not connect") { + t.Errorf("error should say the association does not connect the two: %v", err) + } +} + +// TestPrimitiveTypeIsMappedNotPassedThrough pins the three non-identity +// mappings. A pass-through stores Long and AutoNumber where Mendix stores +// Integer, and Enumeration where it stores String — 279 elements in the corpus, +// and the round trip against ako/TestApp caught it on a Long. +func TestPrimitiveTypeIsMappedNotPassedThrough(t *testing.T) { + c, err := runCreate(t, mdCreate(&ast.MessageDefinitionDef{ + Name: "M", + Entity: ast.QualifiedName{Module: "Sales", Name: "Order"}, + Members: []*ast.MessageMemberDef{ + attrMember("OrderId"), attrMember("Status"), attrMember("Total"), + }, + })) + if err != nil { + t.Fatalf("create: %v", err) + } + want := map[string]string{"OrderId": "Integer", "Status": "String", "Total": "Decimal"} + for _, m := range c.Definitions[0].Root.Children { + if got := m.PrimitiveType; got != want[m.OriginalName] { + t.Errorf("%s PrimitiveType = %q, want %q", m.OriginalName, got, want[m.OriginalName]) + } + } +} + +// TestInheritedAttributeResolvesToItsDeclaringEntity pins the other +// model-dependent resolution. 398 of 3,697 exposed attributes in the corpus are +// inherited, and qualifying one against the entity that merely uses it is +// CE1613. +func TestInheritedAttributeResolvesToItsDeclaringEntity(t *testing.T) { + c, err := runCreate(t, mdCreate(&ast.MessageDefinitionDef{ + Name: "M", + Entity: ast.QualifiedName{Module: "Sales", Name: "Order"}, + Members: []*ast.MessageMemberDef{attrMember("Code")}, + })) + if err != nil { + t.Fatalf("create: %v", err) + } + if got := c.Definitions[0].Root.Children[0].Attribute; got != "Sales.Base.Code" { + t.Errorf("Attribute = %q, want Sales.Base.Code — Code is declared by Base, not Order", got) + } +} + +// TestUnknownAttributeNamesWhatExists pins the shape #882 established: a typo +// says what would have worked. +func TestUnknownAttributeNamesWhatExists(t *testing.T) { + _, err := runCreate(t, mdCreate(&ast.MessageDefinitionDef{ + Name: "M", + Entity: ast.QualifiedName{Module: "Sales", Name: "Customer"}, + Members: []*ast.MessageMemberDef{attrMember("Nope")}, + })) + if err == nil { + t.Fatal("accepted an attribute the entity does not have") + } + if !strings.Contains(err.Error(), "Name") { + t.Errorf("error should list the attributes that exist: %v", err) + } +} + +// TestRootExposedNameDefaultsToTheEntityName pins that mxcli does not guess +// English. Studio Pro pluralises here; reproducing that needs -y -> -ies and an +// already-plural detector, so `as 'Orders'` says it instead — the same +// conclusion as array-item naming. +func TestRootExposedNameDefaultsToTheEntityName(t *testing.T) { + c, err := runCreate(t, mdCreate(&ast.MessageDefinitionDef{ + Name: "M", + Entity: ast.QualifiedName{Module: "Sales", Name: "Order"}, + Members: []*ast.MessageMemberDef{attrMember("OrderId")}, + })) + if err != nil { + t.Fatalf("create: %v", err) + } + root := c.Definitions[0].Root + if root.ExposedName != "Order" || root.ExposedItemName != "Order" { + t.Errorf("root exposed=%q item=%q, want Order/Order", root.ExposedName, root.ExposedItemName) + } + if root.MaxOccurs != -1 { + t.Errorf("root MaxOccurs = %d, want -1 — a definition root always repeats (56/56)", root.MaxOccurs) + } +} diff --git a/mdl/executor/cmd_microflows_build.go b/mdl/executor/cmd_microflows_build.go new file mode 100644 index 0000000000..74bd19b1a6 --- /dev/null +++ b/mdl/executor/cmd_microflows_build.go @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package executor - building a flow from a CREATE statement, separately from +// writing it. +package executor + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// buildFlowOpts controls how far a build is allowed to go. +// +// AllowCreate is what separates `exec` from `diff`. exec sets it: the module +// and folders a script names are created on demand, a session-level DROP is +// consumed, and the guard-don't-drop refusals (a queued call, an unwritable +// REST body) run before anything is written. diff clears it: rendering a +// proposed flow must not touch the project, and a refusal that aborted the +// build would leave the user with no diff at all rather than a diff plus the +// warning exec will give them anyway. +type buildFlowOpts struct { + AllowCreate bool +} + +// builtFlow is a Microflow assembled from a statement, plus what the write +// phase needs to place it. +type builtFlow struct { + Microflow *microflows.Microflow + // ContainerID is where the flow should live: the resolved folder, or the + // module when no folder was named. + ContainerID model.ID + // ExistingID and ExistingContainerID are empty for a flow that is not in + // the project yet — which is how diff decides a statement is an addition. + ExistingID model.ID + ExistingContainerID model.ID +} + +// builtNanoflow is builtFlow for the distinct Nanoflow document type. +type builtNanoflow struct { + Nanoflow *microflows.Nanoflow + ContainerID model.ID + ExistingID model.ID + ExistingContainerID model.ID +} + +// buildMicroflowFromStmt assembles a Microflow from a CREATE MICROFLOW +// statement without writing it. +// +// It exists so that `diff` can render its script side through the SAME +// describer the project side goes through. Before this, diff kept a second +// AST-to-MDL renderer whose statement switch covered 18 of 43 activity types +// and had no default case, so every unhandled activity — every java-action +// call, every `download file`, every canvas annotation — silently rendered as +// nothing and appeared in the diff as a deletion. mxcli then reported that a +// script would gut a microflow that exec proved was a no-op (#997). +// +// The lesson is structural rather than a list of missing cases: two renderers +// for one language drift, and the drift surfaces as a confident false report. +// Adding the 25 missing cases would have fixed the symptom and left the +// mechanism in place. +func buildMicroflowFromStmt(ctx *ExecContext, s *ast.CreateMicroflowStmt, opts buildFlowOpts) (*builtFlow, error) { + // Validate name is not empty + if strings.TrimSpace(s.Name.Name) == "" { + return nil, mdlerrors.NewValidation("microflow name must not be empty") + } + + // Refuse the XPath constraints Mendix rejects, before writing anything. + // `mxcli check` already reported these, but exec ran a different validator + // and wrote them anyway, so a script that skipped check produced a project + // the build fails on (issue #833). Same placement as the entity handler's + // ValidateEntity call. + if opts.AllowCreate { + if err := validateMicroflowRules(s); err != nil { + return nil, err + } + } + + // Find the module, and the folder, WITHOUT creating either on a dry run: + // findOrCreateModule and resolveFolder both write, and `diff` must render a + // proposed flow against an unmodified project. + var module *model.Module + if opts.AllowCreate { + var err error + module, err = findOrCreateModule(ctx, s.Name.Module) + if err != nil { + return nil, err + } + } else if m, err := findModule(ctx, s.Name.Module); err == nil { + module = m + } + var moduleID model.ID + if module != nil { + moduleID = module.ID + } + + containerID := moduleID + if s.Folder != "" { + if opts.AllowCreate { + folderID, err := resolveFolder(ctx, moduleID, s.Folder) + if err != nil { + return nil, mdlerrors.NewBackend("resolve folder "+s.Folder, err) + } + containerID = folderID + } else if folderID, ok := lookupFolder(ctx, moduleID, s.Folder); ok { + containerID = folderID + } + } + + // Check if microflow with same name already exists in this module + var existingID model.ID + var existingContainerID model.ID + var existingAllowedRoles []model.ID + preserveAllowedRoles := false + // Excluded is model state, not script state: an absent @excluded must not + // clear a stored exclusion (#914). + existingExcluded := false + var existingDocumentation string + preserveDocumentation := false + var existingActionInfo, existingWorkflowInfo *types.MicroflowActionInfo + existingMicroflows, err := ctx.Backend.ListMicroflows() + if err != nil { + return nil, mdlerrors.NewBackend("check existing microflows", err) + } + // A module may hold several microflows with this name as long as all but one + // are excluded, so target the live one rather than whichever comes first + // (#914). + if existing, ok := pickLive(existingMicroflows, + func(m *microflows.Microflow) bool { + return m.Name == s.Name.Name && getModuleID(ctx, m.ContainerID) == moduleID + }, + func(m *microflows.Microflow) bool { return m.Excluded }, + ); ok { + if !s.CreateOrModify && opts.AllowCreate { + return nil, mdlerrors.NewAlreadyExistsMsg("microflow", s.Name.Module+"."+s.Name.Name, "microflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") + } + existingID = existing.ID + existingContainerID = existing.ContainerID + existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) + preserveAllowedRoles = true + existingExcluded = existing.Excluded + // The toolbox entries hold four PNG bitmaps MDL cannot name, so a + // rewrite carries them rather than rebuilding from the clause. + existingActionInfo = existing.MicroflowActionInfo + // A rewrite that carried no doc comment keeps the stored one; an + // explicitly empty `/** */` clears it (#1018). + existingDocumentation = existing.Documentation + preserveDocumentation = true + existingWorkflowInfo = existing.WorkflowActionInfo + } + + // For CREATE OR REPLACE/MODIFY, reuse the existing ID to preserve references + qualifiedName := s.Name.Module + "." + s.Name.Name + + // Refuse before writing if the stored microflow has a call bound to a task + // queue: the rebuild would null it out and nothing downstream would notice. + if existingID != "" && opts.AllowCreate { + if err := checkNoQueuedCalls(ctx, existingID, qualifiedName, s); err != nil { + return nil, err + } + // Same reasoning for a REST body the writer cannot express: the rebuild + // would drop it, DESCRIBE would not show it missing, and the app would + // still build. + if err := checkNoUnwritableRestBody(ctx, existingID, qualifiedName); err != nil { + return nil, err + } + } + microflowID := model.ID(types.GenerateID()) + if existingID != "" { + microflowID = existingID + // Keep the original folder unless a new folder is explicitly specified + if s.Folder == "" { + containerID = existingContainerID + } + } else if dropped := consumeDroppedMicroflow(ctx, qualifiedName); opts.AllowCreate && dropped != nil { + // A prior DROP MICROFLOW in the same session removed the unit. Reuse + // its original UnitID and (unless a new folder is specified) + // ContainerID so that Studio Pro sees the rewrite as an in-place + // update rather than a delete+insert pair, which produces + // ".mpr does not look like a Mendix Studio Pro project file" errors. + microflowID = dropped.ID + if s.Folder == "" && dropped.ContainerID != "" { + containerID = dropped.ContainerID + } + // consumeDroppedMicroflow removed the cache entry, so we own this + // slice — no need to clone it again. + existingAllowedRoles = dropped.AllowedRoles + preserveAllowedRoles = true + } + + // Build the microflow + mf := µflows.Microflow{ + BaseElement: model.BaseElement{ + ID: microflowID, + }, + ContainerID: containerID, + Name: s.Name.Name, + Documentation: s.Documentation, + AllowConcurrentExecution: true, // Default: allow concurrent execution + MarkAsUsed: false, + Excluded: s.Excluded || existingExcluded, + } + if preserveDocumentation { + mf.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingDocumentation) + } + if preserveAllowedRoles { + mf.AllowedModuleRoles = existingAllowedRoles + } else { + if module != nil { + mf.AllowedModuleRoles = defaultDocumentAccessRoles(ctx, module) + } + } + if mf.MicroflowActionInfo, mf.WorkflowActionInfo, err = applyExposeClauses(ctx, + s.Expose, existingActionInfo, existingWorkflowInfo, exposeWarner(ctx)); err != nil { + return nil, err + } + + // Build entity resolver function for parameter/return types + entityResolver := func(qn ast.QualifiedName) model.ID { + // Get all domain models and build module name map + dms, err := ctx.Backend.ListDomainModels() + if err != nil { + return "" + } + modules, _ := ctx.Backend.ListModules() + moduleNames := make(map[model.ID]string) + for _, m := range modules { + moduleNames[m.ID] = m.Name + } + // Search for entity in all domain models + for _, dm := range dms { + modName := moduleNames[dm.ContainerID] + if modName != qn.Module { + continue + } + for _, ent := range dm.Entities { + if ent.Name == qn.Name { + return ent.ID + } + } + } + return "" + } + + // Validate and add parameters + for i, p := range s.Parameters { + // Validate entity references for List and Entity types. + // Built-in modules (e.g. System) are not stored in the MPR domain models; + // their types are serialized by qualified name and resolved at runtime. + if p.Type.EntityRef != nil && !isBuiltinModuleEntity(p.Type.EntityRef.Module) { + entityID := entityResolver(*p.Type.EntityRef) + if entityID == "" { + // Bare qualified name in microflow context is treated as TypeEntity by the + // visitor, but it may actually be an enumeration. Try enum lookup before failing. + if found := findEnumeration(ctx, p.Type.EntityRef.Module, p.Type.EntityRef.Name); found != nil { + s.Parameters[i].Type = ast.DataType{Kind: ast.TypeEnumeration, EnumRef: p.Type.EntityRef} + p = s.Parameters[i] + } else { + return nil, mdlerrors.NewNotFoundMsg("entity", p.Type.EntityRef.Module+"."+p.Type.EntityRef.Name, + fmt.Sprintf("entity '%s.%s' not found for parameter '%s'", p.Type.EntityRef.Module, p.Type.EntityRef.Name, p.Name)) + } + } + } + // Validate enumeration references for Enumeration types + if p.Type.Kind == ast.TypeEnumeration && p.Type.EnumRef != nil { + if found := findEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name); found == nil { + return nil, mdlerrors.NewNotFoundMsg("enumeration", p.Type.EnumRef.Module+"."+p.Type.EnumRef.Name, + fmt.Sprintf("enumeration '%s.%s' not found for parameter '%s'", p.Type.EnumRef.Module, p.Type.EnumRef.Name, p.Name)) + } + } + param := µflows.MicroflowParameter{ + BaseElement: model.BaseElement{ + ID: model.ID(types.GenerateID()), + }, + ContainerID: mf.ID, + Name: p.Name, + Type: convertASTToMicroflowDataType(p.Type, entityResolver), + Position: positionFromAST(p.Position), + } + mf.Parameters = append(mf.Parameters, param) + } + + // Validate and set return type + if s.ReturnType != nil { + // Validate entity references for return type. + // Built-in modules (e.g. System) are not stored in the MPR domain models; + // their types are serialized by qualified name and resolved at runtime. + if s.ReturnType.Type.EntityRef != nil && !isBuiltinModuleEntity(s.ReturnType.Type.EntityRef.Module) { + entityID := entityResolver(*s.ReturnType.Type.EntityRef) + if entityID == "" { + return nil, mdlerrors.NewNotFoundMsg("entity", s.ReturnType.Type.EntityRef.Module+"."+s.ReturnType.Type.EntityRef.Name, + fmt.Sprintf("entity '%s.%s' not found for return type", s.ReturnType.Type.EntityRef.Module, s.ReturnType.Type.EntityRef.Name)) + } + } + // Validate enumeration references for return type + if s.ReturnType.Type.Kind == ast.TypeEnumeration && s.ReturnType.Type.EnumRef != nil { + if found := findEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name); found == nil { + return nil, mdlerrors.NewNotFoundMsg("enumeration", s.ReturnType.Type.EnumRef.Module+"."+s.ReturnType.Type.EnumRef.Name, + fmt.Sprintf("enumeration '%s.%s' not found for return type", s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name)) + } + } + mf.ReturnType = convertASTToMicroflowDataType(s.ReturnType.Type, entityResolver) + // Set return variable name if provided (AS $VarName) + if s.ReturnType.Variable != "" { + mf.ReturnVariableName = s.ReturnType.Variable + } + } else { + mf.ReturnType = µflows.VoidType{} + } + + // Build flow graph from body statements + // Initialize variable types from parameters + varTypes := make(map[string]string) + declaredVars := make(map[string]string) + + for _, p := range s.Parameters { + if p.Type.EntityRef != nil { + entityQN := p.Type.EntityRef.Module + "." + p.Type.EntityRef.Name + if p.Type.Kind == ast.TypeListOf { + // Store "List of Module.Entity" for list parameters + varTypes[p.Name] = "List of " + entityQN + } else { + // Store "Module.Entity" for single entity parameters + varTypes[p.Name] = entityQN + } + } else { + // Primitive type parameters are also considered declared + declaredVars[p.Name] = p.Type.Kind.String() + } + } + // Get hierarchy for resolving page/microflow references + hierarchy, _ := getHierarchy(ctx) + + restServices, _ := loadRestServices(ctx) + + builder := &flowBuilder{ + textLang: authoringLanguage(ctx), + // Carry over a HAND-PLACED StartEvent position from the microflow being + // replaced, the way the folder and allowed roles already are: a Studio + // Pro flow's 145;200 became 100;200 on a describe→exec round-trip, the + // only coordinate in it that did not survive (#884). A start sitting + // where mxcli's own layout would have put it is not carried over — that + // pinned the start of every rewritten flow, stranding it across the + // canvas from activities the same script had just moved (#951). An + // explicit @start(x, y) on the first statement overrides both. + startPosition: storedStartPosition(ctx, existingID), + posX: 200, + posY: 200, + baseY: 200, // Base Y for happy path + spacing: HorizontalSpacing, + varTypes: varTypes, + declaredVars: declaredVars, + measurer: &layoutMeasurer{varTypes: varTypes}, + backend: ctx.Backend, + hierarchy: hierarchy, + restServices: restServices, + } + + mf.ObjectCollection = builder.buildFlowGraph(s.Body, s.ReturnType) + + // Check for validation errors + if errors := builder.GetErrors(); len(errors) > 0 { + // Report all errors to the user + var errMsg strings.Builder + errMsg.WriteString(fmt.Sprintf("microflow '%s.%s' has validation errors:\n", s.Name.Module, s.Name.Name)) + for _, err := range errors { + errMsg.WriteString(fmt.Sprintf(" - %s\n", err)) + } + return nil, fmt.Errorf("%s", errMsg.String()) + } + return &builtFlow{ + Microflow: mf, + ContainerID: containerID, + ExistingID: existingID, + ExistingContainerID: existingContainerID, + }, nil +} + +// buildNanoflowFromStmt is buildMicroflowFromStmt for nanoflows — same split, +// same reason (#997). +func buildNanoflowFromStmt(ctx *ExecContext, s *ast.CreateNanoflowStmt, opts buildFlowOpts) (*builtNanoflow, error) { + // Validate name is not empty + if strings.TrimSpace(s.Name.Name) == "" { + return nil, mdlerrors.NewValidation("nanoflow name must not be empty") + } + + if err := refuseExposeOnFlavour(s.Expose, "nanoflow", s.Name.Module+"."+s.Name.Name); err != nil { + return nil, err + } + + // Find the module, and the folder, WITHOUT creating either on a dry run: + // findOrCreateModule and resolveFolder both write, and `diff` must render a + // proposed flow against an unmodified project. + var module *model.Module + if opts.AllowCreate { + var err error + module, err = findOrCreateModule(ctx, s.Name.Module) + if err != nil { + return nil, err + } + } else if m, err := findModule(ctx, s.Name.Module); err == nil { + module = m + } + var moduleID model.ID + if module != nil { + moduleID = module.ID + } + + containerID := moduleID + if s.Folder != "" { + if opts.AllowCreate { + folderID, err := resolveFolder(ctx, moduleID, s.Folder) + if err != nil { + return nil, mdlerrors.NewBackend("resolve folder "+s.Folder, err) + } + containerID = folderID + } else if folderID, ok := lookupFolder(ctx, moduleID, s.Folder); ok { + containerID = folderID + } + } + + // Check if nanoflow with same name already exists in this module. + // NOTE: O(n) scan over all nanoflows — consistent with microflow handler pattern. + // Consider catalog-based lookup if this becomes a bottleneck for large projects. + var existingID model.ID + var existingContainerID model.ID + var existingAllowedRoles []model.ID + preserveAllowedRoles := false + // Excluded is model state, not script state: an absent @excluded must not + // clear a stored exclusion (#914). + existingExcluded := false + var existingDocumentation string + preserveDocumentation := false + existingNanoflows, err := ctx.Backend.ListNanoflows() + if err != nil { + return nil, mdlerrors.NewBackend("check existing nanoflows", err) + } + // A module may hold several nanoflows with this name as long as all but one + // are excluded, so target the live one rather than whichever comes first. + if existing, ok := pickLive(existingNanoflows, + func(n *microflows.Nanoflow) bool { + return n.Name == s.Name.Name && getModuleID(ctx, n.ContainerID) == moduleID + }, + func(n *microflows.Nanoflow) bool { return n.Excluded }, + ); ok { + if !s.CreateOrModify && opts.AllowCreate { + return nil, mdlerrors.NewAlreadyExistsMsg("nanoflow", s.Name.Module+"."+s.Name.Name, "nanoflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") + } + existingID = existing.ID + existingContainerID = existing.ContainerID + existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) + preserveAllowedRoles = true + existingExcluded = existing.Excluded + // A rewrite that carried no doc comment keeps the stored one (#1018). + existingDocumentation = existing.Documentation + preserveDocumentation = true + } + + // For CREATE OR REPLACE/MODIFY, reuse the existing ID to preserve references + qualifiedName := s.Name.Module + "." + s.Name.Name + nanoflowID := model.ID(types.GenerateID()) + if existingID != "" { + nanoflowID = existingID + if s.Folder == "" { + containerID = existingContainerID + } + } else if dropped := consumeDroppedNanoflow(ctx, qualifiedName); opts.AllowCreate && dropped != nil { + nanoflowID = dropped.ID + if s.Folder == "" && dropped.ContainerID != "" { + containerID = dropped.ContainerID + } + if len(dropped.AllowedRoles) > 0 { + existingAllowedRoles = dropped.AllowedRoles + preserveAllowedRoles = true + } + } + + // Build the nanoflow + nf := µflows.Nanoflow{ + BaseElement: model.BaseElement{ + ID: nanoflowID, + }, + ContainerID: containerID, + Name: s.Name.Name, + Documentation: s.Documentation, + MarkAsUsed: false, + Excluded: s.Excluded || existingExcluded, + } + if preserveDocumentation { + nf.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingDocumentation) + } + if preserveAllowedRoles { + nf.AllowedModuleRoles = existingAllowedRoles + } else { + if module != nil { + nf.AllowedModuleRoles = defaultDocumentAccessRoles(ctx, module) + } + } + + // Load metadata needed by the entity resolver up front so backend read + // failures are returned as actionable errors instead of being treated as + // "entity not found". + dms, err := ctx.Backend.ListDomainModels() + if err != nil { + return nil, mdlerrors.NewBackend("list domain models", err) + } + modules, err := ctx.Backend.ListModules() + if err != nil { + return nil, mdlerrors.NewBackend("list modules", err) + } + moduleNames := make(map[model.ID]string) + for _, m := range modules { + moduleNames[m.ID] = m.Name + } + + // Build entity resolver function for parameter/return types + entityResolver := func(qn ast.QualifiedName) model.ID { + for _, dm := range dms { + modName := moduleNames[dm.ContainerID] + if modName != qn.Module { + continue + } + for _, ent := range dm.Entities { + if ent.Name == qn.Name { + return ent.ID + } + } + } + return "" + } + + // Validate and add parameters + for i, p := range s.Parameters { + if p.Type.EntityRef != nil && !isBuiltinModuleEntity(p.Type.EntityRef.Module) { + entityID := entityResolver(*p.Type.EntityRef) + if entityID == "" { + // Bare qualified name in microflow context is treated as TypeEntity by the + // visitor, but it may actually be an enumeration. Try enum lookup before failing. + if found := findEnumeration(ctx, p.Type.EntityRef.Module, p.Type.EntityRef.Name); found != nil { + s.Parameters[i].Type = ast.DataType{Kind: ast.TypeEnumeration, EnumRef: p.Type.EntityRef} + p = s.Parameters[i] + } else { + return nil, mdlerrors.NewNotFoundMsg("entity", p.Type.EntityRef.Module+"."+p.Type.EntityRef.Name, + fmt.Sprintf("entity '%s.%s' not found for parameter '%s'", p.Type.EntityRef.Module, p.Type.EntityRef.Name, p.Name)) + } + } + } + if p.Type.Kind == ast.TypeEnumeration && p.Type.EnumRef != nil { + if found := findEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name); found == nil { + return nil, mdlerrors.NewNotFoundMsg("enumeration", p.Type.EnumRef.Module+"."+p.Type.EnumRef.Name, + fmt.Sprintf("enumeration '%s.%s' not found for parameter '%s'", p.Type.EnumRef.Module, p.Type.EnumRef.Name, p.Name)) + } + } + param := µflows.MicroflowParameter{ + BaseElement: model.BaseElement{ + ID: model.ID(types.GenerateID()), + }, + ContainerID: nf.ID, + Name: p.Name, + Type: convertASTToMicroflowDataType(p.Type, entityResolver), + Position: positionFromAST(p.Position), + } + nf.Parameters = append(nf.Parameters, param) + } + + // Validate and set return type + if s.ReturnType != nil { + if s.ReturnType.Type.EntityRef != nil && !isBuiltinModuleEntity(s.ReturnType.Type.EntityRef.Module) { + entityID := entityResolver(*s.ReturnType.Type.EntityRef) + if entityID == "" { + return nil, mdlerrors.NewNotFoundMsg("entity", s.ReturnType.Type.EntityRef.Module+"."+s.ReturnType.Type.EntityRef.Name, + fmt.Sprintf("entity '%s.%s' not found for return type", s.ReturnType.Type.EntityRef.Module, s.ReturnType.Type.EntityRef.Name)) + } + } + if s.ReturnType.Type.Kind == ast.TypeEnumeration && s.ReturnType.Type.EnumRef != nil { + if found := findEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name); found == nil { + return nil, mdlerrors.NewNotFoundMsg("enumeration", s.ReturnType.Type.EnumRef.Module+"."+s.ReturnType.Type.EnumRef.Name, + fmt.Sprintf("enumeration '%s.%s' not found for return type", s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name)) + } + } + nf.ReturnType = convertASTToMicroflowDataType(s.ReturnType.Type, entityResolver) + } else { + nf.ReturnType = µflows.VoidType{} + } + + // Validate nanoflow-specific constraints before building the flow graph + if errMsg := validateNanoflow(qualifiedName, s.Body, s.ReturnType); errMsg != "" { + return nil, fmt.Errorf("%s", errMsg) + } + + // SYNCHRONIZE UNSYNCHRONIZED is the one mode with a floor inside Mendix 9: + // SynchronizationType.Unsynchronized was introduced in 9.4.0, while the + // activity and its other two modes go back further. Gate the mode, not the + // statement, so a 9.0-9.3 project keeps `synchronize all`. + if bodyUsesUnsynchronized(s.Body) { + if err := checkFeature(ctx, "microflows", "synchronize_unsynchronized", + "synchronize unsynchronized", + "use `synchronize all` or `synchronize $Var`, or upgrade the project to 9.4+"); err != nil { + return nil, err + } + } + + // Build flow graph from body statements + varTypes := make(map[string]string) + declaredVars := make(map[string]string) + + for _, p := range s.Parameters { + if p.Type.EntityRef != nil { + entityQN := p.Type.EntityRef.Module + "." + p.Type.EntityRef.Name + if p.Type.Kind == ast.TypeListOf { + varTypes[p.Name] = "List of " + entityQN + } else { + varTypes[p.Name] = entityQN + } + } else { + declaredVars[p.Name] = p.Type.Kind.String() + } + } + + hierarchy, _ := getHierarchy(ctx) // best-effort: builder works without hierarchy + restServices, _ := loadRestServices(ctx) // best-effort: builder works without REST services + + builder := &flowBuilder{ + textLang: authoringLanguage(ctx), + posX: 200, + posY: 200, + baseY: 200, + spacing: HorizontalSpacing, + varTypes: varTypes, + declaredVars: declaredVars, + measurer: &layoutMeasurer{varTypes: varTypes}, + backend: ctx.Backend, + hierarchy: hierarchy, + restServices: restServices, + isNanoflow: true, + } + + nf.ObjectCollection = builder.buildFlowGraph(s.Body, s.ReturnType) + + // Check for validation errors + if errors := builder.GetErrors(); len(errors) > 0 { + var errMsg strings.Builder + errMsg.WriteString(fmt.Sprintf("nanoflow '%s.%s' has validation errors:\n", s.Name.Module, s.Name.Name)) + for _, err := range errors { + errMsg.WriteString(fmt.Sprintf(" - %s\n", err)) + } + return nil, fmt.Errorf("%s", errMsg.String()) + } + return &builtNanoflow{ + Nanoflow: nf, + ContainerID: containerID, + ExistingID: existingID, + ExistingContainerID: existingContainerID, + }, nil +} + +// lookupFolder resolves a folder path to its ID without creating anything — +// the read-only counterpart of resolveFolder, which creates missing folders. +func lookupFolder(ctx *ExecContext, moduleID model.ID, folderPath string) (model.ID, bool) { + if folderPath == "" { + return moduleID, true + } + folders, err := ctx.Backend.ListFolders() + if err != nil { + return "", false + } + current := moduleID + for _, part := range strings.Split(folderPath, "/") { + if part == "" { + continue + } + found := false + for _, f := range folders { + if f.ContainerID == current && f.Name == part { + current = f.ID + found = true + break + } + } + if !found { + return "", false + } + } + return current, true +} diff --git a/mdl/executor/cmd_microflows_builder_calls.go b/mdl/executor/cmd_microflows_builder_calls.go index 5d473d3c9c..f88531983b 100644 --- a/mdl/executor/cmd_microflows_builder_calls.go +++ b/mdl/executor/cmd_microflows_builder_calls.go @@ -617,21 +617,23 @@ func (fb *flowBuilder) resolveMappingRefForWrite(ref string, preferExport bool) // resolveExternalActionReturnKind looks up the called OData action in the // consumed service's cached $metadata and returns the Mendix kind name // ("Boolean", "String", "Integer", "Long", "Decimal", "DateTime", "Binary", -// or "Void") of its return type. Used to populate -// CallExternalAction.ResultDataType so the writer can emit VariableDataType -// BSON; without it Mendix raises CE7269 whenever the schema declares any -// return type. +// "Object", "List" or "Void") of its return type, plus the qualified name of +// the external entity an Object/List is typed on. Used to populate +// CallExternalAction.ResultDataType/ResultEntity so the writer can emit +// VariableDataType BSON; without it Mendix raises CE7269 ("the return type for +// remote action '' has changed") whenever the schema declares any return +// type. // // Returns "" if the service or action can't be resolved — the writer omits // VariableDataType, falling back to the prior (buggy) behavior rather than // emitting a wrong type. -func (fb *flowBuilder) resolveExternalActionReturnKind(serviceRef ast.QualifiedName, actionName string) string { +func (fb *flowBuilder) resolveExternalActionReturnKind(serviceRef ast.QualifiedName, actionName string) (string, string) { if fb.backend == nil { - return "" + return "", "" } services, err := fb.backend.ListConsumedODataServices() if err != nil { - return "" + return "", "" } for _, svc := range services { modName := fb.hierarchy.GetModuleName(fb.hierarchy.FindModuleID(svc.ContainerID)) @@ -639,22 +641,161 @@ func (fb *flowBuilder) resolveExternalActionReturnKind(serviceRef ast.QualifiedN continue } if svc.Metadata == "" { - return "" + return "", "" } doc, err := types.ParseEdmx(svc.Metadata) if err != nil { - return "" + return "", "" } for _, act := range doc.Actions { - if strings.EqualFold(act.Name, actionName) { - return edmReturnTypeToKind(act.ReturnType) + if !strings.EqualFold(act.Name, actionName) { + continue } + kind := edmReturnTypeToKind(act.ReturnType) + if kind != "" { + return kind, "" + } + // Not a primitive. An entity-typed return needs the local external + // entity that was imported for it, because DataTypes$ObjectType and + // DataTypes$ListType store an Entity by qualified name. + return fb.resolveExternalActionReturnEntity(serviceRef, act.ReturnType) } + return "", "" + } + return "", "" +} + +// externalParamKind is one action parameter's resolved Mendix type. +type externalParamKind struct { + kind string // "String", "Object", … — same vocabulary as the return type + entity string // set only for Object/List +} + +// resolveExternalActionParameterKinds types every parameter of the called action +// from the cached contract, keyed by lower-cased parameter name. +// +// Returns an empty map when the service, contract or action cannot be resolved, +// in which case the writer omits ParameterType exactly as before — a wrong type +// is worse than the known-missing one. +func (fb *flowBuilder) resolveExternalActionParameterKinds(serviceRef ast.QualifiedName, actionName string) map[string]externalParamKind { + out := map[string]externalParamKind{} + if fb.backend == nil { + return out + } + services, err := fb.backend.ListConsumedODataServices() + if err != nil { + return out + } + for _, svc := range services { + modName := fb.hierarchy.GetModuleName(fb.hierarchy.FindModuleID(svc.ContainerID)) + if !strings.EqualFold(modName, serviceRef.Module) || !strings.EqualFold(svc.Name, serviceRef.Name) { + continue + } + if svc.Metadata == "" { + return out + } + doc, err := types.ParseEdmx(svc.Metadata) + if err != nil { + return out + } + for _, act := range doc.Actions { + if !strings.EqualFold(act.Name, actionName) { + continue + } + for _, p := range act.Parameters { + if kind := edmReturnTypeToKind(p.Type); kind != "" && kind != "Void" { + out[strings.ToLower(p.Name)] = externalParamKind{kind: kind} + continue + } + // Entity-typed parameter: same resolution as an entity return. + if kind, entity := fb.resolveExternalActionReturnEntity(serviceRef, p.Type); kind != "" { + out[strings.ToLower(p.Name)] = externalParamKind{kind: kind, entity: entity} + } + } + return out + } + return out + } + return out +} + +// resolveExternalActionReturnEntity maps a non-primitive OData return type onto +// the external entity imported for it. +// +// The contract names the type in its own namespace (`Trippin.Person`, or +// `Collection(Trippin.Person)` for a list); the model names it by the entity +// mxcli created for that type, whose RemoteEntityName is the bare type name and +// whose RemoteServiceName is the consumed service. Matching on those two is the +// same linkage `CREATE OR MODIFY EXTERNAL ENTITIES` writes. +// +// Returns ("", "") when no entity has been imported for the type. That is the +// honest outcome: emitting an ObjectType with no Entity is as unaligned as +// emitting nothing, so the writer keeps omitting VariableDataType and +// ValidateExternalActionCalls reports the missing import at check time with the +// statement that fixes it. +func (fb *flowBuilder) resolveExternalActionReturnEntity(serviceRef ast.QualifiedName, returnType string) (string, string) { + typeName, isList := edmBareTypeName(returnType) + if typeName == "" { + return "", "" + } + qn := fb.findExternalEntityFor(serviceRef.String(), typeName) + if qn == "" { + return "", "" + } + if isList { + return "List", qn + } + return "Object", qn +} + +// findExternalEntityFor finds the external entity imported from serviceQN for +// the remote type remoteName, returning its qualified name. +func (fb *flowBuilder) findExternalEntityFor(serviceQN, remoteName string) string { + if fb.backend == nil { + return "" + } + dms, err := fb.backend.ListDomainModels() + if err != nil { return "" } + for _, dm := range dms { + modName := fb.hierarchy.GetModuleName(fb.hierarchy.FindModuleID(dm.ContainerID)) + for _, ent := range dm.Entities { + if !strings.EqualFold(ent.RemoteServiceName, serviceQN) { + continue + } + if strings.EqualFold(ent.RemoteEntityName, remoteName) { + return modName + "." + ent.Name + } + } + } return "" } +// edmBareTypeName strips OData's Collection() wrapper and the type's namespace, +// reporting the bare type name and whether it was a collection. +// +// The namespace has to go: the contract says `Trippin.Person`, while the +// imported entity records `Person` as its RemoteEntityName. Returns "" for a +// primitive or an unparseable type, so only entity-typed returns reach the +// entity lookup. +func edmBareTypeName(edmType string) (string, bool) { + t := strings.TrimSpace(edmType) + isList := false + if strings.HasPrefix(t, "Collection(") && strings.HasSuffix(t, ")") { + t = strings.TrimSuffix(strings.TrimPrefix(t, "Collection("), ")") + isList = true + } + t = strings.TrimSpace(t) + if t == "" || strings.HasPrefix(t, "Edm.") { + return "", isList + } + if i := strings.LastIndex(t, "."); i >= 0 { + t = t[i+1:] + } + return t, isList +} + // edmReturnTypeToKind maps an EDM type name (e.g. "Edm.Boolean") to the // Mendix kind name used by serializeExternalActionReturnType. Returns "Void" // for an empty/unknown return type so action calls with no return still get @@ -678,9 +819,10 @@ func edmReturnTypeToKind(edmType string) string { case "Edm.Binary": return "Binary" default: - // Complex / collection / entity-typed returns aren't yet mapped. - // Leave empty so the writer omits VariableDataType rather than - // emitting a wrong type that would silently mislead Mendix. + // Not a primitive. Entity-typed and collection returns are resolved by + // resolveExternalActionReturnEntity, which needs the project to map the + // contract's type onto the entity imported for it. Complex types + // (ComplexType, not EntityType) are still unmapped and end up here. return "" } } @@ -688,8 +830,13 @@ func edmReturnTypeToKind(edmType string) string { // addCallExternalActionAction creates a CALL EXTERNAL ACTION statement. func (fb *flowBuilder) addCallExternalActionAction(s *ast.CallExternalActionStmt) model.ID { serviceQN := s.ServiceName.Module + "." + s.ServiceName.Name + returnKind, returnEntity := fb.resolveExternalActionReturnKind(s.ServiceName, s.ActionName) - // Build parameter mappings + // Build parameter mappings. Each carries the parameter's TYPE as well as its + // name: generated/metamodel declares ParameterType without omitempty, and a + // mapping without one is CE7252 plus a CE0117 per argument, because Mendix + // cannot type-check an argument against an untyped parameter. + paramKinds := fb.resolveExternalActionParameterKinds(s.ServiceName, s.ActionName) var mappings []*microflows.ExternalActionParameterMapping for _, arg := range s.Arguments { mapping := µflows.ExternalActionParameterMapping{ @@ -697,6 +844,10 @@ func (fb *flowBuilder) addCallExternalActionAction(s *ast.CallExternalActionStmt ParameterName: arg.Name, Argument: fb.exprToString(arg.Value), } + if pk, ok := paramKinds[strings.ToLower(arg.Name)]; ok { + mapping.ParameterDataType = pk.kind + mapping.ParameterEntity = pk.entity + } mappings = append(mappings, mapping) } @@ -708,7 +859,8 @@ func (fb *flowBuilder) addCallExternalActionAction(s *ast.CallExternalActionStmt ParameterMappings: mappings, ResultVariableName: s.OutputVariable, UseReturnVariable: s.OutputVariable != "", - ResultDataType: fb.resolveExternalActionReturnKind(s.ServiceName, s.ActionName), + ResultDataType: returnKind, + ResultEntity: returnEntity, } activityX := fb.posX diff --git a/mdl/executor/cmd_microflows_create.go b/mdl/executor/cmd_microflows_create.go index 2208d143c8..ac5e6b36c0 100644 --- a/mdl/executor/cmd_microflows_create.go +++ b/mdl/executor/cmd_microflows_create.go @@ -5,13 +5,10 @@ package executor import ( "fmt" - "strings" "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" - "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" - "github.com/mendixlabs/mxcli/sdk/microflows" ) // isBuiltinModuleEntity returns true for modules whose entities are defined @@ -37,285 +34,14 @@ func execCreateMicroflow(ctx *ExecContext, s *ast.CreateMicroflowStmt) error { return mdlerrors.NewNotConnectedWrite() } - // Validate name is not empty - if strings.TrimSpace(s.Name.Name) == "" { - return mdlerrors.NewValidation("microflow name must not be empty") - } - - // Refuse the XPath constraints Mendix rejects, before writing anything. - // `mxcli check` already reported these, but exec ran a different validator - // and wrote them anyway, so a script that skipped check produced a project - // the build fails on (issue #833). Same placement as the entity handler's - // ValidateEntity call. - if err := validateMicroflowRules(s); err != nil { - return err - } - - // Find or auto-create module - module, err := findOrCreateModule(ctx, s.Name.Module) + built, err := buildMicroflowFromStmt(ctx, s, buildFlowOpts{AllowCreate: true}) if err != nil { return err } - - // Resolve folder if specified - containerID := module.ID - if s.Folder != "" { - folderID, err := resolveFolder(ctx, module.ID, s.Folder) - if err != nil { - return mdlerrors.NewBackend("resolve folder "+s.Folder, err) - } - containerID = folderID - } - - // Check if microflow with same name already exists in this module - var existingID model.ID - var existingContainerID model.ID - var existingAllowedRoles []model.ID - preserveAllowedRoles := false - // Excluded is model state, not script state: an absent @excluded must not - // clear a stored exclusion (#914). - existingExcluded := false - var existingActionInfo, existingWorkflowInfo *types.MicroflowActionInfo - existingMicroflows, err := ctx.Backend.ListMicroflows() - if err != nil { - return mdlerrors.NewBackend("check existing microflows", err) - } - // A module may hold several microflows with this name as long as all but one - // are excluded, so target the live one rather than whichever comes first - // (#914). - if existing, ok := pickLive(existingMicroflows, - func(m *microflows.Microflow) bool { - return m.Name == s.Name.Name && getModuleID(ctx, m.ContainerID) == module.ID - }, - func(m *microflows.Microflow) bool { return m.Excluded }, - ); ok { - if !s.CreateOrModify { - return mdlerrors.NewAlreadyExistsMsg("microflow", s.Name.Module+"."+s.Name.Name, "microflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") - } - existingID = existing.ID - existingContainerID = existing.ContainerID - existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) - preserveAllowedRoles = true - existingExcluded = existing.Excluded - // The toolbox entries hold four PNG bitmaps MDL cannot name, so a - // rewrite carries them rather than rebuilding from the clause. - existingActionInfo = existing.MicroflowActionInfo - existingWorkflowInfo = existing.WorkflowActionInfo - } - - // For CREATE OR REPLACE/MODIFY, reuse the existing ID to preserve references - qualifiedName := s.Name.Module + "." + s.Name.Name - - // Refuse before writing if the stored microflow has a call bound to a task - // queue: the rebuild would null it out and nothing downstream would notice. - if existingID != "" { - if err := checkNoQueuedCalls(ctx, existingID, qualifiedName, s); err != nil { - return err - } - // Same reasoning for a REST body the writer cannot express: the rebuild - // would drop it, DESCRIBE would not show it missing, and the app would - // still build. - if err := checkNoUnwritableRestBody(ctx, existingID, qualifiedName); err != nil { - return err - } - } - microflowID := model.ID(types.GenerateID()) - if existingID != "" { - microflowID = existingID - // Keep the original folder unless a new folder is explicitly specified - if s.Folder == "" { - containerID = existingContainerID - } - } else if dropped := consumeDroppedMicroflow(ctx, qualifiedName); dropped != nil { - // A prior DROP MICROFLOW in the same session removed the unit. Reuse - // its original UnitID and (unless a new folder is specified) - // ContainerID so that Studio Pro sees the rewrite as an in-place - // update rather than a delete+insert pair, which produces - // ".mpr does not look like a Mendix Studio Pro project file" errors. - microflowID = dropped.ID - if s.Folder == "" && dropped.ContainerID != "" { - containerID = dropped.ContainerID - } - // consumeDroppedMicroflow removed the cache entry, so we own this - // slice — no need to clone it again. - existingAllowedRoles = dropped.AllowedRoles - preserveAllowedRoles = true - } - - // Build the microflow - mf := µflows.Microflow{ - BaseElement: model.BaseElement{ - ID: microflowID, - }, - ContainerID: containerID, - Name: s.Name.Name, - Documentation: s.Documentation, - AllowConcurrentExecution: true, // Default: allow concurrent execution - MarkAsUsed: false, - Excluded: s.Excluded || existingExcluded, - } - if preserveAllowedRoles { - mf.AllowedModuleRoles = existingAllowedRoles - } else { - mf.AllowedModuleRoles = defaultDocumentAccessRoles(ctx, module) - } - if mf.MicroflowActionInfo, mf.WorkflowActionInfo, err = applyExposeClauses(ctx, - s.Expose, existingActionInfo, existingWorkflowInfo, exposeWarner(ctx)); err != nil { - return err - } - - // Build entity resolver function for parameter/return types - entityResolver := func(qn ast.QualifiedName) model.ID { - // Get all domain models and build module name map - dms, err := ctx.Backend.ListDomainModels() - if err != nil { - return "" - } - modules, _ := ctx.Backend.ListModules() - moduleNames := make(map[model.ID]string) - for _, m := range modules { - moduleNames[m.ID] = m.Name - } - // Search for entity in all domain models - for _, dm := range dms { - modName := moduleNames[dm.ContainerID] - if modName != qn.Module { - continue - } - for _, ent := range dm.Entities { - if ent.Name == qn.Name { - return ent.ID - } - } - } - return "" - } - - // Validate and add parameters - for i, p := range s.Parameters { - // Validate entity references for List and Entity types. - // Built-in modules (e.g. System) are not stored in the MPR domain models; - // their types are serialized by qualified name and resolved at runtime. - if p.Type.EntityRef != nil && !isBuiltinModuleEntity(p.Type.EntityRef.Module) { - entityID := entityResolver(*p.Type.EntityRef) - if entityID == "" { - // Bare qualified name in microflow context is treated as TypeEntity by the - // visitor, but it may actually be an enumeration. Try enum lookup before failing. - if found := findEnumeration(ctx, p.Type.EntityRef.Module, p.Type.EntityRef.Name); found != nil { - s.Parameters[i].Type = ast.DataType{Kind: ast.TypeEnumeration, EnumRef: p.Type.EntityRef} - p = s.Parameters[i] - } else { - return mdlerrors.NewNotFoundMsg("entity", p.Type.EntityRef.Module+"."+p.Type.EntityRef.Name, - fmt.Sprintf("entity '%s.%s' not found for parameter '%s'", p.Type.EntityRef.Module, p.Type.EntityRef.Name, p.Name)) - } - } - } - // Validate enumeration references for Enumeration types - if p.Type.Kind == ast.TypeEnumeration && p.Type.EnumRef != nil { - if found := findEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name); found == nil { - return mdlerrors.NewNotFoundMsg("enumeration", p.Type.EnumRef.Module+"."+p.Type.EnumRef.Name, - fmt.Sprintf("enumeration '%s.%s' not found for parameter '%s'", p.Type.EnumRef.Module, p.Type.EnumRef.Name, p.Name)) - } - } - param := µflows.MicroflowParameter{ - BaseElement: model.BaseElement{ - ID: model.ID(types.GenerateID()), - }, - ContainerID: mf.ID, - Name: p.Name, - Type: convertASTToMicroflowDataType(p.Type, entityResolver), - } - mf.Parameters = append(mf.Parameters, param) - } - - // Validate and set return type - if s.ReturnType != nil { - // Validate entity references for return type. - // Built-in modules (e.g. System) are not stored in the MPR domain models; - // their types are serialized by qualified name and resolved at runtime. - if s.ReturnType.Type.EntityRef != nil && !isBuiltinModuleEntity(s.ReturnType.Type.EntityRef.Module) { - entityID := entityResolver(*s.ReturnType.Type.EntityRef) - if entityID == "" { - return mdlerrors.NewNotFoundMsg("entity", s.ReturnType.Type.EntityRef.Module+"."+s.ReturnType.Type.EntityRef.Name, - fmt.Sprintf("entity '%s.%s' not found for return type", s.ReturnType.Type.EntityRef.Module, s.ReturnType.Type.EntityRef.Name)) - } - } - // Validate enumeration references for return type - if s.ReturnType.Type.Kind == ast.TypeEnumeration && s.ReturnType.Type.EnumRef != nil { - if found := findEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name); found == nil { - return mdlerrors.NewNotFoundMsg("enumeration", s.ReturnType.Type.EnumRef.Module+"."+s.ReturnType.Type.EnumRef.Name, - fmt.Sprintf("enumeration '%s.%s' not found for return type", s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name)) - } - } - mf.ReturnType = convertASTToMicroflowDataType(s.ReturnType.Type, entityResolver) - // Set return variable name if provided (AS $VarName) - if s.ReturnType.Variable != "" { - mf.ReturnVariableName = s.ReturnType.Variable - } - } else { - mf.ReturnType = µflows.VoidType{} - } - - // Build flow graph from body statements - // Initialize variable types from parameters - varTypes := make(map[string]string) - declaredVars := make(map[string]string) - - for _, p := range s.Parameters { - if p.Type.EntityRef != nil { - entityQN := p.Type.EntityRef.Module + "." + p.Type.EntityRef.Name - if p.Type.Kind == ast.TypeListOf { - // Store "List of Module.Entity" for list parameters - varTypes[p.Name] = "List of " + entityQN - } else { - // Store "Module.Entity" for single entity parameters - varTypes[p.Name] = entityQN - } - } else { - // Primitive type parameters are also considered declared - declaredVars[p.Name] = p.Type.Kind.String() - } - } - // Get hierarchy for resolving page/microflow references - hierarchy, _ := getHierarchy(ctx) - - restServices, _ := loadRestServices(ctx) - - builder := &flowBuilder{ - textLang: authoringLanguage(ctx), - // Carry over a HAND-PLACED StartEvent position from the microflow being - // replaced, the way the folder and allowed roles already are: a Studio - // Pro flow's 145;200 became 100;200 on a describe→exec round-trip, the - // only coordinate in it that did not survive (#884). A start sitting - // where mxcli's own layout would have put it is not carried over — that - // pinned the start of every rewritten flow, stranding it across the - // canvas from activities the same script had just moved (#951). An - // explicit @start(x, y) on the first statement overrides both. - startPosition: storedStartPosition(ctx, existingID), - posX: 200, - posY: 200, - baseY: 200, // Base Y for happy path - spacing: HorizontalSpacing, - varTypes: varTypes, - declaredVars: declaredVars, - measurer: &layoutMeasurer{varTypes: varTypes}, - backend: ctx.Backend, - hierarchy: hierarchy, - restServices: restServices, - } - - mf.ObjectCollection = builder.buildFlowGraph(s.Body, s.ReturnType) - - // Check for validation errors - if errors := builder.GetErrors(); len(errors) > 0 { - // Report all errors to the user - var errMsg strings.Builder - errMsg.WriteString(fmt.Sprintf("microflow '%s.%s' has validation errors:\n", s.Name.Module, s.Name.Name)) - for _, err := range errors { - errMsg.WriteString(fmt.Sprintf(" - %s\n", err)) - } - return fmt.Errorf("%s", errMsg.String()) - } + mf := built.Microflow + containerID := built.ContainerID + existingID := built.ExistingID + existingContainerID := built.ExistingContainerID // Create or update the microflow if existingID != "" { diff --git a/mdl/executor/cmd_microflows_parameter_position.go b/mdl/executor/cmd_microflows_parameter_position.go new file mode 100644 index 0000000000..f015dba593 --- /dev/null +++ b/mdl/executor/cmd_microflows_parameter_position.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package executor - parameter placement across a flow rewrite. +package executor + +import ( + "fmt" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// positionFromAST converts an `@position(x, y)` written on a parameter into the +// semantic model's point. nil in, nil out: no annotation means the layout +// places the parameter, which is what microflows.DerivedParameterPosition does. +func positionFromAST(p *ast.Position) *model.Point { + if p == nil { + return nil + } + return &model.Point{X: p.X, Y: p.Y} +} + +// parameterPositionAnnotation returns the `@position(x, y)` line a description +// needs for the parameter at index idx, or "" when the layout would put it +// exactly where it is. +// +// Emitting only an authored position is what keeps a described flow +// round-tripping: a line restating mxcli's own arithmetic would pin every +// parameter of every rewritten flow to the grid it happened to be on, so +// inserting a parameter would strand the others (the #951 lesson, which the +// StartEvent learned first — see authoredStartPosition). +// +// Readers already normalise, so Position is non-nil only when it is intent; +// the derived comparison here is belt-and-braces for a model assembled in +// memory rather than read from disk. +func parameterPositionAnnotation(p *microflows.MicroflowParameter, idx int, indent string) string { + if p == nil || p.Position == nil { + return "" + } + if *p.Position == microflows.DerivedParameterPosition(idx) { + return "" + } + return fmt.Sprintf("%s@position(%d, %d)", indent, p.Position.X, p.Position.Y) +} + +// describeMicroflowParameters renders the parenthesised parameter list of a +// flow header — one parameter per line, each preceded by its `@position` when +// it has one. +// +// Shared by all four describers (microflow, nanoflow, the generic flow +// describer and the rule describer). They were four copies of the same six +// lines, and a describer that emits the annotation while its twins do not makes +// the round-trip depend on which command the author happened to run — the same +// trap startAnnotationLines calls out. +func describeMicroflowParameters(params []*microflows.MicroflowParameter, formatType func(*microflows.MicroflowParameter) string) []string { + lines := make([]string, 0, len(params)*2) + for i, param := range params { + if ann := parameterPositionAnnotation(param, i, " "); ann != "" { + lines = append(lines, ann) + } + paramType := "Object" + if param.Type != nil { + paramType = formatType(param) + } + comma := "," + if i == len(params)-1 { + comma = "" + } + lines = append(lines, fmt.Sprintf(" $%s: %s%s", param.Name, paramType, comma)) + } + return lines +} diff --git a/mdl/executor/cmd_microflows_parameter_position_test.go b/mdl/executor/cmd_microflows_parameter_position_test.go new file mode 100644 index 0000000000..ba9af032ec --- /dev/null +++ b/mdl/executor/cmd_microflows_parameter_position_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// The reported symptom (#993): a parameter is a stored node with real geometry, +// and no annotation reached it — so a generated flow's parameter block landed +// wherever the writer put it, and a hand-aligned one was moved back there by any +// rewrite. Measured before the fix on a real project: a nanoflow parameter at +// -77;0 came back at 200;53 from a describe → exec of mxcli's own output. +// +// DerivedParameterPosition is the arithmetic both writers used inline. It is +// pinned here because the whole design rests on a reader being able to recognise +// it: a parameter sitting exactly there is mxcli's own layout handed back and +// carries no intent, so it is re-derived rather than pinned. +func TestDerivedParameterPositionMatchesTheWriters(t *testing.T) { + for idx, want := range []model.Point{{X: 200, Y: 53}, {X: 300, Y: 53}, {X: 400, Y: 53}} { + if got := microflows.DerivedParameterPosition(idx); got != want { + t.Errorf("DerivedParameterPosition(%d) = %v, want %v", idx, got, want) + } + } +} + +// A stored position that is the derived one carries no intent and must not be +// carried over. Carrying it UNCONDITIONALLY is the #951 mistake one node family +// over: inserting a parameter would leave the existing ones on the old grid +// while the new one lands on top of them. +func TestAuthoredParameterPositionIgnoresTheDerivedSpot(t *testing.T) { + if p := microflows.AuthoredParameterPosition(model.Point{X: 200, Y: 53}, 0); p != nil { + t.Errorf("index 0 at the derived spot: got %v, want nil", *p) + } + if p := microflows.AuthoredParameterPosition(model.Point{X: 300, Y: 53}, 1); p != nil { + t.Errorf("index 1 at the derived spot: got %v, want nil", *p) + } + // The same point at a DIFFERENT index is not the derived one, so it is intent. + got := microflows.AuthoredParameterPosition(model.Point{X: 200, Y: 53}, 1) + if got == nil || *got != (model.Point{X: 200, Y: 53}) { + t.Errorf("index 1 at index 0's spot: got %v, want 200;53 kept", got) + } +} + +// 0;0 is a position a person can choose — two flows in the reference project use +// it — so "unset" cannot be spelled as the zero value. This is why Position is a +// pointer, and the test exists because a bool-free `if p.Position != (Point{})` +// would pass every other case here. +func TestAuthoredParameterPositionKeepsOrigin(t *testing.T) { + got := microflows.AuthoredParameterPosition(model.Point{X: 0, Y: 0}, 0) + if got == nil { + t.Fatal("0;0 was dropped as if unset; it is a position a person can choose") + } + if *got != (model.Point{}) { + t.Errorf("got %v, want 0;0", *got) + } +} + +// DESCRIBE emits the annotation only for an authored position. Emitting the +// derived one would restate mxcli's own arithmetic and pin every parameter of +// every rewritten flow — the round-trip failure startAnnotationLines documents. +func TestParameterPositionAnnotation(t *testing.T) { + derived := µflows.MicroflowParameter{Position: &model.Point{X: 200, Y: 53}} + if got := parameterPositionAnnotation(derived, 0, " "); got != "" { + t.Errorf("derived position emitted %q, want no line", got) + } + authored := µflows.MicroflowParameter{Position: &model.Point{X: -77, Y: 0}} + if got := parameterPositionAnnotation(authored, 0, " "); got != " @position(-77, 0)" { + t.Errorf("got %q, want \" @position(-77, 0)\"", got) + } + if got := parameterPositionAnnotation(µflows.MicroflowParameter{}, 0, " "); got != "" { + t.Errorf("unset position emitted %q, want no line", got) + } +} + +// The control for the fix: with Position dropped on the way in — which is +// exactly what both readers did before #993 — the describer emits nothing and +// the writer has only the index to go on, so the annotation cannot round-trip. +// Without this the suite would pass against a build that never had the fix. +func TestDescribeMicroflowParametersCarriesAuthoredPositionOnly(t *testing.T) { + fmtType := func(*microflows.MicroflowParameter) string { return "Integer" } + params := []*microflows.MicroflowParameter{ + {Name: "A", Type: µflows.IntegerType{}, Position: &model.Point{X: 300, Y: 100}}, + {Name: "B", Type: µflows.IntegerType{}}, + } + got := describeMicroflowParameters(params, fmtType) + want := []string{" @position(300, 100)", " $A: Integer,", " $B: Integer"} + if len(got) != len(want) { + t.Fatalf("got %d lines %q, want %d %q", len(got), got, len(want), want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("line %d: got %q, want %q", i, got[i], want[i]) + } + } + + // Control: strip the positions, as the pre-fix readers did. + for _, p := range params { + p.Position = nil + } + if got := describeMicroflowParameters(params, fmtType); len(got) != 2 { + t.Errorf("with positions dropped, got %q — the annotation must not appear", got) + } +} + +// An annotation a parameter does not take is refused, not ignored. A typo of +// @position is the case that matters: it parses, does nothing, and discards the +// placement the author was trying to state. +func TestValidateFlowParameterAnnotations(t *testing.T) { + params := []ast.MicroflowParam{ + {Name: "A", UnknownAnnotations: []string{"postion"}}, + {Name: "B", Position: &ast.Position{X: 1, Y: 2}}, + } + got := ValidateFlowParameterAnnotations("nanoflow 'M.NF'", params) + if len(got) != 1 { + t.Fatalf("got %d violations, want 1: %+v", len(got), got) + } + if got[0].RuleID != "MDL059" { + t.Errorf("rule = %s, want MDL059", got[0].RuleID) + } + // Control: a parameter with a valid @position and no unknown names is clean. + if v := ValidateFlowParameterAnnotations("x", params[1:]); len(v) != 0 { + t.Errorf("valid @position flagged: %+v", v) + } +} diff --git a/mdl/executor/cmd_microflows_show.go b/mdl/executor/cmd_microflows_show.go index a4de31ef10..395a98b38c 100644 --- a/mdl/executor/cmd_microflows_show.go +++ b/mdl/executor/cmd_microflows_show.go @@ -251,17 +251,10 @@ func describeMicroflow(ctx *ExecContext, name ast.QualifiedName) error { qualifiedName := name.Module + "." + name.Name if len(targetMf.Parameters) > 0 { lines = append(lines, fmt.Sprintf("create or modify microflow %s (", qualifiedName)) - for i, param := range targetMf.Parameters { - paramType := "Object" - if param.Type != nil { - paramType = formatMicroflowDataType(ctx, param.Type, entityNames) - } - comma := "," - if i == len(targetMf.Parameters)-1 { - comma = "" - } - lines = append(lines, fmt.Sprintf(" $%s: %s%s", param.Name, paramType, comma)) - } + lines = append(lines, describeMicroflowParameters(targetMf.Parameters, + func(p *microflows.MicroflowParameter) string { + return formatMicroflowDataType(ctx, p.Type, entityNames) + })...) lines = append(lines, ")") } else { lines = append(lines, fmt.Sprintf("create or modify microflow %s ()", qualifiedName)) @@ -400,17 +393,10 @@ func describeNanoflow(ctx *ExecContext, name ast.QualifiedName) error { qualifiedName := name.Module + "." + name.Name if len(targetNf.Parameters) > 0 { lines = append(lines, fmt.Sprintf("create or modify nanoflow %s (", qualifiedName)) - for i, param := range targetNf.Parameters { - paramType := "Object" - if param.Type != nil { - paramType = formatMicroflowDataType(ctx, param.Type, entityNames) - } - comma := "," - if i == len(targetNf.Parameters)-1 { - comma = "" - } - lines = append(lines, fmt.Sprintf(" $%s: %s%s", param.Name, paramType, comma)) - } + lines = append(lines, describeMicroflowParameters(targetNf.Parameters, + func(p *microflows.MicroflowParameter) string { + return formatMicroflowDataType(ctx, p.Type, entityNames) + })...) lines = append(lines, ")") } else { lines = append(lines, fmt.Sprintf("create or modify nanoflow %s ()", qualifiedName)) @@ -606,17 +592,10 @@ func renderMicroflowMDL( qualifiedName := name.Module + "." + name.Name if len(mf.Parameters) > 0 { lines = append(lines, fmt.Sprintf("create or modify %s %s (", flowType, qualifiedName)) - for i, param := range mf.Parameters { - paramType := "Object" - if param.Type != nil { - paramType = formatMicroflowDataType(ctx, param.Type, entityNames) - } - comma := "," - if i == len(mf.Parameters)-1 { - comma = "" - } - lines = append(lines, fmt.Sprintf(" $%s: %s%s", param.Name, paramType, comma)) - } + lines = append(lines, describeMicroflowParameters(mf.Parameters, + func(p *microflows.MicroflowParameter) string { + return formatMicroflowDataType(ctx, p.Type, entityNames) + })...) lines = append(lines, ")") } else { lines = append(lines, fmt.Sprintf("create or modify %s %s ()", flowType, qualifiedName)) @@ -1535,17 +1514,10 @@ func describeRule(ctx *ExecContext, name ast.QualifiedName) error { qualifiedName := name.Module + "." + name.Name if len(target.Parameters) > 0 { lines = append(lines, fmt.Sprintf("create or modify rule %s (", qualifiedName)) - for i, param := range target.Parameters { - paramType := "Object" - if param.Type != nil { - paramType = formatMicroflowDataType(ctx, param.Type, entityNames) - } - comma := "," - if i == len(target.Parameters)-1 { - comma = "" - } - lines = append(lines, fmt.Sprintf(" $%s: %s%s", param.Name, paramType, comma)) - } + lines = append(lines, describeMicroflowParameters(target.Parameters, + func(p *microflows.MicroflowParameter) string { + return formatMicroflowDataType(ctx, p.Type, entityNames) + })...) lines = append(lines, ")") } else { lines = append(lines, fmt.Sprintf("create or modify rule %s ()", qualifiedName)) diff --git a/mdl/executor/cmd_misc.go b/mdl/executor/cmd_misc.go index 2505910ab9..ffab2109f4 100644 --- a/mdl/executor/cmd_misc.go +++ b/mdl/executor/cmd_misc.go @@ -266,7 +266,7 @@ Navigation: describe navigation Profile; create or replace navigation Profile home page Module.Page - [home page Module.Page for Module.Role] + [home page Module.Page for UserRole] [login page Module.Page] [not found page Module.Page] [menu ( diff --git a/mdl/executor/cmd_nanoflows_create.go b/mdl/executor/cmd_nanoflows_create.go index 0c4dd5ed09..7e56a89092 100644 --- a/mdl/executor/cmd_nanoflows_create.go +++ b/mdl/executor/cmd_nanoflows_create.go @@ -5,13 +5,9 @@ package executor import ( "fmt" - "strings" "github.com/mendixlabs/mxcli/mdl/ast" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" - "github.com/mendixlabs/mxcli/mdl/types" - "github.com/mendixlabs/mxcli/model" - "github.com/mendixlabs/mxcli/sdk/microflows" ) // execCreateNanoflow handles CREATE NANOFLOW statements. @@ -20,247 +16,14 @@ func execCreateNanoflow(ctx *ExecContext, s *ast.CreateNanoflowStmt) error { return mdlerrors.NewNotConnectedWrite() } - // Validate name is not empty - if strings.TrimSpace(s.Name.Name) == "" { - return mdlerrors.NewValidation("nanoflow name must not be empty") - } - - if err := refuseExposeOnFlavour(s.Expose, "nanoflow", s.Name.Module+"."+s.Name.Name); err != nil { - return err - } - - // Find or auto-create module - module, err := findOrCreateModule(ctx, s.Name.Module) + built, err := buildNanoflowFromStmt(ctx, s, buildFlowOpts{AllowCreate: true}) if err != nil { return err } - - // Resolve folder if specified - containerID := module.ID - if s.Folder != "" { - folderID, err := resolveFolder(ctx, module.ID, s.Folder) - if err != nil { - return mdlerrors.NewBackend("resolve folder "+s.Folder, err) - } - containerID = folderID - } - - // Check if nanoflow with same name already exists in this module. - // NOTE: O(n) scan over all nanoflows — consistent with microflow handler pattern. - // Consider catalog-based lookup if this becomes a bottleneck for large projects. - var existingID model.ID - var existingContainerID model.ID - var existingAllowedRoles []model.ID - preserveAllowedRoles := false - // Excluded is model state, not script state: an absent @excluded must not - // clear a stored exclusion (#914). - existingExcluded := false - existingNanoflows, err := ctx.Backend.ListNanoflows() - if err != nil { - return mdlerrors.NewBackend("check existing nanoflows", err) - } - // A module may hold several nanoflows with this name as long as all but one - // are excluded, so target the live one rather than whichever comes first. - if existing, ok := pickLive(existingNanoflows, - func(n *microflows.Nanoflow) bool { - return n.Name == s.Name.Name && getModuleID(ctx, n.ContainerID) == module.ID - }, - func(n *microflows.Nanoflow) bool { return n.Excluded }, - ); ok { - if !s.CreateOrModify { - return mdlerrors.NewAlreadyExistsMsg("nanoflow", s.Name.Module+"."+s.Name.Name, "nanoflow '"+s.Name.Module+"."+s.Name.Name+"' already exists (use create or modify to overwrite)") - } - existingID = existing.ID - existingContainerID = existing.ContainerID - existingAllowedRoles = cloneRoleIDs(existing.AllowedModuleRoles) - preserveAllowedRoles = true - existingExcluded = existing.Excluded - } - - // For CREATE OR REPLACE/MODIFY, reuse the existing ID to preserve references - qualifiedName := s.Name.Module + "." + s.Name.Name - nanoflowID := model.ID(types.GenerateID()) - if existingID != "" { - nanoflowID = existingID - if s.Folder == "" { - containerID = existingContainerID - } - } else if dropped := consumeDroppedNanoflow(ctx, qualifiedName); dropped != nil { - nanoflowID = dropped.ID - if s.Folder == "" && dropped.ContainerID != "" { - containerID = dropped.ContainerID - } - if len(dropped.AllowedRoles) > 0 { - existingAllowedRoles = dropped.AllowedRoles - preserveAllowedRoles = true - } - } - - // Build the nanoflow - nf := µflows.Nanoflow{ - BaseElement: model.BaseElement{ - ID: nanoflowID, - }, - ContainerID: containerID, - Name: s.Name.Name, - Documentation: s.Documentation, - MarkAsUsed: false, - Excluded: s.Excluded || existingExcluded, - } - if preserveAllowedRoles { - nf.AllowedModuleRoles = existingAllowedRoles - } else { - nf.AllowedModuleRoles = defaultDocumentAccessRoles(ctx, module) - } - - // Load metadata needed by the entity resolver up front so backend read - // failures are returned as actionable errors instead of being treated as - // "entity not found". - dms, err := ctx.Backend.ListDomainModels() - if err != nil { - return mdlerrors.NewBackend("list domain models", err) - } - modules, err := ctx.Backend.ListModules() - if err != nil { - return mdlerrors.NewBackend("list modules", err) - } - moduleNames := make(map[model.ID]string) - for _, m := range modules { - moduleNames[m.ID] = m.Name - } - - // Build entity resolver function for parameter/return types - entityResolver := func(qn ast.QualifiedName) model.ID { - for _, dm := range dms { - modName := moduleNames[dm.ContainerID] - if modName != qn.Module { - continue - } - for _, ent := range dm.Entities { - if ent.Name == qn.Name { - return ent.ID - } - } - } - return "" - } - - // Validate and add parameters - for i, p := range s.Parameters { - if p.Type.EntityRef != nil && !isBuiltinModuleEntity(p.Type.EntityRef.Module) { - entityID := entityResolver(*p.Type.EntityRef) - if entityID == "" { - // Bare qualified name in microflow context is treated as TypeEntity by the - // visitor, but it may actually be an enumeration. Try enum lookup before failing. - if found := findEnumeration(ctx, p.Type.EntityRef.Module, p.Type.EntityRef.Name); found != nil { - s.Parameters[i].Type = ast.DataType{Kind: ast.TypeEnumeration, EnumRef: p.Type.EntityRef} - p = s.Parameters[i] - } else { - return mdlerrors.NewNotFoundMsg("entity", p.Type.EntityRef.Module+"."+p.Type.EntityRef.Name, - fmt.Sprintf("entity '%s.%s' not found for parameter '%s'", p.Type.EntityRef.Module, p.Type.EntityRef.Name, p.Name)) - } - } - } - if p.Type.Kind == ast.TypeEnumeration && p.Type.EnumRef != nil { - if found := findEnumeration(ctx, p.Type.EnumRef.Module, p.Type.EnumRef.Name); found == nil { - return mdlerrors.NewNotFoundMsg("enumeration", p.Type.EnumRef.Module+"."+p.Type.EnumRef.Name, - fmt.Sprintf("enumeration '%s.%s' not found for parameter '%s'", p.Type.EnumRef.Module, p.Type.EnumRef.Name, p.Name)) - } - } - param := µflows.MicroflowParameter{ - BaseElement: model.BaseElement{ - ID: model.ID(types.GenerateID()), - }, - ContainerID: nf.ID, - Name: p.Name, - Type: convertASTToMicroflowDataType(p.Type, entityResolver), - } - nf.Parameters = append(nf.Parameters, param) - } - - // Validate and set return type - if s.ReturnType != nil { - if s.ReturnType.Type.EntityRef != nil && !isBuiltinModuleEntity(s.ReturnType.Type.EntityRef.Module) { - entityID := entityResolver(*s.ReturnType.Type.EntityRef) - if entityID == "" { - return mdlerrors.NewNotFoundMsg("entity", s.ReturnType.Type.EntityRef.Module+"."+s.ReturnType.Type.EntityRef.Name, - fmt.Sprintf("entity '%s.%s' not found for return type", s.ReturnType.Type.EntityRef.Module, s.ReturnType.Type.EntityRef.Name)) - } - } - if s.ReturnType.Type.Kind == ast.TypeEnumeration && s.ReturnType.Type.EnumRef != nil { - if found := findEnumeration(ctx, s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name); found == nil { - return mdlerrors.NewNotFoundMsg("enumeration", s.ReturnType.Type.EnumRef.Module+"."+s.ReturnType.Type.EnumRef.Name, - fmt.Sprintf("enumeration '%s.%s' not found for return type", s.ReturnType.Type.EnumRef.Module, s.ReturnType.Type.EnumRef.Name)) - } - } - nf.ReturnType = convertASTToMicroflowDataType(s.ReturnType.Type, entityResolver) - } else { - nf.ReturnType = µflows.VoidType{} - } - - // Validate nanoflow-specific constraints before building the flow graph - if errMsg := validateNanoflow(qualifiedName, s.Body, s.ReturnType); errMsg != "" { - return fmt.Errorf("%s", errMsg) - } - - // SYNCHRONIZE UNSYNCHRONIZED is the one mode with a floor inside Mendix 9: - // SynchronizationType.Unsynchronized was introduced in 9.4.0, while the - // activity and its other two modes go back further. Gate the mode, not the - // statement, so a 9.0-9.3 project keeps `synchronize all`. - if bodyUsesUnsynchronized(s.Body) { - if err := checkFeature(ctx, "microflows", "synchronize_unsynchronized", - "synchronize unsynchronized", - "use `synchronize all` or `synchronize $Var`, or upgrade the project to 9.4+"); err != nil { - return err - } - } - - // Build flow graph from body statements - varTypes := make(map[string]string) - declaredVars := make(map[string]string) - - for _, p := range s.Parameters { - if p.Type.EntityRef != nil { - entityQN := p.Type.EntityRef.Module + "." + p.Type.EntityRef.Name - if p.Type.Kind == ast.TypeListOf { - varTypes[p.Name] = "List of " + entityQN - } else { - varTypes[p.Name] = entityQN - } - } else { - declaredVars[p.Name] = p.Type.Kind.String() - } - } - - hierarchy, _ := getHierarchy(ctx) // best-effort: builder works without hierarchy - restServices, _ := loadRestServices(ctx) // best-effort: builder works without REST services - - builder := &flowBuilder{ - textLang: authoringLanguage(ctx), - posX: 200, - posY: 200, - baseY: 200, - spacing: HorizontalSpacing, - varTypes: varTypes, - declaredVars: declaredVars, - measurer: &layoutMeasurer{varTypes: varTypes}, - backend: ctx.Backend, - hierarchy: hierarchy, - restServices: restServices, - isNanoflow: true, - } - - nf.ObjectCollection = builder.buildFlowGraph(s.Body, s.ReturnType) - - // Check for validation errors - if errors := builder.GetErrors(); len(errors) > 0 { - var errMsg strings.Builder - errMsg.WriteString(fmt.Sprintf("nanoflow '%s.%s' has validation errors:\n", s.Name.Module, s.Name.Name)) - for _, err := range errors { - errMsg.WriteString(fmt.Sprintf(" - %s\n", err)) - } - return fmt.Errorf("%s", errMsg.String()) - } + nf := built.Nanoflow + containerID := built.ContainerID + existingID := built.ExistingID + existingContainerID := built.ExistingContainerID // Create or update the nanoflow if existingID != "" { diff --git a/mdl/executor/cmd_navigation.go b/mdl/executor/cmd_navigation.go index bad8945700..9d9a32f358 100644 --- a/mdl/executor/cmd_navigation.go +++ b/mdl/executor/cmd_navigation.go @@ -19,6 +19,14 @@ func execAlterNavigation(ctx *ExecContext, s *ast.AlterNavigationStmt) error { return mdlerrors.NewNotConnectedWrite() } + // Resolve `FOR ` before anything is written — including the + // profile this may add below. A module-qualified role here produces a project + // Mendix cannot load at all, so refusing is the only useful outcome + // (mendixlabs/mxcli#1001). Same function `check` calls. + if err := validateNavigationRoleForExec(ctx, s); err != nil { + return err + } + nav, err := ctx.Backend.GetNavigation() if err != nil { return mdlerrors.NewBackend("get navigation", err) diff --git a/mdl/executor/cmd_odata.go b/mdl/executor/cmd_odata.go index 653442f757..85ebee85ac 100644 --- a/mdl/executor/cmd_odata.go +++ b/mdl/executor/cmd_odata.go @@ -885,9 +885,11 @@ func execCreateExternalEntity(ctx *ExecContext, s *ast.CreateExternalEntityStmt) if len(attrs) > 0 { existingEntity.Attributes = attrs } - if s.Documentation != "" { - existingEntity.Documentation = s.Documentation - } + // A rewrite that carried no doc comment keeps the stored one; an + // explicitly empty `/** */` clears it (#1018). The `!= ""` form this + // replaces preserved but could not clear. + existingEntity.Documentation = carriedDocumentation( + s.DocumentationSet, s.Documentation, existingEntity.Documentation) if err := ctx.Backend.UpdateEntity(dm.ID, existingEntity); err != nil { return mdlerrors.NewBackend("update external entity", err) } @@ -975,7 +977,10 @@ func createODataClient(ctx *ExecContext, stmt *ast.CreateODataClientStmt) error modName := h.GetModuleName(modID) if strings.EqualFold(modName, stmt.Name.Module) && strings.EqualFold(svc.Name, stmt.Name.Name) { if stmt.CreateOrModify { - svc.Documentation = stmt.Documentation + // svc is the STORED service being mutated, so its own value is + // what a silent rewrite preserves (#1018). + svc.Documentation = carriedDocumentation( + stmt.DocumentationSet, stmt.Documentation, svc.Documentation) if stmt.Version != "" { svc.Version = stmt.Version } @@ -1457,7 +1462,12 @@ func createODataService(ctx *ExecContext, stmt *ast.CreateODataServiceStmt) erro if stmt.CreateOrModify { // Snapshot the grants before anything below can clear them. existingRoles := append([]string(nil), svc.AllowedModuleRoles...) - svc.Documentation = stmt.Documentation + // svc is the STORED service being mutated, so its own value + // is what a silent rewrite preserves (#1018). There are two + // update paths for this doctype and patching only the first + // left the defect fully intact — the test caught it. + svc.Documentation = carriedDocumentation( + stmt.DocumentationSet, stmt.Documentation, svc.Documentation) if stmt.Path != "" { svc.Path = stmt.Path } diff --git a/mdl/executor/cmd_pages_create_v3.go b/mdl/executor/cmd_pages_create_v3.go index 4a746dd150..d27a5f0c62 100644 --- a/mdl/executor/cmd_pages_create_v3.go +++ b/mdl/executor/cmd_pages_create_v3.go @@ -51,6 +51,8 @@ func execCreatePageV3(ctx *ExecContext, s *ast.CreatePageStmtV3) error { // Where the page being rewritten currently sits, so a statement that says // nothing about folders leaves it there (#932). var existingContainerID model.ID + var existingDocumentation string + haveExistingPage := false var excludedMatches []*pages.Page matches := 0 for _, p := range existingPages { @@ -71,6 +73,8 @@ func execCreatePageV3(ctx *ExecContext, s *ast.CreatePageStmtV3) error { existingAllowedRoles = cloneRoleIDs(p.AllowedRoles) preserveAllowedRoles = true existingContainerID = p.ContainerID + existingDocumentation = p.Documentation + haveExistingPage = true } pagesToDelete = append(pagesToDelete, p.ID) } @@ -83,6 +87,8 @@ func execCreatePageV3(ctx *ExecContext, s *ast.CreatePageStmtV3) error { preserveAllowedRoles = true existingExcluded = true existingContainerID = p.ContainerID + existingDocumentation = p.Documentation + haveExistingPage = true pagesToDelete = append(pagesToDelete, p.ID) } @@ -106,6 +112,10 @@ func execCreatePageV3(ctx *ExecContext, s *ast.CreatePageStmtV3) error { return mdlerrors.NewBackend("build page", err) } page.Excluded = page.Excluded || existingExcluded + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExistingPage { + page.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingDocumentation) + } if preserveAllowedRoles { page.AllowedRoles = existingAllowedRoles } else if len(page.AllowedRoles) == 0 { @@ -185,6 +195,8 @@ func execCreateSnippetV3(ctx *ExecContext, s *ast.CreateSnippetStmtV3) error { // snippet back into the module root whenever a script that says nothing // about folders was re-run (#932). var existingContainerID model.ID + var existingSnipDoc string + haveExistingSnip := false var excludedSnippets []*pages.Snippet matches := 0 for _, snip := range existingSnippets { @@ -203,12 +215,16 @@ func execCreateSnippetV3(ctx *ExecContext, s *ast.CreateSnippetStmtV3) error { } if len(snippetsToDelete) == 0 { existingContainerID = snip.ContainerID + existingSnipDoc = snip.Documentation + haveExistingSnip = true } snippetsToDelete = append(snippetsToDelete, snip.ID) } if len(snippetsToDelete) == 0 && len(excludedSnippets) > 0 { existingExcluded = true existingContainerID = excludedSnippets[0].ContainerID + existingSnipDoc = excludedSnippets[0].Documentation + haveExistingSnip = true snippetsToDelete = append(snippetsToDelete, excludedSnippets[0].ID) } @@ -231,6 +247,11 @@ func execCreateSnippetV3(ctx *ExecContext, s *ast.CreateSnippetStmtV3) error { if err != nil { return mdlerrors.NewBackend("build snippet", err) } + + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExistingSnip { + snippet.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingSnipDoc) + } snippet.Excluded = snippet.Excluded || existingExcluded if s.Folder == "" && existingContainerID != "" { snippet.ContainerID = existingContainerID diff --git a/mdl/executor/cmd_pages_layout_v3.go b/mdl/executor/cmd_pages_layout_v3.go index 3c76287561..dccdd29bcc 100644 --- a/mdl/executor/cmd_pages_layout_v3.go +++ b/mdl/executor/cmd_pages_layout_v3.go @@ -113,6 +113,8 @@ func execCreateLayout(ctx *ExecContext, s *ast.CreateLayoutStmt) error { existing, _ := ctx.Backend.ListLayouts() var toDelete []model.ID + var existingLayoutDoc string + haveExistingLayout := false for _, l := range existing { modName := getModuleName(ctx, getModuleID(ctx, l.ContainerID)) if modName != s.Name.Module || l.Name != s.Name.Name { @@ -121,6 +123,10 @@ func execCreateLayout(ctx *ExecContext, s *ast.CreateLayoutStmt) error { if !s.IsReplace && !s.IsModify { return mdlerrors.NewAlreadyExists("layout", s.Name.String()) } + if len(toDelete) == 0 { + existingLayoutDoc = l.Documentation + haveExistingLayout = true + } toDelete = append(toDelete, l.ID) } @@ -145,6 +151,11 @@ func execCreateLayout(ctx *ExecContext, s *ast.CreateLayoutStmt) error { return err } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExistingLayout { + layout.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingLayoutDoc) + } + for _, id := range toDelete { if err := ctx.Backend.DeleteLayout(id); err != nil { return mdlerrors.NewBackend("delete existing layout", err) diff --git a/mdl/executor/cmd_queues.go b/mdl/executor/cmd_queues.go index 16a7934b23..3816ce4d6c 100644 --- a/mdl/executor/cmd_queues.go +++ b/mdl/executor/cmd_queues.go @@ -78,6 +78,10 @@ func execCreateQueue(ctx *ExecContext, s *ast.CreateQueueStmt) error { ClusterWide: s.ClusterWide, ExportLevel: s.ExportLevel, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + q.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existing.Documentation) + } if existing != nil { q.ID = existing.ID diff --git a/mdl/executor/cmd_regularexpressions.go b/mdl/executor/cmd_regularexpressions.go index 0035089e33..2ca5e9121d 100644 --- a/mdl/executor/cmd_regularexpressions.go +++ b/mdl/executor/cmd_regularexpressions.go @@ -80,6 +80,10 @@ func execCreateRegularExpression(ctx *ExecContext, s *ast.CreateRegularExpressio Expression: s.Expression, ExportLevel: s.ExportLevel, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if existing != nil { + re.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existing.Documentation) + } if existing != nil { re.ID = existing.ID re.Excluded = existing.Excluded diff --git a/mdl/executor/cmd_rest_clients.go b/mdl/executor/cmd_rest_clients.go index c4ac0f7429..664ee5ece1 100644 --- a/mdl/executor/cmd_rest_clients.go +++ b/mdl/executor/cmd_rest_clients.go @@ -378,6 +378,7 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { // delete+create, so its container is re-applied on every statement and an // unset value files a foldered service back into the module root (#932). var preservedContainerID model.ID + var preservedDocumentation string wasModified := false for _, existing := range existingServices { existModID := h.FindModuleID(existing.ContainerID) @@ -387,6 +388,9 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { // Preserve the existing ID so SEND REST REQUEST references stay valid after replace. preservedID = existing.ID preservedContainerID = existing.ContainerID + // The rewrite is a delete+create, so the stored documentation + // has to be captured before the delete or it is gone (#1018). + preservedDocumentation = existing.Documentation wasModified = true if err := ctx.Backend.DeleteConsumedRestService(existing.ID); err != nil { return mdlerrors.NewBackend("delete existing rest client", err) @@ -419,6 +423,9 @@ func createRestClient(ctx *ExecContext, stmt *ast.CreateRestClientStmt) error { // Preserve the existing ID on OR MODIFY so SEND REST REQUEST references stay valid. if preservedID != "" { svc.ID = preservedID + // A rewrite that carried no doc comment keeps the stored one (#1018). + svc.Documentation = carriedDocumentation( + stmt.DocumentationSet, stmt.Documentation, preservedDocumentation) } // Authentication — Mendix requires Rest$ConstantValue for BASIC auth credentials diff --git a/mdl/executor/cmd_rules_create.go b/mdl/executor/cmd_rules_create.go index 5bc2108295..97d9272b00 100644 --- a/mdl/executor/cmd_rules_create.go +++ b/mdl/executor/cmd_rules_create.go @@ -59,6 +59,8 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { // Check whether a rule of this name already exists in the module. var existingID model.ID + var existingDocumentation string + haveExisting := false var existingContainerID model.ID // Excluded is model state, not script state: an absent @excluded must not // clear a stored exclusion (#914). @@ -85,6 +87,8 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { existingID = existing.ID existingContainerID = existing.ContainerID existingExcluded = existing.Excluded + existingDocumentation = existing.Documentation + haveExisting = true // MDL has no surface for ReturnVariableName, and Studio Pro writes one // ("Variable" on both reference rules), so carry the stored value rather // than blanking it on every rewrite. @@ -114,6 +118,11 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { ReturnVariableName: existingReturnVariableName, } + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExisting { + rule.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingDocumentation) + } + // Load metadata needed by the entity resolver up front so backend read // failures are returned as actionable errors instead of being treated as // "entity not found". @@ -175,6 +184,7 @@ func execCreateRule(ctx *ExecContext, s *ast.CreateRuleStmt) error { ContainerID: rule.ID, Name: p.Name, Type: convertASTToMicroflowDataType(p.Type, entityResolver), + Position: positionFromAST(p.Position), } rule.Parameters = append(rule.Parameters, param) } diff --git a/mdl/executor/cmd_scheduledevents.go b/mdl/executor/cmd_scheduledevents.go index 4294144f08..76744e7b35 100644 --- a/mdl/executor/cmd_scheduledevents.go +++ b/mdl/executor/cmd_scheduledevents.go @@ -126,6 +126,9 @@ func execCreateScheduledEvent(ctx *ExecContext, s *ast.CreateScheduledEventStmt) // Carry the stored values so a modify does not invent new ones. ev.Interval = existing.Interval ev.IntervalType = existing.IntervalType + // Same rule, one property over: a rewrite that carried no doc comment + // keeps the stored documentation (#1018). + ev.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existing.Documentation) if err := ctx.Backend.UpdateScheduledEvent(ev); err != nil { return mdlerrors.NewBackend("update scheduled event", err) } diff --git a/mdl/executor/cmd_workflows_write.go b/mdl/executor/cmd_workflows_write.go index 5313e8e612..279bb372b0 100644 --- a/mdl/executor/cmd_workflows_write.go +++ b/mdl/executor/cmd_workflows_write.go @@ -85,6 +85,8 @@ func execCreateWorkflow(ctx *ExecContext, s *ast.CreateWorkflowStmt) error { // excluded twin of this name — target the live workflow and carry its // exclusion forward (#914). existingExcluded := false + var existingDocumentation string + haveExistingWf := false if existing, ok := pickLive(existingWorkflows, func(w *workflows.Workflow) bool { return h.GetModuleName(h.FindModuleID(w.ContainerID)) == s.Name.Module && w.Name == s.Name.Name @@ -97,6 +99,8 @@ func execCreateWorkflow(ctx *ExecContext, s *ast.CreateWorkflowStmt) error { existingID = existing.ID existingExcluded = existing.Excluded existingContainer = existing.ContainerID + existingDocumentation = existing.Documentation + haveExistingWf = true // Refuse a rewrite that would delete a stored construct this statement // does not restate (guard-don't-drop, ADR-0005) — issue #948. @@ -115,6 +119,10 @@ func execCreateWorkflow(ctx *ExecContext, s *ast.CreateWorkflowStmt) error { wf.ContainerID = containerID wf.Name = s.Name.Name wf.Documentation = s.Documentation + // A rewrite that carried no doc comment keeps the stored one (#1018). + if haveExistingWf { + wf.Documentation = carriedDocumentation(s.DocumentationSet, s.Documentation, existingDocumentation) + } // Parameter if s.ParameterEntity.Module != "" { diff --git a/mdl/executor/documentation_carry.go b/mdl/executor/documentation_carry.go new file mode 100644 index 0000000000..0c154d5eff --- /dev/null +++ b/mdl/executor/documentation_carry.go @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +// carriedDocumentation returns the documentation a rewrite should store. +// +// A statement that carried a `/** … */` comment — even an empty one — states its +// intent and wins. A statement that carried none said nothing about +// documentation, so the stored value is preserved. +// +// Before this, every rewrite path wrote the statement's value unconditionally, +// so a statement with no doc comment overwrote stored prose with "" and reported +// success (mendixlabs/mxcli#1018). It is an empty overwrite, not a drop, which is +// why nothing downstream could see it: the resulting document is valid, and +// `mx check` has no code for "this used to say something". +// +// Preserve-always was not an option. `SET DOCUMENTATION` / `SET COMMENT` exist +// only in the domain-model grammar — there is no ALTER for a microflow, a queue +// or a workflow — so an explicitly empty comment is the only clearing spelling +// most doctypes have. +func carriedDocumentation(set bool, stated, stored string) string { + if set { + return stated + } + return stored +} diff --git a/mdl/executor/documentation_coverage_test.go b/mdl/executor/documentation_coverage_test.go new file mode 100644 index 0000000000..2043df1b95 --- /dev/null +++ b/mdl/executor/documentation_coverage_test.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// mendixlabs/mxcli#1018 has 26 statement types in scope — every one that accepts +// a `/** … */` doc comment AND has a rewrite path. Three are fixed. This test +// exists so the other 23 are a number somebody can read rather than a thing that +// was quietly not done: patching the reported instances and leaving the class is +// the failure the wiki calls duplicate-resolver-drift. +// +// It fails when a statement type gains a rewrite flag and a Documentation field +// without either carrying the presence bit or being listed below. Adding a new +// doctype therefore has to make a decision about documentation, rather than +// inheriting the bug by default. + +// docCarryDone are the statement types whose rewrite path preserves an unstated +// doc comment. Move a type here when its executor carry lands AND +// TestDocumentation_SurvivesRewrite covers it. +var docCarryDone = map[string]bool{ + "CreateEntityStmt": true, + "CreateMicroflowStmt": true, + "CreateEnumerationStmt": true, + "CreateQueueStmt": true, + "CreateRegularExpressionStmt": true, + "CreateScheduledEventStmt": true, + "CreateNanoflowStmt": true, + "CreateRuleStmt": true, + "CreateJsonStructureStmt": true, + "CreateImageCollectionStmt": true, + "CreateWorkflowStmt": true, + "CreateConstantStmt": true, + "CreateAssociationStmt": true, + "CreateViewEntityStmt": true, + "CreateBusinessEventServiceStmt": true, + "CreateJavaActionStmt": true, + "CreateJavaScriptActionStmt": true, + "CreateMenuStmt": true, + "CreateODataServiceStmt": true, + "CreatePageStmtV3": true, + "CreateSnippetStmtV3": true, + "CreateLayoutStmt": true, + "CreateODataClientStmt": true, + "CreateExternalEntityStmt": true, + "CreateRestClientStmt": true, + "CreateModelStmt": true, + "CreateKnowledgeBaseStmt": true, + "CreateConsumedMCPServiceStmt": true, + "CreateAgentStmt": true, +} + +// docCarryPending is empty: every statement type in scope is carried AND +// covered by TestDocumentation_SurvivesRewrite. +// +// It is kept rather than deleted because it is the mechanism that made the +// remaining work visible while there was any, and because the next doctype +// added has to land in one list or the other. +// +// Worth recording why it emptied. The last seven entries carried BLOCKERS — +// "agent editor needs AgentEditorCommons and Mendix 11.9+", "needs a reachable +// $metadata", "needs an OpenAPI spec" — and every one of them was an assumption +// written down as if it were a measurement. Tested directly, all seven author +// fine offline against an ordinary 11.13 project. A blocker nobody has tried is +// a guess with a citation. +var docCarryPending = map[string]string{} + +var ( + // `(?:V\d)?` is load-bearing: CreatePageStmtV3 and CreateSnippetStmtV3 were + // invisible to the first version of this pattern, so two in-scope doctypes + // were absent from a list whose entire job is to be complete. + stmtRe = regexp.MustCompile(`(?s)type (\w+Stmt(?:V\d)?) struct \{(.*?)\n\}`) + // A rewrite flag is spelled CreateOrModify / CreateOrReplace on most + // statements and IsReplace / IsModify on a few (CreateLayoutStmt). Matching + // only the first spelling made this test disagree with the enumeration it + // was built from, which is the kind of drift it exists to catch. + rewriteRe = regexp.MustCompile(`CreateOr(Modify|Replace)|Is(Replace|Modify)\s+bool`) +) + +func TestDocumentationCarry_EveryRewritableDoctypeIsAccountedFor(t *testing.T) { + entries, err := os.ReadDir("../ast") + if err != nil { + t.Fatal(err) + } + var unaccounted []string + seen := map[string]bool{} + for _, e := range entries { + if !strings.HasPrefix(e.Name(), "ast_") || !strings.HasSuffix(e.Name(), ".go") || + strings.HasSuffix(e.Name(), "_test.go") { + continue + } + src, err := os.ReadFile(filepath.Join("../ast", e.Name())) + if err != nil { + t.Fatal(err) + } + for _, m := range stmtRe.FindAllStringSubmatch(string(src), -1) { + name, body := m[1], m[2] + // Image collections spell it `Comment`; everything else `Documentation`. + if (!strings.Contains(body, "Documentation") && !strings.Contains(body, "Comment")) || + !rewriteRe.MatchString(body) { + continue + } + seen[name] = true + if docCarryDone[name] || docCarryPending[name] != "" { + continue + } + unaccounted = append(unaccounted, name) + } + } + sort.Strings(unaccounted) + for _, n := range unaccounted { + t.Errorf("%s takes a doc comment and has a rewrite path, but is in neither "+ + "docCarryDone nor docCarryPending — its rewrite silently deletes documentation "+ + "(#1018). Carry the stored value when !DocumentationSet, or list it as pending.", n) + } + + // A type that no longer exists must not linger in either list, or the + // remaining-work count is fiction. + for name := range docCarryPending { + if !seen[name] { + t.Errorf("docCarryPending lists %s, which no longer matches a statement type", name) + } + } + for name := range docCarryDone { + if !seen[name] { + t.Errorf("docCarryDone lists %s, which no longer matches a statement type", name) + } + } + t.Logf("#1018 documentation carry: %d done, %d pending (of %d in scope)", + len(docCarryDone), len(docCarryPending), len(seen)) +} + +// A type in docCarryDone must actually carry the bit, or the list is a claim +// rather than a record. +func TestDocumentationCarry_DoneTypesHaveThePresenceBit(t *testing.T) { + entries, _ := os.ReadDir("../ast") + found := map[string]bool{} + for _, e := range entries { + if !strings.HasPrefix(e.Name(), "ast_") || !strings.HasSuffix(e.Name(), ".go") { + continue + } + src, _ := os.ReadFile(filepath.Join("../ast", e.Name())) + for _, m := range stmtRe.FindAllStringSubmatch(string(src), -1) { + if docCarryDone[m[1]] && strings.Contains(m[2], "DocumentationSet") { + found[m[1]] = true + } + } + } + for name := range docCarryDone { + if !found[name] { + t.Errorf("%s is in docCarryDone but has no DocumentationSet field", name) + } + } +} diff --git a/mdl/executor/documentation_preserved_test.go b/mdl/executor/documentation_preserved_test.go new file mode 100644 index 0000000000..da5fab8db9 --- /dev/null +++ b/mdl/executor/documentation_preserved_test.go @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration + +package executor + +import ( + "bytes" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" +) + +// mendixlabs/mxcli#1018 — a rewrite that does not restate a `/** … */` doc +// comment must not delete the stored one. +// +// Measured before the fix: `create or replace microflow` and `create or modify +// entity` both wrote an empty Documentation over the stored value, with the run +// reporting success and mx check clean (a document with no documentation is +// valid). `ALTER ENTITY … ADD ATTRIBUTE` preserved it, which is what localises +// the defect to the rewrite paths. +// +// Table-driven over every doctype that accepts a doc comment, because the +// carry-forward has to be added per rewrite path and fixing only the two +// doctypes in the report would leave the class open — the shape the wiki calls +// duplicate-resolver-drift. + +type docPreserveCase struct { + name string + // storedOnly marks a doctype whose documentation DESCRIBE does not render, + // so the assertion reads the stored units instead. + storedOnly bool + // modelsdk marks a doctype the legacy engine refuses to author (rules, and + // anything else modelsdk-only). The harness defaults to legacy, so without + // this the case fails at its own precondition and says nothing about #1018. + modelsdk bool + // create carries a doc comment; rewrite deliberately does not. + create string + rewrite string + describe string +} + +const docMarker = "DOC-MARKER-PRESERVE-ME" + +// storedContains searches the project's units for text. Some doctypes — +// associations, view entities — store documentation that DESCRIBE does not +// render, so the describe-based assertion cannot see them. Reading the stored +// bytes is what the original #1018 measurement did and is the more faithful +// check anyway: it asks what is in the model, not what the reader reports. +func storedContains(t *testing.T, projectPath, needle string) bool { + t.Helper() + dir := filepath.Join(filepath.Dir(projectPath), "mprcontents") + found := false + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || found { + return nil //nolint:nilerr // a partial walk is reported as not-found + } + b, readErr := os.ReadFile(p) + if readErr == nil && bytes.Contains(b, []byte(needle)) { + found = true + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + return found +} + +func docPreserveCases() []docPreserveCase { + doc := "/** " + docMarker + " */\n" + return []docPreserveCase{ + { + name: "entity", + create: doc + "create entity TestModule.DocEnt ( Label: String );", + rewrite: "create or modify entity TestModule.DocEnt ( Label: String, Extra: Integer );", + describe: "describe entity TestModule.DocEnt", + }, + { + name: "microflow", + create: doc + "create microflow TestModule.DocMf ()\nbegin\nend;", + rewrite: "create or replace microflow TestModule.DocMf ()\nbegin\nend;", + describe: "describe microflow TestModule.DocMf", + }, + { + name: "queue", + create: doc + "create queue TestModule.DocQ ( Parallelism: 2 );", + rewrite: "create or modify queue TestModule.DocQ ( Parallelism: 3 );", + describe: "describe queue TestModule.DocQ", + }, + { + name: "regular expression", + create: doc + "create regular expression TestModule.DocRe ( Expression: '[0-9]+' );", + rewrite: "create or modify regular expression TestModule.DocRe ( Expression: '[0-9]{2}' );", + describe: "describe regular expression TestModule.DocRe", + }, + { + name: "scheduled event", + create: "create microflow TestModule.DocSeFlow ()\nbegin\nend;\n" + doc + + "create scheduled event TestModule.DocSe ( Microflow: TestModule.DocSeFlow, Repeat: Daily, HourOfDay: 4 );", + rewrite: "create or modify scheduled event TestModule.DocSe ( Microflow: TestModule.DocSeFlow, Repeat: Daily, HourOfDay: 5 );", + describe: "describe scheduled event TestModule.DocSe", + }, + { + name: "nanoflow", + create: doc + "create nanoflow TestModule.DocNf ()\nbegin\nend;", + rewrite: "create or replace nanoflow TestModule.DocNf ()\nbegin\nend;", + describe: "describe nanoflow TestModule.DocNf", + }, + { + name: "rule", + modelsdk: true, + create: doc + "create rule TestModule.DocRule ()\nreturns Boolean\nbegin\n return true;\nend;", + rewrite: "create or modify rule TestModule.DocRule ()\nreturns Boolean\nbegin\n return false;\nend;", + describe: "describe rule TestModule.DocRule", + }, + { + name: "json structure", + create: doc + "create json structure TestModule.DocJs\n snippet $${\"a\": 1}$$;", + rewrite: "create or modify json structure TestModule.DocJs\n snippet $${\"a\": 1, \"b\": 2}$$;", + describe: "describe json structure TestModule.DocJs", + }, + { + name: "image collection", + create: doc + "create image collection TestModule.DocIc;", + rewrite: "create or modify image collection TestModule.DocIc;", + describe: "describe image collection TestModule.DocIc", + }, + { + name: "workflow", + create: "create entity TestModule.DocWfCtx ( Label: String );\n" + doc + + "create workflow TestModule.DocWf\n parameter $WorkflowContext: TestModule.DocWfCtx\nbegin\nend workflow;", + rewrite: "create or modify workflow TestModule.DocWf\n parameter $WorkflowContext: TestModule.DocWfCtx\nbegin\nend workflow;", + describe: "describe workflow TestModule.DocWf", + }, + { + name: "constant", + create: doc + "create constant TestModule.DocConst type string default 'a';", + rewrite: "create or modify constant TestModule.DocConst type string default 'b';", + describe: "describe constant TestModule.DocConst", + }, + { + name: "association", + storedOnly: true, + create: "create entity TestModule.DocA ( L: String );\ncreate entity TestModule.DocB ( L: String );\n" + doc + + "create association TestModule.DocA_DocB from TestModule.DocA to TestModule.DocB;", + rewrite: "create or modify association TestModule.DocA_DocB from TestModule.DocA to TestModule.DocB;", + }, + { + name: "view entity", + storedOnly: true, + create: "create entity TestModule.DocSrc ( L: String );\n" + doc + + "create view entity TestModule.DocView as ( select s.L as L from TestModule.DocSrc as s );", + rewrite: "create or modify view entity TestModule.DocView as ( select s.L as Label from TestModule.DocSrc as s );", + }, + { + name: "business event service", + storedOnly: true, + create: "create entity TestModule.DocBePayload ( L: String );\n" + doc + + "create business event service TestModule.DocBes\n( ServiceName: 'DocBes', EventNamePrefix: 'com.example' )\n{\n message DocCreated (OrderId: long) publish\n entity TestModule.DocBePayload;\n};", + rewrite: "create or modify business event service TestModule.DocBes\n( ServiceName: 'DocBes2', EventNamePrefix: 'com.example' )\n{\n message DocCreated (OrderId: long) publish\n entity TestModule.DocBePayload;\n};", + }, + { + name: "java action", + create: doc + "create java action MyFirstModule.DocJa () returns String as $$\npublic String executeAction() { return \"x\"; }\n$$;", + rewrite: "create or modify java action MyFirstModule.DocJa () returns String as $$\npublic String executeAction() { return \"y\"; }\n$$;", + describe: "describe java action MyFirstModule.DocJa", + }, + { + name: "menu", + modelsdk: true, + create: "create microflow TestModule.DocMenuMf ()\nbegin\nend;\n" + doc + "create menu TestModule.DocMenu (\n menu item 'Home' microflow TestModule.DocMenuMf;\n);", + rewrite: "create or modify menu TestModule.DocMenu (\n menu item 'Home2' microflow TestModule.DocMenuMf;\n);", + describe: "describe menu TestModule.DocMenu", + }, + { + name: "odata service", + storedOnly: true, + create: "create entity TestModule.DocOdEnt ( L: String );\n" + doc + + "create odata service TestModule.DocOd (\n path: 'odata/doc/',\n version: '1.0.0',\n ODataVersion: OData4,\n namespace: 'TestModule.Doc'\n)\nauthentication basic\n{\n publish entity TestModule.DocOdEnt as 'Ents' (\n ReadMode: source\n )\n expose (*);\n};", + rewrite: "create or modify odata service TestModule.DocOd (\n path: 'odata/doc2/',\n version: '1.0.1',\n ODataVersion: OData4,\n namespace: 'TestModule.Doc'\n)\nauthentication basic\n{\n publish entity TestModule.DocOdEnt as 'Ents' (\n ReadMode: source\n )\n expose (*);\n};", + }, + { + name: "page", + storedOnly: true, + create: doc + "create page TestModule.DocPage ( Title: 'T', Layout: Atlas_Core.Atlas_Default ) {\n container c { }\n};", + rewrite: "create or replace page TestModule.DocPage ( Title: 'T2', Layout: Atlas_Core.Atlas_Default ) {\n container c2 { }\n};", + }, + { + name: "snippet", + storedOnly: true, + create: doc + "create snippet TestModule.DocSnip {\n container c { }\n};", + rewrite: "create or replace snippet TestModule.DocSnip {\n container c2 { }\n};", + }, + { + name: "layout", + storedOnly: true, + modelsdk: true, + create: doc + "create layout TestModule.DocLayout (\n layouttype: 'Responsive'\n) {\n scrollcontainer layoutContainer {\n region center {\n placeholder Main\n }\n }\n};", + rewrite: "create or replace layout TestModule.DocLayout (\n layouttype: 'Responsive'\n) {\n scrollcontainer layoutContainer {\n region center (class: 'x') {\n placeholder Main\n }\n }\n};", + }, + { + name: "javascript action", + storedOnly: true, + create: doc + "create javascript action MyFirstModule.DocJs ()\nreturns String\nas $$\nreturn Promise.resolve('a');\n$$;", + rewrite: "create or modify javascript action MyFirstModule.DocJs ()\nreturns String\nas $$\nreturn Promise.resolve('b');\n$$;", + }, + { + name: "odata client", + storedOnly: true, + create: doc + "create odata client MyFirstModule.DocOdc (\n MetadataUrl: 'http://127.0.0.1:9/nope/$metadata'\n);", + rewrite: "create or modify odata client MyFirstModule.DocOdc (\n MetadataUrl: 'http://127.0.0.1:9/other/$metadata'\n);", + }, + { + name: "external entity", + storedOnly: true, + create: "create odata client MyFirstModule.DocExtOdc (\n MetadataUrl: 'http://127.0.0.1:9/nope/$metadata'\n);\n" + doc + + "create external entity MyFirstModule.DocExt\nfrom odata client MyFirstModule.DocExtOdc\n(\n EntitySet: 'Things',\n Countable: Yes\n);", + rewrite: "create or modify external entity MyFirstModule.DocExt\nfrom odata client MyFirstModule.DocExtOdc\n(\n EntitySet: 'Things',\n Countable: No\n);", + }, + { + name: "rest client", + storedOnly: true, + create: doc + "create rest client MyFirstModule.DocRc (\n BaseUrl: 'http://localhost:3001/api',\n Authentication: none\n)\n{\n operation \"Ping\" {\n Method: get,\n Path: '/ping'\n }\n};", + rewrite: "create or modify rest client MyFirstModule.DocRc (\n BaseUrl: 'http://localhost:3001/api2',\n Authentication: none\n)\n{\n operation \"Ping\" {\n Method: get,\n Path: '/ping'\n }\n};", + }, + { + name: "ai model", + storedOnly: true, + create: doc + "create model TestModule.DocModel ( Provider: MxCloudGenAI );", + rewrite: "create or modify model TestModule.DocModel ( Provider: MxCloudGenAI );", + }, + { + name: "knowledge base", + storedOnly: true, + create: doc + "create knowledge base TestModule.DocKb ( Provider: MxCloudGenAI );", + rewrite: "create or modify knowledge base TestModule.DocKb ( Provider: MxCloudGenAI );", + }, + { + name: "consumed mcp service", + storedOnly: true, + create: doc + "create consumed mcp service TestModule.DocMcp ( ProtocolVersion: 'v2025_03_26' );", + rewrite: "create or modify consumed mcp service TestModule.DocMcp ( ProtocolVersion: 'v2025_03_26' );", + }, + { + name: "agent", + storedOnly: true, + create: "create model TestModule.DocAgentModel ( Provider: MxCloudGenAI );\n" + doc + + "create agent TestModule.DocAgent ( UsageType: Task, Model: TestModule.DocAgentModel, SystemPrompt: 'p' );", + rewrite: "create or modify agent TestModule.DocAgent ( UsageType: Task, Model: TestModule.DocAgentModel, SystemPrompt: 'p2' );", + }, + { + name: "enumeration", + create: doc + "create enumeration TestModule.DocEnum ( A 'A', B 'B' );", + rewrite: "create or replace enumeration TestModule.DocEnum ( A 'A', B 'B', C 'C' );", + describe: "describe enumeration TestModule.DocEnum", + }, + } +} + +// TestDocumentation_SurvivesRewrite is the defect itself. +func TestDocumentation_SurvivesRewrite(t *testing.T) { + for _, tc := range docPreserveCases() { + t.Run(tc.name, func(t *testing.T) { + env := setupTestEnv(t) + if tc.modelsdk { + env = setupTestEnvWithBackend(t, func() backend.FullBackend { return modelsdkbackend.New() }) + } + defer env.teardown() + + if err := env.executeMDL(tc.create); err != nil { + t.Fatalf("create: %v", err) + } + if tc.storedOnly { + if !storedContains(t, env.projectPath, docMarker) { + t.Fatalf("precondition failed: the doc comment did not reach the stored model") + } + if err := env.executeMDL(tc.rewrite); err != nil { + t.Fatalf("rewrite: %v", err) + } + if !storedContains(t, env.projectPath, docMarker) { + t.Errorf("the rewrite deleted the documentation it never mentioned (#1018)") + } + return + } + before, err := env.describeMDL(tc.describe) + if err != nil { + t.Fatalf("describe after create: %v", err) + } + if !strings.Contains(before, docMarker) { + t.Fatalf("precondition failed: the doc comment did not survive CREATE, so this test "+ + "cannot say anything about the rewrite:\n%s", before) + } + + if err := env.executeMDL(tc.rewrite); err != nil { + t.Fatalf("rewrite: %v", err) + } + after, err := env.describeMDL(tc.describe) + if err != nil { + t.Fatalf("describe after rewrite: %v", err) + } + if !strings.Contains(after, docMarker) { + t.Errorf("the rewrite deleted the documentation it never mentioned (#1018):\n%s", after) + } + }) + } +} + +// TestDocumentation_UntouchedObjectKeepsItsOwn is the control that separates +// "rewrites drop documentation" from "writes drop documentation". Without it a +// green run proves only that nothing was written at all. +func TestDocumentation_UntouchedObjectKeepsItsOwn(t *testing.T) { + env := setupTestEnv(t) + defer env.teardown() + + if err := env.executeMDL( + "/** " + docMarker + " */\ncreate entity TestModule.DocBystander ( Label: String );", + ); err != nil { + t.Fatalf("create bystander: %v", err) + } + if err := env.executeMDL( + "/** unrelated */\ncreate microflow TestModule.DocOther ()\nbegin\nend;", + ); err != nil { + t.Fatalf("create other: %v", err) + } + // Rewrite the OTHER document. + if err := env.executeMDL( + "create or replace microflow TestModule.DocOther ()\nbegin\nend;", + ); err != nil { + t.Fatalf("rewrite other: %v", err) + } + + out, err := env.describeMDL("describe entity TestModule.DocBystander") + if err != nil { + t.Fatalf("describe bystander: %v", err) + } + if !strings.Contains(out, docMarker) { + t.Errorf("a bystander lost its documentation when a different document was rewritten:\n%s", out) + } +} + +// TestDocumentation_EmptyCommentClears pins the other half of the decision: +// an ABSENT doc comment preserves, an explicitly EMPTY one clears. Without this +// the fix makes documentation unclearable — and for a microflow there is no +// `ALTER MICROFLOW … SET DOCUMENTATION` to fall back on, so the empty comment is +// the only spelling available. +func TestDocumentation_EmptyCommentClears(t *testing.T) { + env := setupTestEnv(t) + defer env.teardown() + + if err := env.executeMDL( + "/** " + docMarker + " */\ncreate microflow TestModule.DocClear ()\nbegin\nend;", + ); err != nil { + t.Fatalf("create: %v", err) + } + if err := env.executeMDL( + "/** */\ncreate or replace microflow TestModule.DocClear ()\nbegin\nend;", + ); err != nil { + t.Fatalf("rewrite with empty doc comment: %v", err) + } + out, err := env.describeMDL("describe microflow TestModule.DocClear") + if err != nil { + t.Fatalf("describe: %v", err) + } + if strings.Contains(out, docMarker) { + t.Errorf("an explicitly empty /** */ did not clear the documentation:\n%s", out) + } +} diff --git a/mdl/executor/executor_query.go b/mdl/executor/executor_query.go index 4a42a93b07..e8cbfc72ae 100644 --- a/mdl/executor/executor_query.go +++ b/mdl/executor/executor_query.go @@ -155,6 +155,8 @@ func execShow(ctx *ExecContext, s *ast.ShowStmt) error { return listContractMessages(ctx, s.Name) case ast.ShowJsonStructures: return listJsonStructures(ctx, s.InModule) + case ast.ShowMessageDefinitionCollections: + return listMessageDefinitionCollections(ctx, s.InModule) case ast.ShowImportMappings: return listImportMappings(ctx, s.InModule) case ast.ShowExportMappings: @@ -277,6 +279,8 @@ func execDescribe(ctx *ExecContext, s *ast.DescribeStmt) error { return describeContractMessage(ctx, s.Name) case ast.DescribeJsonStructure: return describeJsonStructure(ctx, s.Name) + case ast.DescribeMessageDefinitionCollection: + return execDescribeMessageDefinitionCollection(ctx, s.Name) case ast.DescribeImportMapping: return describeImportMapping(ctx, s.Name) case ast.DescribeExportMapping: @@ -372,6 +376,8 @@ func describeObjectTypeLabel(t ast.DescribeObjectType) string { return "contractmessage" case ast.DescribeJsonStructure: return "jsonstructure" + case ast.DescribeMessageDefinitionCollection: + return "messagedefinitioncollection" case ast.DescribeImportMapping: return "importmapping" case ast.DescribeExportMapping: diff --git a/mdl/executor/mapping_original_value.go b/mdl/executor/mapping_original_value.go new file mode 100644 index 0000000000..7e6cbc26a2 --- /dev/null +++ b/mdl/executor/mapping_original_value.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Carrying a mapping value element's OriginalValue through a rewrite +// (ako/mxcli#379). +// +// OriginalValue is the sample parsed out of the JSON structure's snippet +// ("42", "\"Widget\""). mxcli wrote it empty on every element, on the strength +// of a measurement over two mappings a blank app ships (#882) — and a rewrite +// therefore DELETED it from every mapping that had one. +// +// The wider measurement says neither "always empty" nor "always copy the +// sample" is right. Across 3,042 value elements whose structure carries a +// sample, 2,322 (76%) store it and 720 do not — and the split is PER DOCUMENT, +// not per element: +// +// 145 mappings carry the sample on EVERY element +// 107 carry it on NONE +// 2 are mixed +// +// So which one a mapping gets is a property of how and when it was authored, +// which mxcli cannot compute. Choosing a global default is wrong for roughly +// half the corpus either way. +// +// A REWRITE does not have to choose. It knows what was stored, so it carries it +// — guard-don't-drop, ADR-0005. That leaves #882's actual decision intact: a +// NEWLY created mapping still writes empty, which is what that issue was about. +package executor + +import "github.com/mendixlabs/mxcli/model" + +// carryImportOriginalValues copies each stored element's OriginalValue onto the +// rebuilt element at the same JsonPath. +// +// Matching is by JsonPath because that is what identifies an element against +// the schema: names can be renamed and order can change, but a value element +// bound to a different path is a different element. +func carryImportOriginalValues(rebuilt, stored *model.ImportMapping) { + if rebuilt == nil || stored == nil { + return + } + byPath := map[string]string{} + collectImportOriginalValues(stored.Elements, byPath) + if len(byPath) == 0 { + return + } + applyImportOriginalValues(rebuilt.Elements, byPath) +} + +func collectImportOriginalValues(elems []*model.ImportMappingElement, out map[string]string) { + for _, e := range elems { + if e == nil { + continue + } + if e.OriginalValue != "" { + out[e.JsonPath] = e.OriginalValue + } + collectImportOriginalValues(e.Children, out) + } +} + +func applyImportOriginalValues(elems []*model.ImportMappingElement, byPath map[string]string) { + for _, e := range elems { + if e == nil { + continue + } + // Only fill an element the rebuild left empty: a statement that somehow + // set one should win over what was stored. + if e.OriginalValue == "" { + if v, ok := byPath[e.JsonPath]; ok { + e.OriginalValue = v + } + } + applyImportOriginalValues(e.Children, byPath) + } +} + +// carryExportOriginalValues is the export twin. An export mapping's value +// elements hardcoded "" in the codec writer rather than carrying the field at +// all, so this needed the semantic type to reach the writer as well. +func carryExportOriginalValues(rebuilt, stored *model.ExportMapping) { + if rebuilt == nil || stored == nil { + return + } + byPath := map[string]string{} + collectExportOriginalValues(stored.Elements, byPath) + if len(byPath) == 0 { + return + } + applyExportOriginalValues(rebuilt.Elements, byPath) +} + +func collectExportOriginalValues(elems []*model.ExportMappingElement, out map[string]string) { + for _, e := range elems { + if e == nil { + continue + } + if e.OriginalValue != "" { + out[e.JsonPath] = e.OriginalValue + } + collectExportOriginalValues(e.Children, out) + } +} + +func applyExportOriginalValues(elems []*model.ExportMappingElement, byPath map[string]string) { + for _, e := range elems { + if e == nil { + continue + } + if e.OriginalValue == "" { + if v, ok := byPath[e.JsonPath]; ok { + e.OriginalValue = v + } + } + applyExportOriginalValues(e.Children, byPath) + } +} diff --git a/mdl/executor/mapping_original_value_test.go b/mdl/executor/mapping_original_value_test.go new file mode 100644 index 0000000000..bfc3a822ba --- /dev/null +++ b/mdl/executor/mapping_original_value_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// OriginalValue is the sample parsed out of a JSON structure's snippet. mxcli +// wrote it empty on every element, so a rewrite DELETED it from every mapping +// that had one — 2,322 of 3,042 value elements in the demo corpus carry one +// (ako/mxcli#379). +// +// Neither global default is right. The split is PER DOCUMENT: 145 mappings +// carry the sample on every element, 107 on none, 2 mixed. So a rewrite +// preserves what was stored instead of choosing, and a NEWLY authored mapping +// still writes empty — which is what #882 actually decided. + +func importElem(path, value string, kids ...*model.ImportMappingElement) *model.ImportMappingElement { + return &model.ImportMappingElement{JsonPath: path, OriginalValue: value, Children: kids} +} + +func TestImportOriginalValuesAreCarriedThroughARewrite(t *testing.T) { + stored := &model.ImportMapping{Elements: []*model.ImportMappingElement{ + importElem("(Object)", "", + importElem("(Object)|title", `"hello"`), + importElem("(Object)|nested", "", + importElem("(Object)|nested|qty", `"3"`)), + ), + }} + rebuilt := &model.ImportMapping{Elements: []*model.ImportMappingElement{ + importElem("(Object)", "", + importElem("(Object)|title", ""), + importElem("(Object)|nested", "", + importElem("(Object)|nested|qty", "")), + ), + }} + + carryImportOriginalValues(rebuilt, stored) + + root := rebuilt.Elements[0] + if got := root.Children[0].OriginalValue; got != `"hello"` { + t.Errorf("title = %q, want \"hello\"", got) + } + // Nesting matters: a mapping's samples are not all at the top level. + if got := root.Children[1].Children[0].OriginalValue; got != `"3"` { + t.Errorf("nested qty = %q, want \"3\"", got) + } +} + +// TestOriginalValuesMatchOnJsonPath pins the matching key. Names can be renamed +// and order can change; a value element bound to a different path is a +// different element, so the path is what identifies it against the schema. +func TestOriginalValuesMatchOnJsonPath(t *testing.T) { + stored := &model.ImportMapping{Elements: []*model.ImportMappingElement{ + importElem("(Object)", "", importElem("(Object)|title", `"hello"`)), + }} + rebuilt := &model.ImportMapping{Elements: []*model.ImportMappingElement{ + importElem("(Object)", "", importElem("(Object)|somethingElse", "")), + }} + + carryImportOriginalValues(rebuilt, stored) + + if got := rebuilt.Elements[0].Children[0].OriginalValue; got != "" { + t.Errorf("carried %q onto a different path — the sample belongs to the element it was measured on", got) + } +} + +// TestARewriteDoesNotOverwriteAnExplicitValue pins the precedence: what the +// rebuild produced wins, and the stored value only fills a gap. +func TestARewriteDoesNotOverwriteAnExplicitValue(t *testing.T) { + stored := &model.ImportMapping{Elements: []*model.ImportMappingElement{ + importElem("(Object)|x", `"old"`), + }} + rebuilt := &model.ImportMapping{Elements: []*model.ImportMappingElement{ + importElem("(Object)|x", `"new"`), + }} + + carryImportOriginalValues(rebuilt, stored) + + if got := rebuilt.Elements[0].OriginalValue; got != `"new"` { + t.Errorf("OriginalValue = %q, want the rebuilt value", got) + } +} + +// TestANewMappingKeepsEmptyOriginalValues is the control for #882. With no +// stored document there is nothing to carry, so a newly authored mapping still +// writes empty — the decision that issue made, left intact. +func TestANewMappingKeepsEmptyOriginalValues(t *testing.T) { + rebuilt := &model.ImportMapping{Elements: []*model.ImportMappingElement{ + importElem("(Object)|title", ""), + }} + + carryImportOriginalValues(rebuilt, nil) + + if got := rebuilt.Elements[0].OriginalValue; got != "" { + t.Errorf("OriginalValue = %q on a new mapping, want empty", got) + } +} + +// TestExportOriginalValuesAreCarriedToo pins the export twin, whose codec +// writer hardcoded "" rather than carrying the field at all. +func TestExportOriginalValuesAreCarriedToo(t *testing.T) { + stored := &model.ExportMapping{Elements: []*model.ExportMappingElement{ + {JsonPath: "(Object)", Children: []*model.ExportMappingElement{ + {JsonPath: "(Object)|title", OriginalValue: `"hello"`}, + }}, + }} + rebuilt := &model.ExportMapping{Elements: []*model.ExportMappingElement{ + {JsonPath: "(Object)", Children: []*model.ExportMappingElement{ + {JsonPath: "(Object)|title"}, + }}, + }} + + carryExportOriginalValues(rebuilt, stored) + + if got := rebuilt.Elements[0].Children[0].OriginalValue; got != `"hello"` { + t.Errorf("title = %q, want \"hello\"", got) + } +} + +// TestSameJSONContentIgnoresFormatting pins the snippet half. describe +// pretty-prints, so describe -> exec rewrote a one-line snippet into a +// multi-line one — same JSON, different bytes, a diff for nothing. +func TestSameJSONContentIgnoresFormatting(t *testing.T) { + cases := []struct { + name string + a, b string + want bool + }{ + {"formatting only", `{"title": "hello", "qty": "3"}`, "{\n \"title\": \"hello\",\n \"qty\": \"3\"\n}", true}, + {"key order", `{"a":1,"b":2}`, `{"b":2,"a":1}`, true}, + {"different value", `{"a":1}`, `{"a":2}`, false}, + {"added key", `{"a":1}`, `{"a":1,"b":2}`, false}, + // Anything that does not parse counts as different, so a malformed + // snippet is replaced rather than silently kept. + {"malformed", `{"a":`, `{"a":1}`, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := sameJSONContent(tc.a, tc.b); got != tc.want { + t.Errorf("sameJSONContent = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/mdl/executor/register_stubs.go b/mdl/executor/register_stubs.go index 5f4246897f..c68378a8de 100644 --- a/mdl/executor/register_stubs.go +++ b/mdl/executor/register_stubs.go @@ -360,6 +360,21 @@ func registerJSONStructureHandlers(r *Registry) { }) } +func registerMessageDefinitionHandlers(r *Registry) { + r.Register(&ast.CreateMessageDefinitionCollectionStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { + return execCreateMessageDefinitionCollection(ctx, stmt.(*ast.CreateMessageDefinitionCollectionStmt)) + }) + r.Register(&ast.DropMessageDefinitionCollectionStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { + return execDropMessageDefinitionCollection(ctx, stmt.(*ast.DropMessageDefinitionCollectionStmt)) + }) + r.Register(&ast.AlterMessageDefinitionCollectionStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { + return execAlterMessageDefinitionCollection(ctx, stmt.(*ast.AlterMessageDefinitionCollectionStmt)) + }) + r.Register(&ast.AlterMessageDefinitionStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { + return execAlterMessageDefinition(ctx, stmt.(*ast.AlterMessageDefinitionStmt)) + }) +} + func registerMappingHandlers(r *Registry) { r.Register(&ast.CreateImportMappingStmt{}, func(ctx *ExecContext, stmt ast.Statement) error { return execCreateImportMapping(ctx, stmt.(*ast.CreateImportMappingStmt)) diff --git a/mdl/executor/registry.go b/mdl/executor/registry.go index 56b3cfd5d1..53a3895a16 100644 --- a/mdl/executor/registry.go +++ b/mdl/executor/registry.go @@ -48,6 +48,7 @@ func NewRegistry() *Registry { registerSettingsHandlers(r) registerODataHandlers(r) registerJSONStructureHandlers(r) + registerMessageDefinitionHandlers(r) registerMappingHandlers(r) registerRESTHandlers(r) registerDataTransformerHandlers(r) diff --git a/mdl/executor/registry_test.go b/mdl/executor/registry_test.go index 653c5141d6..98d0cabeb3 100644 --- a/mdl/executor/registry_test.go +++ b/mdl/executor/registry_test.go @@ -162,6 +162,8 @@ func allKnownStatements() []ast.Statement { &ast.AlterAssociationStmt{}, &ast.AlterConsumedMCPServiceStmt{}, &ast.AlterEntityStmt{}, + &ast.AlterMessageDefinitionCollectionStmt{}, + &ast.AlterMessageDefinitionStmt{}, &ast.AlterEnumerationStmt{}, &ast.AlterKnowledgeBaseStmt{}, &ast.AlterModelStmt{}, @@ -203,6 +205,7 @@ func allKnownStatements() []ast.Statement { &ast.CreateJavaActionStmt{}, &ast.CreateJavaScriptActionStmt{}, &ast.CreateJsonStructureStmt{}, + &ast.CreateMessageDefinitionCollectionStmt{}, &ast.CreateKnowledgeBaseStmt{}, &ast.CreateLayoutStmt{}, &ast.CreateMicroflowStmt{}, @@ -251,6 +254,7 @@ func allKnownStatements() []ast.Statement { &ast.DropJavaActionStmt{}, &ast.DropJavaScriptActionStmt{}, &ast.DropJsonStructureStmt{}, + &ast.DropMessageDefinitionCollectionStmt{}, &ast.DropKnowledgeBaseStmt{}, &ast.DropMicroflowStmt{}, &ast.DropNanoflowStmt{}, diff --git a/mdl/executor/roundtrip_doctype_test.go b/mdl/executor/roundtrip_doctype_test.go index bfd584d237..df4a208bd1 100644 --- a/mdl/executor/roundtrip_doctype_test.go +++ b/mdl/executor/roundtrip_doctype_test.go @@ -71,6 +71,13 @@ var engineScriptSkip = map[string]string{ // for the tree to go. Reads (SHOW/DESCRIBE LAYOUT) work on both engines. "legacy/layouts.mdl": "layout authoring is modelsdk-only by design; the legacy backend refuses it", "legacy/navigation-profiles.mdl": "creating a navigation profile is modelsdk-only by design; the legacy backend refuses it", + // Same shape again: message definition collection authoring is + // modelsdk-only. The legacy writer has no serializer for the document, and + // building one would duplicate a shape the codec already gets right — + // including a typed-array marker of 2 and an empty-but-present Children + // list, neither of which is the codec's default. The legacy backend refuses + // create/modify/drop rather than emitting one. Reads work on both engines. + "legacy/40-message-definition-examples.mdl": "message definition authoring is modelsdk-only by design; the legacy backend refuses it", // Enabling a language writes Settings$LanguageSettings.Languages, which the // legacy serializer carries through from the stored document rather than // writing — so the list cannot change on that engine. The backend refuses it diff --git a/mdl/executor/split_indentation_test.go b/mdl/executor/split_indentation_test.go index 4b0d060423..4362bdebf0 100644 --- a/mdl/executor/split_indentation_test.go +++ b/mdl/executor/split_indentation_test.go @@ -8,6 +8,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/ast" "github.com/mendixlabs/mxcli/mdl/visitor" + "github.com/mendixlabs/mxcli/sdk/microflows" ) // indentOf returns the number of leading spaces on the first line containing want. @@ -32,11 +33,16 @@ func renderMicroflowBody(t *testing.T, src string) []string { if !ok { t.Fatalf("statement 0: got %T, want *ast.CreateMicroflowStmt", prog.Statements[0]) } - var lines []string - for _, stmt := range mf.Body { - lines = append(lines, microflowStatementToMDL(nil, stmt, 1)...) - } - return lines + // Renders through the LIVE describer. This test used to drive the diff's + // own AST-to-MDL renderer, which #997 deleted: two renderers for one + // language drift, and diff's had fallen far enough behind to report + // activities it could not print as deletions. The indentation rule below + // is now the describer's, which is the only place it still exists. + fb := &flowBuilder{posX: 100, posY: 100, spacing: HorizontalSpacing, measurer: &layoutMeasurer{}} + oc := fb.buildFlowGraph(mf.Body, nil) + e := newTestExecutor() + built := µflows.Microflow{ObjectCollection: oc} + return formatMicroflowActivities(e.newExecContext(t.Context()), built, nil, nil) } // Both splits used to render a branch body at the SAME column as its branch @@ -44,7 +50,8 @@ func renderMicroflowBody(t *testing.T, src string) []string { // anything nested: a nested `if`'s `else` landed exactly where a reader expects // a case branch, in output where `else` on a `case` is an MDL008 error. (#913) // -// Reverting either `indent+2` in cmd_diff_mdl.go fails this test. +// The describer indents a branch body from its branch keyword; flattening it +// again fails this test. func TestSplitBranchBodiesIndentFromTheirBranchKeyword(t *testing.T) { t.Run("enum split", func(t *testing.T) { lines := renderMicroflowBody(t, `CREATE MICROFLOW Sample.F ($Status: Enumeration(Sample.Status)) diff --git a/mdl/executor/validate.go b/mdl/executor/validate.go index ed4ec3c5a1..112c8ce68d 100644 --- a/mdl/executor/validate.go +++ b/mdl/executor/validate.go @@ -285,6 +285,15 @@ func validateProgram(ctx *ExecContext, prog *ast.Program) []error { // the same reason and at the same tier: MxBuild otherwise reports the typo // as CE1613, a whole build later (ako/mxcli#259). errors = append(errors, validateMappingSources(ctx, prog)...) + // Resolve `HOME PAGE … FOR `. Also project-resolved, and a tier + // worse than the two above: a module-qualified role here makes the project + // unloadable rather than merely failing the build (mendixlabs/mxcli#1001). + errors = append(errors, validateNavigationRoles(ctx, prog)...) + // Resolve CALL EXTERNAL ACTION against the consumed service's cached + // contract. MxBuild otherwise reports the drift as CE7252/CE7269 on the + // microflow — errors whose wording sends people to the entity import, which + // cannot fix either of them (mendixlabs/mxcli#1020). + errors = append(errors, validateExternalActionCalls(ctx, prog)...) return errors } diff --git a/mdl/executor/validate_external_action_calls.go b/mdl/executor/validate_external_action_calls.go new file mode 100644 index 0000000000..fbe3a5bdba --- /dev/null +++ b/mdl/executor/validate_external_action_calls.go @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Reference validation for CALL EXTERNAL ACTION against the consumed OData +// service's cached $metadata. +// +// Mendix raises two errors on Microflows$CallExternalAction when the stored call +// stops matching the contract, and neither has anything to do with the domain +// model — both are defined on CallExternalAction.cs (Mendix 11.13, +// Mendix.Modeler.Texts.dll): +// +// CE7252 ACTION_PARAMETERS_UNALIGNED +// "The parameters for remote action '{ACTION}' have changed." +// CE7269 ACTION_RETURN_TYPE_UNALIGNED +// "The return type for remote action '{ACTION}' has changed." +// +// That is worth stating plainly, because the natural reading is the opposite: +// the errors mention a remote action, so re-running +// CREATE OR MODIFY EXTERNAL ENTITIES looks like the remedy. It never is — that +// statement writes entities, and these errors are raised by the microflow. +// mendixlabs/mxcli#1020 lost a debugging session to exactly that. +// +// Both are decidable here, against the same cached $metadata the writer reads, +// so they become an mxcli error naming the fix instead of a build-time surprise. +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/mdl/types" + "github.com/mendixlabs/mxcli/model" +) + +// externalCall is one CALL EXTERNAL ACTION and the flow it was written in. +type externalCall struct { + stmt *ast.CallExternalActionStmt + flow string +} + +// externalActionCallsIn collects every CALL EXTERNAL ACTION in a statement, +// including those nested inside loops and splits. +func externalActionCallsIn(stmt ast.Statement) []externalCall { + var body []ast.MicroflowStatement + var flow string + switch s := stmt.(type) { + case *ast.CreateMicroflowStmt: + body, flow = s.Body, s.Name.String() + case *ast.CreateNanoflowStmt: + body, flow = s.Body, s.Name.String() + default: + return nil + } + + var out []externalCall + var walk func([]ast.MicroflowStatement) + walk = func(stmts []ast.MicroflowStatement) { + for _, s := range stmts { + switch st := s.(type) { + case *ast.CallExternalActionStmt: + out = append(out, externalCall{stmt: st, flow: flow}) + case *ast.LoopStmt: + walk(st.Body) + case *ast.WhileStmt: + walk(st.Body) + case *ast.IfStmt: + walk(st.ThenBody) + walk(st.ElseBody) + case *ast.EnumSplitStmt: + for _, c := range st.Cases { + walk(c.Body) + } + walk(st.ElseBody) + case *ast.InheritanceSplitStmt: + for _, c := range st.Cases { + walk(c.Body) + } + walk(st.ElseBody) + } + } + } + walk(body) + return out +} + +// validateExternalActionCalls resolves every CALL EXTERNAL ACTION in the program +// against the cached contract. +func validateExternalActionCalls(ctx *ExecContext, prog *ast.Program) []error { + if !ctx.Connected() { + return nil + } + + type located struct { + call externalCall + index int + } + var calls []located + for i, stmt := range prog.Statements { + for _, c := range externalActionCallsIn(stmt) { + calls = append(calls, located{call: c, index: i + 1}) + } + } + if len(calls) == 0 { + return nil + } + + services, err := ctx.Backend.ListConsumedODataServices() + if err != nil { + return nil + } + h, err := getHierarchy(ctx) + if err != nil { + return nil + } + + var errs []error + for _, c := range calls { + if err := checkExternalActionCall(ctx, h, services, c.call); err != nil { + errs = append(errs, fmt.Errorf("statement %d: %w", c.index, err)) + } + } + return errs +} + +func checkExternalActionCall(ctx *ExecContext, h *ContainerHierarchy, services []*model.ConsumedODataService, c externalCall) error { + svcQN := c.stmt.ServiceName.String() + + var doc *types.EdmxDocument + for _, svc := range services { + modName := h.GetModuleName(h.FindModuleID(svc.ContainerID)) + if !strings.EqualFold(modName, c.stmt.ServiceName.Module) || !strings.EqualFold(svc.Name, c.stmt.ServiceName.Name) { + continue + } + if svc.Metadata == "" { + // No cached contract: nothing to resolve against. Refresh is a + // separate operation, so this is not an error here. + return nil + } + parsed, err := types.ParseEdmx(svc.Metadata) + if err != nil { + return nil + } + doc = parsed + break + } + if doc == nil { + // An unknown service is reported by the existing reference validation. + return nil + } + + var action *types.EdmAction + var known []string + for _, act := range doc.Actions { + known = append(known, act.Name) + if strings.EqualFold(act.Name, c.stmt.ActionName) { + action = act + } + } + if action == nil { + sort.Strings(known) + return mdlerrors.NewNotFoundMsg("external action", c.stmt.ActionName, fmt.Sprintf( + "%s: external action %q does not exist in %s's cached contract.\n Actions in this service: %s", + c.flow, c.stmt.ActionName, svcQN, joinOrNone(known))) + } + + if err := checkExternalActionParameters(c, action); err != nil { + return err + } + return checkExternalActionReturn(ctx, h, c, action, svcQN) +} + +// checkExternalActionParameters compares the arguments written in the statement +// with the contract's parameter list. A mismatch is CE7252. +// +// A BOUND action's first parameter is its binding parameter, which Mendix +// supplies from the object the action is called on rather than from a mapping, +// so it is not something the statement names. +func checkExternalActionParameters(c externalCall, action *types.EdmAction) error { + want := map[string]string{} // lower -> declared spelling + var wantNames []string + for i, p := range action.Parameters { + if action.IsBound && i == 0 { + continue + } + want[strings.ToLower(p.Name)] = p.Name + wantNames = append(wantNames, p.Name) + } + + got := map[string]bool{} + var unknown []string + for _, arg := range c.stmt.Arguments { + declared, ok := want[strings.ToLower(arg.Name)] + if !ok { + unknown = append(unknown, arg.Name) + continue + } + got[declared] = true + } + + var missing []string + for _, name := range wantNames { + if !got[name] { + missing = append(missing, name) + } + } + if len(missing) == 0 && len(unknown) == 0 { + return nil + } + + sort.Strings(missing) + sort.Strings(unknown) + var parts []string + if len(unknown) > 0 { + parts = append(parts, fmt.Sprintf("not declared by the action: %s", strings.Join(unknown, ", "))) + } + if len(missing) > 0 { + parts = append(parts, fmt.Sprintf("declared but not supplied: %s", strings.Join(missing, ", "))) + } + return mdlerrors.NewValidation(fmt.Sprintf( + "%s: the arguments to external action %q do not match the service contract (%s).\n"+ + " The action declares: %s\n"+ + " Mendix reports this as CE7252 \"The parameters for remote action '%s' have changed\"", + c.flow, action.Name, strings.Join(parts, "; "), + joinOrNone(wantNames), action.Name)) +} + +// checkExternalActionReturn reports an entity-typed return whose external entity +// has not been imported. Without it the writer cannot type the result variable, +// and Mendix reports CE7269. +// +// This is the one case where CREATE EXTERNAL ENTITIES really is the remedy — so +// the message says so, with the statement to run. +func checkExternalActionReturn(ctx *ExecContext, h *ContainerHierarchy, c externalCall, action *types.EdmAction, svcQN string) error { + if edmReturnTypeToKind(action.ReturnType) != "" { + return nil // primitive or void: always representable + } + typeName, isList := edmBareTypeName(action.ReturnType) + if typeName == "" { + return nil // a complex type mxcli does not model; not decidable here + } + if externalEntityFor(ctx, h, svcQN, typeName) != "" { + return nil + } + + shape := "an object" + if isList { + shape = "a list" + } + return mdlerrors.NewValidation(fmt.Sprintf( + "%s: external action %q returns %s of %s, but no external entity has been imported "+ + "for that type, so the call's return type cannot be set.\n"+ + " Mendix reports this as CE7269 \"The return type for remote action '%s' has changed\".\n"+ + " Import it first: create or modify external entities from %s entities (%s)", + c.flow, action.Name, shape, action.ReturnType, action.Name, svcQN, typeName)) +} + +// externalEntityFor finds the external entity imported from svcQN for the remote +// type remoteName, returning its qualified name. Mirrors the resolution the flow +// builder does at write time. +func externalEntityFor(ctx *ExecContext, h *ContainerHierarchy, svcQN, remoteName string) string { + dms, err := ctx.Backend.ListDomainModels() + if err != nil { + return "" + } + for _, dm := range dms { + modName := h.GetModuleName(h.FindModuleID(dm.ContainerID)) + for _, ent := range dm.Entities { + if strings.EqualFold(ent.RemoteServiceName, svcQN) && strings.EqualFold(ent.RemoteEntityName, remoteName) { + return modName + "." + ent.Name + } + } + } + return "" +} + +func joinOrNone(names []string) string { + if len(names) == 0 { + return "(none)" + } + return strings.Join(names, ", ") +} diff --git a/mdl/executor/validate_external_action_calls_test.go b/mdl/executor/validate_external_action_calls_test.go new file mode 100644 index 0000000000..3e5995ef9a --- /dev/null +++ b/mdl/executor/validate_external_action_calls_test.go @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/types" +) + +// tripPinMetadata is a minimal OData 4 $metadata with the two shapes the repo +// had no fixture for and that mendixlabs/mxcli#1020 is about: an action +// returning an ENTITY and one returning a COLLECTION of entities. GetNearestAirport +// and its parameters are modelled on the public TripPin service. +const tripPinMetadata = ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +` + +func parseTripPin(t *testing.T) *types.EdmxDocument { + t.Helper() + doc, err := types.ParseEdmx(tripPinMetadata) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + return doc +} + +func actionNamed(t *testing.T, doc *types.EdmxDocument, name string) *types.EdmAction { + t.Helper() + for _, a := range doc.Actions { + if a.Name == name { + return a + } + } + t.Fatalf("fixture has no action %q", name) + return nil +} + +// TestEdmBareTypeName pins the type-name normalisation the entity lookup rests +// on: the contract says `Trippin.Airport`, the imported entity records `Airport`. +func TestEdmBareTypeName(t *testing.T) { + tests := []struct { + in string + wantName string + wantList bool + }{ + {"Trippin.Airport", "Airport", false}, + {"Collection(Trippin.Person)", "Person", true}, + // A namespace with dots in it still yields the last segment. + {"Some.Deep.Namespace.Thing", "Thing", false}, + {"Collection(Some.Deep.Ns.Thing)", "Thing", true}, + // Primitives never reach the entity lookup. + {"Edm.String", "", false}, + {"Collection(Edm.String)", "", true}, + {"", "", false}, + // Unqualified type name (no namespace) is still a name. + {"Airport", "Airport", false}, + } + for _, tt := range tests { + gotName, gotList := edmBareTypeName(tt.in) + if gotName != tt.wantName || gotList != tt.wantList { + t.Errorf("edmBareTypeName(%q) = (%q, %v), want (%q, %v)", + tt.in, gotName, gotList, tt.wantName, tt.wantList) + } + } +} + +// TestEdmReturnTypeToKindLeavesEntitiesToTheEntityLookup documents the split: +// primitives resolve here, entity-typed returns deliberately do not, because +// they need the project to find the imported entity. +func TestEdmReturnTypeToKindLeavesEntitiesToTheEntityLookup(t *testing.T) { + if got := edmReturnTypeToKind("Edm.Int32"); got != "Integer" { + t.Errorf("Edm.Int32 = %q, want Integer", got) + } + if got := edmReturnTypeToKind(""); got != "Void" { + t.Errorf("empty = %q, want Void", got) + } + for _, entityReturn := range []string{"Trippin.Airport", "Collection(Trippin.Person)"} { + if got := edmReturnTypeToKind(entityReturn); got != "" { + t.Errorf("edmReturnTypeToKind(%q) = %q, want \"\" — entity returns are resolved against the project", + entityReturn, got) + } + } +} + +// TestCheckExternalActionParameters covers CE7252. The arguments a statement +// writes must match the action's declared parameter list. +func TestCheckExternalActionParameters(t *testing.T) { + doc := parseTripPin(t) + action := actionNamed(t, doc, "GetNearestAirport") + + call := func(args ...string) externalCall { + var as []ast.CallArgument + for _, a := range args { + as = append(as, ast.CallArgument{Name: a}) + } + return externalCall{flow: "M.Flow", stmt: &ast.CallExternalActionStmt{ + ActionName: "GetNearestAirport", + Arguments: as, + }} + } + + if err := checkExternalActionParameters(call("lat", "lon"), action); err != nil { + t.Errorf("both parameters supplied should pass: %v", err) + } + // Order is not part of the contract — these are named mappings. + if err := checkExternalActionParameters(call("lon", "lat"), action); err != nil { + t.Errorf("order must not matter: %v", err) + } + + err := checkExternalActionParameters(call("lat"), action) + if err == nil { + t.Fatal("a missing parameter must be reported") + } + for _, want := range []string{"declared but not supplied", "lon", "CE7252"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err.Error(), want) + } + } + + err = checkExternalActionParameters(call("lat", "lon", "altitude"), action) + if err == nil { + t.Fatal("an undeclared argument must be reported") + } + if !strings.Contains(err.Error(), "not declared by the action") || !strings.Contains(err.Error(), "altitude") { + t.Errorf("error %q should name the undeclared argument", err.Error()) + } + + // An action with no parameters and no arguments is fine. + if err := checkExternalActionParameters( + externalCall{flow: "M.Flow", stmt: &ast.CallExternalActionStmt{ActionName: "ResetDataSource"}}, + actionNamed(t, doc, "ResetDataSource"), + ); err != nil { + t.Errorf("a parameterless action should pass: %v", err) + } +} + +// TestCheckExternalActionParametersBoundAction covers the binding parameter: a +// bound action's first parameter is supplied by Mendix from the object the +// action is called on, so a statement that does not name it is correct. +func TestCheckExternalActionParametersBoundAction(t *testing.T) { + bound := &types.EdmAction{ + Name: "Rate", + IsBound: true, + Parameters: []*types.EdmActionParameter{ + {Name: "bindingParameter", Type: "Trippin.Person"}, + {Name: "rating", Type: "Edm.Int32"}, + }, + } + c := externalCall{flow: "M.Flow", stmt: &ast.CallExternalActionStmt{ + ActionName: "Rate", + Arguments: []ast.CallArgument{{Name: "rating"}}, + }} + if err := checkExternalActionParameters(c, bound); err != nil { + t.Errorf("the binding parameter must not be demanded from the statement: %v", err) + } +} + +// TestExternalActionCallsIn pins collection, including from nested bodies — a +// call the walker misses is a call nothing checks. +func TestExternalActionCallsIn(t *testing.T) { + mk := func(action string) *ast.CallExternalActionStmt { + return &ast.CallExternalActionStmt{ActionName: action} + } + + tests := []struct { + name string + stmt ast.Statement + want []string + }{ + { + name: "top-level call", + stmt: &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "M", Name: "F"}, + Body: []ast.MicroflowStatement{mk("A")}, + }, + want: []string{"A"}, + }, + { + name: "inside a loop and an if", + stmt: &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "M", Name: "F"}, + Body: []ast.MicroflowStatement{ + &ast.LoopStmt{Body: []ast.MicroflowStatement{mk("A")}}, + &ast.IfStmt{ + ThenBody: []ast.MicroflowStatement{mk("B")}, + ElseBody: []ast.MicroflowStatement{mk("C")}, + }, + }, + }, + want: []string{"A", "B", "C"}, + }, + { + name: "nanoflows carry them too", + stmt: &ast.CreateNanoflowStmt{ + Name: ast.QualifiedName{Module: "M", Name: "N"}, + Body: []ast.MicroflowStatement{mk("A")}, + }, + want: []string{"A"}, + }, + { + name: "an unrelated statement contributes nothing", + stmt: &ast.CreateUserRoleStmt{Name: "Administrator"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := externalActionCallsIn(tt.stmt) + if len(got) != len(tt.want) { + t.Fatalf("got %d calls, want %d", len(got), len(tt.want)) + } + for i, w := range tt.want { + if got[i].stmt.ActionName != w { + t.Errorf("call[%d] = %q, want %q", i, got[i].stmt.ActionName, w) + } + if got[i].flow == "" { + t.Errorf("call[%d] has no flow name for the message", i) + } + } + }) + } +} diff --git a/mdl/executor/validate_flow_parameters.go b/mdl/executor/validate_flow_parameters.go new file mode 100644 index 0000000000..fce9f9dc30 --- /dev/null +++ b/mdl/executor/validate_flow_parameters.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// ValidateFlowParameterAnnotations refuses an annotation written on a flow +// parameter that mxcli does not implement there. +// +// Same reasoning as checkUnknownAnnotations one node family over (#884): +// `@position` is the only canvas property a parameter has, so `@postion(300, +// 100)` or `@size(30, 30)` on one would parse, do nothing, and discard exactly +// the placement the author was trying to state — which is the whole reason +// someone writes the annotation at all (#993). +// +// A free function rather than a microflowValidator method because the three +// flow flavours do not share a validator: ValidateMicroflow takes a +// *ast.CreateMicroflowStmt, so a check living there would leave nanoflows and +// rules — which share the parameter grammar — unguarded. +func ValidateFlowParameterAnnotations(flow string, params []ast.MicroflowParam) []linter.Violation { + var out []linter.Violation + for _, p := range params { + for _, name := range p.UnknownAnnotations { + out = append(out, linter.Violation{ + RuleID: "MDL059", + Severity: linter.SeverityError, + Message: fmt.Sprintf("%s: unknown annotation `@%s` on parameter `$%s` — it parses "+ + "but does nothing, so whatever it was meant to express is silently lost", + flow, name, p.Name), + Suggestion: fmt.Sprintf("`@position(x, y)` is the only annotation a parameter takes; "+ + "it places the parameter box on the canvas. If `@%s` is a typo of it, correct it.", name), + }) + } + } + return out +} diff --git a/mdl/executor/validate_icon_refs.go b/mdl/executor/validate_icon_refs.go index e02632d1f9..1aea9a9f44 100644 --- a/mdl/executor/validate_icon_refs.go +++ b/mdl/executor/validate_icon_refs.go @@ -101,7 +101,14 @@ type iconRef struct { } // iconRefsInStatement collects icon references from the statements that can -// carry one: page/snippet widget trees and navigation menus. +// carry one: page/snippet/layout widget trees, navigation menus and menu +// documents. +// +// Every widget-bearing field has to be walked, not just the obvious one. A +// reference the walker misses is a reference nothing checks, and the escape is +// silent both ways: `mxcli check --references` passes and `exec` succeeds, so +// the first sign is CE1613 from MxBuild — exactly the deferral this validator +// exists to prevent (mendixlabs/mxcli#1008). func iconRefsInStatement(stmt ast.Statement) []iconRef { var out []iconRef switch s := stmt.(type) { @@ -109,6 +116,33 @@ func iconRefsInStatement(stmt ast.Statement) []iconRef { for _, w := range s.Widgets { out = append(out, iconRefsInWidget(w)...) } + // Widgets is the bare body only. Content bound to a named layout + // placeholder is held apart in Placeholders, so walking Widgets alone + // misses every icon inside a `placeholder X { … }` block. + for _, ph := range s.Placeholders { + if ph == nil { + continue + } + for _, w := range ph.Widgets { + out = append(out, iconRefsInWidget(w)...) + } + } + case *ast.CreateSnippetStmtV3: + for _, w := range s.Widgets { + out = append(out, iconRefsInWidget(w)...) + } + case *ast.CreateLayoutStmt: + // A layout's icons are the costliest to get wrong: its topbar is shared, + // so one bad reference is an error on every page using the layout. + for _, w := range s.Widgets { + out = append(out, iconRefsInWidget(w)...) + } + case *ast.CreateMenuStmt: + // A standalone menu document carries the same NavMenuItemDef as a + // profile menu, sub-items included. + for _, item := range s.Items { + out = append(out, iconRefsInMenu(item)...) + } case *ast.AlterPageStmt: for _, op := range s.Operations { switch o := op.(type) { diff --git a/mdl/executor/validate_icon_refs_test.go b/mdl/executor/validate_icon_refs_test.go index feca888618..34b8d1b3bf 100644 --- a/mdl/executor/validate_icon_refs_test.go +++ b/mdl/executor/validate_icon_refs_test.go @@ -154,6 +154,60 @@ func TestIconRefsInStatement(t *testing.T) { }}, want: []string{"Atlas_Core.Atlas.home", "Atlas_Core.Atlas.pencil"}, }, + // The four shapes below all reached MxBuild as CE1613 with `mxcli check + // --references` reporting nothing (mendixlabs/mxcli#1008). Each holds its + // widgets in a field the walk did not visit, so the icon was never seen. + { + // A page's Widgets field is only the bare body. Content bound to a + // named layout placeholder lives in Placeholders. + name: "create page, placeholder block", + stmt: &ast.CreatePageStmtV3{Placeholders: []*ast.PagePlaceholderV3{ + {Name: "Main", Widgets: []*ast.WidgetV3{btn("btnP", "Atlas_Core.Atlas.home")}}, + }}, + want: []string{"Atlas_Core.Atlas.home"}, + }, + { + // Both halves of a page, so fixing one does not silently drop the other. + name: "create page, body and placeholder together", + stmt: &ast.CreatePageStmtV3{ + Widgets: []*ast.WidgetV3{btn("btnBody", "Atlas_Core.Atlas.home")}, + Placeholders: []*ast.PagePlaceholderV3{ + {Name: "Topbar", Widgets: []*ast.WidgetV3{btn("btnPh", "Atlas_Core.Atlas.pencil")}}, + }, + }, + want: []string{"Atlas_Core.Atlas.home", "Atlas_Core.Atlas.pencil"}, + }, + { + name: "create snippet", + stmt: &ast.CreateSnippetStmtV3{Widgets: []*ast.WidgetV3{ + {Type: "CONTAINER", Name: "c1", Children: []*ast.WidgetV3{ + btn("btnS", "Atlas_Core.Atlas.home"), + }}, + }}, + want: []string{"Atlas_Core.Atlas.home"}, + }, + { + // The costliest of the four: a layout's topbar is shared, so one bad + // icon there is an error on every page that uses the layout. + name: "create layout", + stmt: &ast.CreateLayoutStmt{Widgets: []*ast.WidgetV3{ + {Type: "SCROLLCONTAINER", Name: "layoutContainer", Children: []*ast.WidgetV3{ + btn("btnL", "Atlas_Core.Atlas.home"), + }}, + }}, + want: []string{"Atlas_Core.Atlas.home"}, + }, + { + // A menu document carries the same NavMenuItemDef as a profile menu, + // so it needs the same recursion into sub-items. + name: "menu document, including sub-items", + stmt: &ast.CreateMenuStmt{Items: []ast.NavMenuItemDef{ + {Caption: "Home", Icon: "Atlas_Core.Atlas.home", Items: []ast.NavMenuItemDef{ + {Caption: "Nested", Icon: "Atlas_Core.Atlas.pencil"}, + }}, + }}, + want: []string{"Atlas_Core.Atlas.home", "Atlas_Core.Atlas.pencil"}, + }, { name: "no icons at all", stmt: &ast.CreatePageStmtV3{Widgets: []*ast.WidgetV3{ diff --git a/mdl/executor/validate_navigation_roles.go b/mdl/executor/validate_navigation_roles.go new file mode 100644 index 0000000000..2e5eec73ef --- /dev/null +++ b/mdl/executor/validate_navigation_roles.go @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Reference validation for the user role in `HOME PAGE … FOR `. +// +// The role was written through to BSON verbatim, and the damage depends on the +// shape of what was written (measured on a blank Mendix 11.13 app): +// +// - `for Administrator` — a real user role — builds at 0 errors. +// +// - `for Supervisor` — bare, nonexistent — is CE1613 "The selected user role +// 'Supervisor' no longer exists", a normal build error. +// +// - `for MyFirstModule.Administrator` — module-qualified — does not reach the +// checker at all. Mendix fails to LOAD the project. +// +// The load failure verbatim (indented so gofmt does not reflow the quoting): +// +// StorageLoadException: Role based home page in has an invalid value '' +// for property UserRole. The text 'MyFirstModule.Administrator' is not a +// valid UserRoleIdentifier. +// +// The third is the one this exists for, and it was the form mxcli's own +// documentation recommended (mendixlabs/mxcli#1001). A user role is +// project-level, so its identifier is a bare name; a module role is +// module-scoped and looks almost identical — a blank app has a user role +// `Administrator` and module roles named `Administrator` in three modules, so +// the wrong one reads as correct. +// +// A load failure is worse than a build error: it happens before any checking +// runs, so there is no error code and no location, and `mx check` exits 1 +// without the "The app contains: N errors" line that people read as the verdict. +// +// The roles live in the project, so this runs in the --references pass. It is +// also called from the navigation handler, so `exec` refuses what `check` +// refuses rather than writing a project Mendix cannot open. +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" +) + +// navRoleRef is one `FOR ` reference and where it was written. +type navRoleRef struct { + role ast.QualifiedName + profile string +} + +// navigationRoleRefs collects the role references in one statement. +func navigationRoleRefs(stmt ast.Statement) []navRoleRef { + s, ok := stmt.(*ast.AlterNavigationStmt) + if !ok { + return nil + } + var out []navRoleRef + for _, hp := range s.HomePages { + if hp.ForRole == nil { + continue + } + out = append(out, navRoleRef{role: *hp.ForRole, profile: s.ProfileName}) + } + return out +} + +// scriptUserRoles collects the user roles the script itself creates. +// +// Without this a script that creates a role and then uses it — the ordinary way +// to write one — would be refused for naming a role that does not exist *yet*. +// The whole-script scriptContext does not track user roles, so they are +// gathered here. +func scriptUserRoles(prog *ast.Program) []string { + var out []string + for _, stmt := range prog.Statements { + if s, ok := stmt.(*ast.CreateUserRoleStmt); ok && s.Name != "" { + out = append(out, s.Name) + } + } + return out +} + +// validateNavigationRoles resolves every `HOME PAGE … FOR` role in the program. +func validateNavigationRoles(ctx *ExecContext, prog *ast.Program) []error { + if !ctx.Connected() { + return nil + } + + // Collect first: a program with no role-based home page must not pay for + // reading project security. + type located struct { + ref navRoleRef + index int + } + var refs []located + for i, stmt := range prog.Statements { + for _, ref := range navigationRoleRefs(stmt) { + refs = append(refs, located{ref: ref, index: i + 1}) + } + } + if len(refs) == 0 { + return nil + } + + known, err := projectUserRoles(ctx) + if err != nil { + // Reading security failed. Reporting every role as unknown on the back of + // that would be worse than not checking. + return nil + } + known = append(known, scriptUserRoles(prog)...) + + var errs []error + for _, r := range refs { + if err := checkNavigationRole(r.ref, known); err != nil { + errs = append(errs, fmt.Errorf("statement %d: %w", r.index, err)) + } + } + return errs +} + +// validateNavigationRoleForExec is the same check at execution time. By the time +// a navigation statement runs, a role the script creates is already in the +// project, so the stored roles alone are the right set. +func validateNavigationRoleForExec(ctx *ExecContext, stmt *ast.AlterNavigationStmt) error { + refs := navigationRoleRefs(stmt) + if len(refs) == 0 { + return nil + } + known, err := projectUserRoles(ctx) + if err != nil { + return nil + } + for _, ref := range refs { + if err := checkNavigationRole(ref, known); err != nil { + return err + } + } + return nil +} + +func projectUserRoles(ctx *ExecContext) ([]string, error) { + ps, err := ctx.Backend.GetProjectSecurity() + if err != nil { + return nil, err + } + names := make([]string, 0, len(ps.UserRoles)) + for _, ur := range ps.UserRoles { + names = append(names, ur.Name) + } + return names, nil +} + +// checkNavigationRole resolves one reference against the project's user roles. +// +// The two failures need different messages because they need different fixes: a +// module-qualified name is the right role written the wrong way, and an unknown +// bare name is the wrong role. +func checkNavigationRole(ref navRoleRef, known []string) error { + where := "home page" + if ref.profile != "" { + where = fmt.Sprintf("navigation profile %q home page", ref.profile) + } + + if ref.role.Module != "" { + msg := fmt.Sprintf( + "%s: %s is a module-qualified name, but FOR takes a USER role, which is "+ + "project-level and has no module part", + where, ref.role.String()) + // The bare half is usually the role they meant — a blank app has a user + // role Administrator and module roles of the same name in three modules. + if match := matchRole(ref.role.Name, known); match != "" { + msg += fmt.Sprintf(". Write: for %s", match) + } else { + msg += fmt.Sprintf(". Project user roles: %s", listRoles(known)) + } + msg += ".\n Mendix cannot LOAD a project with a qualified name here " + + "(StorageLoadException: not a valid UserRoleIdentifier) — it fails before " + + "checking runs, so there is no error code and no line number" + return mdlerrors.NewValidation(msg) + } + + match := matchRole(ref.role.Name, known) + if match == ref.role.Name { + return nil + } + if match != "" { + // Only the casing differs. Mendix matches the role name exactly: measured, + // `for administrator` is CE1613 on an app whose role is `Administrator`. + return mdlerrors.NewValidation(fmt.Sprintf( + "%s: user role %q differs in case from %q — Mendix matches the name exactly "+ + "(MxBuild reports this as CE1613). Write: for %s", + where, ref.role.Name, match, match)) + } + return mdlerrors.NewNotFoundMsg("user role", ref.role.Name, fmt.Sprintf( + "%s: user role %q does not exist (MxBuild reports this as CE1613).\n"+ + " Project user roles: %s", + where, ref.role.Name, listRoles(known))) +} + +// matchRole finds the declared spelling of a role, ignoring case. Returning the +// project's spelling rather than a bool is what lets the caller say which +// casing is wanted. +func matchRole(name string, known []string) string { + for _, k := range known { + if strings.EqualFold(k, name) { + return k + } + } + return "" +} + +func listRoles(known []string) string { + if len(known) == 0 { + return "(none defined)" + } + sorted := append([]string(nil), known...) + sort.Strings(sorted) + return strings.Join(sorted, ", ") +} diff --git a/mdl/executor/validate_navigation_roles_test.go b/mdl/executor/validate_navigation_roles_test.go new file mode 100644 index 0000000000..30a2f14196 --- /dev/null +++ b/mdl/executor/validate_navigation_roles_test.go @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func navRole(module, name string) *ast.QualifiedName { + return &ast.QualifiedName{Module: module, Name: name} +} + +// TestCheckNavigationRole covers the three outcomes, which were measured on a +// blank Mendix 11.13 app and are three different severities: +// +// for Administrator → 0 errors +// for Supervisor → CE1613, a normal build error +// for MyFirstModule.Administrator → StorageLoadException, project unloadable +func TestCheckNavigationRole(t *testing.T) { + known := []string{"Administrator", "User"} + + tests := []struct { + name string + role *ast.QualifiedName + wantErr bool + wantParts []string + }{ + { + name: "a real user role resolves", + role: navRole("", "Administrator"), + }, + { + name: "the other real user role resolves", + role: navRole("", "User"), + }, + { + // The reported case. The module part is the whole defect, so the message + // has to name the bare form to write instead. + name: "module-qualified role names the bare form to use", + role: navRole("MyFirstModule", "Administrator"), + wantErr: true, + wantParts: []string{ + "MyFirstModule.Administrator", + "USER role", + "for Administrator", + // The consequence is worth stating: this one is not a build error. + "cannot LOAD", + }, + }, + { + // A qualified name whose bare half is not a role either: there is no + // single form to suggest, so list what exists. + name: "module-qualified unknown role lists the real ones", + role: navRole("MyFirstModule", "Supervisor"), + wantErr: true, + wantParts: []string{"USER role", "Administrator, User"}, + }, + { + name: "bare unknown role is reported as CE1613", + role: navRole("", "Supervisor"), + wantErr: true, + wantParts: []string{"Supervisor", "CE1613", "Administrator, User"}, + }, + { + // Measured: Mendix matches the name exactly, so this really is CE1613 + // and not a harmless spelling. + name: "case mismatch names the declared casing", + role: navRole("", "administrator"), + wantErr: true, + wantParts: []string{"differs in case", "for Administrator", "CE1613"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkNavigationRole(navRoleRef{role: *tt.role, profile: "Responsive"}, known) + if !tt.wantErr { + if err != nil { + t.Fatalf("unexpected error for %q: %v", tt.role.String(), err) + } + return + } + if err == nil { + t.Fatalf("expected an error for %q", tt.role.String()) + } + // The profile is the only location a navigation error has. + if !strings.Contains(err.Error(), "Responsive") { + t.Errorf("error %q does not name the profile", err.Error()) + } + for _, want := range tt.wantParts { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err.Error(), want) + } + } + }) + } +} + +// TestCheckNavigationRoleNoRolesDefined covers a project with security off and +// no user roles: the message must still be readable rather than trailing an +// empty list. +func TestCheckNavigationRoleNoRolesDefined(t *testing.T) { + err := checkNavigationRole(navRoleRef{role: *navRole("", "Administrator")}, nil) + if err == nil { + t.Fatal("expected an error when the project has no user roles") + } + if !strings.Contains(err.Error(), "(none defined)") { + t.Errorf("error %q should say the project defines no user roles", err.Error()) + } +} + +// TestNavigationRoleRefs pins collection: a reference the walker misses is a +// reference nothing checks. +func TestNavigationRoleRefs(t *testing.T) { + tests := []struct { + name string + stmt ast.Statement + want []string + }{ + { + name: "default home page carries no role", + stmt: &ast.AlterNavigationStmt{ProfileName: "Responsive", HomePages: []ast.NavHomePageDef{ + {IsPage: true, Target: ast.QualifiedName{Module: "M", Name: "Home"}}, + }}, + }, + { + name: "role-based home page", + stmt: &ast.AlterNavigationStmt{ProfileName: "Responsive", HomePages: []ast.NavHomePageDef{ + {IsPage: true, Target: ast.QualifiedName{Module: "M", Name: "Home"}}, + {IsPage: true, Target: ast.QualifiedName{Module: "M", Name: "Admin"}, ForRole: navRole("", "Administrator")}, + }}, + want: []string{"Administrator"}, + }, + { + // Several roles in one profile: checking only the first would let the + // rest through. + name: "every role-based home page is collected", + stmt: &ast.AlterNavigationStmt{ProfileName: "Responsive", HomePages: []ast.NavHomePageDef{ + {ForRole: navRole("", "Administrator")}, + {ForRole: navRole("M", "Manager")}, + }}, + want: []string{"Administrator", "M.Manager"}, + }, + { + // HOME MICROFLOW takes a role the same way HOME PAGE does. + name: "home microflow carries a role too", + stmt: &ast.AlterNavigationStmt{ProfileName: "Phone", HomePages: []ast.NavHomePageDef{ + {IsPage: false, Target: ast.QualifiedName{Module: "M", Name: "MF"}, ForRole: navRole("", "User")}, + }}, + want: []string{"User"}, + }, + { + name: "an unrelated statement contributes nothing", + stmt: &ast.CreateUserRoleStmt{Name: "Administrator"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + refs := navigationRoleRefs(tt.stmt) + if len(refs) != len(tt.want) { + t.Fatalf("got %d refs %v, want %d %v", len(refs), refs, len(tt.want), tt.want) + } + for i, w := range tt.want { + if got := refs[i].role.String(); got != w { + t.Errorf("ref[%d] = %q, want %q", i, got, w) + } + } + }) + } +} + +// TestScriptUserRoles covers the over-reach guard: a script that creates a role +// and then uses it is the ordinary way to write one, and must not be refused for +// naming a role that does not exist in the project yet. +func TestScriptUserRoles(t *testing.T) { + prog := &ast.Program{Statements: []ast.Statement{ + &ast.CreateUserRoleStmt{Name: "Supervisor"}, + &ast.CreateUserRoleStmt{Name: ""}, // must not become an empty known role + &ast.AlterNavigationStmt{ProfileName: "Responsive", HomePages: []ast.NavHomePageDef{ + {ForRole: navRole("", "Supervisor")}, + }}, + }} + + roles := scriptUserRoles(prog) + if len(roles) != 1 || roles[0] != "Supervisor" { + t.Fatalf("scriptUserRoles = %v, want [Supervisor]", roles) + } + // Resolving against the script's own roles must succeed even though the + // project (here, empty) has none. + if err := checkNavigationRole(navRoleRef{role: *navRole("", "Supervisor")}, roles); err != nil { + t.Errorf("a role the script creates must resolve: %v", err) + } +} diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index 3dae0e6638..792dd753ee 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -59,6 +59,18 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { // Check microflow body for common issues if mfStmt, ok := stmt.(*ast.CreateMicroflowStmt); ok { violations = append(violations, ValidateMicroflow(mfStmt)...) + violations = append(violations, + ValidateFlowParameterAnnotations("microflow '"+mfStmt.Name.String()+"'", mfStmt.Parameters)...) + } + // Parameter annotations for the two flow flavours that do not go through + // ValidateMicroflow but share the parameter grammar. + if nfStmt, ok := stmt.(*ast.CreateNanoflowStmt); ok { + violations = append(violations, + ValidateFlowParameterAnnotations("nanoflow '"+nfStmt.Name.String()+"'", nfStmt.Parameters)...) + } + if ruleStmt, ok := stmt.(*ast.CreateRuleStmt); ok { + violations = append(violations, + ValidateFlowParameterAnnotations("rule '"+ruleStmt.Name.String()+"'", ruleStmt.Parameters)...) } // Check workflow for constructs MxBuild rejects (missing page, // single-outcome-with-activities, invalid decision outcome names) diff --git a/mdl/executor/validate_webservice_mapping.go b/mdl/executor/validate_webservice_mapping.go new file mode 100644 index 0000000000..31520ae4bc --- /dev/null +++ b/mdl/executor/validate_webservice_mapping.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Guard-don't-drop for a mapping sourced from an imported web service (SOAP). +// +// A mapping can be sourced four ways — a JSON structure, an XML schema, a +// message definition, or an IMPORTED WEB SERVICE. MDL can spell the first +// three. The fourth is a WSDL binding (which service, which operation, which +// root element), and a CREATE OR REPLACE/MODIFY rebuilds the mapping from the +// statement, so anything the script does not restate is gone. +// +// Measured on mxbuild 11.13.0 against a mapping carrying the four properties a +// WSDL import writes. describe → exec — which is how a document is copied — +// left ServiceName and OperationName blank and removed ImportedWebService and +// RootElementName outright: +// +// [error] [CE6896] "A mapping must have exactly one schema source." +// [error] [CE0270] "No root element could be found in the schema." +// +// So a working SOAP integration became an unbuildable one, and the diff blames +// the statement the user ran rather than the source they never mentioned. That +// is the same class as the queued-call refusal (ADR-0005) and worse than +// ako/mxcli#259's dangling reference, which at least wrote a bad name through +// rather than deleting a good one. +// +// This REFUSES rather than preserving. Carrying the binding through a rebuild +// would mean claiming the rest of the document survives it too, and mxcli +// cannot check that: a SOAP mapping's elements resolve against the WSDL's +// inline schema entries, which mxcli does not read. Refusing is the honest +// answer until `with web service …` exists. +package executor + +import ( + "fmt" + + mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" + "github.com/mendixlabs/mxcli/model" +) + +// checkNoWebServiceSource refuses a rewrite of a mapping whose stored source is +// an imported web service. kind is "import" or "export". +func checkNoWebServiceSource(kind, qualifiedName string, src model.WebServiceMappingSource) error { + if !src.IsSet() { + return nil + } + return mdlerrors.NewValidation(fmt.Sprintf( + "%s mapping %s is sourced from the imported web service %s%s, which MDL cannot "+ + "express — rewriting it would drop the binding and leave the mapping with no "+ + "schema source at all (CE6896). Edit it in Studio Pro, or drop and re-import "+ + "the web service", + kind, qualifiedName, src.ImportedWebService, webServiceDetail(src))) +} + +// webServiceDetail names the part of the service the mapping covers, so the +// refusal says what would have been lost rather than only that something would. +func webServiceDetail(src model.WebServiceMappingSource) string { + switch { + case src.ServiceName != "" && src.OperationName != "": + return fmt.Sprintf(" (service %s, operation %s)", src.ServiceName, src.OperationName) + case src.OperationName != "": + return fmt.Sprintf(" (operation %s)", src.OperationName) + case src.ServiceName != "": + return fmt.Sprintf(" (service %s)", src.ServiceName) + } + return "" +} diff --git a/mdl/executor/validate_webservice_mapping_test.go b/mdl/executor/validate_webservice_mapping_test.go new file mode 100644 index 0000000000..cd6b17b205 --- /dev/null +++ b/mdl/executor/validate_webservice_mapping_test.go @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// A mapping can be sourced from an imported web service (SOAP) — a fourth kind +// beside JSON structure, XML schema and message definition. MDL cannot spell it, +// so a CREATE OR REPLACE/MODIFY rebuilt the mapping without it and a working +// integration became an unbuildable one (ako/mxcli#365): +// +// [error] [CE6896] "A mapping must have exactly one schema source." +// [error] [CE0270] "No root element could be found in the schema." + +func soapSource() model.WebServiceMappingSource { + return model.WebServiceMappingSource{ + ImportedWebService: "Legacy.WS_Orders", + ServiceName: "OrderService", + OperationName: "GetOrder", + RootElementName: "GetOrderResponse", + } +} + +func TestWebServiceSourcedMappingIsRefused(t *testing.T) { + err := checkNoWebServiceSource("import", "Legacy.IMM_Order", soapSource()) + if err == nil { + t.Fatal("accepted a rewrite that would drop the web-service binding") + } + // The refusal must name what would have been lost, not merely that + // something would: the source is the one thing the statement never mentions. + for _, want := range []string{"Legacy.WS_Orders", "OrderService", "GetOrder", "CE6896"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not mention %q", err, want) + } + } +} + +// TestNonWebServiceMappingIsUntouched is the control. Every mapping mxcli can +// author has an empty source here, so a guard that fired on it would refuse +// every rewrite in the repo. +func TestNonWebServiceMappingIsUntouched(t *testing.T) { + if err := checkNoWebServiceSource("import", "M.IMM_Json", model.WebServiceMappingSource{}); err != nil { + t.Fatalf("refused an ordinary mapping: %v", err) + } +} + +// TestWebServiceSourceIsKeyedOnTheService pins the discriminator. ServiceName +// and OperationName only qualify WHICH part of a service the mapping covers; +// a mapping carrying them without a service is not SOAP-sourced, and refusing +// it would block rewrites on the strength of a leftover empty string. +func TestWebServiceSourceIsKeyedOnTheService(t *testing.T) { + partial := model.WebServiceMappingSource{ServiceName: "OrderService", OperationName: "GetOrder"} + if partial.IsSet() { + t.Error("IsSet true without an imported web service") + } + if err := checkNoWebServiceSource("import", "M.IMM_X", partial); err != nil { + t.Errorf("refused on qualifiers alone: %v", err) + } +} + +// TestWebServiceDetailDegradesGracefully pins that a partially-populated +// binding still produces a readable message rather than empty parentheses. +func TestWebServiceDetailDegradesGracefully(t *testing.T) { + cases := []struct { + name string + src model.WebServiceMappingSource + want string + }{ + {"both", soapSource(), " (service OrderService, operation GetOrder)"}, + {"operation only", model.WebServiceMappingSource{ + ImportedWebService: "L.WS", OperationName: "GetOrder"}, " (operation GetOrder)"}, + {"service only", model.WebServiceMappingSource{ + ImportedWebService: "L.WS", ServiceName: "OrderService"}, " (service OrderService)"}, + {"neither", model.WebServiceMappingSource{ImportedWebService: "L.WS"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := webServiceDetail(tc.src); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + }) + } +} + +// TestExportMappingIsRefusedToo pins that both mapping kinds are covered. An +// export mapping has the same binding plus two SOAP-only properties +// (ParameterName, IsHeader), so leaving it out would drop more, not less. +func TestExportMappingIsRefusedToo(t *testing.T) { + src := soapSource() + src.ParameterName = "body" + src.IsHeader = false + + err := checkNoWebServiceSource("export", "Legacy.EXM_Order", src) + if err == nil { + t.Fatal("accepted an export-mapping rewrite that would drop the binding") + } + if !strings.Contains(err.Error(), "export mapping Legacy.EXM_Order") { + t.Errorf("error does not name the export mapping: %v", err) + } +} diff --git a/mdl/grammar/MDLLexer.g4 b/mdl/grammar/MDLLexer.g4 index a135029834..25beda41c0 100644 --- a/mdl/grammar/MDLLexer.g4 +++ b/mdl/grammar/MDLLexer.g4 @@ -657,7 +657,9 @@ OF: O F; OVER: O V E R; FOR: F O R; REPLACE: R E P L A C E; -MEMBERS: M E M B E R S; +EXAMPLE: E X A M P L E; // `example '...'` on a message definition member +MEMBER: M E M B E R; // `add member X` — ANTLR prefers the longest match, +MEMBERS: M E M B E R S; // so "members" still lexes as MEMBERS ATTRIBUTE_NAME: A T T R I B U T E N A M E; FORMAT: F O R M A T; SQL: S Q L; diff --git a/mdl/grammar/MDLParser.g4 b/mdl/grammar/MDLParser.g4 index 8b13b5426d..b6d1bb06c6 100644 --- a/mdl/grammar/MDLParser.g4 +++ b/mdl/grammar/MDLParser.g4 @@ -120,6 +120,7 @@ createStatement | createScheduledEventStatement | createRegularExpressionStatement | createJsonStructureStatement + | createMessageDefinitionCollectionStatement | createImportMappingStatement | createExportMappingStatement | createConfigurationStatement @@ -154,6 +155,8 @@ alterStatement | ALTER LAYOUT qualifiedName LBRACE alterPageOperation+ RBRACE | ALTER SNIPPET qualifiedName LBRACE alterPageOperation+ RBRACE | ALTER WORKFLOW qualifiedName alterWorkflowAction+ SEMICOLON? + | alterMessageDefinitionCollectionStatement + | alterMessageDefinitionStatement | ALTER PUBLISHED REST SERVICE qualifiedName alterPublishedRestServiceAction (COMMA? alterPublishedRestServiceAction)* | ALTER MODEL qualifiedName SET agentEditorAlterAssignment (COMMA agentEditorAlterAssignment)* | ALTER KNOWLEDGE BASE qualifiedName SET agentEditorAlterAssignment (COMMA agentEditorAlterAssignment)* @@ -373,6 +376,7 @@ dropStatement | DROP ANNOTATION STRING_LITERAL IN identifierOrKeyword | DROP ANNOTATION AT_KW LPAREN NUMBER_LITERAL COMMA NUMBER_LITERAL RPAREN IN identifierOrKeyword | DROP JSON STRUCTURE qualifiedName + | DROP MESSAGE DEFINITION COLLECTION qualifiedName | DROP IMPORT MAPPING qualifiedName | DROP EXPORT MAPPING qualifiedName | DROP REST CLIENT qualifiedName diff --git a/mdl/grammar/domains/MDLCatalog.g4 b/mdl/grammar/domains/MDLCatalog.g4 index cdac6f80b4..5eb25b5225 100644 --- a/mdl/grammar/domains/MDLCatalog.g4 +++ b/mdl/grammar/domains/MDLCatalog.g4 @@ -48,6 +48,7 @@ showStatement | showOrList KNOWLEDGE BASES (IN (qualifiedName | IDENTIFIER))? | showOrList CONSUMED MCP SERVICES (IN (qualifiedName | IDENTIFIER))? | showOrList JSON STRUCTURES (IN (qualifiedName | IDENTIFIER))? + | showOrList MESSAGE DEFINITION COLLECTION (IN (qualifiedName | IDENTIFIER))? | showOrList IMPORT MAPPINGS (IN (qualifiedName | IDENTIFIER))? | showOrList EXPORT MAPPINGS (IN (qualifiedName | IDENTIFIER))? | showOrList ENTITY qualifiedName @@ -188,6 +189,7 @@ describeStatement | DESCRIBE KNOWLEDGE BASE qualifiedName // DESCRIBE KNOWLEDGE BASE Module.Name | DESCRIBE CONSUMED MCP SERVICE qualifiedName // DESCRIBE CONSUMED MCP SERVICE Module.Name | DESCRIBE JSON STRUCTURE qualifiedName // DESCRIBE JSON STRUCTURE Module.Name + | DESCRIBE MESSAGE DEFINITION COLLECTION qualifiedName | DESCRIBE IMPORT MAPPING qualifiedName // DESCRIBE IMPORT MAPPING Module.Name | DESCRIBE EXPORT MAPPING qualifiedName // DESCRIBE EXPORT MAPPING Module.Name | DESCRIBE REST CLIENT qualifiedName // DESCRIBE REST CLIENT Module.Name diff --git a/mdl/grammar/domains/MDLDomainModel.g4 b/mdl/grammar/domains/MDLDomainModel.g4 index a45981adc6..cf9f4516fd 100644 --- a/mdl/grammar/domains/MDLDomainModel.g4 +++ b/mdl/grammar/domains/MDLDomainModel.g4 @@ -492,6 +492,124 @@ customNameMapping | ITEM OF STRING_LITERAL AS STRING_LITERAL // ITEM OF 'jsonKey' AS 'CustomName' ; +// ============================================================================= +// MESSAGE DEFINITION COLLECTION +// ============================================================================= + +/** + * CREATE [OR MODIFY] MESSAGE DEFINITION COLLECTION Module.Name + * FOLDER 'Private/Messages' + * ( + * definition Order for Sales.Order as 'Orders' ( + * OrderId, + * Sales.Order_Line/Sales.Line as 'Lines' ( Sku, Quantity ) + * ) + * ); + * + * A message definition is a SELECTION OVER THE DOMAIN MODEL — every element + * names an entity, an attribute or an association — which is what makes it + * authorable where an XML schema or a WSDL is not (they hold an imported file). + * It is the source for 74 of the 327 mappings in the demo corpus, and the only + * one of the four a script could not create. + * + * A collection holds one or more definitions; 28 of 36 real collections hold + * exactly one, but the reference a mapping uses is three parts either way + * (Module.Collection.Definition), so the collection is never implicit. + */ +createMessageDefinitionCollectionStatement + : MESSAGE DEFINITION COLLECTION qualifiedName + (FOLDER STRING_LITERAL)? + LPAREN messageDefinitionDef (COMMA messageDefinitionDef)* COMMA? RPAREN + ; + +/** + * `definition for [as ''] ( members )` + * + * The definition's Name and its root element's exposed name are independent — + * measured, 19 of 56 definitions are named something other than their entity. + */ +messageDefinitionDef + : DEFINITION identifierOrKeyword FOR qualifiedName messageExposedName? + LPAREN messageMember (COMMA messageMember)* COMMA? RPAREN + ; + +/** + * A member is an ATTRIBUTE (a bare name) or an ASSOCIATION (Assoc/Module.Entity + * with its own member list) — the same discriminator import and export mappings + * use, so there is nothing new to learn. + * + * The association's TARGET entity is spelled out rather than inferred, and that + * is load-bearing: MaxOccurs is not a function of the association's type (all + * 927 resolvable ones in the corpus are `Reference`, yet 526 store 1 and 401 + * store -1). It tracks the DIRECTION of traversal — holder is the FROM entity + * gives 1, holder is the TO entity gives -1, with zero counter-examples. Naming + * the target makes the direction explicit in the source text instead of + * something a reader has to work out. + */ +messageMember + : qualifiedName SLASH qualifiedName messageExposedName? + LPAREN messageMember (COMMA messageMember)* COMMA? RPAREN // association + | identifierOrKeyword messageExposedName? messageExample? // attribute + ; + +// `example 'text'` sets the element's Example — author-set sample text. Only a +// value member carries one in any document measured. +messageExample + : EXAMPLE STRING_LITERAL + ; + +// `as 'Name'` sets ExposedName. A name mapped to a name takes `as`, not `:`. +messageExposedName + : AS STRING_LITERAL + ; + +/** + * ALTER MESSAGE DEFINITION COLLECTION Module.Name ; + * + * Definitions within a collection. Whole-document CREATE OR MODIFY is not + * enough on its own: definitions nest to depth 7 in the corpus, so restating + * one to add a leaf is the diff-unfriendliness ADR-0003 argues against. + */ +alterMessageDefinitionCollectionStatement + : ALTER MESSAGE DEFINITION COLLECTION qualifiedName alterMessageCollectionOperation + ; + +alterMessageCollectionOperation + : ADD DEFINITION (IF NOT EXISTS)? identifierOrKeyword FOR qualifiedName messageExposedName? + LPAREN messageMember (COMMA messageMember)* COMMA? RPAREN + | DROP DEFINITION (IF EXISTS)? identifierOrKeyword + | RENAME DEFINITION identifierOrKeyword TO identifierOrKeyword + ; + +/** + * ALTER MESSAGE DEFINITION Module.Collection.Definition ; + * + * Members within one definition, addressed by the SAME three-part reference + * `WITH MESSAGE DEFINITION` takes, so the two cannot drift apart. + * + * `SET member X AS 'Name'` changes only the element's ExposedName. It is not + * RENAME: ALTER ENTITY's RENAME ATTRIBUTE renames the attribute in the model and + * rewrites every reference to it, and borrowing the verb here would promise + * something far larger than this does. + * + * `IN ` reaches a nested member, written in exposed names. `/` is not used + * as the path separator because it already means "association to entity" inside + * a member. + */ +alterMessageDefinitionStatement + : ALTER MESSAGE DEFINITION qualifiedName alterMessageDefinitionOperation + ; + +alterMessageDefinitionOperation + : ADD MEMBER (IF NOT EXISTS)? messageMember (IN messageMemberPath)? + | DROP MEMBER (IF EXISTS)? identifierOrKeyword (IN messageMemberPath)? + | SET MEMBER identifierOrKeyword (IN messageMemberPath)? messageExposedName + ; + +messageMemberPath + : identifierOrKeyword (SLASH identifierOrKeyword)* + ; + // ============================================================================= // IMPORT / EXPORT MAPPING CREATION // ============================================================================= diff --git a/mdl/grammar/domains/MDLMicroflow.g4 b/mdl/grammar/domains/MDLMicroflow.g4 index 54667f3992..067aca6d59 100644 --- a/mdl/grammar/domains/MDLMicroflow.g4 +++ b/mdl/grammar/domains/MDLMicroflow.g4 @@ -125,8 +125,11 @@ microflowParameterList : microflowParameter (COMMA microflowParameter)* ; +// Annotations on a parameter: `@position(x, y)` places the parameter box on the +// canvas. Parameters are stored nodes with real geometry, so this is the same +// annotation the statements below it take, attached to the thing it positions. microflowParameter - : (parameterName | VARIABLE) COLON dataType + : annotation* (parameterName | VARIABLE) COLON dataType ; // Allow reserved keywords as parameter names (similar to attributeName) diff --git a/mdl/grammar/domains/MDLSettings.g4 b/mdl/grammar/domains/MDLSettings.g4 index c71af8aa40..62c67beab3 100644 --- a/mdl/grammar/domains/MDLSettings.g4 +++ b/mdl/grammar/domains/MDLSettings.g4 @@ -708,5 +708,5 @@ keyword | TRANSFORM | TRANSFORMER | TRANSFORMERS | JSLT | XSLT // Import/Export mapping / SQL generate - | ATTRIBUTE_NAME | CONNECTOR | MEMBERS | OVER | JAVA | XPATH + | ATTRIBUTE_NAME | CONNECTOR | EXAMPLE | MEMBER | MEMBERS | OVER | JAVA | XPATH ; diff --git a/mdl/linter/rules/missing_translations.go b/mdl/linter/rules/missing_translations.go index 3c2e0a8ec6..e9ea544ffb 100644 --- a/mdl/linter/rules/missing_translations.go +++ b/mdl/linter/rules/missing_translations.go @@ -57,12 +57,18 @@ func (r *MissingTranslationsRule) Check(ctx *linter.LintContext) []linter.Violat } // Step 2: Find elements that have translations in some languages but not all. - // Group by (QualifiedName, StringContext) — each group should have all languages. + // Group by (QualifiedName, StringContext, ElementId) — each group should have + // all languages. + // + // ElementId is load-bearing: sibling elements of one type share a + // QualifiedName and a StringContext (an enumeration's twelve values, a page's + // action buttons), so grouping without it folds them into one group and a + // single translated value makes the whole set look complete. rows, err := db.Query(` - SELECT QualifiedName, ObjectType, StringContext, Language, StringValue + SELECT QualifiedName, ObjectType, StringContext, Language, StringValue, ElementId FROM strings WHERE Language != '' - ORDER BY QualifiedName, StringContext, Language + ORDER BY QualifiedName, StringContext, ElementId, Language `) if err != nil { return nil @@ -73,6 +79,7 @@ func (r *MissingTranslationsRule) Check(ctx *linter.LintContext) []linter.Violat type elementKey struct { QualifiedName string StringContext string + ElementID string } type elementInfo struct { ObjectType string @@ -82,11 +89,11 @@ func (r *MissingTranslationsRule) Check(ctx *linter.LintContext) []linter.Violat elements := make(map[elementKey]*elementInfo) for rows.Next() { - var qn, objType, sctx, lang, value string - if err := rows.Scan(&qn, &objType, &sctx, &lang, &value); err != nil { + var qn, objType, sctx, lang, value, elemID string + if err := rows.Scan(&qn, &objType, &sctx, &lang, &value, &elemID); err != nil { continue } - key := elementKey{qn, sctx} + key := elementKey{qn, sctx, elemID} info, ok := elements[key] if !ok { info = &elementInfo{ObjectType: objType, Languages: make(map[string]bool)} diff --git a/mdl/linter/rules/missing_translations_test.go b/mdl/linter/rules/missing_translations_test.go index 381a1e5eee..1c99afdafa 100644 --- a/mdl/linter/rules/missing_translations_test.go +++ b/mdl/linter/rules/missing_translations_test.go @@ -4,6 +4,7 @@ package rules import ( "database/sql" + "strings" "testing" "github.com/mendixlabs/mxcli/mdl/catalog" @@ -160,3 +161,76 @@ func TestMissingTranslationsRuleMetadata(t *testing.T) { t.Errorf("expected severity Warning, got %v", rule.DefaultSeverity()) } } + +// setupTranslationsDBWithIDs is setupTranslationsDB with the ElementId given +// rather than synthesized, which is what lets a test put two sibling elements in +// one document. +// Rows are [QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName]. +func setupTranslationsDBWithIDs(t *testing.T, rows [][]string) catalog.CatalogDB { + t.Helper() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("failed to open in-memory db: %v", err) + } + if _, err := db.Exec(`CREATE VIRTUAL TABLE strings USING fts5( + QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName + )`); err != nil { + t.Fatalf("failed to create strings table: %v", err) + } + stmt, err := db.Prepare(`INSERT INTO strings (QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName) + VALUES (?, ?, ?, ?, ?, ?, ?)`) + if err != nil { + t.Fatalf("failed to prepare insert: %v", err) + } + defer stmt.Close() + for _, row := range rows { + if len(row) != 7 { + t.Fatalf("expected 7 columns, got %d", len(row)) + } + if _, err := stmt.Exec(row[0], row[1], row[2], row[3], row[4], row[5], row[6]); err != nil { + t.Fatalf("failed to insert row: %v", err) + } + } + return catalog.WrapSqlDB(db) +} + +// Sibling elements of one type share a QualifiedName and a StringContext — an +// enumeration's twelve values, a page's action buttons. Grouping without the +// ElementId folds them into one, so translating a single value makes the whole +// set look complete and eleven missing translations go unreported. +func TestMissingTranslations_SiblingElementsAreNotOneGroup(t *testing.T) { + db := setupTranslationsDBWithIDs(t, [][]string{ + // "Low" is translated. + {"MyModule.Priority", "ENUMERATION", "Low", "Enumerations$EnumerationValue.Caption", "en_US", "id-low", "MyModule"}, + {"MyModule.Priority", "ENUMERATION", "Laag", "Enumerations$EnumerationValue.Caption", "nl_NL", "id-low", "MyModule"}, + // "High" is not. + {"MyModule.Priority", "ENUMERATION", "High", "Enumerations$EnumerationValue.Caption", "en_US", "id-high", "MyModule"}, + }) + defer db.Close() + + violations := NewMissingTranslationsRule().Check(linter.NewLintContextFromDB(db)) + + if len(violations) != 1 { + t.Fatalf("want 1 violation for the untranslated value, got %d: %v", len(violations), violations) + } + if !strings.Contains(violations[0].Message, "High") { + t.Errorf("the violation names the wrong value: %q", violations[0].Message) + } +} + +// The control for the test above: with every sibling translated there is nothing +// to report, so the violation there is the missing translation and not an +// artifact of splitting the group. +func TestMissingTranslations_SiblingElementsAllTranslatedIsClean(t *testing.T) { + db := setupTranslationsDBWithIDs(t, [][]string{ + {"MyModule.Priority", "ENUMERATION", "Low", "Enumerations$EnumerationValue.Caption", "en_US", "id-low", "MyModule"}, + {"MyModule.Priority", "ENUMERATION", "Laag", "Enumerations$EnumerationValue.Caption", "nl_NL", "id-low", "MyModule"}, + {"MyModule.Priority", "ENUMERATION", "High", "Enumerations$EnumerationValue.Caption", "en_US", "id-high", "MyModule"}, + {"MyModule.Priority", "ENUMERATION", "Hoog", "Enumerations$EnumerationValue.Caption", "nl_NL", "id-high", "MyModule"}, + }) + defer db.Close() + + if v := NewMissingTranslationsRule().Check(linter.NewLintContextFromDB(db)); len(v) != 0 { + t.Fatalf("want 0 violations, got %d: %v", len(v), v) + } +} diff --git a/mdl/translations/sites.go b/mdl/translations/sites.go new file mode 100644 index 0000000000..50ce37b5e7 --- /dev/null +++ b/mdl/translations/sites.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 + +package translations + +import ( + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/modelsdk/codec" +) + +// Site is one translatable text together with where it sits: the nearest +// enclosing storage object and the property the text hangs off. +// +// It exists so a consumer can name a text's location without knowing the +// document type. The catalog's string index used to reach five sites because +// each one was hand-written against a typed reader (page titles, enum captions, +// three microflow activities); the walk below reaches every site in the project +// with no per-type code — 17 distinct ones in a stock 11.13 app, and whatever a +// future Mendix version adds, for free. +type Site struct { + // OwnerType is the $Type of the nearest enclosing storage object, e.g. + // "Forms$ActionButton". It is the unit's own $Type for a text on the root. + OwnerType string + // Property is the key the Texts$Text hangs off, e.g. "Caption". + Property string + // ElementID is the owner's $ID as a UUID string, empty when it has none. + // + // Load-bearing for grouping: several sibling elements of one type carry the + // same OwnerType and Property, so a consumer that groups without this folds + // an enumeration's twelve values into one and calls the set complete as soon + // as any one value is translated. + ElementID string + // Targets is language code → text, exactly as stored. A language present + // with an empty string is a text that exists but is not translated yet. + Targets map[string]string +} + +// SitesIn returns every translatable text in a decoded document, in document +// order. Order is stable so a caller writing rows gets a deterministic result. +func SitesIn(doc bson.D) []Site { + var out []Site + var walk func(v any, ownerType, ownerID, prop string) + walk = func(v any, ownerType, ownerID, prop string) { + switch n := v.(type) { + case bson.D: + if ty, _ := lookup(n, "$Type").(string); ty == "Texts$Text" { + out = append(out, Site{ + OwnerType: ownerType, + Property: prop, + ElementID: ownerID, + Targets: translationsOf(n), + }) + return + } + // A node with its own $Type becomes the owner for everything below + // it. A node without one (an anonymous sub-document) leaves the + // owner as it was, so the text is still attributed to a real + // element rather than to nothing. + nt, nid := ownerType, ownerID + if ty, _ := lookup(n, "$Type").(string); ty != "" { + nt = ty + nid = elementIDOf(n) + } + for _, e := range n { + walk(e.Value, nt, nid, e.Key) + } + case bson.A: + // An array element inherits the property its array hangs off, so a + // text inside `Widgets` is not reported as living at "Widgets". + for _, e := range n { + walk(e, ownerType, ownerID, prop) + } + } + } + walk(doc, "", "", "") + return out +} + +// SitesInUnit is SitesIn over a unit's raw stored bytes. +func SitesInUnit(raw []byte) ([]Site, error) { + var doc bson.D + if err := bson.Unmarshal(raw, &doc); err != nil { + return nil, err + } + return SitesIn(doc), nil +} + +// elementIDOf reads a storage object's $ID as a UUID string. Real documents +// store it as a 16-byte binary in .NET field order; tests and a few synthetic +// documents use a plain string, which is passed through unchanged. +func elementIDOf(d bson.D) string { + switch v := lookup(d, "$ID").(type) { + case bson.Binary: + return codec.BinaryToUUID(v.Data) + case string: + return v + default: + return "" + } +} diff --git a/mdl/translations/sites_test.go b/mdl/translations/sites_test.go new file mode 100644 index 0000000000..ad0841df4e --- /dev/null +++ b/mdl/translations/sites_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +package translations + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/modelsdk/mpr" +) + +// The point of the walk: it names the site a text lives at without knowing +// anything about the document type. A widget caption and a page title are found +// by the same code, which is what the hand-written catalog extractor could not do. +func TestSitesIn_NamesTheOwnerAndPropertyOfEveryText(t *testing.T) { + pageID := "11111111-1111-1111-1111-111111111111" + btnID := "22222222-2222-2222-2222-222222222222" + + doc := bson.D{ + {Key: "$ID", Value: mpr.IDToBsonBinary(pageID)}, + {Key: "$Type", Value: "Forms$Page"}, + {Key: "Title", Value: text(tr("en_US", "Home"))}, + {Key: "Widgets", Value: bson.A{ + bson.D{ + {Key: "$ID", Value: mpr.IDToBsonBinary(btnID)}, + {Key: "$Type", Value: "Forms$ActionButton"}, + {Key: "Caption", Value: text(tr("en_US", "Save"), tr("nl_NL", "Opslaan"))}, + {Key: "Tooltip", Value: text(tr("en_US", "Save this record"))}, + }, + }}, + } + + got := SitesIn(doc) + if len(got) != 3 { + t.Fatalf("want 3 sites, got %d: %+v", len(got), got) + } + + byProp := map[string]Site{} + for _, s := range got { + byProp[s.OwnerType+"."+s.Property] = s + } + + title, ok := byProp["Forms$Page.Title"] + if !ok { + t.Fatalf("page title site missing; got %v", siteKeys(byProp)) + } + if title.Targets["en_US"] != "Home" { + t.Errorf("title en_US = %q, want %q", title.Targets["en_US"], "Home") + } + if title.ElementID != pageID { + t.Errorf("title ElementID = %q, want the page's %q", title.ElementID, pageID) + } + + // The caption is the one the hand-written builder never reached. + capt, ok := byProp["Forms$ActionButton.Caption"] + if !ok { + t.Fatalf("action button caption site missing; got %v", siteKeys(byProp)) + } + if capt.Targets["nl_NL"] != "Opslaan" { + t.Errorf("caption nl_NL = %q, want %q", capt.Targets["nl_NL"], "Opslaan") + } + if capt.ElementID != btnID { + t.Errorf("caption ElementID = %q, want the button's %q, not the page's", capt.ElementID, btnID) + } + + if _, ok := byProp["Forms$ActionButton.Tooltip"]; !ok { + t.Errorf("tooltip site missing; got %v", siteKeys(byProp)) + } +} + +// Two texts on sibling elements of the same type must stay distinguishable, or a +// consumer grouping by (document, property) folds them into one — which is the +// QUAL005 defect the ElementID exists to let a caller avoid. +func TestSitesIn_SiblingElementsOfOneTypeKeepSeparateIDs(t *testing.T) { + lowID := "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + highID := "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" + + doc := bson.D{ + {Key: "$Type", Value: "Enumerations$Enumeration"}, + {Key: "Values", Value: bson.A{ + bson.D{ + {Key: "$ID", Value: mpr.IDToBsonBinary(lowID)}, + {Key: "$Type", Value: "Enumerations$EnumerationValue"}, + {Key: "Caption", Value: text(tr("en_US", "Low"), tr("nl_NL", "Laag"))}, + }, + bson.D{ + {Key: "$ID", Value: mpr.IDToBsonBinary(highID)}, + {Key: "$Type", Value: "Enumerations$EnumerationValue"}, + {Key: "Caption", Value: text(tr("en_US", "High"))}, // untranslated + }, + }}, + } + + got := SitesIn(doc) + if len(got) != 2 { + t.Fatalf("want 2 sites, got %d", len(got)) + } + if got[0].ElementID == got[1].ElementID { + t.Fatalf("sibling values share ElementID %q — a consumer cannot tell them apart", got[0].ElementID) + } + if got[0].ElementID != lowID || got[1].ElementID != highID { + t.Errorf("ElementIDs = %q, %q; want %q, %q", got[0].ElementID, got[1].ElementID, lowID, highID) + } +} + +// A text directly on the unit root has no enclosing element but is still a real +// site — dropping it would silently lose the document's own title. +func TestSitesIn_TextOnTheUnitRootIsStillASite(t *testing.T) { + doc := bson.D{ + {Key: "$Type", Value: "Forms$Page"}, + {Key: "Title", Value: text(tr("en_US", "Home"))}, + } + got := SitesIn(doc) + if len(got) != 1 { + t.Fatalf("want 1 site, got %d", len(got)) + } + if got[0].OwnerType != "Forms$Page" || got[0].Property != "Title" { + t.Errorf("site = %s.%s, want Forms$Page.Title", got[0].OwnerType, got[0].Property) + } +} + +func siteKeys(m map[string]Site) []string { + var out []string + for k := range m { + out = append(out, k) + } + return out +} diff --git a/mdl/visitor/microflow_parameter_position_test.go b/mdl/visitor/microflow_parameter_position_test.go new file mode 100644 index 0000000000..f107c36fc0 --- /dev/null +++ b/mdl/visitor/microflow_parameter_position_test.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// #993: `@position(x, y)` on a parameter declaration. Before the grammar took +// `annotation*` there, the reporter's script was three parse errors starting at +// "extraneous input '@'". +func TestParameterPositionAnnotationParses(t *testing.T) { + script := `create nanoflow M.NF ( + @position(300, 100) + $A: Integer, + @position(200, 100) + $B: Integer, + $C: Integer +) +returns Integer as $R +begin + @position(300, 200) + declare $R Integer = $A + $B; + @position(500, 200) + return $R; +end;` + prog, errs := Build(script) + if len(errs) != 0 { + t.Fatalf("parse errors: %s", errsText(errs)) + } + stmt, ok := prog.Statements[0].(*ast.CreateNanoflowStmt) + if !ok { + t.Fatalf("statement = %T, want *ast.CreateNanoflowStmt", prog.Statements[0]) + } + if len(stmt.Parameters) != 3 { + t.Fatalf("got %d parameters, want 3", len(stmt.Parameters)) + } + if p := stmt.Parameters[0].Position; p == nil || *p != (ast.Position{X: 300, Y: 100}) { + t.Errorf("$A position = %v, want 300,100", p) + } + if p := stmt.Parameters[1].Position; p == nil || *p != (ast.Position{X: 200, Y: 100}) { + t.Errorf("$B position = %v, want 200,100", p) + } + // Control: an unannotated parameter stays unset, so the layout places it. + // Without this the test would pass against a visitor that stamped every + // parameter with the same point. + if p := stmt.Parameters[2].Position; p != nil { + t.Errorf("$C position = %v, want nil — no annotation was written", *p) + } +} + +// An annotation a parameter does not take is recorded, not dropped, so MDL059 +// can refuse it. `@postion` is the case that matters: silently ignoring it +// discards exactly the placement the author asked for. +func TestUnknownParameterAnnotationIsRecorded(t *testing.T) { + prog, errs := Build(`create nanoflow M.NF ( + @postion(300, 100) + $A: Integer +) begin @position(1, 2) return; end;`) + if len(errs) != 0 { + t.Fatalf("parse errors: %s", errsText(errs)) + } + stmt := prog.Statements[0].(*ast.CreateNanoflowStmt) + got := stmt.Parameters[0].UnknownAnnotations + if len(got) != 1 || got[0] != "postion" { + t.Errorf("UnknownAnnotations = %v, want [postion]", got) + } + if stmt.Parameters[0].Position != nil { + t.Errorf("a typo set a position: %v", *stmt.Parameters[0].Position) + } +} + +// `@position` with too few arguments is not a position — it is recorded as +// unknown rather than silently producing 0;0, which would be a real coordinate +// and would look deliberate. +func TestMalformedParameterPositionIsNotSilentlyZero(t *testing.T) { + prog, errs := Build(`create nanoflow M.NF ( + @position(300) + $A: Integer +) begin @position(1, 2) return; end;`) + if len(errs) != 0 { + t.Fatalf("parse errors: %s", errsText(errs)) + } + p := prog.Statements[0].(*ast.CreateNanoflowStmt).Parameters[0] + if p.Position != nil { + t.Errorf("position = %v, want nil", *p.Position) + } + if len(p.UnknownAnnotations) != 1 { + t.Errorf("UnknownAnnotations = %v, want one entry", p.UnknownAnnotations) + } +} diff --git a/mdl/visitor/visitor_agenteditor.go b/mdl/visitor/visitor_agenteditor.go index e316ec106d..1d152eb1f5 100644 --- a/mdl/visitor/visitor_agenteditor.go +++ b/mdl/visitor/visitor_agenteditor.go @@ -18,7 +18,7 @@ func (b *Builder) ExitCreateModelStatement(ctx *parser.CreateModelStatementConte stmt := &ast.CreateModelStmt{ Name: buildQualifiedName(ctx.QualifiedName()), } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) if lit := ctx.STRING_LITERAL(); lit != nil { stmt.Folder = unquoteString(lit.GetText()) } @@ -158,7 +158,7 @@ func (b *Builder) ExitCreateConsumedMCPServiceStatement(ctx *parser.CreateConsum stmt := &ast.CreateConsumedMCPServiceStmt{ Name: buildQualifiedName(ctx.QualifiedName()), } - stmt.OuterDocumentation = findDocCommentText(ctx) + stmt.OuterDocumentation, stmt.DocumentationSet = findDocComment(ctx) if lit := ctx.STRING_LITERAL(); lit != nil { stmt.Folder = unquoteString(lit.GetText()) } @@ -184,7 +184,7 @@ func (b *Builder) ExitCreateKnowledgeBaseStatement(ctx *parser.CreateKnowledgeBa stmt := &ast.CreateKnowledgeBaseStmt{ Name: buildQualifiedName(ctx.QualifiedName()), } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) if lit := ctx.STRING_LITERAL(); lit != nil { stmt.Folder = unquoteString(lit.GetText()) } @@ -215,7 +215,7 @@ func (b *Builder) ExitCreateAgentStatement(ctx *parser.CreateAgentStatementConte stmt := &ast.CreateAgentStmt{ Name: buildQualifiedName(ctx.QualifiedName()), } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) if lit := ctx.STRING_LITERAL(); lit != nil { stmt.Folder = unquoteString(lit.GetText()) } diff --git a/mdl/visitor/visitor_businessevents.go b/mdl/visitor/visitor_businessevents.go index 1922d6392c..ee4a57bd4d 100644 --- a/mdl/visitor/visitor_businessevents.go +++ b/mdl/visitor/visitor_businessevents.go @@ -87,7 +87,7 @@ func (b *Builder) ExitCreateBusinessEventServiceStatement(ctx *parser.CreateBusi stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } diff --git a/mdl/visitor/visitor_entity.go b/mdl/visitor/visitor_entity.go index 96377216fd..96aa7d5cb7 100644 --- a/mdl/visitor/visitor_entity.go +++ b/mdl/visitor/visitor_entity.go @@ -57,7 +57,7 @@ func (b *Builder) ExitCreateEntityStatement(ctx *parser.CreateEntityStatementCon } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) // Generalization clause (EXTENDS/GENERALIZATION before entity body) if genClause := ctx.GeneralizationClause(); genClause != nil { @@ -177,7 +177,7 @@ func (b *Builder) buildViewEntity(ctx *parser.CreateEntityStatementContext) { } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) // Entity body (view attributes) if body := ctx.EntityBody(); body != nil { @@ -450,19 +450,33 @@ func buildViewAttributes(attrList parser.IAttributeDefinitionListContext) []ast. // findDocCommentText extracts the documentation comment text for a create statement. // The docComment can appear either on the createStatement itself or on the parent statement rule. func findDocCommentText(ctx antlr.RuleContext) string { + text, _ := findDocComment(ctx) + return text +} + +// findDocComment returns the statement's doc comment and whether one was +// written at all. The two are different facts and the caller needs both: +// an ABSENT comment on a rewrite must preserve whatever is stored, while an +// explicitly EMPTY one (`/** */`) must clear it. +// +// Without the second return value they are indistinguishable — both yield "" — +// and a rewrite that says nothing about documentation silently deletes it +// (mendixlabs/mxcli#1018). There is no `ALTER MICROFLOW … SET DOCUMENTATION` to +// fall back on, so the empty comment is the only clearing spelling available. +func findDocComment(ctx antlr.RuleContext) (string, bool) { createStmt := findParentCreateStatement(ctx) if createStmt != nil { if docCtx := createStmt.DocComment(); docCtx != nil { - return extractDocComment(docCtx.GetText()) + return extractDocComment(docCtx.GetText()), true } } stmtCtx := findParentStatement(ctx) if stmtCtx != nil { if docCtx := stmtCtx.DocComment(); docCtx != nil { - return extractDocComment(docCtx.GetText()) + return extractDocComment(docCtx.GetText()), true } } - return "" + return "", false } // findParentCreateStatement navigates up the parse tree to find the CreateStatement parent. @@ -922,6 +936,10 @@ func (b *Builder) ExitDropStatement(ctx *parser.DropStatementContext) { b.statements = append(b.statements, &ast.DropJsonStructureStmt{ Name: buildQualifiedName(names[0]), }) + } else if ctx.MESSAGE() != nil && ctx.DEFINITION() != nil && ctx.COLLECTION() != nil { + b.statements = append(b.statements, &ast.DropMessageDefinitionCollectionStmt{ + Name: buildQualifiedName(names[0]), + }) } else if ctx.IMPORT() != nil && ctx.MAPPING() != nil { b.statements = append(b.statements, &ast.DropImportMappingStmt{ Name: buildQualifiedName(names[0]), diff --git a/mdl/visitor/visitor_enumeration.go b/mdl/visitor/visitor_enumeration.go index bfd94f6a5a..8624910675 100644 --- a/mdl/visitor/visitor_enumeration.go +++ b/mdl/visitor/visitor_enumeration.go @@ -31,7 +31,7 @@ func (b *Builder) ExitCreateEnumerationStatement(ctx *parser.CreateEnumerationSt stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } @@ -131,7 +131,7 @@ func (b *Builder) ExitCreateConstantStatement(ctx *parser.CreateConstantStatemen stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } diff --git a/mdl/visitor/visitor_imagecollection.go b/mdl/visitor/visitor_imagecollection.go index 4d8f9c51a5..c39f5cc72e 100644 --- a/mdl/visitor/visitor_imagecollection.go +++ b/mdl/visitor/visitor_imagecollection.go @@ -18,7 +18,7 @@ func (b *Builder) ExitCreateImageCollectionStatement(ctx *parser.CreateImageColl } // Extract /** ... */ doc comment (same as other create statements) - stmt.Comment = findDocCommentText(ctx) + stmt.Comment, stmt.DocumentationSet = findDocComment(ctx) if opts := ctx.ImageCollectionOptions(); opts != nil { optsCtx := opts.(*parser.ImageCollectionOptionsContext) diff --git a/mdl/visitor/visitor_javaaction.go b/mdl/visitor/visitor_javaaction.go index 74a67b1e03..a6be0a27d3 100644 --- a/mdl/visitor/visitor_javaaction.go +++ b/mdl/visitor/visitor_javaaction.go @@ -94,7 +94,7 @@ func (b *Builder) ExitCreateJavaActionStatement(ctx *parser.CreateJavaActionStat // Check for documentation comment and OR MODIFY/REPLACE from parent createStatement if parent, ok := ctx.GetParent().(*parser.CreateStatementContext); ok { if docComment := parent.DocComment(); docComment != nil { - stmt.Documentation = extractDocComment(docComment.GetText()) + stmt.Documentation, stmt.DocumentationSet = extractDocComment(docComment.GetText()), true } if parent.OR() != nil && (parent.MODIFY() != nil || parent.REPLACE() != nil) { stmt.CreateOrModify = true @@ -105,7 +105,7 @@ func (b *Builder) ExitCreateJavaActionStatement(ctx *parser.CreateJavaActionStat if stmt.Documentation == "" { if stmtCtx := findParentStatement(ctx); stmtCtx != nil { if docCtx := stmtCtx.DocComment(); docCtx != nil { - stmt.Documentation = extractDocComment(docCtx.GetText()) + stmt.Documentation, stmt.DocumentationSet = extractDocComment(docCtx.GetText()), true } } } @@ -194,7 +194,7 @@ func (b *Builder) ExitCreateJavaScriptActionStatement(ctx *parser.CreateJavaScri if parent, ok := ctx.GetParent().(*parser.CreateStatementContext); ok { if docComment := parent.DocComment(); docComment != nil { - stmt.Documentation = extractDocComment(docComment.GetText()) + stmt.Documentation, stmt.DocumentationSet = extractDocComment(docComment.GetText()), true } if parent.OR() != nil && (parent.MODIFY() != nil || parent.REPLACE() != nil) { stmt.CreateOrModify = true @@ -203,7 +203,7 @@ func (b *Builder) ExitCreateJavaScriptActionStatement(ctx *parser.CreateJavaScri if stmt.Documentation == "" { if stmtCtx := findParentStatement(ctx); stmtCtx != nil { if docCtx := stmtCtx.DocComment(); docCtx != nil { - stmt.Documentation = extractDocComment(docCtx.GetText()) + stmt.Documentation, stmt.DocumentationSet = extractDocComment(docCtx.GetText()), true } } } diff --git a/mdl/visitor/visitor_menu.go b/mdl/visitor/visitor_menu.go index 5d85999f44..6353c5e36a 100644 --- a/mdl/visitor/visitor_menu.go +++ b/mdl/visitor/visitor_menu.go @@ -19,6 +19,10 @@ func (b *Builder) ExitCreateMenuStatement(ctx *parser.CreateMenuStatementContext } stmt := &ast.CreateMenuStmt{Name: buildQualifiedName(qn)} + // A menu's doc comment was parsed and then read by nobody, so the + // documentation never reached the model at all — a write gap rather than + // the rewrite gap of #1018, found while testing the latter. + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) if lit := ctx.STRING_LITERAL(); lit != nil { stmt.Folder = unquoteString(lit.GetText()) } diff --git a/mdl/visitor/visitor_messagedefinition.go b/mdl/visitor/visitor_messagedefinition.go new file mode 100644 index 0000000000..960e385043 --- /dev/null +++ b/mdl/visitor/visitor_messagedefinition.go @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/grammar/parser" +) + +// ExitCreateMessageDefinitionCollectionStatement builds +// +// CREATE [OR MODIFY] MESSAGE DEFINITION COLLECTION Module.Name +// [FOLDER 'path'] +// ( definition Name for Module.Entity [as 'Exposed'] ( members ), ... ); +func (b *Builder) ExitCreateMessageDefinitionCollectionStatement(ctx *parser.CreateMessageDefinitionCollectionStatementContext) { + stmt := &ast.CreateMessageDefinitionCollectionStmt{ + Name: buildQualifiedName(ctx.QualifiedName()), + } + if ctx.FOLDER() != nil { + if lit := ctx.STRING_LITERAL(); lit != nil { + stmt.Folder = unquoteString(lit.GetText()) + } + } + for _, d := range ctx.AllMessageDefinitionDef() { + if def := b.buildMessageDefinitionDef(d); def != nil { + stmt.Definitions = append(stmt.Definitions, def) + } + } + if createStmt := findParentCreateStatement(ctx); createStmt != nil { + if createStmt.OR() != nil && (createStmt.REPLACE() != nil || createStmt.MODIFY() != nil) { + stmt.CreateOrModify = true + } + } + b.statements = append(b.statements, stmt) +} + +// buildMessageDefinitionDef builds one `definition for ` block. +func (b *Builder) buildMessageDefinitionDef(c parser.IMessageDefinitionDefContext) *ast.MessageDefinitionDef { + ctx, ok := c.(*parser.MessageDefinitionDefContext) + if ctx == nil || !ok { + return nil + } + def := &ast.MessageDefinitionDef{ + Name: identifierOrKeywordText(ctx.IdentifierOrKeyword()), + Entity: buildQualifiedName(ctx.QualifiedName()), + ExposedName: exposedNameOf(ctx.MessageExposedName()), + } + for _, m := range ctx.AllMessageMember() { + if mem := b.buildMessageMember(m); mem != nil { + def.Members = append(def.Members, mem) + } + } + return def +} + +// buildMessageMember builds an exposed attribute or an exposed association. +// +// The discriminator is the association form's two qualified names separated by a +// slash — the same one import and export mappings use, so a reader who knows one +// knows the other. +func (b *Builder) buildMessageMember(c parser.IMessageMemberContext) *ast.MessageMemberDef { + ctx, ok := c.(*parser.MessageMemberContext) + if ctx == nil || !ok { + return nil + } + mem := &ast.MessageMemberDef{ExposedName: exposedNameOf(ctx.MessageExposedName())} + + if qns := ctx.AllQualifiedName(); len(qns) == 2 { + // Association: Assoc/Module.Entity ( members ). The target entity is + // spelled out because MaxOccurs tracks the DIRECTION of traversal, not + // the association's type — see the AST doc comment. + mem.Association = buildQualifiedName(qns[0]) + mem.Entity = buildQualifiedName(qns[1]) + for _, sub := range ctx.AllMessageMember() { + if child := b.buildMessageMember(sub); child != nil { + mem.Members = append(mem.Members, child) + } + } + return mem + } + + mem.Attribute = identifierOrKeywordText(ctx.IdentifierOrKeyword()) + if ex, ok := ctx.MessageExample().(*parser.MessageExampleContext); ok && ex != nil { + if lit := ex.STRING_LITERAL(); lit != nil { + mem.Example = unquoteString(lit.GetText()) + } + } + return mem +} + +// exposedNameOf reads the optional `as 'Name'` clause. +func exposedNameOf(c parser.IMessageExposedNameContext) string { + ctx, ok := c.(*parser.MessageExposedNameContext) + if ctx == nil || !ok { + return "" + } + if lit := ctx.STRING_LITERAL(); lit != nil { + return unquoteString(lit.GetText()) + } + return "" +} + +// ExitAlterMessageDefinitionCollectionStatement builds the collection-level +// ALTER: add, drop or rename a DEFINITION. +func (b *Builder) ExitAlterMessageDefinitionCollectionStatement(ctx *parser.AlterMessageDefinitionCollectionStatementContext) { + stmt := &ast.AlterMessageDefinitionCollectionStmt{ + Name: buildQualifiedName(ctx.QualifiedName()), + } + op, ok := ctx.AlterMessageCollectionOperation().(*parser.AlterMessageCollectionOperationContext) + if !ok || op == nil { + return + } + ids := op.AllIdentifierOrKeyword() + + switch { + case op.ADD() != nil: + stmt.Op = "ADD" + stmt.IfNotExist = op.NOT() != nil + def := &ast.MessageDefinitionDef{ + Entity: buildQualifiedName(op.QualifiedName()), + ExposedName: exposedNameOf(op.MessageExposedName()), + } + if len(ids) > 0 { + def.Name = identifierOrKeywordText(ids[0]) + } + for _, m := range op.AllMessageMember() { + if mem := b.buildMessageMember(m); mem != nil { + def.Members = append(def.Members, mem) + } + } + stmt.Definition = def + case op.RENAME() != nil: + stmt.Op = "RENAME" + if len(ids) > 0 { + stmt.Target = identifierOrKeywordText(ids[0]) + } + if len(ids) > 1 { + stmt.NewName = identifierOrKeywordText(ids[1]) + } + case op.DROP() != nil: + stmt.Op = "DROP" + stmt.IfExists = op.IF() != nil + if len(ids) > 0 { + stmt.Target = identifierOrKeywordText(ids[0]) + } + default: + return + } + b.statements = append(b.statements, stmt) +} + +// ExitAlterMessageDefinitionStatement builds the member-level ALTER. +// +// The definition is addressed as Module.Collection.Definition — the same +// three-part reference `WITH MESSAGE DEFINITION` takes — so the last segment is +// split off here rather than being a separate clause. +func (b *Builder) ExitAlterMessageDefinitionStatement(ctx *parser.AlterMessageDefinitionStatementContext) { + full := buildQualifiedName(ctx.QualifiedName()) + collection, definition, ok := splitDefinitionRef(full) + if !ok { + b.addErrorWithExample( + "ALTER MESSAGE DEFINITION takes a three-part name — Module.Collection.Definition, "+ + "the same reference WITH MESSAGE DEFINITION uses", + "alter message definition Sales.MD_Order.Order add member Total;") + return + } + stmt := &ast.AlterMessageDefinitionStmt{Collection: collection, Definition: definition} + + op, opOK := ctx.AlterMessageDefinitionOperation().(*parser.AlterMessageDefinitionOperationContext) + if !opOK || op == nil { + return + } + if p, pathOK := op.MessageMemberPath().(*parser.MessageMemberPathContext); pathOK && p != nil { + for _, seg := range p.AllIdentifierOrKeyword() { + stmt.Path = append(stmt.Path, identifierOrKeywordText(seg)) + } + } + // The path's segments live inside the MessageMemberPath sub-rule, so the + // operation's own identifier is the only direct IdentifierOrKeyword child — + // no slicing needed to tell them apart. + target := identifierOrKeywordText(op.IdentifierOrKeyword()) + + switch { + case op.ADD() != nil: + stmt.Op = "ADD" + stmt.IfNotExist = op.NOT() != nil + stmt.Member = b.buildMessageMember(op.MessageMember()) + case op.SET() != nil: + stmt.Op = "SET" + stmt.Target = target + stmt.ExposedName = exposedNameOf(op.MessageExposedName()) + case op.DROP() != nil: + stmt.Op = "DROP" + stmt.IfExists = op.IF() != nil + stmt.Target = target + default: + return + } + b.statements = append(b.statements, stmt) +} + +// splitDefinitionRef splits Module.Collection.Definition into the collection's +// qualified name and the definition's name. +func splitDefinitionRef(qn ast.QualifiedName) (ast.QualifiedName, string, bool) { + // buildQualifiedName puts everything after the module into Name, so a + // three-part reference arrives as Module="Sales", Name="MD_Order.Order". + if qn.Module == "" || !strings.Contains(qn.Name, ".") { + return ast.QualifiedName{}, "", false + } + idx := strings.LastIndex(qn.Name, ".") + return ast.QualifiedName{Module: qn.Module, Name: qn.Name[:idx]}, qn.Name[idx+1:], true +} + +// DROP MESSAGE DEFINITION COLLECTION is built alongside the other DROP forms, in +// visitor_entity.go's dropStatement switch. diff --git a/mdl/visitor/visitor_messagedefinition_test.go b/mdl/visitor/visitor_messagedefinition_test.go new file mode 100644 index 0000000000..628adbd074 --- /dev/null +++ b/mdl/visitor/visitor_messagedefinition_test.go @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func buildOne[T ast.Statement](t *testing.T, src string) T { + t.Helper() + prog, errs := Build(src) + if len(errs) > 0 { + t.Fatalf("parse error: %v", errs[0]) + } + if len(prog.Statements) != 1 { + t.Fatalf("got %d statements, want 1", len(prog.Statements)) + } + stmt, ok := prog.Statements[0].(T) + if !ok { + t.Fatalf("got %T, want %T", prog.Statements[0], *new(T)) + } + return stmt +} + +// A message definition collection is a selection over the domain model, which is +// what makes it authorable where a mapping's other non-JSON sources are not +// (ako/mxcli#272). It is the source for 74 of the 327 mappings in the demo +// corpus. + +func TestCreateMessageDefinitionCollection(t *testing.T) { + s := buildOne[*ast.CreateMessageDefinitionCollectionStmt](t, `create message definition collection Sales.MD_Order + folder 'Private/Messages' +( + definition Order for Sales.Order as 'Orders' ( + OrderId, + Total as 'GrandTotal', + Sales.Order_Line/Sales.Line as 'Lines' ( Sku, Quantity ), + Sales.Order_Customer/Sales.Customer ( Name ) + ), + definition Line for Sales.Line ( Sku ) +);`) + + if s.Name.String() != "Sales.MD_Order" || s.Folder != "Private/Messages" { + t.Fatalf("name=%q folder=%q", s.Name.String(), s.Folder) + } + if len(s.Definitions) != 2 { + t.Fatalf("got %d definitions, want 2", len(s.Definitions)) + } + + d := s.Definitions[0] + // The definition's Name and its root's ExposedName are independent — 19 of + // 56 real definitions are named something other than their entity. + if d.Name != "Order" || d.Entity.String() != "Sales.Order" || d.ExposedName != "Orders" { + t.Errorf("definition = %+v", d) + } + if len(d.Members) != 4 { + t.Fatalf("got %d members, want 4", len(d.Members)) + } + + if d.Members[0].IsAssociation() || d.Members[0].Attribute != "OrderId" || d.Members[0].ExposedName != "" { + t.Errorf("member 0 = %+v, want the attribute OrderId with no rename", d.Members[0]) + } + if d.Members[1].ExposedName != "GrandTotal" { + t.Errorf("member 1 exposed name = %q", d.Members[1].ExposedName) + } + + // An association names its TARGET entity. That is load-bearing rather than + // decorative: the stored MaxOccurs tracks the DIRECTION of traversal, not + // the association's type, so the target is what makes the direction explicit. + assoc := d.Members[2] + if !assoc.IsAssociation() { + t.Fatalf("member 2 is not an association: %+v", assoc) + } + if assoc.Association.String() != "Sales.Order_Line" || assoc.Entity.String() != "Sales.Line" { + t.Errorf("association = %s -> %s", assoc.Association.String(), assoc.Entity.String()) + } + if assoc.ExposedName != "Lines" || len(assoc.Members) != 2 { + t.Errorf("association exposed=%q members=%d", assoc.ExposedName, len(assoc.Members)) + } + if d.Members[3].ExposedName != "" || len(d.Members[3].Members) != 1 { + t.Errorf("member 3 = %+v", d.Members[3]) + } +} + +func TestCreateMessageDefinitionCollectionOrModify(t *testing.T) { + s := buildOne[*ast.CreateMessageDefinitionCollectionStmt](t, + `create or modify message definition collection S.MD ( definition A for S.A ( X ) );`) + if !s.CreateOrModify { + t.Error("OR MODIFY not recorded — a fresh document would break every WITH MESSAGE DEFINITION") + } +} + +func TestAlterMessageDefinitionCollection(t *testing.T) { + add := buildOne[*ast.AlterMessageDefinitionCollectionStmt](t, + `alter message definition collection S.MD add definition X for S.X as 'Xs' ( A );`) + if add.Op != "ADD" || add.Definition == nil { + t.Fatalf("add = %+v", add) + } + if add.Definition.Name != "X" || add.Definition.ExposedName != "Xs" || len(add.Definition.Members) != 1 { + t.Errorf("added definition = %+v", add.Definition) + } + + drop := buildOne[*ast.AlterMessageDefinitionCollectionStmt](t, + `alter message definition collection S.MD drop definition if exists X;`) + if drop.Op != "DROP" || drop.Target != "X" || !drop.IfExists { + t.Errorf("drop = %+v", drop) + } + + ren := buildOne[*ast.AlterMessageDefinitionCollectionStmt](t, + `alter message definition collection S.MD rename definition X to Y;`) + if ren.Op != "RENAME" || ren.Target != "X" || ren.NewName != "Y" { + t.Errorf("rename = %+v", ren) + } +} + +// TestAlterMessageDefinitionSplitsTheThreePartName pins the address. A +// definition is referred to as Module.Collection.Definition — the same reference +// WITH MESSAGE DEFINITION takes — so the two cannot drift apart. +func TestAlterMessageDefinitionSplitsTheThreePartName(t *testing.T) { + s := buildOne[*ast.AlterMessageDefinitionStmt](t, + `alter message definition Sales.MD_Order.Order add member Total;`) + if s.Collection.String() != "Sales.MD_Order" || s.Definition != "Order" { + t.Fatalf("collection=%q definition=%q", s.Collection.String(), s.Definition) + } + if s.Op != "ADD" || s.Member == nil || s.Member.Attribute != "Total" { + t.Errorf("stmt = %+v member=%+v", s, s.Member) + } +} + +func TestAlterMessageDefinitionRejectsATwoPartName(t *testing.T) { + _, errs := Build(`alter message definition Sales.MD_Order add member Total;`) + if len(errs) == 0 { + t.Fatal("accepted a two-part name — the collection and definition would be indistinguishable") + } +} + +// TestAlterMessageDefinitionMemberPath pins the nested address. Members nest to +// depth 7 in the corpus, so reaching one is the common case, not an edge case. +func TestAlterMessageDefinitionMemberPath(t *testing.T) { + drop := buildOne[*ast.AlterMessageDefinitionStmt](t, + `alter message definition S.MD.D drop member Sku in Lines/Prices;`) + if drop.Op != "DROP" || drop.Target != "Sku" { + t.Fatalf("drop = %+v", drop) + } + // The path's segments and the operation's own identifier must not be + // confused: both are IdentifierOrKeyword, and the path lives in its own + // sub-rule precisely so they stay apart. + if len(drop.Path) != 2 || drop.Path[0] != "Lines" || drop.Path[1] != "Prices" { + t.Errorf("path = %v, want [Lines Prices]", drop.Path) + } +} + +// TestAlterMessageDefinitionSetIsNotARename pins that SET carries an exposed +// name and no model rename. ALTER ENTITY's RENAME ATTRIBUTE changes the model +// and rewrites every reference; this changes one element's ExposedName. +func TestAlterMessageDefinitionSetIsNotARename(t *testing.T) { + s := buildOne[*ast.AlterMessageDefinitionStmt](t, + `alter message definition S.MD.D set member Total as 'GrandTotal';`) + if s.Op != "SET" || s.Target != "Total" || s.ExposedName != "GrandTotal" { + t.Errorf("stmt = %+v", s) + } +} + +func TestDropMessageDefinitionCollection(t *testing.T) { + s := buildOne[*ast.DropMessageDefinitionCollectionStmt](t, + `drop message definition collection Sales.MD_Order;`) + if s.Name.String() != "Sales.MD_Order" { + t.Errorf("name = %q", s.Name.String()) + } +} diff --git a/mdl/visitor/visitor_microflow.go b/mdl/visitor/visitor_microflow.go index 2d964ebdac..f60e52b783 100644 --- a/mdl/visitor/visitor_microflow.go +++ b/mdl/visitor/visitor_microflow.go @@ -57,7 +57,7 @@ func (b *Builder) ExitCreateMicroflowStatement(ctx *parser.CreateMicroflowStatem } } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } @@ -109,7 +109,7 @@ func (b *Builder) ExitCreateNanoflowStatement(ctx *parser.CreateNanoflowStatemen } } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } @@ -154,7 +154,7 @@ func (b *Builder) ExitCreateRuleStatement(ctx *parser.CreateRuleStatementContext } } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } @@ -310,12 +310,46 @@ func buildMicroflowParameters(ctx parser.IMicroflowParameterListContext) []ast.M param.Type = buildMicroflowDataType(dt) } + applyParameterAnnotations(¶m, p.AllAnnotation()) + params = append(params, param) } return params } +// applyParameterAnnotations reads the annotations written on a parameter. +// +// Only @position(x, y) means anything on a parameter — it is a stored node with +// its own coordinates, and nothing else about it is a canvas property. Every +// other name is recorded rather than ignored, so MDL059 can refuse it; an +// annotation that parses and does nothing is the failure mode #884 was about, +// and a typo of `@position` is exactly the case that has to be caught. +func applyParameterAnnotations(param *ast.MicroflowParam, annotations []parser.IAnnotationContext) { + for _, annCtx := range annotations { + ann := annCtx.(*parser.AnnotationContext) + name := strings.ToLower(ann.AnnotationName().GetText()) + if name != "position" { + param.UnknownAnnotations = append(param.UnknownAnnotations, name) + continue + } + params := ann.AnnotationParams() + if params == nil { + param.UnknownAnnotations = append(param.UnknownAnnotations, name) + continue + } + all := params.(*parser.AnnotationParamsContext).AllAnnotationParam() + if len(all) < 2 { + param.UnknownAnnotations = append(param.UnknownAnnotations, name) + continue + } + param.Position = &ast.Position{ + X: parseAnnotationParamInt(all[0]), + Y: parseAnnotationParamInt(all[1]), + } + } +} + // buildMicroflowReturnType converts return type context to MicroflowReturnType. func buildMicroflowReturnType(ctx parser.IMicroflowReturnTypeContext) *ast.MicroflowReturnType { if ctx == nil { diff --git a/mdl/visitor/visitor_odata.go b/mdl/visitor/visitor_odata.go index bbddd13ae1..3a1d23f401 100644 --- a/mdl/visitor/visitor_odata.go +++ b/mdl/visitor/visitor_odata.go @@ -89,7 +89,7 @@ func (b *Builder) ExitCreateODataClientStatement(ctx *parser.CreateODataClientSt stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } @@ -161,7 +161,7 @@ func (b *Builder) ExitCreateODataServiceStatement(ctx *parser.CreateODataService stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } @@ -222,7 +222,7 @@ func (b *Builder) ExitCreateExternalEntityStatement(ctx *parser.CreateExternalEn stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } diff --git a/mdl/visitor/visitor_page_v3.go b/mdl/visitor/visitor_page_v3.go index 490f2e36ea..4713b5a2d6 100644 --- a/mdl/visitor/visitor_page_v3.go +++ b/mdl/visitor/visitor_page_v3.go @@ -55,7 +55,7 @@ func (b *Builder) buildPageV3(ctx *parser.CreatePageStatementContext) *ast.Creat stmt.IsModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) for _, ann := range createStmt.AllAnnotation() { annCtx := ann.(*parser.AnnotationContext) if strings.EqualFold(annCtx.AnnotationName().GetText(), "excluded") { @@ -208,7 +208,7 @@ func (b *Builder) buildSnippetV3(ctx *parser.CreateSnippetStatementContext) *ast stmt.IsModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) } // Parse V3 header @@ -1801,7 +1801,7 @@ func (b *Builder) buildLayoutV3(ctx *parser.CreateLayoutStatementContext) *ast.C stmt.IsReplace = createStmt.REPLACE() != nil stmt.IsModify = createStmt.MODIFY() != nil } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) } if props := ctx.WidgetPropertiesV3(); props != nil { holder := &ast.WidgetV3{Properties: map[string]any{}} diff --git a/mdl/visitor/visitor_query.go b/mdl/visitor/visitor_query.go index 142a82a305..88e9b6a460 100644 --- a/mdl/visitor/visitor_query.go +++ b/mdl/visitor/visitor_query.go @@ -711,6 +711,17 @@ func (b *Builder) ExitShowStatement(ctx *parser.ShowStatementContext) { } } b.statements = append(b.statements, stmt) + } else if ctx.MESSAGE() != nil && ctx.DEFINITION() != nil && ctx.COLLECTION() != nil { + // SHOW MESSAGE DEFINITION COLLECTIONS [IN module] + stmt := &ast.ShowStmt{ObjectType: ast.ShowMessageDefinitionCollections} + if ctx.IN() != nil { + if qn := ctx.QualifiedName(); qn != nil { + stmt.InModule = getQualifiedNameText(qn) + } else if id := ctx.IDENTIFIER(); id != nil { + stmt.InModule = id.GetText() + } + } + b.statements = append(b.statements, stmt) } else if ctx.IMPORT() != nil && ctx.MAPPINGS() != nil { // SHOW IMPORT MAPPINGS [IN module] stmt := &ast.ShowStmt{ObjectType: ast.ShowImportMappings} @@ -1214,6 +1225,11 @@ func (b *Builder) ExitDescribeStatement(ctx *parser.DescribeStatementContext) { ObjectType: ast.DescribeDataTransformer, Name: name, }) + } else if ctx.MESSAGE() != nil && ctx.DEFINITION() != nil && ctx.COLLECTION() != nil { + b.statements = append(b.statements, &ast.DescribeStmt{ + ObjectType: ast.DescribeMessageDefinitionCollection, + Name: name, + }) } else if ctx.JSON() != nil && ctx.STRUCTURE() != nil { b.statements = append(b.statements, &ast.DescribeStmt{ ObjectType: ast.DescribeJsonStructure, diff --git a/mdl/visitor/visitor_rest.go b/mdl/visitor/visitor_rest.go index f156a3bc13..8d33a73587 100644 --- a/mdl/visitor/visitor_rest.go +++ b/mdl/visitor/visitor_rest.go @@ -96,7 +96,7 @@ func (b *Builder) ExitCreateRestClientStatement(ctx *parser.CreateRestClientStat stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) b.statements = append(b.statements, stmt) } diff --git a/mdl/visitor/visitor_workflow.go b/mdl/visitor/visitor_workflow.go index 3b6a727b3d..fcccebdec7 100644 --- a/mdl/visitor/visitor_workflow.go +++ b/mdl/visitor/visitor_workflow.go @@ -81,7 +81,7 @@ func (b *Builder) ExitCreateWorkflowStatement(ctx *parser.CreateWorkflowStatemen stmt.CreateOrModify = true } } - stmt.Documentation = findDocCommentText(ctx) + stmt.Documentation, stmt.DocumentationSet = findDocComment(ctx) // Parse body if body := ctx.WorkflowBody(); body != nil { diff --git a/model/types.go b/model/types.go index 777ab21915..1395509e3a 100644 --- a/model/types.go +++ b/model/types.go @@ -1070,6 +1070,31 @@ type DistributionSettings struct { // ============================================================================ // ImportMapping represents an ImportMappings$ImportMapping document. +// WebServiceMappingSource is a mapping's SOAP binding: the imported web service +// document plus which service, operation and root element of it the mapping +// covers. +// +// It is read-only. mxcli cannot author a SOAP mapping, and the point of reading +// it is precisely that it cannot: a rewrite that dropped these turned a working +// integration into CE6896 "A mapping must have exactly one schema source" and +// CE0270 "No root element could be found in the schema" (ako/mxcli#365). +// +// ParameterName and IsHeader exist on EXPORT mappings only — which SOAP message +// part the mapping produces, and whether it is a header — and are carried for +// the same reason. +type WebServiceMappingSource struct { + ImportedWebService string `json:"importedWebService,omitempty"` // stored as wsdlFile + ServiceName string `json:"serviceName,omitempty"` + OperationName string `json:"operationName,omitempty"` + RootElementName string `json:"rootElementName,omitempty"` // stored as xsdRootElementName + ParameterName string `json:"parameterName,omitempty"` // export only + IsHeader bool `json:"isHeader,omitempty"` // export only +} + +// IsSet reports whether the mapping is sourced from a web service. The imported +// service is the discriminator: the other fields only qualify which part of it. +func (w WebServiceMappingSource) IsSet() bool { return w.ImportedWebService != "" } + type ImportMapping struct { BaseElement ContainerID ID `json:"containerId"` @@ -1085,6 +1110,12 @@ type ImportMapping struct { // CARRIES rather than derives: nil means the stored document does not have // the key, which is not the same as present-and-empty (ako/mxcli#279). MessageDefinition2 *string `json:"messageDefinition2,omitempty"` + // WebServiceSource is the imported web service (SOAP) a mapping can be + // sourced from — a FOURTH source kind beside JSON structure, XML schema and + // message definition. mxcli does not author one, but it must not destroy + // one: the properties are read so a rewrite can be refused rather than + // silently dropping the binding (ako/mxcli#365). + WebServiceSource WebServiceMappingSource `json:"webServiceSource,omitempty"` // ParameterEntity is the entity of the mapping's INPUT object, stored as // ParameterType — a DataTypes$ObjectType naming it. Empty means the mapping // takes none, which Mendix stores as the DataTypes$UnknownType marker rather @@ -1196,9 +1227,15 @@ type MappingMicroflowParameter struct { // MessageDefinition reference is THREE parts: Module.Collection.Definition. type MessageDefinitionCollection struct { BaseElement - ContainerID ID `json:"containerId"` - Name string `json:"name"` - Definitions []*MessageDefinition `json:"definitions,omitempty"` + ContainerID ID `json:"containerId"` + Name string `json:"name"` + // Documentation, Excluded and ExportLevel are carried so a CREATE OR MODIFY + // preserves what the statement does not restate. ExportLevel is "Hidden" on + // every collection measured, but it is read rather than assumed. + Documentation string `json:"documentation,omitempty"` + Excluded bool `json:"excluded,omitempty"` + ExportLevel string `json:"exportLevel,omitempty"` + Definitions []*MessageDefinition `json:"definitions,omitempty"` } // MessageDefinition is one EntityMessageDefinition inside a collection. @@ -1219,14 +1256,26 @@ type MessageDefinition struct { type MessageDefinitionElement struct { // "Entity" or "Attribute". Kind string `json:"kind"` - // Entity/Association are set on an Entity node (Association only when the - // node is reached through one); Attribute on an Attribute node. + // Entity/Association are set on an Entity node; Attribute on an Attribute + // node. Association is set only when the node is reached through one, and + // Entity is then its TARGET — both are needed to rebuild or describe the + // node, because the stored MaxOccurs depends on the direction of traversal + // and cannot be recovered from the association alone. Entity string `json:"entity,omitempty"` Association string `json:"association,omitempty"` Attribute string `json:"attribute,omitempty"` ExposedName string `json:"exposedName,omitempty"` ExposedItemName string `json:"exposedItemName,omitempty"` + // OriginalName is the member's own name — the entity's, the attribute's, or + // for an association node the TARGET entity's. Stored beside ExposedName + // because the two differ routinely (52 of 56 roots, 406 of 933 + // associations) and Mendix keeps both. + OriginalName string `json:"originalName,omitempty"` + // Example is author-set free text. Rare — 1 of 4,707 elements across the + // demo corpus and ako/TestApp — but hardcoding it empty would silently drop + // the one that exists, so it is carried like any other authored value. + Example string `json:"example,omitempty"` // Path is the definition's own path ("Email|From"). It is NOT the mapping's // XmlPath, which is built from the exposed names — the definition root's // path is the ITEM name while the mapping's is "Emails|Email". @@ -1263,8 +1312,14 @@ type ExportMapping struct { // the key, which is not the same as present-and-empty (ako/mxcli#279). MessageDefinition2 *string `json:"messageDefinition2,omitempty"` // NullValueOption controls how null values are serialized: "LeaveOutElement" or "SendAsNil" - NullValueOption string `json:"nullValueOption,omitempty"` - Elements []*ExportMappingElement `json:"elements,omitempty"` + NullValueOption string `json:"nullValueOption,omitempty"` + // WebServiceSource is the imported web service (SOAP) a mapping can be + // sourced from — a FOURTH source kind beside JSON structure, XML schema and + // message definition. mxcli does not author one, but it must not destroy + // one: the properties are read so a rewrite can be refused rather than + // silently dropping the binding (ako/mxcli#365). + WebServiceSource WebServiceMappingSource `json:"webServiceSource,omitempty"` + Elements []*ExportMappingElement `json:"elements,omitempty"` } // GetName returns the export mapping's name. @@ -1304,6 +1359,11 @@ type ExportMappingElement struct { // Shared fields ExposedName string `json:"exposedName,omitempty"` JsonPath string `json:"jsonPath,omitempty"` + // OriginalValue is the sample parsed out of the JSON structure's snippet. + // Carried rather than derived: whether a mapping stores it is a per-document + // property mxcli cannot compute, so a rewrite preserves what was there + // instead of choosing (ako/mxcli#379). + OriginalValue string `json:"originalValue,omitempty"` // XmlPath — see the note on ImportMappingElement. XmlPath string `json:"xmlPath,omitempty"` Children []*ExportMappingElement `json:"children,omitempty"` diff --git a/modelsdk/mpr/nav_patch.go b/modelsdk/mpr/nav_patch.go index 60746b6d05..3a0beb8613 100644 --- a/modelsdk/mpr/nav_patch.go +++ b/modelsdk/mpr/nav_patch.go @@ -139,8 +139,16 @@ func navpPatchWebProfile(doc bson.D, spec types.NavigationProfileSpec) bson.D { // Studio Pro's "Fallback page". The $Type is // Navigation$NotFoundHomePage, not the Navigation$HomePage the home // page slot takes -- measured on ako/TestApp, whose fallback page - // Studio Pro stored as Navigation$NotFoundHomePage/Page. mxbuild - // accepts either, so nothing caught this. + // Studio Pro stored as Navigation$NotFoundHomePage/Page. + // + // The wrong $Type here is not cosmetic: Mendix cannot LOAD the + // project. Both `mx check` and `mxbuild --target=deploy` exit 1 with + // "Object of type '...Navigation.HomePage' cannot be converted to + // type '...Navigation.NotFoundHomePage'" (measured on 11.13, against + // a build of this file emitting the old spelling). Nothing caught it + // because nothing ever BUILT a project with a fallback page set -- + // the automated mx-check coverage runs doctype-tests/ only, and no + // script there sets one. {Key: "$Type", Value: "Navigation$NotFoundHomePage"}, {Key: "Microflow", Value: ""}, {Key: "Page", Value: spec.NotFoundPage}, diff --git a/modelsdk/mpr/nav_patch_notfound_test.go b/modelsdk/mpr/nav_patch_notfound_test.go new file mode 100644 index 0000000000..ae1be17d94 --- /dev/null +++ b/modelsdk/mpr/nav_patch_notfound_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/types" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// navpNotFoundHomepageOf patches a bare web profile with the given spec and +// returns the NotFoundHomepage it wrote. +func navpNotFoundHomepageOf(t *testing.T, spec types.NavigationProfileSpec) (interface{}, bool) { + t.Helper() + return navpTestEntry(navpPatchWebProfile(bson.D{}, spec), "NotFoundHomepage") +} + +// Studio Pro's "Fallback page" is its own type. The NotFoundHomepage property is +// declared Navigation$NotFoundHomePage, NOT the Navigation$HomePage the home-page +// slot takes, and .NET refuses the assignment on load: +// +// System.ArgumentException: Object of type +// 'Mendix.Modeler.WebUI.Navigation.HomePage' cannot be converted to type +// 'Mendix.Modeler.WebUI.Navigation.NotFoundHomePage'. +// +// The project is then unopenable — and because the failure happens while LOADING +// the model, `mx check` prints that trace INSTEAD OF its "The app contains: N +// errors" line, so a caller reading the count rather than the exit status sees a +// run that reported nothing at all (mendixlabs/mxcli#1000). +// +// This writer had no test of its own: reverting just this $Type left the whole +// suite green. +func TestNavpPatchWebProfile_NotFoundPageUsesItsOwnType(t *testing.T) { + nfp, present := navpNotFoundHomepageOf(t, types.NavigationProfileSpec{ + NotFoundPage: "MyFirstModule.NotFound", + }) + if !present { + t.Fatal("NotFoundHomepage key missing") + } + d, ok := nfp.(bson.D) + if !ok { + t.Fatalf("NotFoundHomepage = %#v, want a document", nfp) + } + if typ, _ := navpTestEntry(d, "$Type"); typ != "Navigation$NotFoundHomePage" { + t.Errorf("$Type = %v, want Navigation$NotFoundHomePage (NOT Navigation$HomePage)", typ) + } + if page, _ := navpTestEntry(d, "Page"); page != "MyFirstModule.NotFound" { + t.Errorf("Page = %v", page) + } + if _, present := navpTestEntry(d, "$ID"); !present { + t.Error("every stored element needs its own $ID") + } +} + +// The two slots are adjacent, take the same Page/Microflow pair, and differ only +// in $Type — so a fix applied one `sed` too wide silently converts the home page +// as well. HomePage keeps Navigation$HomePage. +func TestNavpPatchWebProfile_HomePageKeepsTheHomePageType(t *testing.T) { + doc := navpPatchWebProfile(bson.D{}, types.NavigationProfileSpec{ + HomePages: []types.NavHomePageSpec{{IsPage: true, Target: "MyFirstModule.Home"}}, + NotFoundPage: "MyFirstModule.NotFound", + }) + hp, present := navpTestEntry(doc, "HomePage") + if !present { + t.Fatal("HomePage key missing") + } + d, ok := hp.(bson.D) + if !ok { + t.Fatalf("HomePage = %#v, want a document", hp) + } + if typ, _ := navpTestEntry(d, "$Type"); typ != "Navigation$HomePage" { + t.Errorf("HomePage $Type = %v, want Navigation$HomePage", typ) + } +} + +// No fallback page is an explicit null, not an element with a blank Page: a +// NotFoundHomePage pointing at "" is a dangling reference where absent is the +// modelled default. +func TestNavpPatchWebProfile_NoNotFoundPageStaysNull(t *testing.T) { + nfp, present := navpNotFoundHomepageOf(t, types.NavigationProfileSpec{}) + if !present { + t.Fatal("the NotFoundHomepage key must be written even when unset") + } + if nfp != nil { + t.Errorf("NotFoundHomepage = %#v, want nil", nfp) + } +} diff --git a/sdk/microflows/microflows.go b/sdk/microflows/microflows.go index 1ebe1514bf..562f83a2fa 100644 --- a/sdk/microflows/microflows.go +++ b/sdk/microflows/microflows.go @@ -131,6 +131,46 @@ type MicroflowParameter struct { Name string `json:"name"` Documentation string `json:"documentation,omitempty"` Type DataType `json:"type"` + + // Position is the parameter's place on the canvas, set only when those + // coordinates say something DerivedParameterPosition would not have + // produced — see that function for why the distinction is the whole point. + // nil means "wherever the layout puts it", which is why this is a pointer: + // 0;0 is a position a person can choose, and two flows in the reference + // project use it. + Position *model.Point `json:"position,omitempty"` +} + +// DerivedParameterPosition returns where mxcli's own layout puts the parameter +// at index idx: a row of boxes along the top of the canvas, one spacing unit +// apart. Both writers used to compute this inline and unconditionally, which is +// why a hand-placed parameter did not survive a rewrite (#993). +// +// It exists so that both readers can tell an authored position from mxcli's own +// arithmetic handed back. A parameter sitting exactly here carries no intent and +// is re-derived on the next write; one anywhere else was put there by a person +// and is kept. This is the arbitration authoredStartPosition makes for the +// StartEvent, and it is made for the same reason: carrying stored coordinates +// over UNCONDITIONALLY pins the node, so inserting a parameter would leave the +// existing ones stranded on the old grid while the new one lands on top of them +// (the shape of #951, one node family over). +// +// A person who places a parameter exactly where the layout would have is +// indistinguishable from the layout — and re-deriving gives back the same point, +// so the ambiguity costs nothing. +func DerivedParameterPosition(idx int) model.Point { + return model.Point{X: 200 + idx*100, Y: 53} +} + +// AuthoredParameterPosition returns stored, or nil when stored is the point the +// layout would have derived for index idx. Readers call this so that everything +// downstream can treat a non-nil Position as intent. +func AuthoredParameterPosition(stored model.Point, idx int) *model.Point { + if stored == DerivedParameterPosition(idx) { + return nil + } + p := stored + return &p } // GetName returns the parameter's name. diff --git a/sdk/microflows/microflows_actions.go b/sdk/microflows/microflows_actions.go index b8517f8ff6..3c1ac5fcc8 100644 --- a/sdk/microflows/microflows_actions.go +++ b/sdk/microflows/microflows_actions.go @@ -739,6 +739,12 @@ type CallExternalAction struct { // triggers Mendix's CE7269 "return type has changed" when the schema // declares any return type. ResultDataType string `json:"resultDataType,omitempty"` + // ResultEntity is the qualified name of the external entity a returned + // object or list is typed on. Required when ResultDataType is "Object" or + // "List" — DataTypes$ObjectType and DataTypes$ListType both store an + // Entity, and one without it is as unaligned as no type at all. Empty for + // every primitive kind. + ResultEntity string `json:"resultEntity,omitempty"` } func (CallExternalAction) isMicroflowAction() {} @@ -749,6 +755,15 @@ type ExternalActionParameterMapping struct { ParameterName string `json:"parameterName,omitempty"` Argument string `json:"argument,omitempty"` // Expression CanBeEmpty bool `json:"canBeEmpty,omitempty"` + // ParameterDataType / ParameterEntity type the parameter, resolved from the + // consumed service's cached $metadata, and are written as the mapping's + // ParameterType sub-document. generated/metamodel declares ParameterType + // WITHOUT omitempty — it is not optional — and omitting it is CE7252 "the + // parameters for remote action '' have changed" plus a CE0117 + // "Error(s) in expression" per argument, because an argument cannot be + // type-checked against a parameter that has no type (mendixlabs/mxcli#1020). + ParameterDataType string `json:"parameterDataType,omitempty"` + ParameterEntity string `json:"parameterEntity,omitempty"` } // WebServiceCallAction calls a web service. diff --git a/sdk/mpr/microflow_parameter_position_test.go b/sdk/mpr/microflow_parameter_position_test.go new file mode 100644 index 0000000000..ea7a0731b7 --- /dev/null +++ b/sdk/mpr/microflow_parameter_position_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "testing" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +func rmp(t *testing.T, doc bson.D) string { + t.Helper() + v, _ := doc.Map()["RelativeMiddlePoint"].(string) + return v +} + +// #993: a hand-placed parameter must be written where it was placed. Before the +// fix the legacy serializer computed the position from the index and ignored +// anything stored, so a describe → exec of mxcli's own output moved a real +// parameter from -77;0 to 200;53. +func TestSerializeMicroflowParameterKeepsAuthoredPosition(t *testing.T) { + authored := µflows.MicroflowParameter{ + Name: "Feedback", + Position: &model.Point{X: -77, Y: 0}, + } + if got := rmp(t, serializeMicroflowParameter(authored, 0, 11)); got != "-77;0" { + t.Errorf("authored position = %q, want -77;0", got) + } + + // Control: with no authored position the parameter goes where the layout + // puts it — the behaviour every unannotated flow still relies on. Without + // this the test would pass against a writer that had simply stopped + // deriving. + derived := µflows.MicroflowParameter{Name: "Feedback"} + if got := rmp(t, serializeMicroflowParameter(derived, 0, 11)); got != "200;53" { + t.Errorf("derived position at index 0 = %q, want 200;53", got) + } + if got := rmp(t, serializeMicroflowParameter(derived, 2, 11)); got != "400;53" { + t.Errorf("derived position at index 2 = %q, want 400;53", got) + } +} + +// The reader is where the derived/authored arbitration happens, so that +// everything downstream can treat a non-nil Position as intent. A parameter +// stored on the derived grid must come back unset — carrying it over would pin +// it, and inserting a parameter would then strand the others (#951's shape). +func TestParseMicroflowParameterNormalizesDerivedPosition(t *testing.T) { + raw := func(pos string) map[string]any { + return map[string]any{"Name": "A", "RelativeMiddlePoint": pos} + } + if p := parseMicroflowParameter(raw("200;53"), 0); p.Position != nil { + t.Errorf("derived position came back as %v, want nil", *p.Position) + } + if p := parseMicroflowParameter(raw("300;53"), 1); p.Position != nil { + t.Errorf("derived position at index 1 came back as %v, want nil", *p.Position) + } + p := parseMicroflowParameter(raw("-77;0"), 0) + if p.Position == nil { + t.Fatal("authored position was dropped — this is the #993 read-side loss") + } + if *p.Position != (model.Point{X: -77, Y: 0}) { + t.Errorf("position = %v, want -77;0", *p.Position) + } +} diff --git a/sdk/mpr/parser_export_mapping.go b/sdk/mpr/parser_export_mapping.go index 90e3233de1..62b64b26e7 100644 --- a/sdk/mpr/parser_export_mapping.go +++ b/sdk/mpr/parser_export_mapping.go @@ -56,6 +56,7 @@ func (r *Reader) parseExportMapping(unitID, containerID string, contents []byte) if v, ok := raw["NullValueOption"].(string); ok { em.NullValueOption = v } + em.WebServiceSource = parseWebServiceSource(raw) // Parse top-level mapping elements (array with int32 version prefix) if elements, ok := raw["Elements"].(bson.A); ok { @@ -150,6 +151,9 @@ func parseExportValueMappingElement(raw map[string]any) *model.ExportMappingElem if v, ok := raw["Converter"].(string); ok { elem.Converter = v } + if v, ok := raw["OriginalValue"].(string); ok { + elem.OriginalValue = v + } // Extract the primitive type from the nested Type object if typeObj, ok := raw["Type"].(map[string]any); ok { diff --git a/sdk/mpr/parser_import_mapping.go b/sdk/mpr/parser_import_mapping.go index 2d00bdc455..ceacdd1aeb 100644 --- a/sdk/mpr/parser_import_mapping.go +++ b/sdk/mpr/parser_import_mapping.go @@ -48,6 +48,7 @@ func (r *Reader) parseImportMapping(unitID, containerID string, contents []byte) if v, ok := raw["MessageDefinition"].(string); ok { im.MessageDefinition = v } + im.WebServiceSource = parseWebServiceSource(raw) // MessageDefinition2 is version-introduced (11.10+) and carried, not derived: // nil means the stored document does not have the key (ako/mxcli#279). if v, ok := raw["MessageDefinition2"].(string); ok { @@ -212,3 +213,32 @@ func extractPrimitiveTypeName(typeObj map[string]any) string { return "String" } } + +// parseWebServiceSource reads a mapping's SOAP binding. +// +// Read-only, and read for one reason: a rewrite that dropped these keys turned a +// working integration into CE6896 + CE0270. ImportedWebService is stored under +// `wsdlFile`'s SDK name — the BSON key is ImportedWebService — and the root +// element under RootElementName (`xsdRootElementName` in the SDK). +func parseWebServiceSource(raw map[string]any) model.WebServiceMappingSource { + var w model.WebServiceMappingSource + if v, ok := raw["ImportedWebService"].(string); ok { + w.ImportedWebService = v + } + if v, ok := raw["ServiceName"].(string); ok { + w.ServiceName = v + } + if v, ok := raw["OperationName"].(string); ok { + w.OperationName = v + } + if v, ok := raw["RootElementName"].(string); ok { + w.RootElementName = v + } + if v, ok := raw["ParameterName"].(string); ok { + w.ParameterName = v + } + if v, ok := raw["IsHeader"].(bool); ok { + w.IsHeader = v + } + return w +} diff --git a/sdk/mpr/parser_microflow.go b/sdk/mpr/parser_microflow.go index c61edee650..1da092d909 100644 --- a/sdk/mpr/parser_microflow.go +++ b/sdk/mpr/parser_microflow.go @@ -82,7 +82,7 @@ func ParseMicroflowFromRaw(raw map[string]any, unitID, containerID model.ID) *mi } for _, p := range extractBsonSlice(paramsArray) { if paramMap := extractBsonMap(p); paramMap != nil { - param := parseMicroflowParameter(paramMap) + param := parseMicroflowParameter(paramMap, len(mf.Parameters)) mf.Parameters = append(mf.Parameters, param) } } @@ -109,7 +109,7 @@ func ParseMicroflowFromRaw(raw map[string]any, unitID, containerID model.ID) *mi for _, obj := range extractBsonSlice(ocRaw["Objects"]) { if objMap := extractBsonMap(obj); objMap != nil { if typeName, _ := objMap["$Type"].(string); typeName == "Microflows$MicroflowParameter" { - param := parseMicroflowParameter(objMap) + param := parseMicroflowParameter(objMap, len(mf.Parameters)) mf.Parameters = append(mf.Parameters, param) } } @@ -237,7 +237,11 @@ func parseCaseValue(raw any) microflows.CaseValue { return nil } -func parseMicroflowParameter(raw map[string]any) *microflows.MicroflowParameter { +// parseMicroflowParameter reads one Microflows$MicroflowParameter. idx is the +// parameter's ordinal in its flow, needed to tell a stored position that says +// something from one that is mxcli's own layout arithmetic handed back — see +// microflows.AuthoredParameterPosition. +func parseMicroflowParameter(raw map[string]any, idx int) *microflows.MicroflowParameter { param := µflows.MicroflowParameter{} // Use extractBsonID to handle binary IDs @@ -255,6 +259,9 @@ func parseMicroflowParameter(raw map[string]any) *microflows.MicroflowParameter } else if pt := extractBsonMap(raw["ParameterType"]); pt != nil { param.Type = parseMicroflowDataType(pt) } + if rmp, ok := raw["RelativeMiddlePoint"]; ok { + param.Position = microflows.AuthoredParameterPosition(parsePoint(rmp), idx) + } return param } diff --git a/sdk/mpr/parser_nanoflow.go b/sdk/mpr/parser_nanoflow.go index 11557eced0..bf60f1b930 100644 --- a/sdk/mpr/parser_nanoflow.go +++ b/sdk/mpr/parser_nanoflow.go @@ -62,7 +62,7 @@ func (r *Reader) parseNanoflow(unitID, containerID string, contents []byte) (*mi } for _, p := range extractBsonSlice(paramsArray) { if paramMap := extractBsonMap(p); paramMap != nil { - param := parseMicroflowParameter(paramMap) + param := parseMicroflowParameter(paramMap, len(nf.Parameters)) nf.Parameters = append(nf.Parameters, param) } } @@ -83,7 +83,7 @@ func (r *Reader) parseNanoflow(unitID, containerID string, contents []byte) (*mi for _, obj := range extractBsonSlice(ocRaw["Objects"]) { if objMap := extractBsonMap(obj); objMap != nil { if typeName, _ := objMap["$Type"].(string); typeName == "Microflows$MicroflowParameter" { - param := parseMicroflowParameter(objMap) + param := parseMicroflowParameter(objMap, len(nf.Parameters)) nf.Parameters = append(nf.Parameters, param) } } diff --git a/sdk/mpr/parser_rule.go b/sdk/mpr/parser_rule.go index 50650ee10c..3ba1f01697 100644 --- a/sdk/mpr/parser_rule.go +++ b/sdk/mpr/parser_rule.go @@ -66,7 +66,7 @@ func (r *Reader) parseRule(unitID, containerID string, contents []byte) (*microf for _, obj := range extractBsonSlice(oc["Objects"]) { if objMap := extractBsonMap(obj); objMap != nil { if typeName, _ := objMap["$Type"].(string); typeName == "Microflows$MicroflowParameter" { - rule.Parameters = append(rule.Parameters, parseMicroflowParameter(objMap)) + rule.Parameters = append(rule.Parameters, parseMicroflowParameter(objMap, len(rule.Parameters))) } } } diff --git a/sdk/mpr/parser_webservice_source_test.go b/sdk/mpr/parser_webservice_source_test.go new file mode 100644 index 0000000000..6d4a8e3c24 --- /dev/null +++ b/sdk/mpr/parser_webservice_source_test.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import "testing" + +// A mapping's SOAP binding must survive the read, because that is the only way +// a rewrite can be refused rather than silently dropping it (ako/mxcli#365). +// Before this, model.ImportMapping carried three source fields and a comment +// saying "at most one is set" — the fourth was not in the type at all. + +func TestParseWebServiceSourceReadsTheBinding(t *testing.T) { + got := parseWebServiceSource(map[string]any{ + "ImportedWebService": "Legacy.WS_Orders", + "ServiceName": "OrderService", + "OperationName": "GetOrder", + "RootElementName": "GetOrderResponse", + "ParameterName": "body", + "IsHeader": true, + }) + + if !got.IsSet() { + t.Fatal("IsSet false for a mapping with an imported web service") + } + if got.ImportedWebService != "Legacy.WS_Orders" { + t.Errorf("ImportedWebService = %q", got.ImportedWebService) + } + if got.ServiceName != "OrderService" || got.OperationName != "GetOrder" { + t.Errorf("service/operation = %q/%q", got.ServiceName, got.OperationName) + } + // RootElementName is stored under that key; the SDK calls it + // xsdRootElementName, which is what makes it easy to bind wrongly. + if got.RootElementName != "GetOrderResponse" { + t.Errorf("RootElementName = %q", got.RootElementName) + } + // Export-only, and carried for the same reason as the rest. + if got.ParameterName != "body" || !got.IsHeader { + t.Errorf("ParameterName/IsHeader = %q/%v", got.ParameterName, got.IsHeader) + } +} + +// TestParseWebServiceSourceIsEmptyForAnOrdinaryMapping is the control: every +// mapping mxcli can author reaches this with none of the keys present, and must +// come back not-set or the guard would refuse every rewrite. +func TestParseWebServiceSourceIsEmptyForAnOrdinaryMapping(t *testing.T) { + got := parseWebServiceSource(map[string]any{ + "Name": "IMM_Order", + "JsonStructure": "Shop.JSON_Order", + }) + if got.IsSet() { + t.Errorf("IsSet true for a JSON-sourced mapping: %+v", got) + } +} diff --git a/sdk/mpr/writer_export_mapping.go b/sdk/mpr/writer_export_mapping.go index ebb67d482e..a1e7d767f3 100644 --- a/sdk/mpr/writer_export_mapping.go +++ b/sdk/mpr/writer_export_mapping.go @@ -199,7 +199,7 @@ func serializeExportValueElement(id string, elem *model.ExportMappingElement, pa {Key: "IsKey", Value: elem.IsKey}, {Key: "IsContent", Value: false}, {Key: "IsXmlAttribute", Value: false}, - {Key: "OriginalValue", Value: ""}, + {Key: "OriginalValue", Value: elem.OriginalValue}, {Key: "XmlPrimitiveType", Value: xmlPrimitiveTypeName(elem.DataType)}, } } diff --git a/sdk/mpr/writer_external_action_returntype_test.go b/sdk/mpr/writer_external_action_returntype_test.go new file mode 100644 index 0000000000..d924790787 --- /dev/null +++ b/sdk/mpr/writer_external_action_returntype_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import "testing" + +// TestSerializeExternalActionReturnType covers the DataTypes$ element written +// into CallExternalAction.VariableDataType. +// +// Object and List were unreachable before: the resolver mapped only EDM +// primitives and returned "" for anything else, so an action returning an entity +// (or a collection of them) got NO VariableDataType at all, and Mendix reported +// CE7269 "The return type for remote action '' has changed" +// (mendixlabs/mxcli#1020). Both carry an Entity — a DataTypes$ObjectType without +// one is as unaligned as no type at all. +func TestSerializeExternalActionReturnType(t *testing.T) { + tests := []struct { + name string + kind string + entity string + wantType string + wantEntity string // "" = the key must be absent + }{ + {name: "object return", kind: "Object", entity: "Trippin.Airport", + wantType: "DataTypes$ObjectType", wantEntity: "Trippin.Airport"}, + {name: "list return", kind: "List", entity: "Trippin.Person", + wantType: "DataTypes$ListType", wantEntity: "Trippin.Person"}, + {name: "boolean", kind: "Boolean", wantType: "DataTypes$BooleanType"}, + {name: "string", kind: "String", wantType: "DataTypes$StringType"}, + {name: "integer", kind: "Integer", wantType: "DataTypes$IntegerType"}, + {name: "long is an integer", kind: "Long", wantType: "DataTypes$IntegerType"}, + {name: "decimal", kind: "Decimal", wantType: "DataTypes$DecimalType"}, + {name: "datetime", kind: "DateTime", wantType: "DataTypes$DateTimeType"}, + {name: "binary", kind: "Binary", wantType: "DataTypes$BinaryType"}, + {name: "void", kind: "Void", wantType: "DataTypes$VoidType"}, + {name: "empty is void", kind: "", wantType: "DataTypes$VoidType"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + doc := serializeExternalActionReturnType(tt.kind, tt.entity) + + var gotType, gotEntity string + var hasEntity, hasID bool + for _, e := range doc { + switch e.Key { + case "$Type": + gotType, _ = e.Value.(string) + case "Entity": + gotEntity, _ = e.Value.(string) + hasEntity = true + case "$ID": + hasID = true + } + } + + if gotType != tt.wantType { + t.Errorf("$Type = %q, want %q", gotType, tt.wantType) + } + if !hasID { + t.Error("every DataTypes$ element needs its own $ID") + } + if tt.wantEntity == "" { + if hasEntity { + t.Errorf("a primitive return must not carry an Entity (got %q)", gotEntity) + } + return + } + if gotEntity != tt.wantEntity { + t.Errorf("Entity = %q, want %q", gotEntity, tt.wantEntity) + } + }) + } +} diff --git a/sdk/mpr/writer_microflow.go b/sdk/mpr/writer_microflow.go index de6fa6a22c..8215263984 100644 --- a/sdk/mpr/writer_microflow.go +++ b/sdk/mpr/writer_microflow.go @@ -325,8 +325,13 @@ func serializeAnnotationFlow(af *microflows.AnnotationFlow, majorVersion int) bs // DefaultValue and IsRequired were introduced in Mendix 10; emitting them on a // Mendix 9 project trips the Studio Pro metamodel checker, so they are gated. func serializeMicroflowParameter(p *microflows.MicroflowParameter, posX int, majorVersion int) bson.D { - // Calculate position based on index - parameters appear at the top of the microflow - relativeMiddlePoint := fmt.Sprintf("%d;53", 200+posX*100) + // An authored position is written as given; without one the parameter goes + // where the layout puts it — a row of boxes along the top of the canvas. + pos := microflows.DerivedParameterPosition(posX) + if p.Position != nil { + pos = *p.Position + } + relativeMiddlePoint := pointToString(pos) doc := bson.D{ {Key: "$ID", Value: idToBsonBinary(string(p.ID))}, diff --git a/sdk/mpr/writer_microflow_actions.go b/sdk/mpr/writer_microflow_actions.go index b06dccff97..a9896cce5f 100644 --- a/sdk/mpr/writer_microflow_actions.go +++ b/sdk/mpr/writer_microflow_actions.go @@ -190,7 +190,7 @@ func serializeMicroflowAction(action microflows.MicroflowAction) bson.D { // VariableDataType when we know the schema kind — the executor // resolves it from the consumed service's cached $metadata. if a.ResultDataType != "" { - doc = append(doc, bson.E{Key: "VariableDataType", Value: serializeExternalActionReturnType(a.ResultDataType)}) + doc = append(doc, bson.E{Key: "VariableDataType", Value: serializeExternalActionReturnType(a.ResultDataType, a.ResultEntity)}) } // Serialize parameter mappings if len(a.ParameterMappings) > 0 { @@ -204,6 +204,14 @@ func serializeMicroflowAction(action microflows.MicroflowAction) bson.D { {Key: "Argument", Value: pm.Argument}, {Key: "CanBeEmpty", Value: pm.CanBeEmpty}, } + // generated/metamodel declares ParameterType without omitempty. + // Omitting it is CE7252 + a CE0117 per argument. + if pm.ParameterDataType != "" { + mapping = append(mapping, bson.E{ + Key: "ParameterType", + Value: serializeExternalActionReturnType(pm.ParameterDataType, pm.ParameterEntity), + }) + } mappings = append(mappings, mapping) } doc = append(doc, bson.E{Key: "ParameterMappings", Value: mappings}) @@ -1663,7 +1671,10 @@ func serializeExportXmlAction(a *microflows.ExportXmlAction) bson.D { // suitable for ODataPublish$CallExternalAction.VariableDataType. Mendix's // CE7269 fires when this field's $Type doesn't match what the cached schema // declares for the action's return. -func serializeExternalActionReturnType(kind string) bson.D { +// An Object or List return also carries the entity it is typed on: both +// DataTypes$ObjectType and DataTypes$ListType store an Entity by qualified +// name, and one without it is as unaligned as no type at all. +func serializeExternalActionReturnType(kind, entity string) bson.D { typeID := idToBsonBinary(generateUUID()) bsonType := "DataTypes$VoidType" switch kind { @@ -1679,6 +1690,18 @@ func serializeExternalActionReturnType(kind string) bson.D { bsonType = "DataTypes$DateTimeType" case "Binary": bsonType = "DataTypes$BinaryType" + case "Object": + return bson.D{ + {Key: "$ID", Value: typeID}, + {Key: "$Type", Value: "DataTypes$ObjectType"}, + {Key: "Entity", Value: entity}, + } + case "List": + return bson.D{ + {Key: "$ID", Value: typeID}, + {Key: "$Type", Value: "DataTypes$ListType"}, + {Key: "Entity", Value: entity}, + } case "Void", "": bsonType = "DataTypes$VoidType" } diff --git a/sdk/mpr/writer_navigation.go b/sdk/mpr/writer_navigation.go index 8bc077693c..57c3b3e305 100644 --- a/sdk/mpr/writer_navigation.go +++ b/sdk/mpr/writer_navigation.go @@ -161,8 +161,16 @@ func patchWebProfile(doc bson.D, spec NavigationProfileSpec) bson.D { // Studio Pro's "Fallback page". The $Type is // Navigation$NotFoundHomePage, not the Navigation$HomePage the home // page slot takes -- measured on ako/TestApp, whose fallback page - // Studio Pro stored as Navigation$NotFoundHomePage/Page. mxbuild - // accepts either, so nothing caught this. + // Studio Pro stored as Navigation$NotFoundHomePage/Page. + // + // The wrong $Type here is not cosmetic: Mendix cannot LOAD the + // project. Both `mx check` and `mxbuild --target=deploy` exit 1 with + // "Object of type '...Navigation.HomePage' cannot be converted to + // type '...Navigation.NotFoundHomePage'" (measured on 11.13, against + // a build of this file emitting the old spelling). Nothing caught it + // because nothing ever BUILT a project with a fallback page set -- + // the automated mx-check coverage runs doctype-tests/ only, and no + // script there sets one. {Key: "$Type", Value: "Navigation$NotFoundHomePage"}, {Key: "Microflow", Value: ""}, {Key: "Page", Value: spec.NotFoundPage}, diff --git a/sdk/mpr/writer_navigation_notfound_test.go b/sdk/mpr/writer_navigation_notfound_test.go new file mode 100644 index 0000000000..7f977d4259 --- /dev/null +++ b/sdk/mpr/writer_navigation_notfound_test.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mpr + +import ( + "testing" + + "go.mongodb.org/mongo-driver/bson" +) + +// navNotFoundEntry returns the value of a key and whether it was present, +// separating an absent key from an explicitly null one. +func navNotFoundEntry(d bson.D, key string) (interface{}, bool) { + for _, e := range d { + if e.Key == key { + return e.Value, true + } + } + return nil, false +} + +// notFoundHomepageOf patches a bare web profile with the given spec and returns +// the NotFoundHomepage it wrote. +func notFoundHomepageOf(t *testing.T, spec NavigationProfileSpec) (interface{}, bool) { + t.Helper() + return navNotFoundEntry(patchWebProfile(bson.D{}, spec), "NotFoundHomepage") +} + +// Studio Pro's "Fallback page" is its own type. The NotFoundHomepage property is +// declared Navigation$NotFoundHomePage, NOT the Navigation$HomePage the home-page +// slot takes, and .NET refuses the assignment on load: +// +// System.ArgumentException: Object of type +// 'Mendix.Modeler.WebUI.Navigation.HomePage' cannot be converted to type +// 'Mendix.Modeler.WebUI.Navigation.NotFoundHomePage'. +// +// The project is then unopenable — and because the failure happens while LOADING +// the model, `mx check` prints that trace INSTEAD OF its "The app contains: N +// errors" line, so a caller reading the count rather than the exit status sees a +// run that reported nothing at all (mendixlabs/mxcli#1000). +// +// This is the writer the default engine uses, and the one that produced the +// unopenable project in #1000's report. It had no test: reverting just this +// $Type left the whole suite green. +func TestPatchWebProfile_NotFoundPageUsesItsOwnType(t *testing.T) { + nfp, present := notFoundHomepageOf(t, NavigationProfileSpec{ + NotFoundPage: "MyFirstModule.NotFound", + }) + if !present { + t.Fatal("NotFoundHomepage key missing") + } + d, ok := nfp.(bson.D) + if !ok { + t.Fatalf("NotFoundHomepage = %#v, want a document", nfp) + } + if typ, _ := navNotFoundEntry(d, "$Type"); typ != "Navigation$NotFoundHomePage" { + t.Errorf("$Type = %v, want Navigation$NotFoundHomePage (NOT Navigation$HomePage)", typ) + } + if page, _ := navNotFoundEntry(d, "Page"); page != "MyFirstModule.NotFound" { + t.Errorf("Page = %v", page) + } + if _, present := navNotFoundEntry(d, "$ID"); !present { + t.Error("every stored element needs its own $ID") + } +} + +// The two slots are adjacent, take the same Page/Microflow pair, and differ only +// in $Type — so a fix applied one `sed` too wide silently converts the home page +// as well. HomePage keeps Navigation$HomePage. +func TestPatchWebProfile_HomePageKeepsTheHomePageType(t *testing.T) { + doc := patchWebProfile(bson.D{}, NavigationProfileSpec{ + HomePages: []NavHomePageSpec{{IsPage: true, Target: "MyFirstModule.Home"}}, + NotFoundPage: "MyFirstModule.NotFound", + }) + hp, present := navNotFoundEntry(doc, "HomePage") + if !present { + t.Fatal("HomePage key missing") + } + d, ok := hp.(bson.D) + if !ok { + t.Fatalf("HomePage = %#v, want a document", hp) + } + if typ, _ := navNotFoundEntry(d, "$Type"); typ != "Navigation$HomePage" { + t.Errorf("HomePage $Type = %v, want Navigation$HomePage", typ) + } +} + +// No fallback page is an explicit null, not an element with a blank Page: a +// NotFoundHomePage pointing at "" is a dangling reference where absent is the +// modelled default. +func TestPatchWebProfile_NoNotFoundPageStaysNull(t *testing.T) { + nfp, present := notFoundHomepageOf(t, NavigationProfileSpec{}) + if !present { + t.Fatal("the NotFoundHomepage key must be written even when unset") + } + if nfp != nil { + t.Errorf("NotFoundHomepage = %#v, want nil", nfp) + } +}