diff --git a/.github/workflows/macos-diag.yml b/.github/workflows/macos-diag.yml index cad62c24..fabd88fe 100644 --- a/.github/workflows/macos-diag.yml +++ b/.github/workflows/macos-diag.yml @@ -4,6 +4,8 @@ name: macos-diag # per-assertion output and, if they stall, dumps thread stacks. on: workflow_dispatch: + push: + branches: [mtfix23-sync] jobs: diag: diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..e3b64e20 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,4 @@ +title = "OpenADS Gitleaks configuration" + +[extend] +useDefault = true diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 00000000..0544730a --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,4 @@ +src/session/connection.cpp:generic-api-key:525 +src/session/connection.cpp:generic-api-key:1457 +tests/unit/aes_test.cpp:generic-api-key:41 +third_party/tinyaes/aes.c:generic-api-key:18 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e33af96..a5caed99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,14 @@ project(OpenADS # a .git tree falls back to ${PROJECT_VERSION} from project() above. # The previous hard-coded "1.0.0-rc1" string drifted six releases # behind reality and surprised users running rc12. +# A pipeline that knows the release tag passes it explicitly: the tag +# is the source of truth for a release build's stamp, not the local +# clone's git state (shallow/container checkouts may not resolve +# `git describe` - the glibc231 container leg shipped a plain +# "1.09.68" banner for exactly that reason). +if(OPENADS_VERSION_FORCE) + set(OPENADS_VERSION_STR "${OPENADS_VERSION_FORCE}") +else() set(OPENADS_VERSION_STR "${PROJECT_VERSION}") find_package(Git QUIET) if(GIT_FOUND AND EXISTS "${CMAKE_SOURCE_DIR}/.git") @@ -29,6 +37,7 @@ if(GIT_FOUND AND EXISTS "${CMAKE_SOURCE_DIR}/.git") string(REGEX REPLACE "-dirty$" "" OPENADS_VERSION_STR "${OPENADS_VERSION_STR}") endif() endif() +endif() message(STATUS "OpenADS version: ${OPENADS_VERSION_STR}") # Deliver the version through a generated header instead of a global # compile definition: a new commit used to change the command line of @@ -219,9 +228,11 @@ if(OPENADS_WITH_TLS) set(MBEDTLS_INSTALL OFF CACHE BOOL "" FORCE) FetchContent_Declare( mbedtls - GIT_REPOSITORY https://github.com/Mbed-TLS/mbedtls.git - GIT_TAG v3.6.2 - GIT_SHALLOW TRUE + GIT_REPOSITORY https://github.com/Mbed-TLS/mbedtls.git + GIT_TAG v3.6.2 + GIT_SHALLOW TRUE + GIT_SUBMODULES framework + GIT_SUBMODULES_RECURSE TRUE ) FetchContent_MakeAvailable(mbedtls) message(STATUS "OpenADS: TLS enabled via vendored mbedtls 3.6.2 (statically linked)") @@ -246,15 +257,13 @@ if(OPENADS_WITH_HTTP) set(JSON_Install OFF CACHE BOOL "" FORCE) FetchContent_Declare( cpp_httplib - GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git - GIT_TAG v0.18.5 - GIT_SHALLOW TRUE + URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.18.5.tar.gz + URL_HASH SHA256=731190e97acd63edce57cc3dacd496f57e7743bfc7933da7137cb3e93ec6c9a0 ) FetchContent_Declare( nlohmann_json - GIT_REPOSITORY https://github.com/nlohmann/json.git - GIT_TAG v3.11.3 - GIT_SHALLOW TRUE + URL https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz + URL_HASH SHA256=0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406 ) set(JSON_BuildTests OFF CACHE INTERNAL "") FetchContent_MakeAvailable(cpp_httplib nlohmann_json) @@ -278,8 +287,8 @@ if(NOT EXISTS "${_openads_zlib_dir}/zlib.h") include(FetchContent) FetchContent_Declare( zlib_src - URL https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz - DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz + URL_HASH SHA256=9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 ) set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) set(SKIP_INSTALL_ALL ON CACHE BOOL "" FORCE) @@ -334,8 +343,8 @@ if(OPENADS_WITH_SQLITE) include(FetchContent) FetchContent_Declare( sqlite_amalgamation - URL https://www.sqlite.org/2024/sqlite-amalgamation-3460100.zip - DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://www.sqlite.org/2024/sqlite-amalgamation-3460100.zip + URL_HASH SHA256=77823cb110929c2bcb0f5d48e4833b5c59a8a6e40cdea3936b99e199dbbe5784 ) FetchContent_MakeAvailable(sqlite_amalgamation) set(_openads_sqlite_src "${sqlite_amalgamation_SOURCE_DIR}/sqlite3.c") diff --git a/README.md b/README.md index 97b45307..c7b7244a 100644 --- a/README.md +++ b/README.md @@ -103,13 +103,13 @@ generated baseline; OpenADS clears every line of the regenerated ADS-flavoured baseline. The session that closed the last gap is recorded across 28 incremental commits ending at `28be1be`. -**Current release: [v1.8.4](https://github.com/FiveTechSoft/OpenADS/releases/tag/v1.8.4) (2026-07-09).** -Harbour `rddads` + FiveWin `TDataBase` / `xBrowse` over `tcp://` is -production-ready: remote scope (`OrdScope`), index navigation, key counts, -date fields, and field I/O by ordinal all work end-to-end. Full Data -Dictionary enforcement, Studio web console, SAP DD import, SQL backends -(PostgreSQL / MariaDB / MSSQL / ODBC / SQLite), and CDX bulk-load index -build are in production. `openads_serverd` serves the OpenADS wire +**Current release: [v1.09.68](https://github.com/FiveTechSoft/OpenADS/releases/tag/v1.09.68) (2026-09-20).** +OpenADS has broad compatibility coverage, but support varies by API and +backend. Before production deployment, review [`TODO.parity.md`](TODO.parity.md) +and [`docs/known-issues.md`](docs/known-issues.md), test the exact workload, +and apply normal database security and backup controls. In particular, +documented Data Dictionary and access-control parity gaps make untrusted or +multi-user deployments experimental until those gaps are closed. `openads_serverd` serves the OpenADS wire protocol; clients connect with `AdsConnect60("tcp://host:port/path.add", ...)` and no application code changes. Docs: @@ -120,9 +120,10 @@ including [migrating from ADS](https://fivetechsoft.github.io/OpenADS/en/migrati (CDX rollback / error 7017 caveat). `docs/wire-protocol.md` is the formal spec for non-C++ clients (Python, Go, Rust, Harbour AEP). -Cross-platform CI is **green on all three runners** -(`ubuntu-24.04 / ninja-clang`, `macos-14 / default`, -`windows-2022 / msvc-x64`). +Cross-platform CI runs on Ubuntu, macOS, and Windows. Check the current +[Actions results](https://github.com/FiveTechSoft/OpenADS/actions/workflows/ci.yml) +before relying on a branch or release; this document does not claim that the +latest run is green. Release timeline: diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..bbb0aeb7 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported versions + +Security fixes are applied to the latest published release and the default branch. Older releases may not receive fixes. + +## Reporting a vulnerability + +Do not open a public issue for a vulnerability or suspected credential exposure. Use GitHub's private vulnerability reporting for this repository: + +1. Open the repository's **Security** tab. +2. Select **Advisories** and **Report a vulnerability**. +3. Include affected versions, reproduction steps, impact, and any suggested mitigation. + +If private vulnerability reporting is unavailable, contact a repository maintainer privately and ask for a secure reporting channel. Do not include secrets or exploit details in ordinary email, discussions, issues, pull requests, or chat. + +You should receive an acknowledgement within 7 days. A maintainer will coordinate validation, remediation, disclosure, and any CVE request. Please allow time for a fix before public disclosure. + +## Credential exposure + +Treat a committed credential as compromised even after the file is deleted. Rotate or revoke it first, then remove it from the current tree and purge it from Git history. Avoid copying the credential into issues, pull requests, commit messages, or logs. diff --git a/contrib/oads_hb/oads_hb.c b/contrib/oads_hb/oads_hb.c index e43a2ad2..6c040ffb 100644 --- a/contrib/oads_hb/oads_hb.c +++ b/contrib/oads_hb/oads_hb.c @@ -29,11 +29,12 @@ * hbmk2 myproject.hbp oads_hb.c -L/path/to/openads/importlib -lace64 * * Connection management: - * OAds_SetConnection( hConn ) -- set default connection for this thread - * OAds_GetConnection() -> hConn -- get current default connection + * OAds_SetConnection( hConn ) -- set the shared OAds default handle + * OAds_GetConnection() -> hConn -- read the shared OAds default * * All OAds_F* functions accept hConn as the first (optional) parameter. - * When omitted, the thread-local default connection is used: + * When omitted, the handle set by OAds_SetConnection is used. If none + * has been set, the ACE default connection is used for compatibility: * OAds_FOpen( cFileName, nMode ) -- uses default connection * OAds_FOpen( hConn, cFileName, nMode ) -- uses explicit connection */ @@ -44,6 +45,48 @@ #include "ace.h" #include +/* The no-handle OAds_* overloads belong to this Harbour glue, not to + * rddads. ACE's default is thread-local; the app can set it on its login + * thread then make an OAds_* call on a different thread, accidentally + * falling back to ACE's local cwd connection. Keep an OAds-specific shared + * handle as the source of truth once OAds_SetConnection has been called. + * Atomic access protects MT Harbour callers; explicit-handle overloads + * bypass this value. An app with multiple independent connections must pass + * the handle explicitly rather than relying on one process-wide default. + */ +#if defined( _WIN32 ) +# include +static volatile LONG s_oads_conn = 0; +static void oads_store_default( ADSHANDLE hConn ) +{ + InterlockedExchange( &s_oads_conn, ( LONG ) hConn ); +} +static ADSHANDLE oads_load_default( void ) +{ + return ( ADSHANDLE ) InterlockedCompareExchange( &s_oads_conn, 0, 0 ); +} +#else +static ADSHANDLE s_oads_conn = 0; +static void oads_store_default( ADSHANDLE hConn ) +{ + __atomic_store_n( &s_oads_conn, hConn, __ATOMIC_RELEASE ); +} +static ADSHANDLE oads_load_default( void ) +{ + return __atomic_load_n( &s_oads_conn, __ATOMIC_ACQUIRE ); +} +#endif + +static ADSHANDLE oads_default_connection( void ) +{ + ADSHANDLE hConn = 0; + hConn = oads_load_default(); + if( hConn != 0 ) + return hConn; + AdsGetDefaultConnection( &hConn ); + return hConn; +} + /* AdsGetServerVersion is an OpenADS extension (v1.09.28+). Declared here as well so this file still compiles against an older ace.h; an identical redeclaration is legal C when the new ace.h is used. */ @@ -53,23 +96,30 @@ extern UNSIGNED32 ENTRYPOINT AdsGetServerVersion( ADSHANDLE hConnect, /* ------------------------------------------------------------------ */ /* OADS_SETCONNECTION( hConn ) -> lOk */ -/* Set the default connection for the calling thread. */ +/* Set both the ACE current-thread default and the shared OAds default. */ /* ------------------------------------------------------------------ */ HB_FUNC( OADS_SETCONNECTION ) { ADSHANDLE hConn = ( ADSHANDLE ) hb_parnint( 1 ); - hb_retl( AdsSetDefaultConnection( hConn ) == 0 ); + UNSIGNED32 rc; + if( hConn == 0 ) + { + hb_retl( 0 ); + return; + } + rc = AdsSetDefaultConnection( hConn ); + if( rc == 0 ) + oads_store_default( hConn ); + hb_retl( rc == 0 ); } /* ------------------------------------------------------------------ */ /* OADS_GETCONNECTION() -> hConn */ -/* Get the current default connection for the calling thread. */ +/* Report the same handle that no-handle OAds_* calls will use. */ /* ------------------------------------------------------------------ */ HB_FUNC( OADS_GETCONNECTION ) { - ADSHANDLE hConn = 0; - AdsGetDefaultConnection( &hConn ); - hb_retnint( ( HB_MAXINT ) hConn ); + hb_retnint( ( HB_MAXINT ) oads_default_connection() ); } /* ------------------------------------------------------------------ */ @@ -103,7 +153,7 @@ HB_FUNC( OADS_FCREATE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); usAttr = ( UNSIGNED16 ) hb_parni( 2 ); } @@ -133,7 +183,7 @@ HB_FUNC( OADS_FOPEN ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); usMode = ( UNSIGNED16 ) hb_parni( 2 ); } @@ -248,7 +298,7 @@ HB_FUNC( OADS_CHECKEXISTENCE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -274,7 +324,7 @@ HB_FUNC( OADS_DELETEFILE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -302,7 +352,7 @@ HB_FUNC( OADS_RENAMEFILE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szOld = hb_parc( 1 ); szNew = hb_parc( 2 ); } @@ -330,7 +380,7 @@ HB_FUNC( OADS_GETFILESIZE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -357,7 +407,7 @@ HB_FUNC( OADS_GETFILEDATE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -384,7 +434,7 @@ HB_FUNC( OADS_GETFILETIME ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -410,7 +460,7 @@ HB_FUNC( OADS_DIRMAKE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szPath = hb_parc( 1 ); } @@ -436,7 +486,7 @@ HB_FUNC( OADS_DIRREMOVE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szPath = hb_parc( 1 ); } @@ -462,7 +512,7 @@ HB_FUNC( OADS_DIREXIST ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szPath = hb_parc( 1 ); } @@ -498,7 +548,7 @@ HB_FUNC( OADS_DIRECTORY ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szMask = hb_parc( 1 ); usAttr = ( UNSIGNED16 ) hb_parni( 2 ); } @@ -601,7 +651,7 @@ HB_FUNC( OADS_FEXIST ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -632,7 +682,7 @@ HB_FUNC( OADS_MUTEXCREATE ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -661,7 +711,7 @@ HB_FUNC( OADS_MUTEXLOCK ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); ulTimeOut = ( UNSIGNED32 ) hb_parnint( 2 ); } @@ -689,7 +739,7 @@ HB_FUNC( OADS_MUTEXTRYLOCK ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -716,7 +766,7 @@ HB_FUNC( OADS_MUTEXUNLOCK ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -742,7 +792,7 @@ HB_FUNC( OADS_MUTEXDESTROY ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szName = hb_parc( 1 ); } @@ -806,7 +856,7 @@ HB_FUNC( OADS_SERVERVERSION ) if( hb_pcount() >= 1 ) hConn = ( ADSHANDLE ) hb_parnint( 1 ); else - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); ulRc = AdsGetServerVersion( hConn, ( UNSIGNED8 * ) szVer, &usLen ); szVer[ sizeof( szVer ) - 1 ] = '\0'; @@ -928,7 +978,7 @@ HB_FUNC( OADS_ZIP ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); base = 0; } if( hb_pcount() < base + 4 ) @@ -1023,7 +1073,7 @@ HB_FUNC( OADS_UNZIP ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); base = 0; } if( hb_pcount() < base + 1 ) @@ -1089,7 +1139,7 @@ HB_FUNC( OADS_ZIPFILECOUNT ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); szZip = hb_parc( 1 ); } @@ -1179,7 +1229,7 @@ HB_FUNC( OADS_ZIPFILELIST ) } else { - AdsGetDefaultConnection( &hConn ); + hConn = oads_default_connection(); base = 0; } if( hb_pcount() < base + 1 ) diff --git a/docs/abi-split-plan.md b/docs/abi-split-plan.md new file mode 100644 index 00000000..42ad8047 --- /dev/null +++ b/docs/abi-split-plan.md @@ -0,0 +1,30 @@ +# ACE export translation-unit split plan + +`src/abi/ace_exports.cpp` contains the public C ABI and tightly coupled process state. Splitting it is a refactor, not a behavior change. The work must preserve exported names, ordinals, calling conventions, handle lifetime, last-error behavior, and recursive-lock re-entry. + +## Boundary introduced in this release + +Shared process state now lives in `abi/runtime.{h,cpp}` under `openads::abi::detail`. There is still exactly one recursive mutex, handle registry, connection map, and remote-find registry. This small extraction creates a stable internal seam without moving public entry points. + +## Planned modules + +1. `backend_dispatch` and `remote_state` +2. low-coupling export groups: management, filesystem, compatibility, and stubs +3. connection and teardown paths +4. table, field, index, and transaction exports +5. dictionary and access-control exports +6. SQL statement state, catalogs, procedures, scripts, triggers, and execution + +SQL moves last because execution deliberately re-enters public SQL entry points while the recursive ABI lock is held. + +## Rules for each move + +- Keep C signatures and `ENTRYPOINT` declarations byte-for-byte compatible. +- Do not combine relocation with behavior changes or cleanup. +- Keep one definition of every registry, mutex, and thread-local. +- Preserve `.def` manifests and x86 stdcall aliases. +- Compare produced binary exports against the release baseline in CI. +- Run Windows MSVC and MinGW plus Linux and macOS checks for every move. +- Remove `/bigobj` and `-Wa,-mbig-obj` only after the remaining translation units no longer need them. + +Prefer reviewable moves of roughly 500-1,500 lines. Access-control semantics and known parity gaps are separate feature work and must not be hidden inside this refactor. diff --git a/docs/builds/mtfix22.md b/docs/builds/mtfix22.md new file mode 100644 index 00000000..36183753 --- /dev/null +++ b/docs/builds/mtfix22.md @@ -0,0 +1,23 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +# v1.09.70-mtfix22 - index focus test build + +Bug report supplied by Tim Stone and relayed by Pritpal Bedi. This is a test build, not a production recommendation. Use matching client and server versions and back up binaries and data before testing. No fork-main or upstream merge is part of this build. Tim's exact ECLMST test remains the application gate. + +## Changes + +LOCAL tag lookup now compares names without regard to case, matching REMOTE. Stored/displayed spelling and numeric tag order are unchanged. This covers CDX, native ADI and ADI routed through CDX. + +Stock Harbour rddads sets order 0 or empty focus by changing its current handle to the table handle, without sending an ACE reset. Table-handle GoTop, GoBottom and Skip now stop inheriting an order activated only by index-handle navigation. Index handles remain open, and record edits/appends still maintain all open tags. Explicit AdsSetIndexOrder/ByHandle retains ordered table-handle navigation. Newly created indexes keep the existing immediate table-navigation convention until index-handle navigation takes over; the stock zero-focus path then becomes natural. + +Parking a tag retains its scope and runtime direction. REMOTE direct natural Skip reanchors at the client-visible physical record when an order reset or locally served boundary pair left the server cursor elsewhere. Boundary-pair answers remain available; the next natural Skip synchronizes before stepping. + +## Validation + +Local Linux GCC used -O0/-g0 with warnings-as-errors disabled because of existing SQL/cache warnings; shipped platform flags are not changed. Focus/order/scope/navigation checks passed 152 cases and 61,760 assertions. The CDX-format ADI focused navigation run passed 23 cases and 2,395 assertions. New regression covers stored/requested tag cases, numeric lookup, repeated implicit zero/empty-focus call sequences, direct Skip, explicit APIs, retained scopes, creation-to-index-handle transition, and append maintenance across LOCAL/REMOTE ADT/CDX. + +Full-suite chunks covered 1,605 enabled cases and 16 disabled cases. MT reader/appender, Exclusive-held OpenIndex return-code, and remote-create storm tests failed intermittently; these failure classes also reproduced on mtfix21. Some tests were corrected to select an explicit order before navigating by table handle, rather than depending on implicit index focus. The normal CTest suite passes under its existing slow/flaky exclusions. No result from Tim's actual Windows 64-bit binary or ECLMST files is claimed. + +## Application check + +Repeat Tim's report with CDX-format ECLMST.adi (adt_cdx_index=1): select eclcom by name and number, check OrdName/IndexOrd, then set order 0 and empty focus and verify the physical record walk. Repeat LOCAL and REMOTE, switch back to a tag, test scopes and append/write maintenance. Report exact handles, recnos and return codes for any mismatch. Pritpal does not use ADT and is not providing application feedback for this fix. diff --git a/release-notes-1.09.68-diag-createindex.md b/release-notes-1.09.68-diag-createindex.md new file mode 100644 index 00000000..978579b9 --- /dev/null +++ b/release-notes-1.09.68-diag-createindex.md @@ -0,0 +1,9 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-diag-createindex - temporary client-side index-create probe + +This diagnostic build starts from v1.09.68-mtfix12 and adds an opt-in trace at the ACE client ABI boundary. It is for one fresh VouchADS org creation, not normal use. No server replacement is needed. + +Set `OPENADS_CREATE_INDEX_DIAG_FILE=C:\tmp\createindex_diag.log` before starting VouchADS.exe. Create C:\tmp first if needed. The DLL writes ENTRY records for AdsCreateIndex, AdsCreateIndex61 and AdsCreateIndex90, including the handle, file, tag, expression, optional condition and options; it records null-argument and unknown-handle 5000 exits, native or remote routing, and CreateIndex wire send and ACK/error. Only index-create calls are recorded. The log may contain application table paths and index expressions; share it privately, not in a public issue. Unset the environment variable after the run and restore the original DLL. + +Use the Windows x86 archive's ace32.dll for a 32-bit VouchADS.exe, or Windows x64 archive's ace64.dll only for a 64-bit executable. Put the diagnostic DLL alongside VouchADS.exe in place of its existing ACE DLL, after saving a backup. The version string in this build is 1.09.68-diag-createindex. Do not switch the server binary for this client-side test. diff --git a/release-notes-1.09.68-mtfix10.md b/release-notes-1.09.68-mtfix10.md new file mode 100644 index 00000000..5539bdc1 --- /dev/null +++ b/release-notes-1.09.68-mtfix10.md @@ -0,0 +1,49 @@ +# v1.09.68-mtfix10 — WAN voucher save, round 10 (TEST BUILD) + +**Built for Pritpal Bedi — bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS.** + +TEST ONLY — drop `ace32.dll` (Windows x86 zip) next to VouchADS.exe. Not for production. + +## What changed (client-side only) + +- **AdsGetNumLocks answers from the client lock ledger** while the ledger is + provably complete (no append auto-lock outstanding, no table lock) — the + same fast path mtfix8 gave AdsGetAllLocks. rddads polls lock counts after + every commit/unlock, and these calls still rode the wire opcode even right + after a real UnlockTable had provably emptied the ledger: ~9 RTTs per + startup+save cycle in the mtfix9 trace. +- **FileExists probe for a table open on the connection answers locally.** + A live table proves its own .dbf exists; the existing production-bag + short-circuit now covers the table's own path too. +- **Existence/dir cache keys unify slash direction** (the server + separator-normalizes, so the spellings are the same file). Case is + deliberately NOT folded: the Linux server fs is case-sensitive. +- **Trace: every FileExists probe now logs its real path and outcome** + (pos-cache / neg-cache / wire). The wire-op line's tid is a truncated + hash that cannot tell same-length paths apart; this line can. The next + trace will show exactly which files probe and why they miss. + +## Expected effect + +~0.5s off time-to-readiness (mtfix9 measured 37.9s). The lock-count polls +are the certain part; FileExists savings depend on how many probes hit the +new short-circuit — the new trace lines will quantify it. + +## Locking discipline (unchanged) + +No implicit locking anywhere: Seek stays pure, and only the app's explicit +dbRLock()/FLock() calls lock. mtfix10 changes only how lock *queries* are +answered, never when locks are taken. + +## Validation + +- New unit tests: GetNumLocks ledger fast path (incl. fallback to wire while + the append auto-lock makes the ledger uncertain), open-table existence + short-circuit. +- Full CI-tier suite green (1577/1577). + +## Verification asks + +1. Time-to-readiness number, same as before. +2. The new trace will contain `probe -> yes/no (pos-cache|neg-cache|wire)` + lines — send it over; those lines drive the next round. diff --git a/release-notes-1.09.68-mtfix11.md b/release-notes-1.09.68-mtfix11.md new file mode 100644 index 00000000..b63ad47e --- /dev/null +++ b/release-notes-1.09.68-mtfix11.md @@ -0,0 +1,12 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix11 - two ADS-compatibility fixes + +### 1. AdsIsRecordLocked now sees other connections' locks +Previously it answered from the caller's own lock list only (per connection), so a guard that QUERIES a marker record instead of attempting a lock stayed blind to other instances. Now: own list first, then a real OS byte-range lock probe (F_OFD_GETLK with F_GETLK fallback on Linux/macOS, try-LockFileEx on Windows). CDX/VFP nuance handled: a file lock's byte range covers every record lock, so another owner's FLock is correctly reported for any record, without false positives. SAP's own doc text for AdsIsRecordLocked was not independently re-verified - the behavioral evidence from production traces is the compat signal here. + +### 2. ADS_EXCLUSIVE opens enforced across sessions +Exclusive opens were a silent no-op (POSIX has no share modes), so a second instance's opens sailed through an exclusive hold that real ADS denies. The server now keeps an open registry keyed by the resolved file path: an exclusive holder denies other sessions both shared and exclusive opens; a shared-opened table denies another session's exclusive open; reopening/upgrading on the SAME connection stays legal. Denied opens return 7040 AE_FILE_IN_USE. Close and disconnect release the registration. Wire-level only: SQL-internal and engine-internal opens bypass it (same as before). Note for our own client: a closed shared table may stay parked for fast reopen, which keeps its server registration until eviction or disconnect - documented in the new test. + +### Tests +Full CI-tier suite green: 1579/1579 cases, 560,890 assertions. New: M13.1 cross-connection IsRecordLocked (record lock, FLock visibility, unlock-to-free), mtfix11 exclusive-open matrix (denial both directions, same-session reopen, upgrade rules, teardown sweep). One pre-existing test helper fixed (it had been opening ADS_EXCLUSIVE under the name open_shared - masked while exclusive was a no-op). diff --git a/release-notes-1.09.68-mtfix12.md b/release-notes-1.09.68-mtfix12.md new file mode 100644 index 00000000..8be0449e --- /dev/null +++ b/release-notes-1.09.68-mtfix12.md @@ -0,0 +1,23 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix12 - navigation fusion: boundary pairs, self-goto suppression, record-number anchoring + +One voucher save replayed from the mtfix11 production trace: 893 wire round trips. This release takes 155 of them off the wire in the default configuration (-17%), up to 201 (-23%) with the opt-in below. At the WAN rate he measured (~40 ms/RTT) that is about 6 s off the 38.5 s run by default, about 8.5 s with the opt-in. Zero behavior change: every suppressed call answers from state the server already certified, under the same freshness envelope the shipped duplicate-suppression uses. + +### 1. Boundary-pair certification (protocol extension, caps-gated) +GotoTop and GotoBottom requests can now ask the server to certify BOTH boundaries in one visit (new capability bit kCapNavBoundaryPair; old clients and old servers interoperate unchanged - the byte is only sent when both sides advertise it). The app's dbGoTop(); dbGoBottom() ritual - 102 adjacent pairs in the trace - now costs one frame instead of two. Bonus: top and bottom read in the same server visit are more consistent with each other than two separate frames ever were. + +### 2. Self-GotoRecord suppression +Re-issuing GotoRecord for the record the server cursor already sits on (the xBrowse bookmark-restore pattern - 89 of 126 GotoRecords in the trace are self-gotos) now answers locally. The default envelope is connection-wide (53 served in the replay); setting OPENADS_NAV_SELF_GOTO=table widens freshness to per-table (96 served in the replay) for apps that want the extra savings. + +### 3. GetRecordNum from the certified cursor +AdsGetRecordNum answers from the server-certified cursor anchor when the freshness envelope holds, instead of asking the server where it already told us it is. + +### Validation +- Full unit suite: 1588/1591 green; the 3 failures are the known pre-existing flaky tests that fail identically on the clean tree (mt contention, openindex race, create storm). +- New boundary-pair unit tests (82 assertions): pair serve in both directions, write invalidation, cross-table expiry, self-goto conditions, record-number anchor. +- Analytical replay of the mtfix11 production trace with the shipped invalidation rules: -102 (boundary pairs), -53 default / -96 opt-in (self-goto), GetRecordNum serves on top. Predicted wire totals: 738 default, 692 with OPENADS_NAV_SELF_GOTO=table (from 893). +- The replay caught one real bug during validation: the self-goto serve initially kept the prefetch queue a wire GotoRecord would have cleared; fixed, covered by the ramp test. + +### A/B +Same harness as mtfix9-11: wire_trace=1 in openads.ini (or OPENADS_WIRE_TRACE=1), OPENADS_WIRE_TRACE_FILE picks the log path. Run the voucher save on the mtfix11 build, then on mtfix12, count ` wire ` lines. Expect ~17% fewer by default, ~23% with OPENADS_NAV_SELF_GOTO=table; zero errors either way. diff --git a/release-notes-1.09.68-mtfix13.md b/release-notes-1.09.68-mtfix13.md new file mode 100644 index 00000000..4349a772 --- /dev/null +++ b/release-notes-1.09.68-mtfix13.md @@ -0,0 +1,7 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix13 - correct index-bag existence checks + +An optimization added in mtfix10 answered that `brwcon01.z01` existed if `brwcon01.dbf` was open, because it compared filenames after removing their extensions. With the bag absent, this could tell the application to skip creating its index. This build restricts the table-name shortcut to the exact table path. Checking for an unopened bag now goes through the normal file-existence path; already-open index bags retain their earlier shortcut. A regression test checks that a live DBF cannot make a missing bag of the same stem appear present, and that a repeated miss uses the negative cache. The prior index-create diagnostic remains available when `OPENADS_CREATE_INDEX_DIAG_FILE` is set; it is off by default. + +This is a targeted test build, not yet proof that it fixes Vouch's early exit. Test by starting a fresh org with BRWCON01.dbf and BRWCON01.z01 absent, then check that BRWCON01.z01 is created and the app continues. Save a backup of your old ace32.dll, replace only the client DLL in your application folder from the Windows x86 archive, and restore it after the test if needed. No server update is needed for this client-side change. diff --git a/release-notes-1.09.68-mtfix14.md b/release-notes-1.09.68-mtfix14.md new file mode 100644 index 00000000..0624060c --- /dev/null +++ b/release-notes-1.09.68-mtfix14.md @@ -0,0 +1,9 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix14 - B_BIG boundary-pair diagnostic + +This is a diagnostic test build, not a performance fix. Pritpal's 10-thread B_BIG trace on mtfix13 showed that ten GotoTop requests took roughly 0.9-1.2 seconds apiece, while 100 appends and field writes were individually fast. All worker traffic used one TCP lane. The mtfix12 boundary-pair feature makes each GotoTop also visit the opposite end of the table and count index keys, and it is the leading unproven source of the delay. + +This build disables only boundary-pair negotiation and requests on both sides of the wire. It retains mtfix13's index-bag existence correction, client lane pinning for detached-workarea lock identity, write/commit behavior and other navigation optimizations. It works with an existing server or client: a diagnostic client will not request a pair even from an older server advertising it, and a diagnostic server will not advertise pairs to older clients. The opposite-boundary call simply falls back to its normal wire request. + +**What to test:** Replace only the x86 client `ace32.dll` with the DLL in the Windows x86 archive, keeping the mtfix13 server unchanged. With no client trace enabled, run the same 10-thread, 100-record B_BIG job under the same conditions as the 185 ms upstream and 8.4 s mtfix13 comparisons. Record the job time and whether the first browse appears while workers are still running. A short trace on a separate run can confirm GotoTop returns without the 0x02 pair request; trace I/O itself can change timings. Back up the original DLL and restore it afterward. If this closes the gap, the boundary-pair server subphases need timing before a permanent fix is chosen. A remaining delay would point back to lane pinning or another interaction. Do not deploy this diagnostic build as a production fix. diff --git a/release-notes-1.09.68-mtfix15.md b/release-notes-1.09.68-mtfix15.md new file mode 100644 index 00000000..8a58ebd2 --- /dev/null +++ b/release-notes-1.09.68-mtfix15.md @@ -0,0 +1,7 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix15 - restore boundary-pair navigation without eager key counting + +The mtfix14 B_BIG diagnostic removed boundary-pair negotiation and cut the 10-thread job from roughly 8.4 seconds on mtfix13 to about 220 milliseconds, versus 185 milliseconds on upstream 1.09.69. This build restores opposite-boundary pair navigation, but removes the unconditional ordered key count from every pair response. The pair still carries the opposite row and boundary/record-count facts when valid; the optional key-count flag remains unset. If an ordered GoBottom actually needs a cold scoped key count, the client obtains it on demand. Tests cover ordered scoped and deleted rows, empty and nonempty pairs, opposite navigation after a write or order change, and the cold-count fallback. The existing workarea lane pin, lock ownership and write paths are unchanged. + +**What to test:** Back up the current x86 client DLL and replace only `ace32.dll` from the Windows x86 archive; keep the server at mtfix13 for the first A/B. With client tracing off, run the same B_BIG 10-thread/100-record job and compare with mtfix14's roughly 220 ms, then try the 10-instance local storm. If the pair traversal is cheap, expect about the mtfix14 time; if it is materially slower, leave mtfix14 in place and disable pair negotiation by default in the next change. Record the total time on each run. Both upstream and our client delay the first browse until the remote writers finish, so browse timing alone does not distinguish the builds. This is a test build, not a production performance claim: the remote one-instance timings still vary (mtfix14 32-60 s versus upstream 13.8 s on the same server), and the Vouch third-instance add/edit failure and fresh-org `uxxyyzza.dbf` CreateTable 5000 are separate open issues. Do not change the server for this client A/B. Restore the backed-up DLL if the test regresses. diff --git a/release-notes-1.09.68-mtfix16.md b/release-notes-1.09.68-mtfix16.md new file mode 100644 index 00000000..b637445c --- /dev/null +++ b/release-notes-1.09.68-mtfix16.md @@ -0,0 +1,14 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix16 - opt-in fresh-organization diagnostics + +This is a test build, not a fix or a production recommendation. It keeps mtfix15 behavior, but can log stages for remote table creation and subsequent server-side opens and appends. The server records DBF write, memo creation, its local post-create reopen, close result, wire OpenTable outcomes, and AppendBlank failures. The client records whether CreateTable failed on the wire or on its own final reopen, and failed OpenTable and AppendRecord calls. Codes are not changed. No table schema or row content is written to this diagnostic log; it contains sanitized table basenames, stage, code and timestamp. The normal `ads_err` log remains separate and may hold additional context. + +Set environment variables on **both** the Vouch client process and the server process before launch: + +- `OPENADS_CREATE_TABLE_DIAG_FILE`: a distinct writable log path on each machine, e.g. `C:\temp\openads-create-client.log` and `/tmp/openads-create-server.log`. Absent means diagnostic off. +- `OPENADS_CREATE_TABLE_DIAG_FILTER=uxxyyzza.dbf` for one table. To investigate several first-login table failures, omit FILTER: every CreateTable and failed OpenTable or AppendBlank can be logged. This broad mode can cause extra disk I/O and stops writing at 2 MB per process; use it for one fresh-org reproduction only, then unset both variables and restart. With a filter, the server additionally logs successful opens and first appends of the named table. + +Keep client/server DLL and daemon from this same build. Collect both logs, the app's exact error or silent symptom, and the table name and time. A create error followed by a file on disk proves the file was written but not where the subsequent failure happened; the stage log is the answer. The test uses a long-path fixture on POSIX and a memo-create failure on Windows to force a post-write failure, then checks that the DBF remains, the stage is logged, and the same ACE code reaches the caller. This diagnostic does not modify locks, retry policy, lane pinning, file format, or client-visible error codes. + +**CI caveat:** Windows x86/x64 and Linux tests, including the new diagnostic test, passed on this commit. macOS compiled, but its full test step exceeded the CI job's 30-minute limit and was cancelled; the same macOS timeout occurred on the prior diagnostic commit and predated mtfix16 on this branch. The release workflow does not use macOS tests as a gate. macOS test status remains unverified. Keep mtfix15 as the recommended working build; use mtfix16 only to capture the fresh-org failure, then restore mtfix15. diff --git a/release-notes-1.09.68-mtfix17.md b/release-notes-1.09.68-mtfix17.md new file mode 100644 index 00000000..493a5e29 --- /dev/null +++ b/release-notes-1.09.68-mtfix17.md @@ -0,0 +1,13 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix17 - fresh-organization directory fix candidate + +This is a test build, not a production recommendation. Keep mtfix15 as the recommended working build while checking this candidate against a fresh Vouch organization. It retains the opt-in table-create diagnostics from mtfix16. + +The remote client no longer caches positive or negative directory-existence answers or skips DirMake after an earlier success. DirExist and DirMake now reach the server every time, so a directory removed or created outside the current connection is not hidden by an old answer. Server-side DirExist also treats a missing path as a normal false answer, not an internal error. File-existence and other optimizations are unchanged. Each directory call may now cost one extra network round trip compared with a cache hit. + +The regression test creates a directory, sees it, removes it directly on the server, checks that the same connection sees it missing, and verifies that DirMake recreates it, including a repeat mkdir after another removal. This matches a plausible cause of the observed first-create failure (`uxxyyzza.dbf`: exclusive open saw a missing parent directory), but the diagnostic trace did not prove the full raw table path or all of the Vouch folder steps. A successful fresh-org Vouch run remains the needed validation. + +**What to test:** Use the mtfix17 client and server from this same build for one new Vouch organization, including its first table create. Report the organization-create result, any `uxxyyzza.dbf` or other missing-folder error, and the client and server diagnostic logs if it still fails. Then return to mtfix15 for regular work until this candidate is validated. The diagnostic environment variables from mtfix16 are optional and should be unset again after capture. + +**CI:** Windows x86/x64 and Linux (including TLS) passed, along with Harbour smoke. macOS configured and built, but its test step was cancelled at the CI timeout, so macOS tests remain unverified. The same timeout affected prior builds; this is not evidence that mtfix17's macOS tests passed. No strict no-symlink change is included here. diff --git a/release-notes-1.09.68-mtfix18.md b/release-notes-1.09.68-mtfix18.md new file mode 100644 index 00000000..e643beaa --- /dev/null +++ b/release-notes-1.09.68-mtfix18.md @@ -0,0 +1,13 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix18 - fresh-organization existence fix candidate + +This is a test build, not a production recommendation. Keep mtfix15 as the recommended working build while checking this candidate against a fresh Vouch organization. mtfix17 was superseded before its package release: it fixed directory probes but missed the separate file-existence cache that Vouch's guard may use. The empty mtfix17 pre-release and tag were removed, and its packaging workflow was cancelled. mtfix18 includes both fixes and retains the opt-in table-create diagnostics from mtfix16. + +Remote DirExist, DirMake, and CheckExistence now reach the server every time. This removes both positive and negative directory and file-existence caches, plus the shortcut that inferred file existence from an open table or similarly named index bag. A folder or file created or removed outside the current connection should no longer be hidden by a stale answer. Server-side DirExist treats a missing path as a normal false answer. Each formerly cached existence call may add one network round trip; no file format, locking, or retry behavior changes here. + +Tests cover DirExist true -> server-side removal -> DirExist false -> DirMake recreates, and file existence in both directions after external server-side changes on the same connection. They also check that an open table or bag does not bypass an exact FileExists probe. The observed first-create error (`uxxyyzza.dbf`: exclusive open saw a missing parent directory) is consistent with a stale existence answer, but the earlier diagnostic trace did not prove the full raw table path or every Vouch folder step. A successful fresh-org Vouch run is still needed to validate this candidate. + +**What to test:** Use mtfix18 client and server from this same build for one new Vouch organization, including its first table create. Report the organization-create result, any `uxxyyzza.dbf` or other missing-folder error, and the client and server diagnostic logs if it still fails. Check whether Vouch's `OAds_FileExist()` wrapper reaches OpenADS's `oads_CheckExistence` export; the fork's Harbour helpers are named `OADS_CHECKEXISTENCE` and `OADS_FEXIST`, and it does not ship an `OADS_FILEEXIST` helper by that exact name. Return to mtfix15 for regular work until mtfix18 is validated. + +**CI:** Windows x86/x64 and Linux (including TLS) passed, along with Harbour smoke. macOS configured and built, but its test step was cancelled at the CI timeout, so macOS tests remain unverified. This timeout occurred on earlier builds too; it is not evidence that mtfix18's macOS tests passed. Strict no-symlink enforcement is separate and not in this test build. diff --git a/release-notes-1.09.68-mtfix19.md b/release-notes-1.09.68-mtfix19.md new file mode 100644 index 00000000..52678703 --- /dev/null +++ b/release-notes-1.09.68-mtfix19.md @@ -0,0 +1,13 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix19 - OAds connection handoff test build + +This is a test build, not a production recommendation. Keep mtfix15 as the recommended working build until Vouch's fresh-organization flow validates this candidate. mtfix19 includes mtfix18's always-live directory and file existence checks, plus a change in the Harbour `contrib/oads_hb/oads_hb.c` glue. It does not change server-side table locking or file formats. + +`OAds_SetConnection(hConn)` now stores a shared OAds default handle, and `OAds_GetConnection()` and all no-handle OAds file, directory, mutex, archive, and server-version helpers use that same handle. Explicit-handle overloads are unchanged. Before the first OAds SetConnection, the ACE default remains the fallback. Zero is rejected as a new OAds default. This targets the Vouch case where `OAds_DirExist(cName)` answered YES for a directory absent on the server but `OAds_DirExist(SetAdsConxn(), cName)` answered NO. That difference establishes a wrong implicit connection in that run; it does not prove whether a thread switch, intervening overwrite, or compiled binding difference caused the old default to diverge. + +**What to test:** `oads_hb.c` is compiled into Vouch. Rebuild Vouch with the mtfix19 copy of that file and restore the original one-argument `OAds_DirExist(cName)` call. Merely replacing the DLL does not replace the compiled glue. Use matching mtfix19 client and server builds for a clean new Vouch organization. Confirm the absent directory returns NO, creation succeeds, and the first tables open. Report the exact return/error and both client/server logs if it fails. In apps concurrently using independent connections or organizations, pass the explicit handle rather than switching one process-wide OAds default mid-operation. + +A separate 700-instance B_BIG WAN storm stalled and remains undiagnosed. This test build does not claim a storm fix or a safe upper bound. + +**Validation:** The source diff passes whitespace checks and a C syntax-only check against mocked Harbour declarations; a real Harbour Vouch rebuild and fresh-org test are still required. Windows x86/x64 and Linux CI status belongs to this release's workflow once complete. macOS configured and built in the prior mtfix18 run, but its Test step timed out/cancelled, so macOS tests remain unverified until a completed run says otherwise. diff --git a/release-notes-1.09.68-mtfix2.md b/release-notes-1.09.68-mtfix2.md new file mode 100644 index 00000000..b924cf9c --- /dev/null +++ b/release-notes-1.09.68-mtfix2.md @@ -0,0 +1,19 @@ +# OpenADS test build 1.09.68-mtfix2 (TEST ONLY - not for production) + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on 1.09.68. Not an official release. + +## What's in it +- Session-pool multi-thread fix 1: the remote table store (park/adopt on close, AdsCheckExistence scan, index-park invalidation, relation check) now always holds the global lock. Fixes a heap corruption crash seen in network_pool_mt_test. +- Server multi-thread fix 2: conn_for_table holds the global lock while scanning connection table maps (ThreadSanitizer race with Connection::open_table). +- Logical fields over remote (contributor patch): remote writes now store the DBF byte 'T'/'F' instead of '1'/'0', so FOR-conditional indexes, DBFCDX and SAP ADS read them correctly. New regression test abi_remote_logical_write_test. +- Patch 1: safe local answers on the remote path (fewer round trips at startup). +- Wire-frame logging kept on for diagnosis. +- network_pool_mt_test repeats its 4-thread phase 25 times; new network_pool_mt_repeat test runs it in 30 fresh processes. + +## What to test +- Several Vouch instances at once, many threads, against openads_serverd from this build (Linux tarball) and the client DLL from the Windows x86 zip. +- LOGGED turns T on login; the license limit holds. +- Startup time over remote. + +## Packages +- openads-1.09.68-mtfix2-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.68-mtfix20.md b/release-notes-1.09.68-mtfix20.md new file mode 100644 index 00000000..582a7733 --- /dev/null +++ b/release-notes-1.09.68-mtfix20.md @@ -0,0 +1,13 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.68-mtfix20 - record-lock contract test build + +This is a test build, not a production recommendation. Keep mtfix15 as the recommended working build until Pritpal checks this candidate in Vouch. mtfix20 includes the earlier mtfix19 changes; this release adds two record-lock fixes without changing LockMgr's internal reference counting or the global lock probe, which remains available internally. + +A repeated RLOCK of the same record by the same Table handle is now a no-op for lock acquisition. In Xbase terms, `RLOCK(n)` followed by another `RLOCK(n)` still holds one record lock, and one successful `UNLOCK(n)` releases it. This addresses Pritpal's observation that repeated locks followed by a single unlock left the record locked. + +`AdsIsRecordLocked` (used by Harbour's `DBRI_LOCKED`) now reports whether the calling table handle owns the record lock, rather than whether any connection has locked that record. That matches the own-handle lock list returned by `AdsGetAllLocks` (used by `dbrLockList()`), so the two Vouch `IsLogged` branches should agree. For an uncertain remote append auto-lock, it asks the same handle's `GetAllLocks` rather than using the global probe. Another connection's lock does not make this handle's `DBRI_LOCKED` true. This answers lock ownership, not whether acquiring a new lock would succeed; the global probe is retained internally for that separate question. + +**What to test:** With matching mtfix20 client and server builds, use the Windows x86 archive for Pritpal's Vouch setup. On one handle, RLOCK a record twice, unlock it once, and check both `DBRI_LOCKED` and `dbrLockList()`. On a second connection, check that it can then acquire the record. Also test Vouch's blocked record-edit sequence and the `LOGIN_A` case, recording the exact handle, record number, return code, and lock list if either remains wrong. No Vouch result is claimed here. Back up the working DLLs and return to mtfix15 for regular work if this candidate regresses. + +**Validation:** The guarded patch run built and tested the Linux configurations before committing. In full CI #36, the first TLS Linux run failed an ordered-prefetch byte-ratio assertion; that one-run fluctuation cleared on the single failed-jobs rerun and is unrelated to the lock changes. The other non-macOS jobs passed. The macOS build finished, but its unit test was canceled by the known 30-minute CI job timeout, which predates these changes. macOS unit tests remain unverified; do not read the canceled overall CI status as a green run. The release workflow separately requires the Windows x86/x64, Linux x64, and macOS universal packaging legs and all four canonical assets before publication. diff --git a/release-notes-1.09.68-mtfix3.md b/release-notes-1.09.68-mtfix3.md new file mode 100644 index 00000000..e17639e2 --- /dev/null +++ b/release-notes-1.09.68-mtfix3.md @@ -0,0 +1,16 @@ +# OpenADS test build 1.09.68-mtfix3 (TEST ONLY - not for production) + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on 1.09.68. Not an official release. + +## What's in it +- Everything in 1.09.68-mtfix2 (session-pool MT fixes 1 and 2, contributor logical-field fix, patch 1 safe local answers). +- Patch 2: still-empty window. After the server confirms a table is empty, Seek / GO n on that table are answered locally for up to 1.5 s instead of a round trip each. A new record added by another station shows up once the window ends; this station's own writes close the window at once. OPENADS_EMPTY_TTL_MS sets the window in ms (0 disables, max 2000). Trace lines mark each local empty answer. +- Wire-frame logging kept on for diagnosis. +- network_pool_mt_test (25x 4-thread phase) and network_pool_mt_repeat (30 fresh processes) pass. + +## What to test +- Several Vouch instances at once, many threads, against openads_serverd from this build (Linux tarball) and the client DLL from the Windows x86 zip. +- Startup time over remote compared with mtfix2. + +## Packages +- openads-1.09.68-mtfix3-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.68-mtfix4.md b/release-notes-1.09.68-mtfix4.md new file mode 100644 index 00000000..4f651cae --- /dev/null +++ b/release-notes-1.09.68-mtfix4.md @@ -0,0 +1,18 @@ +# OpenADS test build 1.09.68-mtfix4 (TEST ONLY - not for production) + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on 1.09.68. Not an official release. + +## What's in it +- Everything in 1.09.68-mtfix3 (session-pool MT fixes 1 and 2, contributor logical-field fix, patch 1 safe local answers, patch 2 still-empty window). +- Remote REINDEX fix: remote AdsReindex / ordListRebuild now really rebuilds the table's .cdx/.z01 bags on the server (before, it rebuilt nothing and still reported success). New test network_remote_reindex_test. +- Patch 2: still-empty window. After the server confirms a table is empty, Seek / GO n on that table are answered locally for up to 1.5 s instead of a round trip each. A new record added by another station shows up once the window ends; this station's own writes close the window at once. OPENADS_EMPTY_TTL_MS sets the window in ms (0 disables, max 2000). Trace lines mark each local empty answer. +- Wire-frame logging kept on for diagnosis. +- network_pool_mt_test (25x 4-thread phase) and network_pool_mt_repeat (30 fresh processes) pass. + +## What to test +- Several Vouch instances at once, many threads, against openads_serverd from this build (Linux tarball) and the client DLL from the Windows x86 zip. +- REINDEX over remote: .z01 file times should change. +- Startup time over remote. + +## Packages +- openads-1.09.68-mtfix4-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.68-mtfix5.md b/release-notes-1.09.68-mtfix5.md new file mode 100644 index 00000000..430812e7 --- /dev/null +++ b/release-notes-1.09.68-mtfix5.md @@ -0,0 +1,17 @@ +# OpenADS test build 1.09.68-mtfix5 (TEST ONLY - not for production) + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on 1.09.68. Not an official release. + +## What's in it +- Everything in 1.09.68-mtfix4 (session-pool MT fixes 1 and 2, contributor logical-field fix, patch 1 safe local answers, patch 2 still-empty window, remote REINDEX fix). +- MT fix 3: a remote table open no longer holds the process-wide OpenADS lock while it waits for the server's reply. Before, every other OpenADS call in the same process (any thread) waited a full round trip whenever one thread opened a remote table, and with an in-process server it could deadlock (rare hang in the repeated pool MT test). Same rule the code already used for the pool flush and the production-index auto-open. +- Wire-frame logging kept on for diagnosis. +- network_pool_mt_test (25x 4-thread phase) and network_pool_mt_repeat (30 fresh processes) pass; a separate diagnostic loop ran the pool MT test 1,800 more times per build with no hang. + +## What to test +- Several Vouch instances at once, many threads, against openads_serverd from this build (Linux tarball) and the client DLL from the Windows x86 zip. +- REINDEX over remote: .z01 file times should change. +- Startup time over remote. + +## Packages +- openads-1.09.68-mtfix5-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.68-mtfix6.md b/release-notes-1.09.68-mtfix6.md new file mode 100644 index 00000000..baec725b --- /dev/null +++ b/release-notes-1.09.68-mtfix6.md @@ -0,0 +1,19 @@ +# OpenADS test build 1.09.68-mtfix6 (TEST ONLY - not for production) + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on 1.09.68. Not an official release. + +## What's in it +- Everything in 1.09.68-mtfix5 (session-pool MT fixes 1-3, contributor logical-field fix, patch 1 safe local answers, patch 2 still-empty window, remote REINDEX fix). +- Single-attempt locks by default. A failed AdsLockRecord / AdsLockTable now returns AE_LOCK_FAILED after ONE attempt (one round trip remote), matching xBase RLOCK()/FLOCK() semantics - fail means fail and the application owns any retry loop. Before, the ACE client kept retrying a contended lock for up to ~1 second (10 x 100 ms) even though the server had already answered on the first attempt, which turned every intentional lock probe (login-semaphore walks, "already logged?" checks) into a ~1 s burn per probe. Opt back into ACE-style waits per install: openads.ini lock_retry_count = 10 (env OPENADS_LOCK_RETRY_COUNT), optional lock_cycle_ms ceiling (default 100). Apps that call AdsSetLockCycle / AdsSetLockRetryCount get exactly what they ask for, unchanged. +- The server's append auto-lock keeps its own internal budget (10 x 100 ms, engine_lock_retry_count / engine_lock_cycle_ms) - it is an internal wait the app cannot do itself, so single-attempt client locks do not break appends under FLock/Browse convoys. +- Lane pinning by alias (lane_pin_alias = 1, default). Every USE of the same alias+path on one logical connection binds to the same server session, from any thread and across close/reopen. Record-lock identity now follows the workarea, so Harbour zero-space workareas (hb_dbRequest / hb_dbDetach, Vouch's LOGIN_A pattern) keep their locks visible and unlockable no matter which thread holds the workarea. Different aliases of the same file keep different lock owners, so cross-owner lock probes (UsrConsole) still conflict as before. lane_pin_alias = 0 restores pure thread-affinity. + +## What to test +- Login semaphore flow from several threads: log in on one thread, IsLogged()/__howManyLogins from others; lock probes should be instant now, not ~1 s each. +- b_devboo / b_backup paths (LogUse closes and re-USEs LOGIN_A): login state must stay consistent afterwards. +- UsrConsole: still shows logged-in users (lock probes across aliases must still fail). +- Saving a record over remote: RLock -> REPLACE -> Unlock should cost only its round trips (about 4-6). +- Startup time over remote; several Vouch instances at once against openads_serverd from this build (Linux tarball) with the client DLL from the Windows x86 zip. + +## Packages +- openads-1.09.68-mtfix6-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.68-mtfix7-diag.md b/release-notes-1.09.68-mtfix7-diag.md new file mode 100644 index 00000000..78199318 --- /dev/null +++ b/release-notes-1.09.68-mtfix7-diag.md @@ -0,0 +1,15 @@ +# OpenADS test build 1.09.68-mtfix7-diag (TEST ONLY - not for production) + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on mtfix6 / 1.09.68. Not an official release. Nothing merged to main or upstream. + +## What's in it +- Everything in 1.09.68-mtfix6, plus a client-only timing trace for remote operations. No Vouch application change or server update is needed for this trace. +- Opt-in `wire_trace = 1` in the `openads.ini` used by Vouch. Set `wire_trace_file = C:/tmp/cli_trace.log` to choose the output path; create `C:/tmp` first. If unset, this is the default path. Or set `OPENADS_WIRE_TRACE=1` and `OPENADS_WIRE_TRACE_FILE` before starting Vouch. Restart Vouch after changing the settings; turn tracing off again when done. +- Each completed remote round trip records a local wall-clock timestamp, milliseconds since trace start, table alias when known, named wire operation (AppendBlank, SetFields, FlushTable, LockRecord, UnlockRecord, Seek, etc.), request/reply sizes, duration in microseconds, table ID, and connection marker. Existing local-answer trace lines remain. Field values and seek keys are not logged by this patch. The log can contain file/table names and existing trace details, so review it before sharing. Tracing adds disk I/O and changes timings: use it to count calls and find slow calls, not as an unmodified benchmark. + +## What to test +- Post one voucher over the remote connection. Compare the voucher append/write/commit/unlock and the two GL seek/lock/write/commit/unlock paths. See whether field writes go as a single SetFields batch or multiple SetField frames. +- If useful, run one remote startup with tracing on. Zip and share the log after reviewing it. No timing points need to be added in Vouch. + +## Packages +- openads-1.09.68-mtfix7-diag-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.68-mtfix8.md b/release-notes-1.09.68-mtfix8.md new file mode 100644 index 00000000..c8ba93ba --- /dev/null +++ b/release-notes-1.09.68-mtfix8.md @@ -0,0 +1,25 @@ +# OpenADS test build 1.09.68-mtfix8 (TEST ONLY - not for production) + +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on mtfix7-diag / 1.09.68. Not an official release. Nothing merged to main or upstream. + +## What's in it +Everything in 1.09.68-mtfix7-diag, plus four client-side round-trip cuts found by your wire trace (no Vouch application change): + +1. Commit slimming. The trace showed each voucher save paying frames that release or flush nothing: + - UnlockTable is skipped when the client knows no locks are held (rddads calls it before every lock and every append - 1 round trip each time). + - GetAllLocks is answered from the client's own lock list when that list is provably complete (1 round trip after every commit). + - COMMIT on a table with no changes since the last flush sends nothing (kills the 16-frame flush storm after every voucher save). + - With an mtfix8 SERVER, FlushTable now includes the full file flush, so the trailing FlushFileBuffers frame goes away too (one frame per commit instead of two). This one saving needs the server updated; the other three work against your current server. Mixed old/new is safe both ways - the old pair of frames is kept automatically when the server is older. +2. Locked-then-released tables park instead of closing. GN_COUNT, FA_ACC01 and USASELOG each paid a full 7-frame reopen per save because they had once held a lock. Locks actually still held at close still force a real close, so no lock can leak to other users. +3. Directory and missing-file answers cache per session. Startup DirExist/DirMake probes and "is this optional file there" checks stop repeating round trips. Note: a file created by another workstation mid-session can be reported missing until your session itself creates or deletes something - tell me if you hit that. + +## What to test +- Turn on wire_trace as before, post one voucher, send the new log. The save should show "skipped: no locks held", "clean: flush skipped" and "served from client lock ledger" lines, far fewer FlushTable/FlushFileBuffers frames, and no full reopen of GN_COUNT. +- Expected: roughly 40-60 fewer round trips per voucher save than the mtfix7-diag trace (about 2-3s at your ~43ms latency), plus up to ~1.5s off startup. +- Also run a normal startup, a reindex, and two or three Vouch instances at once - the lock handling changed, so watch for any user being blocked from a record they just unlocked (that would be a bug, report it). +- For the full saving, update openads_serverd on Lightsail with this build using your usual script. Client-only already gives most of it. + +## Packages +- openads-1.09.68-mtfix8-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.68-mtfix9.md b/release-notes-1.09.68-mtfix9.md new file mode 100644 index 00000000..05777cd7 --- /dev/null +++ b/release-notes-1.09.68-mtfix9.md @@ -0,0 +1,22 @@ +# OpenADS test build 1.09.68-mtfix9 (TEST ONLY - not for production) + +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Based on mtfix8 / 1.09.68. Not an official release. Nothing merged to main or upstream. + +## What's in it +Everything in 1.09.68-mtfix8, plus one client-side round-trip cut: + +Navigation stamps survive the CloseAllIndexes/OpenIndex rotation. rddads closes and reopens the index set around many operations, and each reopen paid an extra GotoTop round trip to reposition (57 of those frames in your mtfix8 trace). The client now parks the position stamp when the index set closes and restores it on the matching reopen when the order is unchanged, so that frame goes away. Client-only change, no wire-protocol difference - works against any server version, mixed old/new is safe. + +## Measured so far (your mtfix8 trace) +- Round trips: 1,079 -> 1,009, wire time 47.1s -> 44.1s. +- Voucher save: 280 -> 230 round trips, 12.4s -> 9.7s wire. +- Flush storm gone: FlushTable 46 -> 15, FlushFileBuffers 19 -> 0. + +## What to test +- Turn on wire_trace as before, post one voucher, send the new log. Expect "unpark hit" lines on reopened indexes and roughly 50-60 fewer round trips than the mtfix8 run (about 2.5s at your ~43ms latency), mostly from dropped GotoTop frames. +- Also run a normal startup, a reindex, and two or three Vouch instances at once. + +## Packages +- openads-1.09.68-mtfix9-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/release-notes-1.09.70-mtfix21.md b/release-notes-1.09.70-mtfix21.md new file mode 100644 index 00000000..988e8258 --- /dev/null +++ b/release-notes-1.09.70-mtfix21.md @@ -0,0 +1,35 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +## v1.09.70-mtfix21 - upstream-sync and Exclusive ADT append test build + +This is a test build, not a production recommendation. Keep mtfix15 as the recommended working build until Pritpal checks this candidate in Vouch. Use matching client and server builds. Back up the working binaries before testing. + +This candidate syncs FiveTechSoft/OpenADS main through 54e237b1 into the fork's test branch with a real two-parent merge. It keeps the fork's mtfix20 lock-contract fixes, client lock-ledger fast paths, engine append retry policy, and CI/release workflows. Upstream's accidental duplicated table.cpp prefix was removed locally so the merged source has one append implementation. No source merge to fork main or upstream is part of this release. + +### Exclusive ADT append + +Exclusive ADT appends no longer add an automatic record lock for each new row. Shared ADT appends still take a record lock, and shared writes still require the caller's lock. The engine change affects Exclusive ADT engine instances, including the server; it is not limited to LOCAL connections. DBF append behavior is unchanged by this condition. + +A new LOCAL ADT regression appends 10,000 rows with field updates and no per-row AdsWriteRecord, checks that Exclusive mode retains zero record locks, closes/reopens the table, and verifies distinct persisted values at rows 1, 5,000 and 10,000. It also checks the Shared write guard and append lock/unlock behavior. + +No measured speed gain on Tim Stone's 300,000-row program or on Pritpal's Vouch workload is claimed. Harbour's ordinary DbUnlock can skip Exclusive tables; direct AdsUnlockRecord reaches the engine. The shown append loop does not establish per-row fsync, and an empty transaction log without an explicit transaction is expected. + +### Other synced upstream work + +The upstream base advances CMake's project version to 1.09.70. The test series steps up from v1.09.68-mtfix20 to v1.09.70-mtfix21 to match that base; use the new version in test download scripts. The tag supplies the packaged binary version. The sync also includes upstream NTX inter-process index locking and MariaDB numeric/logical setters. Those upstream additions need application testing where used. + +The server-side own-lock query recognizes the calling engine handle's table lock and the ABI twin's record locks, with an engine held-list fallback. It does not substitute a global other-connection lock probe for the caller's own lock state. + +### Validation + +Local Linux GCC validation used -O0/-g0 and disabled warnings-as-errors after existing shadow warnings. The focused lock/introspection harness passed 50 cases and 2,120 assertions. The new ADT regression passed 1 case and 20,034 assertions. + +The full local suite ran 1,604 cases: 1,601 passed, 3 failed, and 16 were skipped, with 598,135 assertions and 8 assertion failures. The failures involved the MT reader/appender walk, an Exclusive-held OpenIndex return-code expectation, and remote create/append stress counts. All three failure classes reproduced on a reconstructed pre-sync 807f0694 baseline with the changed core sources and their header consumers recompiled. This was a focused baseline comparison, not a fresh full baseline CI run; it does not make the full suite green. + +CI #41 on 510e3626 passed all 10 non-macOS jobs, including Windows MSVC x86/x64, Linux with and without TLS, Harbour smoke, PHP, and the live SQL jobs. macOS configured and built successfully, then its unit test was canceled by the existing 30-minute CI job cap. macOS unit tests remain unverified; the overall CI status is Cancelled, not green. The release workflow separately requires all four packaging legs and canonical assets before publication. + +CI: https://github.com/bedipritpal/OpenADS/actions/runs/36664053398 + +### Application checks + +Test Tim's Exclusive LOCAL ADT append program against a backed-up dataset and compare elapsed time, record count, persisted values, and lock count. Test Shared append and locked record edits separately. For Vouch, retest repeated RLOCK/unlock, DBRI_LOCKED versus dbrLockList(), blocked edits, and LOGIN_A using matching client/server binaries. Record exact return codes, handles and record numbers if anything disagrees. No Vouch result is claimed here. diff --git a/release-notes-1.09.70-mtfix22.md b/release-notes-1.09.70-mtfix22.md new file mode 100644 index 00000000..f90f34fd --- /dev/null +++ b/release-notes-1.09.70-mtfix22.md @@ -0,0 +1,34 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +# v1.09.70-mtfix22 - index focus test build + +Bug report supplied by Tim Stone and relayed by Pritpal Bedi. This is a test build, not a production recommendation. Use matching client and server versions and back up binaries and data before testing. No fork-main or upstream merge is part of this build. Tim's exact ECLMST test remains the application gate. + +## Changes + +LOCAL tag lookup now compares names without regard to case, matching REMOTE. Stored/displayed spelling and numeric tag order are unchanged. This covers CDX, native ADI and ADI routed through CDX. + +Stock Harbour rddads sets order 0 or empty focus by changing its current handle to the table handle, without sending an ACE reset. Table-handle GoTop, GoBottom and Skip now stop inheriting an order activated only by index-handle navigation. Index handles remain open, and record edits/appends still maintain all open tags. Explicit AdsSetIndexOrder/ByHandle retains ordered table-handle navigation. Newly created indexes keep the existing immediate table-navigation convention until index-handle navigation takes over; the stock zero-focus path then becomes natural. + +Parking a tag retains its scope and runtime direction. REMOTE direct natural Skip reanchors at the client-visible physical record when an order reset or locally served boundary pair left the server cursor elsewhere. Boundary-pair answers remain available; the next natural Skip synchronizes before stepping. + +## Validation + +Local Linux GCC used -O0/-g0 with warnings-as-errors disabled because of existing SQL/cache warnings; shipped platform flags are not changed. Focus/order/scope/navigation checks passed 152 cases and 61,760 assertions. The CDX-format ADI focused navigation run passed 23 cases and 2,395 assertions. New regression covers stored/requested tag cases, numeric lookup, repeated implicit zero/empty-focus call sequences, direct Skip, explicit APIs, retained scopes, creation-to-index-handle transition, and append maintenance across LOCAL/REMOTE ADT/CDX. + +Full-suite chunks covered 1,605 enabled cases and 16 disabled cases. MT reader/appender, Exclusive-held OpenIndex return-code, and remote-create storm tests failed intermittently; these failure classes also reproduced on mtfix21. Some tests were corrected to select an explicit order before navigating by table handle, rather than depending on implicit index focus. The normal CTest suite passes under its existing slow/flaky exclusions. No result from Tim's actual Windows 64-bit binary or ECLMST files is claimed. + +## Application check + +Repeat Tim's report with CDX-format ECLMST.adi (adt_cdx_index=1): select eclcom by name and number, check OrdName/IndexOrd, then set order 0 and empty focus and verify the physical record walk. Repeat LOCAL and REMOTE, switch back to a tag, test scopes and append/write maintenance. Report exact handles, recnos and return codes for any mismatch. Pritpal does not use ADT and is not providing application feedback for this fix. + +## Automated build status + +The guarded patch run passed Linux Clang build/test with and without TLS before committing the working-branch change. [fork CI run 51](https://github.com/bedipritpal/OpenADS/actions/runs/36812469507) passed all 10 non-macOS jobs: Windows x64/x86, Linux Clang with and without TLS, Harbour smoke (Windows), PHP binding, and four SQL jobs. + +macOS unit validation is incomplete. Its Configure and Build steps passed, but the Test step reached the runner's 30-minute job cap and was canceled after printing only "Start 1: openads_unit_tests". No source assertion failure was shown. Investigation is continuing. This limitation does not count as a macOS unit-test pass. Packaging run 33 built every canonical archive successfully. Its macOS diagnostic Test step also timed out at the existing 20-minute step cap. The repeated timeout still does not count as a macOS unit-test pass. Final archives are opened and checked for required files, matching ACE aliases and the mtfix22 version stamp before publication. + +Apply-patch: https://github.com/bedipritpal/OpenADS/actions/runs/36812071504 +CI: https://github.com/bedipritpal/OpenADS/actions/runs/36812469507 + +Packages must include Windows x64/x86, Linux x64, and macOS universal archives. Use the full client/server kit for the chosen platform; do not mix old and new ACE libraries. diff --git a/release-notes-1.09.71-mtfix23.md b/release-notes-1.09.71-mtfix23.md new file mode 100644 index 00000000..fc3ddc8d --- /dev/null +++ b/release-notes-1.09.71-mtfix23.md @@ -0,0 +1,41 @@ +Built for Pritpal Bedi - bedipritpal/OpenADS, forked from FiveTechSoft/OpenADS. + +# v1.09.71-mtfix23 - upstream sync test build + +Test build only. Use matching client and server versions and back up binaries and data before testing. This is not a production recommendation. No fork-main merge is included; Tim Stone's exact ECLMST application test remains the gate for that merge. + +## Upstream boundary and retained fixes + +Merged upstream v1.09.71 at 82ac63c438c155ef8c267f3f2992ca530f9e2908 once into perf/safe-local-answers, preserving both parents in merge 37ceaf51eec6fac69be4395f37c0897b0c05e1ef. Newer upstream main and its macOS compile experiments are excluded. + +The upstream merge itself left src/, include/ and the fork's workflows byte-identical to mtfix22. This test build then adds the diagnostic and packaging changes below. Upstream documentation and release notes were merged. Its ADT exclusive-append regression was added as a separate test rather than replacing the fork's broader distinct-values/shared-guard test. + +Retained fork behavior includes case-insensitive LOCAL tag lookup, stock Harbour order-0/empty natural focus, retained scopes and tag direction, REMOTE natural-Skip reanchoring, engine append retry/probes, alias lane pinning, canonical archive/header checks and the Linux glibc 2.31 kit. Vouch application code is untouched. + +## Validation + +Local Linux Debug -O0/-g0, warnings-as-errors disabled because of inherited warnings. Byte-identical source/include reused mtfix22 build objects after fresh-build resource limits; the new upstream test compiled fresh and core/server/ACE/unit binaries were relinked. + +All 1,606 enabled cases were run across four ranges, with 16 disabled cases: 1,604 passed and 2 failed; 628,676 assertions, 7 failed. Both failure classes independently reproduced on the untouched mtfix22 baseline: Exclusive-held OpenIndex returned 6106 rather than 7040, and the REMOTE create/index storm produced a count mismatch. These are known unresolved failures, not a clean full-suite pass. + +The normal suite under its existing slow/flaky exclusions passed 1,595 cases and 612,903 assertions in 93 seconds. The initial 90-second CTest wrapper timeout did not indicate an assertion failure. Five other CTest entries passed: slow cases, 30-repeat pool stress, SQL URI, SQLite filter and SQLite seek. Both ADT append regressions passed together: 2 cases, 50,055 assertions. + +macOS unit validation remains incomplete from mtfix22: Configure/Build passed but unit-test runs timed out. No macOS unit pass, Tim Windows x64 application pass, or ECLMST-file pass is claimed. Platform packaging results must be checked separately before this draft is published. + +## Release integrity + +mtfix22's tag-push packaging run silently replaced its reviewed archives and notes. The exact run34 archives and reviewed notes were restored and all five public downloads were verified byte-for-byte. This build adds release guards: mtfix tag pushes do not package or upload; draft staging requires a manual dispatch; a manual dispatch refuses to alter an existing public mtfix release and fails closed if release state cannot be read. Production-tag behavior is unchanged. + +## Application check + +Repeat Tim's ECLMST report using CDX-format ECLMST.adi (adt_cdx_index=1): select eclcom by name and number, check OrdName/IndexOrd, set order 0 and empty focus and verify physical record navigation. Repeat LOCAL and REMOTE, switch back to a tag, check scopes and append/write maintenance. Record exact handles, recnos and return codes for mismatches. Pritpal does not personally use ADT tables. + +Use the full Windows x64/x86, Linux x64/glibc231 or macOS universal client/server kit. Do not mix ACE libraries from different builds. + +## Headers and opt-in diagnostics + +All five archives include the ADS client header at include/ace.h and include/openads/ace.h, together with its companion headers. + +CDX lock diagnostics no longer write /tmp/lock_diag.log or /tmp/cdx_diag.log unconditionally. Normal runs create neither file. To collect the three existing lock/open/error diagnostic messages, set OPENADS_LOCK_DIAG to a writable log-file path before starting the process. Unset or empty disables logging; an unavailable path does not change operation results. Diagnostic logs include table/index paths and native handle values, so enable them only when needed and handle them as private data. + +Diagnostic gate validation: rebuilt the modified CDX source; 92 non-slow/non-flaky CDX cases and 317,285 assertions passed with the variable unset, with an explicit log path, and with an unavailable path. The unset run left the legacy log unchanged; the explicit path received 279 diagnostic lines. A broader 96-case run reproduced the already-known REMOTE create/index storm failure; 95 cases passed. diff --git a/release-notes-test-p1-mtfix-1.md b/release-notes-test-p1-mtfix-1.md new file mode 100644 index 00000000..a4b95e4f --- /dev/null +++ b/release-notes-test-p1-mtfix-1.md @@ -0,0 +1,16 @@ +# OpenADS test build test-p1-mtfix-1 (TEST ONLY - not for production) + +Branch: perf/safe-local-answers on bedipritpal/OpenADS. Not an official release. + +## What's in it +- Session-pool multi-thread fix 1: remote table store (park/adopt on close, AdsCheckExistence scan, index-park invalidation, relation check) now always holds the global lock. Fixes a heap corruption crash seen in network_pool_mt_test. +- Server multi-thread fix 2: conn_for_table holds the global lock while scanning connection table maps (ThreadSanitizer race with Connection::open_table). +- Patch 1: safe local answers on the remote path (fewer round trips at startup). +- Wire-frame logging kept on for diagnosis. +- network_pool_mt_test repeats its 4-thread phase 25 times; new network_pool_mt_repeat test runs it in 30 fresh processes. + +## What to test +- Several Vouch instances at once, many threads, against openads_serverd from this build (Linux tarball) and the client DLL from the Windows x86 zip. + +## Packages +- openads-test-p1-mtfix-1-windows-x86.zip, windows-x64.zip, linux-x64.tar.gz, macos-universal.tar.gz diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b2ed37a1..f3a1c404 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(openads_core STATIC platform/proc.cpp engine/server_fs.cpp abi/ace_exports.cpp + abi/runtime.cpp abi/ace_stubs.cpp abi/adsfunc.c abi/openads_sql_c.cpp diff --git a/src/abi/ace_exports.cpp b/src/abi/ace_exports.cpp index 35a246a2..64d65f20 100644 --- a/src/abi/ace_exports.cpp +++ b/src/abi/ace_exports.cpp @@ -18,6 +18,8 @@ using openads::abi::lock_retry_policy; #include "abi/backend_registry.h" #include "abi/charset.h" #include "abi/last_error.h" +#include "abi/create_table_diag.h" +#include "abi/runtime.h" #include "engine/aof_eval.h" #include "engine/aof_expr.h" @@ -157,21 +159,8 @@ extern thread_local bool g_field_read_raw; // registry Handle is 64-bit; centralise the (value-preserving) narrowing. ADSHANDLE to_ads_handle(Handle h) { return static_cast(h); } -struct ProcessState { - // M10.36 -- recursive_mutex so UNION dispatch can re-enter - // AdsExecuteSQLDirect (used to materialise each member's cursor) - // while still holding the outer lock. - std::recursive_mutex mu; - openads::session::HandleRegistry registry; - std::unordered_map> conns; - // M12.33 — remote find handles (owned here, not in a Connection). - std::vector> remote_finds; -}; - -ProcessState& state() { - static ProcessState s; - return s; -} +using openads::abi::detail::ProcessState; +using openads::abi::detail::state; // Serialise the file-creating entry points (AdsCreateTable local paths, // AdsCreateIndex61 native branch) PER TARGET PATH. All serverd sessions @@ -222,6 +211,8 @@ UNSIGNED32 write_new_table_file(const std::string& full, auto fres = openads::platform::File::open( full, openads::platform::OpenMode::CreateExclusive); if (!fres) { + openads::abi::create_diag::log("file-open-fail", fres.error().code, + "exclusive-open", fres.error().sub_code); std::error_code ec; if (fs::exists(full, ec)) { return fail(openads::util::Error{ @@ -235,6 +226,8 @@ UNSIGNED32 write_new_table_file(const std::string& full, auto file = std::move(fres).value(); auto wrote = file.write_at(0, bytes.data(), bytes.size()); if (!wrote || wrote.value() != bytes.size()) { + openads::abi::create_diag::log("file-write-fail", wrote ? 0 : wrote.error().code, + "short-or-failed-write", wrote ? 0 : wrote.error().sub_code); return fail(openads::util::Error{ static_cast(openads::AE_INTERNAL_ERROR), 0, std::string(op) + ": write failed", ""}); @@ -755,12 +748,15 @@ UNSIGNED32 remote_emit_close_all(openads::network::RemoteTable* rt) { } rt->close_all_indexes_pending = false; rt->indexes_parked = false; + rt->parked_nav_which = 0; rt->parked_by_tag.clear(); rt->parked_handles.clear(); rt->index_by_tag.clear(); rt->index_handles.clear(); rt->active_index_id = 0; rt->last_nav = 0; + rt->pair_valid = false; + rt->anchor_ok = false; return ok(); } // Parked-index truth enforcement (see the nav entries): order-dependent @@ -1486,37 +1482,87 @@ bool remote_table_has_index(const openads::network::RemoteTable* rt) { (rt->active_index_id != 0 || !rt->index_by_tag.empty()); } -// Grid log for WAN diagnosis: fixed-width columns (3-space separated, -// like ads_err.log) plus a variable-length detail tail, so both naked -// eyes and analytical tools can parse it: -// -// [+ 123ms] [V_USRCFG ] [AdsAtBOF ] nav=0 row=1 -// | ms since trace start | table alias | operation | detail... -// -// Same-ms runs read as locally served calls; an RTT-sized jump between -// adjacent lines is exactly one wire round-trip. -static void cli_trace_line(long long ms, const char* who, const char* op, - const char* detail) { +// Diagnostic WAN trace. Off unless wire_trace=1; no file opens or timestamps +// on the disabled request path. Payload and seek keys are never logged. +static bool cli_trace_on() { static const bool on = openads::util::client_setting_truthy( "OPENADS_WIRE_TRACE", "wire_trace"); - if (!on) return; - FILE* hf = std::fopen("C:/tmp/cli_trace.log", "a"); - if (hf == nullptr) return; - std::fprintf(hf, "[+%9lldms] [%-12.12s] [%-20.20s] %s\n", ms, - who != nullptr ? who : "-", - op != nullptr ? op : "-", detail != nullptr ? detail : ""); + return on; +} + +static const std::string& cli_trace_path() { + static const std::string path = [] { + auto p = openads::util::client_setting("OPENADS_WIRE_TRACE_FILE", + "wire_trace_file"); + return p.empty() ? std::string("C:/tmp/cli_trace.log") : p; + }(); + return path; +} + +struct CliTraceState { + std::mutex mu; + std::map, std::string> tables; + std::map, std::string> indexes; +}; +static CliTraceState& cli_trace_state() { + static CliTraceState trace; + return trace; +} + +// Called after a successful open; table ID belongs to its connection, +// not globally. Do not hold this mutex while performing any wire request. +static void cli_trace_table_open(const openads::network::RemoteTable* rt) { + if (!cli_trace_on() || !rt || !rt->conn) return; + auto& trace = cli_trace_state(); + std::lock_guard lk(trace.mu); + trace.tables[{rt->conn, rt->id}] = + rt->alias.empty() ? rt->name : rt->alias; +} +static void cli_trace_table_close(const openads::network::RemoteTable* rt) { + if (!cli_trace_on() || !rt || !rt->conn) return; + auto& trace = cli_trace_state(); + std::lock_guard lk(trace.mu); + trace.tables.erase({rt->conn, rt->id}); + for (const auto& entry : rt->index_by_tag) + trace.indexes.erase({rt->conn, entry.second}); +} + +static void cli_trace_write_locked(FILE* hf, long long ms, + const char* who, const char* op, + const char* detail) { + const auto now = std::chrono::system_clock::now(); + const auto epoch_ms = std::chrono::duration_cast( + now.time_since_epoch()).count(); + const auto secs = std::chrono::system_clock::to_time_t(now); + std::tm local{}; +#if defined(_WIN32) + localtime_s(&local, &secs); +#else + localtime_r(&secs, &local); +#endif + char stamp[32]; + std::strftime(stamp, sizeof(stamp), "%Y-%m-%d %H:%M:%S", &local); + std::fprintf(hf, "%s.%03lld [+%9lldms] [%-20.20s] [%-20.20s] %s\n", + stamp, static_cast(epoch_ms % 1000), ms, + who ? who : "-", op ? op : "-", detail ? detail : ""); +} + +static void cli_trace_line(long long ms, const char* who, const char* op, + const char* detail) { + if (!cli_trace_on()) return; + auto& trace = cli_trace_state(); + std::lock_guard lk(trace.mu); + FILE* hf = std::fopen(cli_trace_path().c_str(), "a"); + if (!hf) return; + cli_trace_write_locked(hf, ms, who, op, detail); std::fclose(hf); } static long long cli_trace_ms() { - // Millisecond stamp since the first traced call: adjacent-line gaps - // separate local answers (~0 ms) from wire round-trips (~RTT ms), - // which is the whole point of reading this log. static const auto t0 = std::chrono::steady_clock::now(); return static_cast( std::chrono::duration_cast( - std::chrono::steady_clock::now() - t0) - .count()); + std::chrono::steady_clock::now() - t0).count()); } static void cli_trace(const char* op, const char* fmt, ...) { @@ -1528,16 +1574,11 @@ static void cli_trace(const char* op, const char* fmt, ...) { buf[sizeof(buf) - 1] = 0; cli_trace_line(cli_trace_ms(), "-", op, buf); } -// Table-attributed variant: the alias (file name when the open -// carried none) goes in the grid's table column, so repeated probe -// blocks attribute to the table that paid them. static void cli_trace_tbl(const openads::network::RemoteTable* rt, const char* op, const char* fmt, ...) { - std::string who; - if (rt != nullptr) { - who = !rt->alias.empty() ? rt->alias : rt->name; - } - if (who.empty()) who = "-"; + if (!cli_trace_on()) return; + const std::string who = rt == nullptr ? "-" : + (!rt->alias.empty() ? rt->alias : rt->name); char buf[512]; va_list ap; va_start(ap, fmt); @@ -1547,6 +1588,161 @@ static void cli_trace_tbl(const openads::network::RemoteTable* rt, cli_trace_line(cli_trace_ms(), who.c_str(), op, buf); } +static const char* cli_trace_opcode(std::uint8_t op) { + switch (op) { + case 0x01: return "Hello"; + case 0x10: return "Connect"; + case 0x12: return "Disconnect"; + case 0x20: return "OpenTable"; + case 0x22: return "CloseTable"; + case 0x30: return "ExecuteSQL"; + case 0x32: return "Fetch"; + case 0x40: return "GotoTop"; + case 0x42: return "Skip"; + case 0x44: return "GetField"; + case 0x46: return "GetRecordCount"; + case 0x48: return "AtEOF"; + case 0x4A: return "DescribeTable"; + case 0x4C: return "AtBOF"; + case 0x4E: return "GetRecordNum"; + case 0x62: return "IsRecordDeleted"; + case 0x64: return "GotoBottom"; + case 0x66: return "IsFound"; + case 0x68: return "RefreshRecord"; + case 0x6A: return "GetTableType"; + case 0x6C: return "GetRecordLength"; + case 0x6E: return "GetNumIndexes"; + case 0x70: return "GetLastAutoinc"; + case 0x72: return "LockRecord"; + case 0x74: return "UnlockRecord"; + case 0x76: return "LockTable"; + case 0x78: return "UnlockTable"; + case 0x7A: return "PackTable"; + case 0x7C: return "ZapTable"; + case 0x7E: return "FlushFileBuffers"; + case 0x80: return "CloseAllIndexes"; + case 0x82: return "SetAOF"; + case 0x84: return "ClearAOFRemote"; + case 0x86: return "GetAOFOptLevel"; + case 0x88: return "OpenIndex"; + case 0x8A: return "CloseIndex"; + case 0x8C: return "SetOrder"; + case 0x8E: return "SetOrderByName"; + case 0x90: return "Seek"; + case 0x92: return "SeekLast"; + case 0x94: return "CreateIndex"; + case 0x96: return "SkipUnique"; + case 0x98: return "SetScope"; + case 0x9A: return "ClearScope"; + case 0x9C: return "FetchCurrentRow"; + case 0x50: return "AppendBlank"; + case 0x52: return "SetField"; + case 0x5E: return "SetFields"; + case 0x54: return "DeleteRecord"; + case 0x56: return "RecallRecord"; + case 0x58: return "GotoRecord"; + case 0x5A: return "FlushTable"; + case 0x5C: return "GetKeyType"; + case 0x60: return "Reindex"; + case 0x9E: return "GetLastTableUpdate"; + case 0x13: return "IsRecordLocked"; + case 0x15: return "GetAllLocks"; + case 0xA0: return "MgConnect"; + case 0xA2: return "MgRequest"; + case 0xA4: return "FetchWhere"; + case 0xA6: return "Aggregate"; + case 0xA8: return "GetRecord"; + case 0xAA: return "SetRecord"; + case 0xAC: return "CustomizeAOF"; + case 0xAE: return "GetRecordCRC"; + case 0xB0: return "GetKeyCount"; + case 0x03: return "GetKeyNum"; + case 0x05: return "FindTables"; + case 0x07: return "BeginTransaction"; + case 0x09: return "CommitTransaction"; + case 0x0B: return "RollbackTransaction"; + case 0x0D: return "FindRecord"; + case 0x17: return "ZipArchive"; + case 0x19: return "UnzipArchive"; + case 0x1B: return "ZipList"; + case 0xB2: return "DDGetProperty"; + case 0xB4: return "DDSetProperty"; + case 0xB6: return "DDCreateProc"; + case 0xB8: return "DDCreateFunction"; + case 0xBA: return "DDCreateTrigger"; + case 0xBC: return "DDDropTrigger"; + case 0xBE: return "DDDropView"; + case 0xC0: return "DDDropLink"; + case 0xC2: return "DDCreateUser"; + case 0xC4: return "DDDropObject"; + case 0xC6: return "DDAddUserToGroup"; + case 0xC8: return "DDRemoveUserFromGroup"; + case 0xCA: return "DDCreateLink"; + case 0xCC: return "DDModifyLink"; + case 0xCE: return "DDCreateRefIntegrity"; + case 0xD0: return "DDCreateView"; + case 0xD2: return "DDAddIndexFile"; + case 0xD4: return "DDRemoveIndexFile"; + case 0xD6: return "DDGetPermissions"; + case 0xD8: return "DDGrantPermission"; + case 0xDA: return "ShowDeleted"; + case 0xDC: return "CreateTable"; + case 0xDE: return "DropTable"; + case 0xE0: return "FileExists"; + case 0xE2: return "FileErase"; + case 0xE4: return "FileRename"; + case 0xE6: return "FileSize"; + case 0xE8: return "FileMTime"; + case 0xEA: return "Directory"; + case 0xEC: return "DirExist"; + case 0xEE: return "DirMake"; + case 0xF0: return "DirRemove"; + case 0xF2: return "FOpen"; + case 0xF4: return "FCreate"; + case 0xF6: return "FClose"; + case 0xF8: return "FRead"; + case 0xFA: return "FWrite"; + case 0xFC: return "FSeek"; + case 0xFE: return "Mutex"; + default: return "Other"; + } +} + +// One line per completed request/reply. Connection and table ID identify +// overlapping aliases, duration includes send+receive; no payload bytes. +static void cli_trace_frame_hook(const void* conn, std::uint8_t op, + std::uint32_t tid, std::size_t req_bytes, + std::uint8_t rep_op, std::size_t rep_bytes, + long long us) { + auto& trace = cli_trace_state(); + std::lock_guard lk(trace.mu); + std::string table = "-"; + const bool index_op = op == 0x90 || op == 0x92 || op == 0x8A || + op == 0x8C || op == 0x8E || op == 0x5C; + if (index_op) { + const auto ix = trace.indexes.find({conn, tid}); + if (ix != trace.indexes.end()) table = ix->second; + } else if (op != 0x01 && op != 0x10 && op != 0x20 && + op != 0x30 && op != 0x32 && op != 0x05 && op != 0x07 && + op != 0x09 && op != 0x0B) { + const auto it = trace.tables.find({conn, tid}); + if (it != trace.tables.end()) table = it->second; + } + char buf[200]; + std::snprintf(buf, sizeof(buf), + "wire op=0x%02X tid=%u req=%lu ack=0x%02X ackb=%lu dur_us=%lld conn=%04X", + static_cast(op), static_cast(tid), + static_cast(req_bytes), + static_cast(rep_op), + static_cast(rep_bytes), us, + static_cast(reinterpret_cast(conn) & 0xFFFFu)); + FILE* hf = std::fopen(cli_trace_path().c_str(), "a"); + if (!hf) return; + cli_trace_write_locked(hf, cli_trace_ms(), table.c_str(), + cli_trace_opcode(op), buf); + std::fclose(hf); +} + void remote_clear_nav_boundaries(openads::network::RemoteTable* rt) { if (rt == nullptr) return; rt->nav_at_bof = false; @@ -1656,10 +1852,136 @@ void remote_sync_keyno_gototop(openads::network::RemoteTable* rt) { } } +// True when the server itself certified, on the latest cursor-affecting +// frame, that this table's cursor sits on no row with BOTH limits set +// (the xBase empty-cursor state), and nothing client-side could have +// changed what that means since. Filters are excluded on purpose: the +// server key/record counts do not apply them, so an empty filtered +// cursor says nothing about the counts. +bool remote_cursor_proven_empty(const openads::network::RemoteTable* rt) { + return rt != nullptr && rt->conn != nullptr && + !rt->row_valid && + rt->bound_bof_ok && rt->bound_bof && + rt->bound_eof_ok && rt->bound_eof && + rt->bound_seq == rt->conn->nav_seq() && + !rt->pending_order && + rt->pending_sets.empty() && + rt->filter_expr.empty() && rt->aof_expr.empty(); +} + +// "Still empty" window (user-approved trade-off, 23/09/2026): after the +// server certifies a table physically EMPTY (record count 0) with the +// cursor on no row (BOF+EOF), Seek / GO n on it are answered "not found" +// locally for a short time instead of paying a round trip each. Vouch +// probes its empty per-user settings tables ~280 times per startup. +// Risk accepted: a record another STATION adds inside the window is seen +// only when the window ends. This station's own writes (any append / +// field write / SQL on this connection) close the window at once. +// OPENADS_EMPTY_TTL_MS: window length, default 1500, 0 = off, max 2000. +std::uint32_t remote_empty_ttl_ms() { + static const std::uint32_t v = [] { + const char* e = std::getenv("OPENADS_EMPTY_TTL_MS"); + if (e == nullptr || *e == 0) return 1500u; + long x = std::strtol(e, nullptr, 10); + if (x < 0) x = 0; + if (x > 2000) x = 2000; + return static_cast(x); + }(); + return v; +} + +// mtfix12 R2 opt-in envelope (his explicit flag): conn-wide by +// default — any cursor-affecting frame on the connection expires the +// self-goto anchor. OPENADS_NAV_SELF_GOTO=table scopes freshness to +// the table handle: server cursors are per handle, so only this +// handle's frames can move its cursor, and the per-table generation +// (bumped centrally in request()) sees them all. For his deployment — +// writes coordinated by SEMA4s and physical locks — this is safe; the +// general default stays conn-wide. +bool remote_self_goto_per_table() { + static const bool v = [] { + const char* e = std::getenv("OPENADS_NAV_SELF_GOTO"); + return e != nullptr && std::strcmp(e, "table") == 0; + }(); + return v; +} + +bool remote_empty_window(const openads::network::RemoteTable* rt) { + if (rt == nullptr || rt->conn == nullptr) return false; + const std::uint32_t ttl = remote_empty_ttl_ms(); + if (ttl == 0) return false; + if (rt->row_valid || !rt->pending_sets.empty() || rt->write_dirty) + return false; + if (!(rt->bound_bof_ok && rt->bound_bof && + rt->bound_eof_ok && rt->bound_eof)) + return false; + // The count must come from the same certification as the bounds. + if (!(rt->count_bound_ok && rt->count_bound == 0 && + rt->count_bound_seq == rt->bound_seq)) + return false; + if (rt->bound_data_epoch != rt->conn->data_epoch()) return false; + const auto age = std::chrono::steady_clock::now() - rt->bound_at; + return age >= std::chrono::steady_clock::duration::zero() && + age <= std::chrono::milliseconds(ttl); +} + +// Leave the table exactly as a wire answer on an empty table would: +// no row, both limits, recno/count unchanged, certified at the current +// seq (bound_at is NOT refreshed: the window runs from the last real +// server answer, never extended by local answers). +void remote_empty_restamp(openads::network::RemoteTable* rt) { + const std::uint64_t seq = rt->conn->nav_seq(); + rt->row_valid = false; + rt->invalidate_prefetch(); + rt->nav_at_bof = true; + rt->nav_at_eof = true; + rt->nav_not_bof = false; + rt->nav_not_eof = false; + rt->bound_seq = seq; + rt->recno_bound_seq = seq; + rt->count_bound_seq = seq; + rt->last_nav = 0; + // The serve lands on no row: no anchor, and any certified pair + // landing described a state this serve just replaced. + rt->pair_valid = false; + rt->anchor_ok = false; +} + +// Memo key for AdsGetRecordLength re-opens: lowercased table name plus +// the whole schema. Empty (= no memo) when the schema is not known. +std::string remote_record_length_key(const openads::network::RemoteTable* rt) { + if (rt == nullptr || !rt->fields_cached || rt->fields.empty() || + rt->name.empty()) { + return {}; + } + std::string k = rt->name; + for (auto& c : k) + c = static_cast(std::tolower(static_cast(c))); + for (const auto& fd : rt->fields) { + k.push_back('\x1f'); + k += fd.name; + k.push_back('\x1e'); + k += std::to_string(fd.type) + "," + std::to_string(fd.length) + + "," + std::to_string(fd.decimals); + } + return k; +} + void remote_sync_keyno_gotobottom(openads::network::RemoteTable* rt) { if (rt == nullptr) return; remote_clear_nav_boundaries(rt); if (remote_table_has_index(rt)) { + // An ordered GotoBottom that the server certified empty (no row, + // BOF+EOF) proves the order holds no visible keys in its scope: + // the server key count (scope + SET DELETED aware, filter-blind) + // is 0. Seed it instead of asking — the empty tables of a USE + // otherwise pay one GetKeyCount frame each here. + if (rt->active_index_id != 0 && !rt->key_count_cached && + remote_cursor_proven_empty(rt)) { + rt->cached_key_count = 0; + rt->key_count_cached = true; + rt->key_counts[rt->active_index_id] = 0; + } // Bottom of the ORDER, not of the file: with a scope active the // last key is key #scoped_key_count, not #physical_rec_count. const std::uint32_t kc = remote_ensure_key_count(rt); @@ -1708,6 +2030,22 @@ void remote_nav_stamp(openads::network::RemoteTable* rt, int which, rt->last_nav_seq = rt->conn->nav_seq(); } +// mtfix12 R1 — opposite-boundary certification serve check. True when +// the just-landed GotoTop/GotoBottom carried THIS boundary's full +// landing state (kCapNavBoundaryPair) and the same freshness envelope +// the duplicate check uses still holds — conn-wide nav_seq unchanged — +// plus no write touched this table since certification (the blob's row +// bytes predate any write; a stale blob must never overwrite fresher +// row state). Pending local mutations stay on the wire path. +bool remote_nav_pair(const openads::network::RemoteTable* rt, int which, + std::uint32_t order) { + return rt != nullptr && rt->conn != nullptr && rt->pair_valid && + rt->pair_which == which && rt->pair_order == order && + rt->pair_seq == rt->conn->nav_seq() && + rt->pair_write_gen == rt->conn->tbl_write_gen(rt->id) && + rt->pending_sets.empty() && !rt->pending_order; +} + // Empty-cursor sticky for AdsAtBOF/AdsAtEOF: true when the last wire // nav on this table established an empty cursor with no // cursor-affecting frame since. An empty cursor is simultaneously BOF @@ -4168,6 +4506,12 @@ bool& in_ri_check() { // Find the Connection that owns Table* t. Connection* conn_for_table(Table* t) { auto& s = state(); + // Each Connection's table map is changed under s.mu (open/close), so + // the scan must hold it too. Without it a navigation on one session + // (snapshot_ri_pks) read a map another session was inserting into + // (found by ThreadSanitizer under the session-pool test). Lock order + // s.mu -> registry matches register_object/release. + std::lock_guard lk(s.mu); Connection* found = nullptr; s.registry.for_each_handle([&](Handle, HandleKind k, void* p) { if (k != HandleKind::Connection || found) return; @@ -6654,6 +6998,93 @@ remote_pool_lanes_of(openads::network::RemoteConnection* rc) { return pool->lanes; } +// Lane pinning by (logical connection, path, alias). +// +// The thread-affine lane pool spreads a process's tables over several +// server sessions for parallel wire throughput, but record locks are +// owned per server SESSION. Harbour's zero-space pattern +// (hb_dbRequest/hb_dbDetach - one workarea shared process-wide across +// threads for login semaphores) needs the opposite: a workarea's lock +// identity must survive being driven from any thread, and must survive a +// close + fresh USE of the same alias (Vouch's LogUse). DBFCDX gets this +// for free because the lock list lives in the workarea struct itself. +// +// Pinning gives each (connection, normalized path, alias) one stable +// lane for the connection's lifetime: every USE of that alias+path from +// any thread - including the first USE after a close - binds to the same +// server session, so locks taken through it stay visible and unlockable +// no matter which thread is holding the workarea. Different aliases of +// the same file keep different owners (UsrConsole-style cross-owner lock +// probes still conflict, either on another lane or as another server-side +// Table object on the same lane). Single-threaded callers are unaffected +// (thread affinity already kept them on lane 0). +// +// Pins persist across AdsCloseTable on purpose - a close releases held +// locks (same as DBFCDX closing a workarea); what pinning preserves is +// IDENTITY for the next open, not the locks themselves. Pins are dropped +// at AdsDisconnect. +// +// Default ON; kill switch: openads.ini lane_pin_alias = 0 (or +// OPENADS_LANE_PIN_ALIAS=0) restores pure thread-affinity. +static std::unordered_map& +remote_lane_pins() { + static std::unordered_map m; + return m; +} +static bool remote_lane_pin_enabled() { + static const bool on = [] { + const std::string v = openads::util::client_setting( + "OPENADS_LANE_PIN_ALIAS", "lane_pin_alias"); + if (v.empty()) return true; + std::string l = v; + for (auto& c : l) c = static_cast(::tolower((unsigned char)c)); + return !(l == "0" || l == "false" || l == "off" || l == "no"); + }(); + return on; +} +static std::string remote_lane_pin_key(ADSHANDLE rem_h, + const std::string& name, + const std::string& alias) { + std::string n = name; + for (auto& c : n) { if (c == '\\') c = '/'; } + std::string a = alias; + for (auto& c : a) c = static_cast(::toupper((unsigned char)c)); + return std::to_string(static_cast(rem_h)) + + '\x01' + n + '\x01' + a; +} +// Resolve the lane for a (connection, path, alias) open: pinned lane when +// one is recorded and alive, otherwise the thread-affine pick, which then +// becomes the pin. s.mu must be held. +static openads::network::RemoteConnection* +remote_pool_lane_for_open(ADSHANDLE rem_h, const std::string& name, + const std::string& alias) { + namespace net = openads::network; + if (!remote_lane_pin_enabled()) + return remote_pool_lane_conn(static_cast(rem_h)); + const std::string key = remote_lane_pin_key(rem_h, name, alias); + auto& pins = remote_lane_pins(); + auto it = pins.find(key); + if (it != pins.end()) { + if (it->second != nullptr && it->second->valid()) return it->second; + pins.erase(it); // dead lane: fall through and re-pin + } + net::RemoteConnection* rc = remote_pool_lane_conn(static_cast(rem_h)); + if (rc != nullptr) pins[key] = rc; + return rc; +} +// Drop every pin of a logical connection (AdsDisconnect). s.mu held. +static void remote_lane_pins_clear(ADSHANDLE rem_h) { + const std::string prefix = + std::to_string(static_cast(rem_h)) + '\x01'; + for (auto it = remote_lane_pins().begin(); + it != remote_lane_pins().end();) { + if (it->first.compare(0, prefix.size(), prefix) == 0) + it = remote_lane_pins().erase(it); + else + ++it; + } +} + extern "C" { @@ -6710,6 +7141,9 @@ UNSIGNED32 ENTRYPOINT AdsConnect60(UNSIGNED8* pucServer, UNSIGNED16 usServerType UNSIGNED8* pucUser, UNSIGNED8* pucPwd, UNSIGNED32 /*ulOptions*/, ADSHANDLE* phConnect) { arc2_trace("AdsConnect60"); + if (cli_trace_on()) { + openads::network::set_frame_trace_hook(&cli_trace_frame_hook); + } if (phConnect == nullptr) return fail(openads::AE_INTERNAL_ERROR, "phConnect is null"); auto path = openads::abi::to_internal(pucServer, 0); @@ -7330,6 +7764,12 @@ remote_table_store() { std::unique_ptr remote_table_take(openads::network::RemoteTable* rt) { if (rt == nullptr) return nullptr; + // remote_table_store() is shared by every thread (all lanes of a + // session pool): every access must hold s.mu. The park path in + // AdsCloseTable used to call this unlocked while other threads + // inserted/erased under s.mu -- a data race on the map that corrupted + // the heap under the 4-thread pool test (~1 run in 30 on Linux). + std::lock_guard lk(state().mu); auto& m = remote_table_store(); auto it = m.find(rt); if (it == m.end()) return nullptr; @@ -7340,6 +7780,7 @@ remote_table_take(openads::network::RemoteTable* rt) { void remote_table_forget(openads::network::RemoteTable* rt) { if (rt == nullptr) return; + std::lock_guard lk(state().mu); // see take() remote_table_store().erase(rt); } @@ -7358,6 +7799,7 @@ void remote_invalidate_index_parks(openads::network::RemoteConnection* rc) { if (c == l) return true; return false; }; + std::lock_guard lk(state().mu); // shared store for (auto& kv : remote_table_store()) { auto* rt = kv.first; if (rt == nullptr || !lane_match(rt->conn)) continue; @@ -7383,6 +7825,7 @@ std::string remote_pool_key(const std::string& name, // Parked tables keep no registry handle, so relations addressed by // handle could neither follow nor clean up -- real-close those. bool remote_table_has_relations(ADSHANDLE hTable) { + std::lock_guard lk(state().mu); auto& tbl = relation_map(); if (tbl.find(hTable) != tbl.end()) return true; for (auto& [parent, kids] : tbl) { @@ -7400,12 +7843,41 @@ bool remote_table_has_relations(ADSHANDLE hTable) { // so no peer can invalidate them behind our back — an active order // resumes exactly (Vouch keeps IndexOrd()=1 across USEs; excluding it // would empty the pool). +// wire_trace only: first rule (in remote_table_poolable order, then +// relations) that keeps a closing table out of the park. Mirrors the +// checks below; it never decides anything itself. +const char* remote_table_unpoolable_reason( + openads::network::RemoteTable* rt, ADSHANDLE hTable) { + if (rt == nullptr || rt->conn == nullptr) return "no_connection"; + if (!rt->close_counted) return "sql_cursor"; + if (rt->open_exclusive) return "exclusive"; + if (!rt->pending_sets.empty()) return "pending_sets"; + if (!rt->held_recs.empty() || rt->table_lock_held || + rt->locks_uncertain) return "locks_held"; + if (rt->scope_touched) return "scope"; + if (!rt->aof_expr.empty()) return "aof"; + if (!rt->filter_expr.empty()) return "filter"; + if (rt->flush_file_pending || rt->close_all_indexes_pending) + return "teardown_pending"; + if (rt->pending_order) return "pending_order"; + if (remote_table_has_relations(hTable)) return "relations"; + return "none"; +} + bool remote_table_poolable(openads::network::RemoteTable* rt) { if (rt == nullptr || rt->conn == nullptr) return false; if (!rt->close_counted) return false; // SQL cursors etc. if (rt->open_exclusive) return false; // parked exclusive blocks peers if (!rt->pending_sets.empty()) return false; // flushed before close - if (rt->ever_locked || rt->scope_touched) return false; + // Locks once held no longer bar the park by themselves: the ledger + // proves whether any lock is STILL held (a parked table would + // otherwise pin it against every peer forever). ever_locked stays + // recorded for the trace but stops forcing a real close -- that + // policy cost GN_COUNT/FA_ACC01/USASELOG a full 7-frame reopen per + // voucher save. + if (!rt->held_recs.empty() || rt->table_lock_held || + rt->locks_uncertain) return false; + if (rt->scope_touched) return false; if (!rt->aof_expr.empty() || !rt->filter_expr.empty()) return false; // Deferred teardown (FlushFileBuffers/CloseAllIndexes) is absorbed // by a real close, never by a park (no server close happens) — the @@ -7435,6 +7907,7 @@ void remote_close_table_live(ADSHANDLE hTable, auto* rc = rt->conn; const bool counted = rt->close_counted; if (rc != nullptr) (void)rc->close_table(rt->id); + cli_trace_table_close(rt); if (hTable != 0) forget_relations(hTable); auto& s2 = state(); openads::network::RemoteConnection* fire = nullptr; @@ -7481,9 +7954,6 @@ void remote_close_table_live(ADSHANDLE hTable, // arrange that (disconnect/open paths unlock first). static void remote_flush_pools_one(openads::network::RemoteConnection* rc) { if (rc == nullptr) return; - // File lifecycle changed meaning on disk: drop the existence - // cache alongside the parked handles. - rc->file_exists_invalidate(); std::vector> parked; rc->parked_flush(parked); for (auto& e : parked) { @@ -7616,6 +8086,8 @@ UNSIGNED32 ENTRYPOINT AdsDisconnect(ADSHANDLE hConnect) { // ANY lane still has open tables. auto lanes = remote_pool_lanes_of(rc0); if (lanes.empty()) lanes.push_back(rc0); + // Lane pins name lanes of this connection; drop them all. + remote_lane_pins_clear(hConnect); // Null out rt->conn on any open SQL cursors that reference this // connection so AdsCloseTable can detect the dangling case and // skip the wire op rather than crashing with a use-after-free. @@ -7789,13 +8261,6 @@ UNSIGNED32 ENTRYPOINT AdsOpenTable(ADSHANDLE hConnect, } if (auto* rc0 = s.registry.lookup( rem_h, HandleKind::RemoteConnection)) { - // MT lane: this thread's session for the logical connection. - // Each table pins to its lane via rt->conn below (cursor/order - // bindings are per-session server-side); single-threaded callers - // always get the primary (lane 0) — behaviour unchanged. - openads::network::RemoteConnection* rc = - remote_pool_lane_conn(static_cast(rem_h)); - if (rc == nullptr) rc = rc0; auto name = openads::abi::to_internal(pucName, 0); // M12.33 — strip tcp:// URI prefix that legacy Delphi TAdsTable // components embed in the table name (e.g. @@ -7830,6 +8295,15 @@ UNSIGNED32 ENTRYPOINT AdsOpenTable(ADSHANDLE hConnect, if (alias.empty()) { alias = std::filesystem::path(name).stem().string(); } + // MT lane: this table's session for the logical connection. + // Lane pinning (default ON): same alias+path binds to the same + // lane from any thread and across close/reopen, so record-lock + // identity follows the workarea (zero-space detach/request). + // Without pinning: this thread's affine lane, and single-threaded + // callers always get the primary (lane 0) — behaviour unchanged. + openads::network::RemoteConnection* rc = + remote_pool_lane_for_open(rem_h, name, alias); + if (rc == nullptr) rc = rc0; openads::util::write_remote_open_audit(name, alias); // Pooled re-USE (USE latency): same connection/path/alias/mode // within TTL reuses the parked server handle — no OpenTable wire @@ -7843,6 +8317,39 @@ UNSIGNED32 ENTRYPOINT AdsOpenTable(ADSHANDLE hConnect, adopted = std::move(hit); } } + if (cli_trace_on()) { + // Park diagnostics: hit/miss for this lane, plus whether + // another lane of the same session pool holds it parked. + char pbuf[320]; + if (adopted) { + std::snprintf(pbuf, sizeof(pbuf), "park hit id=%u %.200s", + static_cast(adopted->id), + name.c_str()); + } else { + const std::string pk = remote_pool_key(name, alias, usMode); + bool other_lane = false; + auto pit = remote_lane_pool_of().find(rc); + if (pit != remote_lane_pool_of().end() && + pit->second != nullptr) { + for (auto* ln : pit->second->lanes) { + if (ln != nullptr && ln != rc && + ln->parked_contains(pk)) { + other_lane = true; + break; + } + } + } + std::snprintf(pbuf, sizeof(pbuf), + "park miss%s conn=%04X %.200s", + other_lane ? " (parked on other lane)" : "", + static_cast( + reinterpret_cast(rc) & + 0xFFFFu), + name.c_str()); + } + cli_trace_line(cli_trace_ms(), alias.c_str(), "AdsOpenTable", + pbuf); + } if (adopted) { adopted->row_valid = false; adopted->rec_count_cached = false; @@ -7853,6 +8360,9 @@ UNSIGNED32 ENTRYPOINT AdsOpenTable(ADSHANDLE hConnect, adopted->found_cached = false; adopted->nav_at_bof = adopted->nav_at_eof = false; adopted->last_nav = 0; // re-stamped by the warm GotoTop below + adopted->pair_valid = false; + adopted->anchor_ok = false; + adopted->parked_nav_which = 0; adopted->flush_file_pending = false; adopted->close_all_indexes_pending = false; adopted->pending_order = false; @@ -7906,9 +8416,25 @@ UNSIGNED32 ENTRYPOINT AdsOpenTable(ADSHANDLE hConnect, remote_flush_pools(rc); lk.lock(); } + // Release s.mu across the OpenTable round-trip (same rule as the + // pool flush above and the production auto-open below). Holding + // it here blocks every other ABI call in the process for a full + // RTT, and with an in-process server (local serverd / tests) it + // deadlocks: this thread waits on rc's request lock, the lane + // thread that owns it waits on a reply, and the server handler + // for that reply waits on s.mu. Nothing below touches shared + // state until the lock is re-taken. + lk.unlock(); auto otr = rc->open_table(name, static_cast(map_open_mode(usMode))); - if (!otr) return fail(otr.error()); + lk.lock(); + if (!otr) { + openads::abi::create_diag::Scope diag_scope(name, + openads::abi::create_diag::correlation); + openads::abi::create_diag::log("client-open-fail", otr.error().code, + "wire-open", otr.error().sub_code); + return fail(otr.error()); + } auto& ot = otr.value(); auto rt = std::make_unique(); rt->conn = rc; @@ -7919,6 +8445,25 @@ UNSIGNED32 ENTRYPOINT AdsOpenTable(ADSHANDLE hConnect, rt->open_mode_raw = usMode; rt->open_exclusive = (map_open_mode(usMode) == openads::engine::OpenMode::Exclusive); + // Table type is decided by the server from the opened file's + // extension alone (.adt -> ADS_ADT, anything else -> ADS_CDX; + // see the GetTableType handler). The name we just opened carries + // that extension, so answer it locally. No extension -> leave it + // to the wire (the server may have resolved one). + { + std::string ext = std::filesystem::path(name).extension().string(); + for (auto& c : ext) + c = static_cast(std::tolower( + static_cast(c))); + if (!ext.empty()) { + rt->cached_table_type = (ext == ".adt") ? ADS_ADT : ADS_CDX; + rt->table_type_cached = true; + } + } + cli_trace_table_open(rt.get()); + cli_trace_tbl(rt.get(), "AdsOpenTable", "opened id=%u mode=%u", + static_cast(rt->id), + static_cast(usMode)); rc->deferred_open_tables += 1; Handle gh = s.registry.register_object( HandleKind::RemoteTable, rt.get()); @@ -8580,11 +9125,23 @@ UNSIGNED32 ENTRYPOINT AdsGetRecordLength(ADSHANDLE hTable, UNSIGNED32* pulLen) { *pulLen = rt->cached_record_length; return ok(); } + // Re-open of a table already measured on this connection with + // the identical schema: same file layout, same length. + const std::string memo_key = remote_record_length_key(rt); + if (!memo_key.empty() && + rt->conn->recall_record_length(memo_key, + rt->cached_record_length)) { + rt->record_length_cached = true; + *pulLen = rt->cached_record_length; + return ok(); + } auto r = rt->conn->get_record_length(rt->id); if (!r) return fail(r.error()); *pulLen = r.value(); rt->cached_record_length = r.value(); rt->record_length_cached = true; + if (!memo_key.empty()) + rt->conn->remember_record_length(memo_key, r.value()); return ok(); } Table* t = get_table(hTable); @@ -8972,6 +9529,8 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, } const auto rel = openads::abi::to_internal(pucName, 0); const auto defs = openads::abi::to_internal(pucFields, 0); + openads::abi::create_diag::Scope create_scope(rel, + openads::abi::create_diag::correlation); const bool is_vfp = (usTableType == ADS_VFP); auto fields = parse_rddads_field_defs(defs, is_vfp); if (fields.empty()) { @@ -9050,11 +9609,16 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, // Create-over-existing must see closed files (SAP: overwrite // when closed, 7040 when open) — a parked handle reads as open. remote_flush_pools(rc); + openads::abi::create_diag::log("client-wire-begin"); auto cr = rc->create_table(rel, defs, static_cast(usTableType), static_cast(usCharType), static_cast(usMemoBlockSize)); - if (!cr) return fail(cr.error()); + if (!cr) { + openads::abi::create_diag::log("client-wire-fail", cr.error().code); + return fail(cr.error()); + } + openads::abi::create_diag::log("client-wire-ok"); // Re-open via the normal remote path so production-bag auto-open // and the implicit GotoTop match AdsOpenTable semantics. std::vector namebuf(rel.size() + 1, 0); @@ -9064,8 +9628,13 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, (usTableType == ADS_ADT) ? ADS_ADT : (usTableType == ADS_VFP) ? ADS_VFP : ADS_CDX; - return AdsOpenTable(rc_h, namebuf.data(), pucAlias, + const UNSIGNED32 reopen_rc = AdsOpenTable(rc_h, namebuf.data(), pucAlias, open_type, usCharType, 0, 0, 1, phTable); + const auto reopen_error = reopen_rc ? openads::abi::last_error_code() : 0; + openads::abi::create_diag::log(reopen_rc ? "client-reopen-fail" : "client-reopen-ok", + reopen_rc, reopen_rc && reopen_error != static_cast(reopen_rc) + ? "last-error-differs" : "last-error-matches"); + return reopen_rc; } // Explicit handle to a disconnected remote connection with no // live remote to fall back to: fail fast rather than writing a @@ -9234,9 +9803,11 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, if (const UNSIGNED32 wrc = write_new_table_file( full.string(), adt_file, "AdsCreateTable: ADT"); wrc != openads::AE_SUCCESS) { + openads::abi::create_diag::log("adt-write-fail", wrc); return wrc; } } + openads::abi::create_diag::log("adt-write-ok"); // Create a companion .adm for MEMO/BINARY/IMAGE fields if (has_companion) { @@ -9244,7 +9815,11 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, adm.replace_extension(".adm"); { std::error_code ec; fs::remove(adm, ec); } auto mr = openads::drivers::adm::AdmMemo::create(adm.string()); - if (!mr) return fail(mr.error()); + if (!mr) { + openads::abi::create_diag::log("memo-create-fail", + mr.error().code, "memo", mr.error().sub_code); + return fail(mr.error()); + } } // Open via the standard path so the caller gets a usable handle @@ -9255,8 +9830,12 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, std::size_t adt_nb = std::min(rel_adt.size(), sizeof(adt_namebuf) - 1); std::memcpy(adt_namebuf, rel_adt.data(), adt_nb); - return AdsOpenTable(hConn, adt_namebuf, adt_namebuf, + const auto local_reopen_rc = AdsOpenTable(hConn, adt_namebuf, adt_namebuf, ADS_ADT, usCharType, 0, 0, 1, phTable); + const auto captured_error = local_reopen_rc ? openads::abi::last_error_code() : 0; + openads::abi::create_diag::log(local_reopen_rc ? "server-abi-reopen-fail" : "server-abi-reopen-ok", + local_reopen_rc, captured_error == static_cast(local_reopen_rc) ? "last-error-matches" : "last-error-differs"); + return local_reopen_rc; } if (is_vfp) { @@ -9321,9 +9900,11 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, if (const UNSIGNED32 wrc = write_new_table_file( full.string(), file, "AdsCreateTable: VFP"); wrc != openads::AE_SUCCESS) { + openads::abi::create_diag::log("vfp-write-fail", wrc); return wrc; } } + openads::abi::create_diag::log("vfp-write-ok"); if (has_memo) { fs::path fpt = full; @@ -9331,14 +9912,22 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, { std::error_code ec; fs::remove(fpt, ec); } std::uint16_t bs = usMemoBlockSize != 0 ? usMemoBlockSize : 512; auto mr = openads::drivers::fpt::FptMemo::create(fpt.string(), bs); - if (!mr) return fail(mr.error()); + if (!mr) { + openads::abi::create_diag::log("memo-create-fail", + mr.error().code, "memo", mr.error().sub_code); + return fail(mr.error()); + } } UNSIGNED8 namebuf[260] = {0}; std::size_t nb = std::min(rel.size(), sizeof(namebuf) - 1); std::memcpy(namebuf, rel.data(), nb); - return AdsOpenTable(hConn, namebuf, namebuf, + const auto local_reopen_rc = AdsOpenTable(hConn, namebuf, namebuf, ADS_VFP, usCharType, 0, 0, 1, phTable); + const auto captured_error = local_reopen_rc ? openads::abi::last_error_code() : 0; + openads::abi::create_diag::log(local_reopen_rc ? "server-abi-reopen-fail" : "server-abi-reopen-ok", + local_reopen_rc, captured_error == static_cast(local_reopen_rc) ? "last-error-matches" : "last-error-differs"); + return local_reopen_rc; } // ── DBF creation path (existing) ──────────────────────────────────────── @@ -9405,9 +9994,11 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, if (const UNSIGNED32 wrc = write_new_table_file( full.string(), file, "AdsCreateTable"); wrc != openads::AE_SUCCESS) { + openads::abi::create_diag::log("dbf-write-fail", wrc); return wrc; } } + openads::abi::create_diag::log("dbf-write-ok"); // If the field list declares any memo (M) field, stage an empty // .fpt next to the .dbf -- Connection::open_table auto-attaches it, @@ -9425,7 +10016,11 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, // usMemoBlockSize. std::uint16_t bs = usMemoBlockSize != 0 ? usMemoBlockSize : 512; auto mr = openads::drivers::fpt::FptMemo::create(fpt.string(), bs); - if (!mr) return fail(mr.error()); + if (!mr) { + openads::abi::create_diag::log("memo-create-fail", + mr.error().code, "memo", mr.error().sub_code); + return fail(mr.error()); + } } } @@ -9434,10 +10029,14 @@ UNSIGNED32 ENTRYPOINT AdsCreateTable(ADSHANDLE hConn, UNSIGNED8 namebuf[260] = {0}; std::size_t nb = std::min(rel.size(), sizeof(namebuf) - 1); std::memcpy(namebuf, rel.data(), nb); - return AdsOpenTable(hConn, namebuf, namebuf, + const auto local_reopen_rc = AdsOpenTable(hConn, namebuf, namebuf, ADS_CDX, // table type usCharType, 0, 0, 1, // char/lock/checkrights/mode phTable); + const auto captured_error = local_reopen_rc ? openads::abi::last_error_code() : 0; + openads::abi::create_diag::log(local_reopen_rc ? "server-abi-reopen-fail" : "server-abi-reopen-ok", + local_reopen_rc, captured_error == static_cast(local_reopen_rc) ? "last-error-matches" : "last-error-differs"); + return local_reopen_rc; } UNSIGNED32 ENTRYPOINT AdsDropTable(ADSHANDLE hConnect, @@ -9969,6 +10568,24 @@ UNSIGNED32 ENTRYPOINT AdsRefreshRecord(ADSHANDLE hTable) { arc2_trace("AdsRefreshRecord"); if (auto* rt = get_remote_table(hTable)) { if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) return frc; + // Refresh right after a GotoRecord on this handle, with nothing + // at all sent to the server in between (no lock, write, nav or + // read frame) and the cursor still on the row that ack carried: + // the server's GotoRecord had just re-read that row from disk, + // which is all a refresh does. Serve it from that ack. + if (rt->goto_row_fresh && rt->row_valid && rt->conn != nullptr && + rt->goto_row_frame_seq == rt->conn->frame_seq() && + rt->current_recno == rt->goto_row_recno && + rt->cursor_lag == 0 && rt->pending_sets.empty()) { + rt->goto_row_fresh = false; + cli_trace_tbl(rt, "AdsRefreshRecord", "served from GotoRecord ack"); + rt->invalidate_prefetch(); + rt->rec_count_cached = false; + rt->key_count_cached = false; + rt->key_counts.clear(); + return ok(); + } + rt->goto_row_fresh = false; remote_settle_cursor(rt); // M12.21 option C rt->row_valid = false; // M12.17 cache invalidation rt->rec_count_cached = false; @@ -10041,8 +10658,76 @@ UNSIGNED32 ENTRYPOINT AdsGotoRecord(ADSHANDLE hTable, UNSIGNED32 ulRecord) { // walk on the very next AdsGetRelKeyPos call (~22 sec for 9500 records). const std::uint32_t prev_recno = rt->row_valid ? rt->current_recno : 0u; - auto r = rt->conn->goto_record(rt, ulRecord); - if (!r) return fail(r.error()); + // GO 0 on a table the server certified physically EMPTY with the + // cursor already in the empty (BOF+EOF, no row) state: the server + // keeps that state (Table::goto_record preserves the phantom + // position on GO 0 even if another station appended meanwhile) + // and would send back exactly the bounds/recno we already hold. + // Answer it locally. GO n>0 always goes to the wire (a peer may + // have appended record n). Only for record count 0: on a + // non-empty table GO 0 moves the server's engine cursor. + const bool empty_goto_exact = + ulRecord == 0u && + remote_cursor_proven_empty(rt) && + rt->count_bound_ok && rt->count_bound == 0 && + rt->count_bound_seq == rt->conn->nav_seq(); + const bool empty_goto = + empty_goto_exact || remote_empty_window(rt); + // mtfix12 R2 — self-GotoRecord: the app re-issues GotoRecord + // for the recno the server cursor already sits on (FWH xBrowse + // bookmark restore). Serve locally when the certified anchor + // IS the wanted recno, the row cache holds that very row, no + // prefetch drain is outstanding (lag 0: client logical == + // server cursor), and the freshness envelope holds — + // conn-wide by default, per-table under the explicit opt-in. + // The serve is byte-identical to the wire ack: same row (cache), + // same position (anchor), lag stays 0, keyno preserved below + // (ulRecord == prev_recno), duplicate-nav stamp expired as a + // real frame would. + const bool self_goto = + ulRecord != 0u && rt->row_valid && + rt->current_recno == ulRecord && rt->cursor_lag == 0 && + rt->anchor_ok && rt->anchor_recno == ulRecord && + rt->pending_sets.empty() && + (remote_self_goto_per_table() + ? rt->anchor_tbl_seq == rt->conn->tbl_nav_seq(rt->id) + : rt->anchor_seq == rt->conn->nav_seq()); + rt->goto_row_fresh = false; + if (self_goto) { + cli_trace_tbl(rt, "AdsGotoRecord", "served locally (self-goto)"); + // A wire GotoRecord ack carries the row but NO lookahead + // block, and its trailer parse clears the queue — mirror + // that exactly, or a following Skip drains rows a real + // reposition would have discarded (the ramp test caught + // this: the skip never reached the wire). + rt->invalidate_prefetch(); + rt->goto_row_fresh = true; + rt->goto_row_frame_seq = rt->conn->frame_seq(); + rt->goto_row_recno = rt->current_recno; + rt->last_nav = 0; // a real frame would have expired it + } else if (empty_goto) { + if (empty_goto_exact) { + cli_trace_tbl(rt, "AdsGotoRecord", "served locally (empty table)"); + rt->invalidate_prefetch(); + // A real frame would have expired the duplicate-nav stamp. + rt->last_nav = 0; + rt->pair_valid = false; + rt->anchor_ok = false; // landed on no row + } else { + cli_trace_tbl(rt, "AdsGotoRecord", + "empty window: served locally want=%u", + ulRecord); + remote_empty_restamp(rt); + } + } else { + auto r = rt->conn->goto_record(rt, ulRecord); + if (!r) return fail(r.error()); + if (rt->row_valid) { + rt->goto_row_fresh = true; + rt->goto_row_frame_seq = rt->conn->frame_seq(); + rt->goto_row_recno = rt->current_recno; + } + } if (remote_table_has_index(rt)) { // Only invalidate when the cursor actually moved: same-record // restores (the xbrowse common case) keep the cached key number. @@ -10132,39 +10817,12 @@ UNSIGNED32 ENTRYPOINT AdsCheckExistence(ADSHANDLE hConn, UNSIGNED8* pucName, auto name = openads::abi::to_internal(pucName, 0); auto ctx = resolve_fs_conn(hConn); if (ctx.remote) { - // Open-bag short-circuit (per-USE VouExistIndex probes): a bag - // bound by any live table on this connection trivially exists - // — the app is reading through it. Matched by lowercased stem - // (no directory, no extension; .cdx/.z01 are equivalent - // production spellings). False positives are safe (a real - // open follows and errors properly); false negatives never - // happen here. - auto stem_of = [](const std::string& p) { - std::string b = p; - auto sep = b.find_last_of("/\\"); - if (sep != std::string::npos) b = b.substr(sep + 1); - auto dot = b.find_last_of('.'); - if (dot != std::string::npos) b = b.substr(0, dot); - for (auto& c : b) - c = static_cast(std::tolower( - static_cast(c))); - return b; - }; - const std::string want = stem_of(name); - if (!want.empty()) { - for (auto& kv : remote_table_store()) { - auto* rt = kv.first; - if (rt == nullptr || rt->conn != ctx.remote) continue; - if (stem_of(rt->prod_bag_path) == want || - (!rt->last_open_bag.empty() && - stem_of(rt->last_open_bag) == want)) { - *pbExists = 1; - return ok(); - } - } - } + // External directory/file changes require a wire probe even when an + // open table or index has the same stem: existence is not ownership. auto r = ctx.remote->file_exists(name); if (!r) return fail(r.error()); + cli_trace("FileExists", "probe %s -> %s (wire)", name.c_str(), + r.value() ? "yes" : "no"); *pbExists = r.value() ? 1 : 0; return ok(); } @@ -10861,7 +11519,19 @@ UNSIGNED32 ENTRYPOINT AdsCloseTable(ADSHANDLE hTable) { // reading its members is use-after-move (it crashed). std::string pkey = remote_pool_key(owned->name, owned->alias, owned->open_mode_raw); + openads::network::RemoteTable* traced = owned.get(); + if (cli_trace_on()) { + cli_trace_tbl(traced, "AdsCloseTable", "parked id=%u", + static_cast(traced->id)); + } rc->parked_store(std::move(pkey), std::move(owned), evicted); + if (cli_trace_on()) { + for (auto& e : evicted) { + cli_trace_tbl(e.get(), "AdsCloseTable", + "evicted from park id=%u", + e ? static_cast(e->id) : 0u); + } + } for (auto& e : evicted) remote_close_table_live(0, e.get()); return ok(); } @@ -10870,6 +11540,11 @@ UNSIGNED32 ENTRYPOINT AdsCloseTable(ADSHANDLE hTable) { // server close flushes data via its shadow handle and purges // the table's index bindings, so these frames would be waste. // A parked index snapshot dies with the handle (same purge). + if (cli_trace_on()) { + cli_trace_tbl(rt, "AdsCloseTable", "real close id=%u reason=%s", + static_cast(rt->id), + remote_table_unpoolable_reason(rt, hTable)); + } rt->flush_file_pending = false; rt->close_all_indexes_pending = false; rt->indexes_parked = false; @@ -10921,6 +11596,7 @@ UNSIGNED32 ENTRYPOINT AdsCloseTable(ADSHANDLE hTable) { UNSIGNED32 ENTRYPOINT AdsGotoTop(ADSHANDLE hTable) { arc2_trace("AdsGotoTop"); if (auto* ri = get_remote_index(hTable)) { + if (ri->parent) ri->parent->created_order_focus = false; // Sets + teardown flush here, but NOT a deferred order switch: // that absorbs into the nav below (fused frame) when it targets // this order, or flushes plainly inside remote_index_goto_top's @@ -10967,6 +11643,18 @@ UNSIGNED32 ENTRYPOINT AdsGotoTop(ADSHANDLE hTable) { apply_relations_for_handle(to_ads_handle(th)); return ok(); } + if (remote_nav_pair(ri->parent, 1, ri->id)) { + // mtfix12 R1: the preceding GotoBottom certified this + // landing in the same server visit; apply it verbatim. + auto pr = ri->parent->conn->apply_pair_blob(ri->parent); + if (!pr) return fail(pr.error()); + cli_trace_tbl(ri->parent, "AdsGotoTop(idx)", "pair certified"); + remote_sync_keyno_gototop(ri->parent); + remote_nav_stamp(ri->parent, 1, ri->id); + if (Handle th = handle_for_remote_table(ri->parent)) + apply_relations_for_handle(to_ads_handle(th)); + return ok(); + } auto r = openads::network::remote_index_goto_top(ri); if (!r) return fail(r.error()); remote_sync_keyno_gototop(ri->parent); @@ -10980,6 +11668,15 @@ UNSIGNED32 ENTRYPOINT AdsGotoTop(ADSHANDLE hTable) { return ok(); } if (auto* rt = get_remote_table(hTable)) { + // Stock rddads sets focus 0 by switching from index to table handle, + // without a SetOrder call. Drop only an implicitly inherited order. + if (!rt->explicit_order_focus && !rt->created_order_focus && + (rt->active_index_id != 0 || + (rt->server_order_id != 0 && rt->server_order_id != openads::network::RemoteTable::kOrderUnknown) || + (rt->pending_order && rt->pending_order_id != 0))) { + if (UNSIGNED32 rc = AdsSetIndexOrderByHandle(hTable, 0); rc != 0) + return rc; + } // Sets + teardown flush here; a deferred order switch absorbs // into the nav below (fused frame) instead of going out alone. if (UNSIGNED32 frc = remote_flush_sets(rt); frc != 0) return frc; @@ -11025,6 +11722,15 @@ UNSIGNED32 ENTRYPOINT AdsGotoTop(ADSHANDLE hTable) { apply_relations_for_handle(hTable); return ok(); } + if (remote_nav_pair(rt, 1, rt->server_order_id)) { + auto pr = rt->conn->apply_pair_blob(rt); + if (!pr) return fail(pr.error()); + cli_trace_tbl(rt, "AdsGotoTop", "pair certified"); + remote_sync_keyno_gototop(rt); + remote_nav_stamp(rt, 1, rt->server_order_id); + apply_relations_for_handle(hTable); + return ok(); + } auto r = rt->conn->goto_top(rt); if (!r) return fail(r.error()); remote_sync_keyno_gototop(rt); @@ -11041,6 +11747,11 @@ UNSIGNED32 ENTRYPOINT AdsGotoTop(ADSHANDLE hTable) { } Table* t = get_table(hTable); if (!t) return fail(openads::AE_INTERNAL_ERROR, "unknown table"); + if (lookup_table_by_index(hTable) != nullptr) t->set_created_order_focus(false); + if (state().registry.lookup(hTable, HandleKind::Table) != nullptr && + !t->explicit_order_focus()) { + park_active_order(t); + } auto r = t->goto_top(); if (!r) return fail(r.error()); snapshot_ri_pks(t); @@ -11051,6 +11762,7 @@ UNSIGNED32 ENTRYPOINT AdsGotoTop(ADSHANDLE hTable) { UNSIGNED32 ENTRYPOINT AdsGotoBottom(ADSHANDLE hTable) { arc2_trace("AdsGotoBottom"); if (auto* ri = get_remote_index(hTable)) { + if (ri->parent) ri->parent->created_order_focus = false; // Sets + teardown flush here, but NOT a deferred order switch: // that absorbs into the nav below (fused frame) when it targets // this order, or flushes plainly otherwise. @@ -11093,6 +11805,17 @@ UNSIGNED32 ENTRYPOINT AdsGotoBottom(ADSHANDLE hTable) { apply_relations_for_handle(to_ads_handle(th)); return ok(); } + if (remote_nav_pair(ri->parent, 2, ri->id)) { + // mtfix12 R1: the preceding GotoTop certified this landing. + auto pr = ri->parent->conn->apply_pair_blob(ri->parent); + if (!pr) return fail(pr.error()); + cli_trace_tbl(ri->parent, "AdsGotoBottom(idx)", "pair certified"); + remote_sync_keyno_gotobottom(ri->parent); + remote_nav_stamp(ri->parent, 2, ri->id); + if (Handle th = handle_for_remote_table(ri->parent)) + apply_relations_for_handle(to_ads_handle(th)); + return ok(); + } auto r = openads::network::remote_index_goto_bottom(ri); if (!r) return fail(r.error()); remote_sync_keyno_gotobottom(ri->parent); @@ -11106,6 +11829,15 @@ UNSIGNED32 ENTRYPOINT AdsGotoBottom(ADSHANDLE hTable) { return ok(); } if (auto* rt = get_remote_table(hTable)) { + // Stock rddads sets focus 0 by switching from index to table handle, + // without a SetOrder call. Drop only an implicitly inherited order. + if (!rt->explicit_order_focus && !rt->created_order_focus && + (rt->active_index_id != 0 || + (rt->server_order_id != 0 && rt->server_order_id != openads::network::RemoteTable::kOrderUnknown) || + (rt->pending_order && rt->pending_order_id != 0))) { + if (UNSIGNED32 rc = AdsSetIndexOrderByHandle(hTable, 0); rc != 0) + return rc; + } // Sets + teardown flush here; a deferred order switch absorbs // into the nav below (fused frame) instead of going out alone. if (UNSIGNED32 frc = remote_flush_sets(rt); frc != 0) return frc; @@ -11142,6 +11874,15 @@ UNSIGNED32 ENTRYPOINT AdsGotoBottom(ADSHANDLE hTable) { apply_relations_for_handle(hTable); return ok(); } + if (remote_nav_pair(rt, 2, rt->server_order_id)) { + auto pr = rt->conn->apply_pair_blob(rt); + if (!pr) return fail(pr.error()); + cli_trace_tbl(rt, "AdsGotoBottom", "pair certified"); + remote_sync_keyno_gotobottom(rt); + remote_nav_stamp(rt, 2, rt->server_order_id); + apply_relations_for_handle(hTable); + return ok(); + } auto r = rt->conn->goto_bottom(rt); if (!r) return fail(r.error()); remote_sync_keyno_gotobottom(rt); @@ -11158,6 +11899,11 @@ UNSIGNED32 ENTRYPOINT AdsGotoBottom(ADSHANDLE hTable) { } Table* t = get_table(hTable); if (!t) return fail(openads::AE_INTERNAL_ERROR, "unknown table"); + if (lookup_table_by_index(hTable) != nullptr) t->set_created_order_focus(false); + if (state().registry.lookup
(hTable, HandleKind::Table) != nullptr && + !t->explicit_order_focus()) { + park_active_order(t); + } auto r = t->goto_bottom(); if (!r) return fail(r.error()); snapshot_ri_pks(t); @@ -11169,6 +11915,7 @@ UNSIGNED32 ENTRYPOINT AdsSkip(ADSHANDLE hTable, SIGNED32 lRows) { arc2_trace("AdsSkip"); seek_last_retry_latch() = false; if (auto* ri = get_remote_index(hTable)) { + if (ri->parent) ri->parent->created_order_focus = false; cli_trace_tbl(ri->parent, "AdsSkip(idx)", "rows=%d", (int)lRows); openads::network::RemoteTable* rt = ri->parent; if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) return frc; @@ -11187,7 +11934,26 @@ UNSIGNED32 ENTRYPOINT AdsSkip(ADSHANDLE hTable, SIGNED32 lRows) { return ok(); } if (auto* rt = get_remote_table(hTable)) { + // Stock rddads sets focus 0 by switching from index to table handle, + // without a SetOrder call. Drop only an implicitly inherited order. + const bool natural_transition = !rt->explicit_order_focus && !rt->created_order_focus && + (rt->active_index_id != 0 || + (rt->server_order_id != 0 && rt->server_order_id != openads::network::RemoteTable::kOrderUnknown) || + (rt->pending_order && rt->pending_order_id != 0)); + const std::uint32_t natural_anchor = rt->row_valid ? rt->current_recno : 0; + if (natural_transition) { + if (UNSIGNED32 rc = AdsSetIndexOrderByHandle(hTable, 0); rc != 0) + return rc; + } if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) return frc; + // Cached boundary landings/read-ahead can leave the physical server + // cursor elsewhere. Clearing the order also drops their lag, so + // anchor a direct natural Skip at the client-visible physical row. + if ((natural_transition || (rt->server_order_id == 0 && rt->pair_local_position)) && + natural_anchor != 0) { + auto r = rt->conn->goto_record(rt, natural_anchor); + if (!r) return fail(r.error()); + } if (UNSIGNED32 frc = remote_close_parked_indexes(rt); frc != 0) { return frc; } @@ -11240,6 +12006,11 @@ UNSIGNED32 ENTRYPOINT AdsSkip(ADSHANDLE hTable, SIGNED32 lRows) { } Table* t = get_table(hTable); if (!t) return fail(openads::AE_INTERNAL_ERROR, "unknown table"); + if (lookup_table_by_index(hTable) != nullptr) t->set_created_order_focus(false); + if (state().registry.lookup
(hTable, HandleKind::Table) != nullptr && + !t->explicit_order_focus()) { + park_active_order(t); + } auto r = t->skip(lRows); if (!r) return fail(r.error()); snapshot_ri_pks(t); @@ -12037,6 +12808,17 @@ UNSIGNED32 ENTRYPOINT AdsGetRecordNum(ADSHANDLE hTable, UNSIGNED16 /*bFilterOpti *pulRecordNum = rt->recno_bound; return ok(); } + // mtfix12 R3 — the R2 anchor certifies the server cursor's + // recno even when the local row cache was dropped without a + // wire move. With no drain outstanding (lag 0), the wire answer + // would be exactly anchor_recno. + if (rt->anchor_ok && rt->cursor_lag == 0 && + (remote_self_goto_per_table() + ? rt->anchor_tbl_seq == rt->conn->tbl_nav_seq(rt->id) + : rt->anchor_seq == rt->conn->nav_seq())) { + *pulRecordNum = rt->anchor_recno; + return ok(); + } // Phantom derivation (no frame, no currency): at the EOF // phantom the recno is LastRec+1 by Clipper convention — a // pure function of the certified EOF flag plus the cached @@ -12747,7 +13529,12 @@ bool fire_triggers_(Handle hConn, Connection* conn, UNSIGNED32 ENTRYPOINT AdsAppendRecord(ADSHANDLE hTable) { arc2_trace("AdsAppendRecord"); if (auto* rt = get_remote_table(hTable)) { - if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) return frc; + openads::abi::create_diag::Scope append_scope(rt->name, {}); + openads::abi::create_diag::log("client-append-enter"); + if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) { + openads::abi::create_diag::log("client-append-preflush-fail", frc); + return frc; + } remote_settle_cursor(rt); // M12.21 option C rt->row_valid = false; // M12.17 rt->rec_count_cached = false; @@ -12762,11 +13549,19 @@ UNSIGNED32 ENTRYPOINT AdsAppendRecord(ADSHANDLE hTable) { remote_clear_nav_boundaries(rt); rt->invalidate_prefetch(); auto r = rt->conn->append_blank(rt->id); - if (!r) return fail(r.error()); + if (!r) { + openads::abi::create_diag::log("client-append-fail", r.error().code, + "wire-append", r.error().sub_code); + return fail(r.error()); + } // Fresh appends auto-lock (non-exclusive tables): pooled reuse - // must not resurrect a locked handle. + // must not resurrect a locked handle. The ack carries no + // recno, so the ledger cannot name the auto-lock -- mark the + // lock state uncertain until a full UnlockTable proves clear. rt->ever_locked = true; + rt->locks_uncertain = true; rt->write_dirty = true; + openads::abi::create_diag::log("client-append-ok"); return ok(); } #if defined(OPENADS_WITH_FIREBIRD) @@ -12887,6 +13682,13 @@ UNSIGNED32 ENTRYPOINT AdsAppendRecord(ADSHANDLE hTable) { UNSIGNED32 ENTRYPOINT AdsWriteRecord(ADSHANDLE hTable) { arc2_trace("AdsWriteRecord"); if (auto* rt = get_remote_table(hTable)) { + // Snapshot dirtiness before remote_flush_pending drains the + // buffered sets: a commit with nothing written since the last + // flush (rddads DBCOMMITALL touches every open workarea) owes + // the server no frame at all -- it would only fsync unchanged + // pages, 1 RTT each, 16 of them after a Vouch voucher save. + const bool had_writes = + !rt->pending_sets.empty() || rt->write_dirty; if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) return frc; rt->row_valid = false; // M12.17 cache invalidation // A write can move the row in/out of a conditional order or @@ -12900,9 +13702,18 @@ UNSIGNED32 ENTRYPOINT AdsWriteRecord(ADSHANDLE hTable) { // instead of serving a stale current_keyno. rt->keyno_valid = false; rt->invalidate_prefetch(); + if (!had_writes) { + cli_trace_tbl(rt, "AdsWriteRecord", "clean: flush skipped"); + return ok(); + } auto r = rt->conn->flush_table(rt->id); if (!r) return fail(r.error()); - rt->write_dirty = true; + // kCapFlushTableDurable servers run the full file-buffers + // flush inside the FlushTable handler, so the commit is + // durable right here and a trailing AdsFlushFileBuffers owes + // nothing (1 RTT saved per commit). Old servers keep the + // classic two-frame pair. + rt->write_dirty = !rt->conn->server_flush_table_durable(); return ok(); } #if defined(OPENADS_WITH_FIREBIRD) @@ -14061,6 +14872,13 @@ UNSIGNED32 ENTRYPOINT AdsLockRecord(ADSHANDLE hTable, UNSIGNED32 ulRecord) { auto r = rt->conn->lock_record(rt->id, ulRecord); if (!r) return fail(r.error()); rt->ever_locked = true; + // ulRecord == 0 -> the current record (ACE convention). With + // no valid cursor the recno is unknowable client-side, so the + // ledger marks itself uncertain rather than guessing. + const std::uint32_t lrec = (ulRecord == 0) + ? (rt->row_valid ? rt->current_recno : 0) : ulRecord; + if (lrec == 0) rt->locks_uncertain = true; + else rt->held_recs.insert(lrec); return ok(); } #if defined(OPENADS_WITH_FIREBIRD) @@ -14137,6 +14955,11 @@ UNSIGNED32 ENTRYPOINT AdsUnlockRecord(ADSHANDLE hTable, UNSIGNED32 ulRecord) { if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) return frc; auto r = rt->conn->unlock_record(rt->id, ulRecord); if (!r) return fail(r.error()); + const std::uint32_t urec = (ulRecord == 0) + ? (rt->row_valid ? rt->current_recno : 0) : ulRecord; + if (urec != 0) rt->held_recs.erase(urec); + // locks_uncertain survives: only a full UnlockTable (or the + // close) proves the append auto-lock is gone. return ok(); } #if defined(OPENADS_WITH_FIREBIRD) @@ -14209,6 +15032,7 @@ UNSIGNED32 ENTRYPOINT AdsLockTable(ADSHANDLE hTable) { auto r = rt->conn->lock_table(rt->id); if (!r) return fail(r.error()); rt->ever_locked = true; + rt->table_lock_held = true; return ok(); } #if defined(OPENADS_WITH_FIREBIRD) @@ -14279,8 +15103,24 @@ UNSIGNED32 ENTRYPOINT AdsUnlockTable(ADSHANDLE hTable) { arc2_trace("AdsUnlockTable"); if (auto* rt = get_remote_table(hTable)) { if (UNSIGNED32 frc = remote_flush_pending(rt); frc != 0) return frc; + // Lock ledger: with nothing held on the server, the frame + // would release zero locks. Harbour rddads dbUnlock() has no + // client-side lock list and calls this blindly before every + // lock/append -- that blind call is most of the UnlockTable + // traffic in a Vouch save. + if (rt->held_recs.empty() && !rt->table_lock_held && + !rt->locks_uncertain) { + cli_trace_tbl(rt, "AdsUnlockTable", "skipped: no locks held"); + return ok(); + } auto r = rt->conn->unlock_table(rt->id); if (!r) return fail(r.error()); + // SAP ACE semantics: AdsUnlockTable releases ALL locks (table + // AND record, including the append auto-lock), so the ledger + // is provably empty again. + rt->held_recs.clear(); + rt->table_lock_held = false; + rt->locks_uncertain = false; return ok(); } #if defined(OPENADS_WITH_FIREBIRD) @@ -14399,6 +15239,8 @@ struct IndexBinding { std::string tag_name; std::unique_ptr parked; // nullptr when this is the active binding std::string path; // resolved index file path (M9.14) + openads::engine::Scope parked_scope{}; + std::optional parked_descending{}; }; std::unordered_map& index_bindings() { @@ -14430,6 +15272,10 @@ void park_active_order(Table* t) { auto& m = index_bindings(); auto bit = m.find(act_it->second); if (bit != m.end()) { + if (auto* o = t->order()) { + bit->second.parked_scope = o->scope(); + bit->second.parked_descending = o->descending_traverse(); + } auto taken = t->take_order(); openads::drivers::IIndex* raw = taken.get(); bit->second.parked = std::move(taken); @@ -14499,6 +15345,10 @@ openads::util::Result activate_binding(ADSHANDLE h) { if (act_it != act.end()) { auto prev = m.find(act_it->second); if (prev != m.end()) { + if (auto* o = t->order()) { + prev->second.parked_scope = o->scope(); + prev->second.parked_descending = o->descending_traverse(); + } auto taken = t->take_order(); openads::drivers::IIndex* raw = taken.get(); prev->second.parked = std::move(taken); @@ -14514,6 +15364,9 @@ openads::util::Result activate_binding(ADSHANDLE h) { openads::drivers::IIndex* raw = it->second.parked.get(); t->unregister_extra_index_view(raw); t->set_order(std::move(it->second.parked)); + t->order()->scope() = it->second.parked_scope; + if (it->second.parked_descending.has_value()) + t->order()->set_descending_traverse(*it->second.parked_descending); } act[t] = h; return {}; @@ -14872,6 +15725,21 @@ UNSIGNED32 ENTRYPOINT AdsOpenIndex(ADSHANDLE hTable, UNSIGNED8* pucName, rt->active_index_id = rt->parked_active; rt->indexes_parked = false; rt->close_all_indexes_pending = false; + // Restore the nav stamp parked at CloseAll when it still + // proves the server cursor: same order coming back, no + // cursor-affecting frame since the park (seq gate). The + // post-unpark GotoTop(idx) then dedupes locally instead + // of paying a refresh RTT for a position the server + // never lost. + if (rt->parked_nav_which != 0 && + rt->parked_nav_order == rt->parked_active && + rt->parked_nav_seq == rt->conn->nav_seq()) { + rt->last_nav = rt->parked_nav_which; + rt->last_nav_order = rt->parked_nav_order; + rt->last_nav_row = rt->parked_nav_row; + rt->last_nav_seq = rt->parked_nav_seq; + } + rt->parked_nav_which = 0; cli_trace_tbl(rt, "AdsOpenIndex", "unpark hit %.32s", rt->parked_bag_stem.c_str()); auto& s = state(); @@ -14965,6 +15833,11 @@ UNSIGNED32 ENTRYPOINT AdsOpenIndex(ADSHANDLE hTable, UNSIGNED8* pucName, } ++count; remote_indexes.emplace(gh, std::move(ri)); + if (cli_trace_on()) { + auto& trace = cli_trace_state(); + std::lock_guard trace_lk(trace.mu); + trace.indexes[{rt->conn, ent.id}] = rt->alias.empty() ? rt->name : rt->alias; + } // Dedup by tag: production-CDX auto-open and a later explicit // AdsOpenIndex on the same bag must not register the order // twice, or AdsGetNumIndexes / by-order resolution skew. @@ -15434,6 +16307,14 @@ UNSIGNED32 ENTRYPOINT AdsCloseAllIndexes(ADSHANDLE hTable) { rt->index_handles.clear(); rt->active_index_id = 0; rt->indexes_parked = true; + // Park the nav stamp with the bindings: if the same order + // comes back via an unpark with no intervening wire nav + // (seq-gated), the post-unpark GotoTop dedupes instead of + // paying a refresh frame for state the server never lost. + rt->parked_nav_which = rt->last_nav; + rt->parked_nav_order = rt->last_nav_order; + rt->parked_nav_row = rt->last_nav_row; + rt->parked_nav_seq = rt->last_nav_seq; cli_trace_tbl(rt, "AdsCloseAllIndexes", "parked %u tags", ntags); } else if (rt->indexes_parked) { cli_trace_tbl(rt, "AdsCloseAllIndexes", "repeat clear, park kept"); @@ -15443,6 +16324,8 @@ UNSIGNED32 ENTRYPOINT AdsCloseAllIndexes(ADSHANDLE hTable) { // Order belief changed with no frame: expire the nav stamp so a // subsequent GotoTop cannot dedupe against the old binding. rt->last_nav = 0; + rt->pair_valid = false; + rt->anchor_ok = false; rt->close_all_indexes_pending = true; return ok(); } @@ -15492,6 +16375,27 @@ UNSIGNED32 ENTRYPOINT AdsCloseAllIndexes(ADSHANDLE hTable) { return ok(); } +// One-off client ABI discriminator for Vouch INDEX ON. Off unless explicitly +// enabled. Keep this independent of wire_trace: a rejected call never hits wire. +static void create_index_diag(const char* fmt, ...) { + const char* path = std::getenv("OPENADS_CREATE_INDEX_DIAG_FILE"); + if (!path || !*path) return; + static std::mutex mu; + std::lock_guard lock(mu); + FILE* f = std::fopen(path, "a"); + if (!f) return; + va_list args; + va_start(args, fmt); + std::vfprintf(f, fmt, args); + va_end(args); + std::fputc('\n', f); + std::fclose(f); +} + +static const char* create_index_arg(const UNSIGNED8* p) { + return p ? reinterpret_cast(p) : ""; +} + UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, UNSIGNED8* pucFileName, UNSIGNED8* pucIndexName, @@ -15502,8 +16406,15 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, UNSIGNED16 usPageSize, ADSHANDLE* phIndex) { arc2_trace("AdsCreateIndex61"); + create_index_diag("61 ENTRY h=%llu file=%.160s tag=%.160s expr=%.160s cond=%.160s keyfilter=%.160s opts=0x%08lx page=%u out=%p", + static_cast(hTable), create_index_arg(pucFileName), + create_index_arg(pucIndexName), create_index_arg(pucExpr), + create_index_arg(pucCondition), create_index_arg(pucKeyFilter), + static_cast(ulOptions), static_cast(usPageSize), + static_cast(phIndex)); if (phIndex == nullptr || pucFileName == nullptr || pucIndexName == nullptr || pucExpr == nullptr) { + create_index_diag("61 EXIT null-arg 5000 h=%llu", static_cast(hTable)); return fail(openads::AE_INTERNAL_ERROR, "null arg"); } #if defined(OPENADS_WITH_SQLITE) @@ -15674,6 +16585,7 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, } #endif if (auto* rt = get_remote_table(hTable)) { + create_index_diag("61 ROUTE remote h=%llu table_id=%u", static_cast(hTable), static_cast(rt->id)); std::string path = openads::abi::to_internal(pucFileName, 0); path = normalize_index_path(std::move(path)); std::string tag = openads::abi::to_internal(pucIndexName, 0); @@ -15682,10 +16594,19 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, ? openads::abi::to_internal(pucCondition, 0) : std::string(); std::string kf = pucKeyFilter ? openads::abi::to_internal(pucKeyFilter, 0) : std::string(); + create_index_diag("61 WIRE 0x94 h=%llu table_id=%u path=%.160s tag=%.160s", + static_cast(hTable), static_cast(rt->id), + path.c_str(), tag.c_str()); auto r = rt->conn->create_index(rt->id, path, tag, expr, cond, kf, ulOptions, usPageSize); - if (!r) return fail(r.error()); + if (!r) { + create_index_diag("61 WIRE error h=%llu code=%u (no success ACK)", + static_cast(hTable), static_cast(r.error().code)); + return fail(r.error()); + } + create_index_diag("61 WIRE ACK success h=%llu index_id=%u (0x95)", + static_cast(hTable), static_cast(r.value())); // Creating a tag opens EVERY tag in the bag (ADS semantics -- the // server-side create binds siblings as parked views), so refresh // the client registry from the server: OrdCount() (AdsGetNumIndexes) @@ -15758,7 +16679,30 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, rt->index_by_tag = std::move(keep_tags); rt->index_handles = std::move(keep_handles); } - if (rt->active_index_id == 0) rt->active_index_id = r.value(); + rt->created_order_focus = true; + // OpenIndex can replace the just-created local binding and its + // tag spelling (native ADI uses the expression field name). Use + // the fresh wire id from that bag rather than the stale create id. + std::uint32_t created_focus = r.value(); + if (refr) { + for (const auto& ent : refr.value()) { + if (ent.tag == tag || refr.value().size() == 1) { + created_focus = ent.id; + break; + } + } + } + // The returned handle must reference the re-opened bag binding too. + if (auto* created = get_remote_index(*phIndex)) created->id = created_focus; + for (auto& mapping : rt->index_by_tag) { + if (mapping.second == r.value()) mapping.second = created_focus; + } + rt->active_index_id = created_focus; + auto focus_result = rt->conn->set_order(rt->id, created_focus); + if (!focus_result) return fail(focus_result.error()); + rt->server_order_id = created_focus; + rt->pending_order = false; + rt->invalidate_prefetch(); // Structural change: the bag gained a tag, so a parked snapshot // predates it — drop the park (the maps above were rebuilt fresh // from the server). The pending flag stays: the next flush sends @@ -15770,8 +16714,11 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, } Table* t = get_table(hTable); if (!t) { + create_index_diag("61 EXIT unknown-handle 5000 h=%llu remote_lookup_missed=1", + static_cast(hTable)); return fail(openads::AE_INTERNAL_ERROR, "unknown table"); } + create_index_diag("61 ROUTE native h=%llu", static_cast(hTable)); // Settle any coalesced dirty record first: the build loop below reads // rows straight from disk, so a pending buffer edit would be indexed // from its stale on-disk image (and silently dropped by @@ -16458,6 +17405,10 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, if (prev != act.end()) { auto pit = m.find(prev->second); if (pit != m.end()) { + if (auto* o = t->order()) { + pit->second.parked_scope = o->scope(); + pit->second.parked_descending = o->descending_traverse(); + } auto displaced = t->take_order(); pit->second.parked = std::move(displaced); t->register_extra_index_view(pit->second.parked.get()); @@ -16466,6 +17417,7 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex61(ADSHANDLE hTable, t->set_order(std::move(idx_owner)); m[h] = IndexBinding{t, tag, nullptr, p.string()}; act[t] = h; + t->set_created_order_focus(true); // Clipper / Harbour: a fresh CREATE INDEX leaves the cursor // positioned at the new order's TOP key. (void)t->goto_top(); @@ -16478,11 +17430,18 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex(ADSHANDLE hTable, UNSIGNED8* pucFile, UNSIGNED8* pucCondition, UNSIGNED32 ulOptions, UNSIGNED16 usKeyType, ADSHANDLE* phIndex) { arc2_trace("AdsCreateIndex"); + create_index_diag("legacy ENTRY h=%llu file=%.160s tag=%.160s expr=%.160s cond=%.160s opts=0x%08lx keytype=%u out=%p", + static_cast(hTable), create_index_arg(pucFile), + create_index_arg(pucTag), create_index_arg(pucExpr), + create_index_arg(pucCondition), static_cast(ulOptions), + static_cast(usKeyType), static_cast(phIndex)); if (phIndex == nullptr) { + create_index_diag("legacy EXIT null-out 5000 h=%llu", static_cast(hTable)); return fail(openads::AE_INTERNAL_ERROR, "null index out-param"); } // Legacy API: remote tables route through AdsCreateIndex61 (M12.16). if (get_remote_table(hTable) != nullptr) { + create_index_diag("legacy ROUTE 61 h=%llu", static_cast(hTable)); return AdsCreateIndex61(hTable, pucFile, pucTag, pucExpr, pucCondition, nullptr, ulOptions, usKeyType ? usKeyType @@ -16491,8 +17450,10 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex(ADSHANDLE hTable, UNSIGNED8* pucFile, } Table* t = get_table(hTable); if (!t) { + create_index_diag("legacy EXIT unknown-handle 5000 h=%llu", static_cast(hTable)); return fail(openads::AE_INTERNAL_ERROR, "unknown table or null out"); } + create_index_diag("legacy ROUTE native h=%llu", static_cast(hTable)); // Settle any coalesced dirty record first (see AdsCreateIndex61). if (auto cr = t->commit_dirty_record(); !cr) return fail(cr.error()); auto file = normalize_index_path( @@ -16633,6 +17594,7 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex(ADSHANDLE hTable, UNSIGNED8* pucFile, m[h] = IndexBinding{t, tag, std::move(idx), file}; t->register_extra_index_view(raw); } + t->set_created_order_focus(true); *phIndex = h; return ok(); } @@ -16954,6 +17916,26 @@ UNSIGNED32 ENTRYPOINT AdsGetAllLocks(ADSHANDLE hTable, UNSIGNED32* paRecnos, UNSIGNED16* pusCount) { arc2_trace("AdsGetAllLocks"); if (auto* rt = get_remote_table(hTable)) { + // The client lock ledger is complete unless an append + // auto-lock or an unresolvable current-record lock made it + // uncertain (and a table lock shadows the record list, so + // that case stays on the wire too). rddads polls this after + // every commit -- answering locally saves 1 RTT each time. + if (pusCount != nullptr && !rt->locks_uncertain && + !rt->table_lock_held) { + cli_trace_tbl(rt, "AdsGetAllLocks", + "served from client lock ledger"); + const UNSIGNED16 lcap = *pusCount; + UNSIGNED16 li = 0; + if (paRecnos != nullptr) { + for (std::uint32_t lrn : rt->held_recs) { + if (li >= lcap) break; + paRecnos[li++] = lrn; + } + } + *pusCount = static_cast(rt->held_recs.size()); + return ok(); + } // M12.36 — remote record locks are server-managed; the // GetAllLocks wire opcode returns this connection's held list. if (pusCount == nullptr) return fail(openads::AE_INTERNAL_ERROR, ""); @@ -17001,6 +17983,8 @@ UNSIGNED32 ENTRYPOINT AdsSetIndexOrder(ADSHANDLE hTable, UNSIGNED8* pucName) { if (UNSIGNED32 frc = remote_flush_teardown(rt); frc != 0) return frc; std::string name = pucName ? openads::abi::to_internal(pucName, 0) : std::string(); + rt->explicit_order_focus = !name.empty(); + rt->created_order_focus = false; // Resolve the target locally (case-insensitive, like the mirror // block below). Unmapped names stay kOrderUnknown and always go // out on the wire: the server resolves those, and a wrong local @@ -17193,6 +18177,7 @@ UNSIGNED32 ENTRYPOINT AdsSetIndexOrder(ADSHANDLE hTable, UNSIGNED8* pucName) { // into its slot so a subsequent AdsSetIndexOrder picks the // current Table::order_ up cleanly. park_active_order(t); + t->set_explicit_order_focus(false); return ok(); } auto upper_eq = [](const std::string& a, const std::string& b) { @@ -17213,6 +18198,7 @@ UNSIGNED32 ENTRYPOINT AdsSetIndexOrder(ADSHANDLE hTable, UNSIGNED8* pucName) { if (b.table == t && upper_eq(b.tag_name, name)) { auto r = activate_binding(h); if (!r) return fail(r.error()); + t->set_explicit_order_focus(true); return ok(); } } @@ -17227,6 +18213,8 @@ UNSIGNED32 ENTRYPOINT AdsSetIndexOrderByHandle(ADSHANDLE hTable, ADSHANDLE hInde // (overwritten below — see ByName). if (UNSIGNED32 frc = remote_flush_sets(rt); frc != 0) return frc; if (UNSIGNED32 frc = remote_flush_teardown(rt); frc != 0) return frc; + rt->explicit_order_focus = hIndex != 0; + rt->created_order_focus = false; if (hIndex == 0) { // "Back to natural order" via the explicit API: send the reset // frame so the server drops its ordered_tables_ entry (it used @@ -17305,6 +18293,7 @@ UNSIGNED32 ENTRYPOINT AdsSetIndexOrderByHandle(ADSHANDLE hTable, ADSHANDLE hInde // Natural order (rddads' patched DbSetOrder(0) calls this): park // the active order back into its binding slot. park_active_order(t); + t->set_explicit_order_focus(false); return ok(); } auto& m = index_bindings(); @@ -17317,6 +18306,7 @@ UNSIGNED32 ENTRYPOINT AdsSetIndexOrderByHandle(ADSHANDLE hTable, ADSHANDLE hInde } auto r = activate_binding(hIndex); if (!r) return fail(r.error()); + t->set_explicit_order_focus(true); return ok(); } @@ -19555,7 +20545,11 @@ UNSIGNED32 ENTRYPOINT AdsGetIndexHandle(ADSHANDLE hTable, UNSIGNED8* pucName, // Storm fix: reader must hold index_bindings_mu (binds are unlocked now). std::lock_guard _rh_lk(index_bindings_mu()); for (auto& [h, b] : index_bindings()) { - if (b.table == t && b.tag_name == name) { *phIndex = h; return ok(); } + if (b.table == t && b.tag_name.size() == name.size() && + std::equal(b.tag_name.begin(), b.tag_name.end(), name.begin(), + [](unsigned char a, unsigned char z) { + return std::toupper(a) == std::toupper(z); + })) { *phIndex = h; return ok(); } } return fail(openads::AE_INTERNAL_ERROR, "index name not found"); } @@ -19803,6 +20797,7 @@ static UNSIGNED32 ads_seek_local(ADSHANDLE hIndex, UNSIGNED16* pbFound, bool find_last) { Table* t = table_for_index(hIndex); + if (t) t->set_created_order_focus(false); if (!t) return fail(openads::AE_INTERNAL_ERROR, "unknown index"); std::string key; (void)u16KeyType; @@ -20100,6 +21095,7 @@ UNSIGNED32 ENTRYPOINT AdsSeek(ADSHANDLE hIndex, } #endif if (auto* ri = get_remote_index(hIndex)) { + if (ri->parent) ri->parent->created_order_focus = false; std::string key(reinterpret_cast(pucKey), u16KeyLen); if (ri->parent) { @@ -20114,8 +21110,18 @@ UNSIGNED32 ENTRYPOINT AdsSeek(ADSHANDLE hIndex, // -- nothing on the network to make it look wrong -- and the wire skip // after that sent (step + a lag that no longer applied). ri->parent->invalidate_prefetch(); - cli_trace_tbl(ri->parent, "AdsSeek", "key=%.24s", - key.c_str()); + cli_trace_tbl(ri->parent, "AdsSeek", "key_len=%u", + static_cast(u16KeyLen)); + } + if (ri->parent != nullptr && remote_empty_window(ri->parent)) { + cli_trace_tbl(ri->parent, "AdsSeek", + "empty window: not found (local)"); + remote_empty_restamp(ri->parent); + ri->parent->found_cached = true; + ri->parent->current_found = false; + if (pbFound) *pbFound = 0; + (void)u16KeyType; + return ok(); } // RCB 07/14/2026: M12.24 -- pass the parent so the SeekAck's row trailer // lands straight in the row cache. Without it row_valid stays false and @@ -20281,6 +21287,7 @@ UNSIGNED32 ENTRYPOINT AdsSeekLast(ADSHANDLE hIndex, } #endif if (auto* ri = get_remote_index(hIndex)) { + if (ri->parent) ri->parent->created_order_focus = false; std::string key(reinterpret_cast(pucKey), u16KeyLen); if (ri->parent) { @@ -20289,8 +21296,8 @@ UNSIGNED32 ENTRYPOINT AdsSeekLast(ADSHANDLE hIndex, // RCB 07/14/2026: same stale-queue bug as AdsSeek -- see the note // there for why dropping the block is mandatory after a seek. ri->parent->invalidate_prefetch(); - cli_trace_tbl(ri->parent, "AdsSeekLast", "key=%.24s", - key.c_str()); + cli_trace_tbl(ri->parent, "AdsSeekLast", "key_len=%u", + static_cast(u16KeyLen)); } auto r = ri->conn->seek(ri->id, key, /*soft=*/0, @@ -20777,6 +21784,8 @@ UNSIGNED32 ENTRYPOINT AdsSetAOF(ADSHANDLE hTable, UNSIGNED8* pucCondition, // Replacement filter can narrow to empty or widen to rows: // expire the nav stamp, proven-false answers and cached answers. rt->last_nav = 0; + rt->pair_valid = false; + rt->anchor_ok = false; rt->nav_not_bof = false; rt->nav_not_eof = false; remote_nav_bound_clear(rt); @@ -20948,6 +21957,8 @@ UNSIGNED32 ENTRYPOINT AdsClearAOF(ADSHANDLE hTable) { // cached boundary answers (the wire frame already expired the // seq-gated state; this covers the seq-blind sticky). rt->last_nav = 0; + rt->pair_valid = false; + rt->anchor_ok = false; rt->nav_not_bof = false; rt->nav_not_eof = false; remote_nav_bound_clear(rt); @@ -20968,6 +21979,8 @@ UNSIGNED32 ENTRYPOINT AdsClearFilter(ADSHANDLE hTable) { if (auto* rt = get_remote_table(hTable)) { rt->filter_expr.clear(); rt->last_nav = 0; // local visibility change: expire nav stamp + rt->pair_valid = false; + rt->anchor_ok = false; rt->nav_not_bof = false; // widening-safe to keep; uniformity wins rt->nav_not_eof = false; remote_nav_bound_clear(rt); @@ -37059,6 +38072,16 @@ UNSIGNED32 ENTRYPOINT AdsGetNumLocks(ADSHANDLE hTable, UNSIGNED16* p) { // Remote: route through the wire GetAllLocks op and count (was a stub // returning 0 — lock introspection on remote tables reported nothing). if (auto* rt = get_remote_table(hTable)) { + // Same ledger fast path as AdsGetAllLocks: while the ledger is + // provably complete (no append auto-lock outstanding, no table + // lock), the count is the ledger size. rddads polls this after + // every commit -- answering locally saves 1 RTT each time. + if (!rt->locks_uncertain && !rt->table_lock_held) { + cli_trace_tbl(rt, "AdsGetNumLocks", + "served from client lock ledger"); + *p = static_cast(rt->held_recs.size()); + return ok(); + } auto r = rt->conn->get_all_locks(rt->id); if (!r) return fail(r.error()); *p = static_cast(r.value().size()); @@ -37537,21 +38560,38 @@ UNSIGNED32 ENTRYPOINT AdsIsRecordInAOF(ADSHANDLE, UNSIGNED32, UNSIGNED16* p) { arc2_trace("AdsIsRecordInAOF"); if (p) *p = 1; return openads::AE_SUCCESS; } // ulRecord == 0 means "the current record" (ACE convention). Reports -// whether this table handle owns the lock (its own record locks or its -// table lock), not whether another user does. Remote handles go over the -// wire (M12.36 IsRecordLocked opcode); the server answers atomically from -// the calling session's own state - the engine Table's table lock plus -// the ABI twin's record locks, which include append auto-locks - and -// translates recno 0 to the current record on its side. +// whether this table handle owns the lock, not whether another user does. +// The global OS byte probe remains separately in Table::is_record_locked_any. UNSIGNED32 ENTRYPOINT AdsIsRecordLocked(ADSHANDLE hTable, UNSIGNED32 ulRecord, UNSIGNED16* pbLocked) { arc2_trace("AdsIsRecordLocked"); if (pbLocked == nullptr) return fail(openads::AE_INTERNAL_ERROR, ""); *pbLocked = 0; if (auto* rt = get_remote_table(hTable)) { - auto r = rt->conn->is_record_locked(rt->id, ulRecord); + const std::uint32_t rec = ulRecord == 0 + ? (rt->row_valid ? rt->current_recno : 0) : ulRecord; + // A held file lock or an explicit recno in the client ledger is + // positive proof of ownership, even if an append also made the + // ledger incomplete. A negative answer is safe only when complete. + if (rt->table_lock_held || + (rec != 0 && rt->held_recs.count(rec) != 0)) { + *pbLocked = 1; + return ok(); + } + if (rec != 0 && !rt->locks_uncertain) return ok(); + // Append auto-locks and unknown current recnos need the server's + // GetAllLocks, which enumerates this wire table's owning ABI handle. + auto r = rt->conn->get_all_locks(rt->id); if (!r) return fail(r.error()); - *pbLocked = r.value(); + std::uint32_t target = rec; + if (target == 0) { + auto rn = rt->conn->get_record_num(rt->id); + if (!rn) return fail(rn.error()); + target = rn.value(); + } + const auto& held = r.value(); + *pbLocked = rt->table_lock_held || + std::find(held.begin(), held.end(), target) != held.end(); return ok(); } Table* t = get_table(hTable); @@ -37847,6 +38887,8 @@ UNSIGNED32 ENTRYPOINT AdsSetFilter(ADSHANDLE hTable, UNSIGNED8* pucFilter) { // nav stamp, proven-false answers and cached answers — the // wire-seq rule cannot see any of them. rt->last_nav = 0; + rt->pair_valid = false; + rt->anchor_ok = false; rt->nav_not_bof = false; rt->nav_not_eof = false; remote_nav_bound_clear(rt); @@ -37885,6 +38927,10 @@ UNSIGNED32 ENTRYPOINT AdsSetRecord(ADSHANDLE hTable, UNSIGNED8* pucRecord, auto r = rt->conn->set_record(rt->id, pucRecord, static_cast(ulLen)); if (!r) return fail(r.error()); + // Full-record write: mark dirty so the commit's FlushTable and + // the FlushFileBuffers gate see it (the write_dirty snapshot in + // AdsWriteRecord depends on this). + rt->write_dirty = true; return ok(); } Table* t = get_table(hTable); @@ -38680,9 +39726,17 @@ UNSIGNED32 ENTRYPOINT AdsCreateIndex90(ADSHANDLE hObj, UNSIGNED8* pucFileName, UNSIGNED32 ulOptions, UNSIGNED32 ulPageSize, UNSIGNED8* /*pucCollation*/, ADSHANDLE* phIndex) { arc2_trace("AdsCreateIndex90"); - return AdsCreateIndex61(hObj, pucFileName, pucTag, pucExpr, pucCondition, + create_index_diag("90 ENTRY h=%llu file=%.160s tag=%.160s expr=%.160s cond=%.160s while=%.160s opts=0x%08lx page=%lu out=%p", + static_cast(hObj), create_index_arg(pucFileName), + create_index_arg(pucTag), create_index_arg(pucExpr), + create_index_arg(pucCondition), create_index_arg(pucWhile), + static_cast(ulOptions), static_cast(ulPageSize), + static_cast(phIndex)); + auto rc = AdsCreateIndex61(hObj, pucFileName, pucTag, pucExpr, pucCondition, pucWhile, ulOptions, static_cast(ulPageSize), phIndex); + create_index_diag("90 EXIT h=%llu rc=%lu", static_cast(hObj), static_cast(rc)); + return rc; } UNSIGNED32 ENTRYPOINT AdsDDAddTable90(ADSHANDLE hConnect, UNSIGNED8* pucAlias, diff --git a/src/abi/create_table_diag.h b/src/abi/create_table_diag.h new file mode 100644 index 00000000..e687d5ed --- /dev/null +++ b/src/abi/create_table_diag.h @@ -0,0 +1,92 @@ +#pragma once + +// Opt-in CreateTable stages, shared by the client ABI and server ABI twin. +// Never log schema, row values, usernames, or raw paths. +#include +#include +#include +#include +#include +#include +#include +#include + +namespace openads::abi::create_diag { +inline thread_local std::string target; +inline thread_local std::string correlation; + +inline std::string leaf(std::string_view path) { + auto end = path.find_last_not_of("/\\"); + if (end == std::string_view::npos) return {}; + auto start = path.find_last_of("/\\", end); + std::string out(path.substr(start == std::string_view::npos ? 0 : start + 1, + end - (start == std::string_view::npos ? 0 : start + 1) + 1)); + for (auto& ch : out) { + if (!((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || + (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' || ch == '.')) + ch = '_'; + } + return out; +} +inline bool enabled(std::string_view path) { + const char* dest = std::getenv("OPENADS_CREATE_TABLE_DIAG_FILE"); + if (!dest || !*dest) return false; + const char* filter = std::getenv("OPENADS_CREATE_TABLE_DIAG_FILTER"); + if (!filter || !*filter) return true; + auto a = leaf(path), b = leaf(filter); + if (a.size() != b.size()) return false; + for (std::size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) != + std::tolower(static_cast(b[i]))) return false; + } + return true; +} +inline std::string new_id() { + static std::mutex mu; + static unsigned long long serial = 0; + std::lock_guard lock(mu); + return std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + + "." + std::to_string(++serial); +} +struct Scope { + std::string old_target, old_correlation; + Scope(std::string_view name, std::string id) : old_target(target), old_correlation(correlation) { + if (enabled(name)) { + target = leaf(name); + correlation = id.empty() ? new_id() : std::move(id); + } else { + target.clear(); + correlation.clear(); + } + } + ~Scope() { + target = std::move(old_target); + correlation = std::move(old_correlation); + } +}; +inline void log(std::string_view stage, std::int64_t code = 0, + std::string_view detail = {}, int os_error = 0) { + if (target.empty()) return; + const char* dest = std::getenv("OPENADS_CREATE_TABLE_DIAG_FILE"); + if (!dest || !*dest) return; + static std::mutex mu; + std::lock_guard lock(mu); + auto* fp = std::fopen(dest, "a"); + if (!fp) return; + // Bound each process log even when the filter is omitted for a fresh-org run. + if (std::fseek(fp, 0, SEEK_END) != 0 || + std::ftell(fp) >= 2 * 1024 * 1024) { + std::fclose(fp); + return; + } + // Detail is restricted to a stage label, never arbitrary error text or paths. + const auto epoch_ms = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + std::fprintf(fp, "create-table epoch_ms=%lld id=%s table=%s stage=%.*s code=%lld os_code=%d detail=%.*s\n", + static_cast(epoch_ms), + correlation.c_str(), target.c_str(), static_cast(stage.size()), stage.data(), + static_cast(code), os_error, + static_cast(detail.size()), detail.data()); + std::fclose(fp); +} +} // namespace openads::abi::create_diag diff --git a/src/abi/lock_retry_policy.h b/src/abi/lock_retry_policy.h index f3599555..4e9b3c19 100644 --- a/src/abi/lock_retry_policy.h +++ b/src/abi/lock_retry_policy.h @@ -1,25 +1,28 @@ -#pragma once +#pragma once // Shared lock-retry policy between the ABI layer (AdsLockRecord/AdsLockTable) // and the network session layer (LockRecord/LockTable wire opcodes). #include +#include #include #include #include +#include "util/client_config.h" + namespace openads::abi { struct LockPolicy { - std::uint32_t cycle_ms = 100; // ACE default (AdsSetLockCycle) - std::uint16_t retry_count = 10; // ACE default (AdsSetLockRetryCount) + std::uint32_t cycle_ms = 100; // ACE ceiling (AdsSetLockCycle) + std::uint16_t retry_count = 0; // single attempt by default (see below) // Sleep between attempt i and i+1 (0-based). The ACE contract is a flat // cycle_ms quantum, but a flat 100ms slice per retry multiplies badly // under many-instance contention (Pritpal Bedi's 700-instance B_BIG: // a contended RLock that resolves in 20 retries costs a full 2 s of - // pure sleep). Most contentions resolve within a few milliseconds — - // the holder is inside a REPLACE — so the first attempts recheck after + // pure sleep). Most contentions resolve within a few milliseconds - + // the holder is inside a REPLACE - so the first attempts recheck after // 2/4/8 ms and only the later attempts fall back to the configured // cycle_ms quantum. AdsSetLockCycle still controls the ceiling. std::uint32_t sleep_before_attempt(std::uint32_t attempt) const { @@ -35,20 +38,80 @@ struct LockPolicy { } }; -// Process-global lock policy — shared between ABI entry points and the -// server session. ADS_SETLOCKCYCLE / ADS_SETLOCKRETRYCOUNT modify it. +// Process-global lock policy for the client-facing ACE lock calls. +// +// DEFAULT IS SINGLE-ATTEMPT (retry_count 0): xBase RLOCK()/FLOCK() are +// single-attempt primitives - fail means fail, the application owns any +// retry loop (Vouch polls with its own waits; CacheRDD does the same). +// A hidden client-side wait underneath double-waits the app and, worse, +// turns every intentional lock PROBE (login semaphores, "is anyone +// logged" walks) into a full-budget burn: the server already answered +// fail-fast on the first attempt. (Pritpal Bedi, 2026-09-24: "If lock +// fails it must return immediately. App will decide on what it has to +// do next.") +// +// Opt back into ACE-style waits either per-app (AdsSetLockCycle / +// AdsSetLockRetryCount override these values at runtime, unchanged ABI +// contract) or per-install: +// openads.ini: lock_retry_count = 10 (any value > 0 re-enables waits) +// lock_cycle_ms = 100 (ceiling per wait cycle) +// env: OPENADS_LOCK_RETRY_COUNT / OPENADS_LOCK_CYCLE_MS (env wins). inline LockPolicy& lock_retry_policy() { - static LockPolicy p; + static LockPolicy p = [] { + LockPolicy d; + try { + const std::string rc = openads::util::client_setting( + "OPENADS_LOCK_RETRY_COUNT", "lock_retry_count"); + if (!rc.empty()) + d.retry_count = static_cast(std::stoul(rc)); + const std::string cm = openads::util::client_setting( + "OPENADS_LOCK_CYCLE_MS", "lock_cycle_ms"); + if (!cm.empty()) + d.cycle_ms = static_cast(std::stoul(cm)); + } catch (...) {} + return d; + }(); return p; } -// Sleep for the interval the policy prescribes before retry `attempt`. -inline void lock_retry_sleep(std::uint32_t attempt) { - const auto& p = lock_retry_policy(); +// Engine-internal lock policy for the append auto-lock (Table::append_blank). +// Deliberately decoupled from the client-facing policy above: the auto-lock +// on a freshly appended record is an internal wait the application cannot +// perform itself, and a single attempt there makes appends fail (5012) +// under any FLock/Browse convoy. Keeps the historical 100ms x 10 budget; +// tunable via engine_lock_retry_count / engine_lock_cycle_ms (env +// OPENADS_ENGINE_LOCK_RETRY_COUNT / OPENADS_ENGINE_LOCK_CYCLE_MS). +inline const LockPolicy& engine_append_lock_policy() { + static const LockPolicy p = [] { + LockPolicy d; + d.cycle_ms = 100; + d.retry_count = 10; + try { + const std::string rc = openads::util::client_setting( + "OPENADS_ENGINE_LOCK_RETRY_COUNT", "engine_lock_retry_count"); + if (!rc.empty()) + d.retry_count = static_cast(std::stoul(rc)); + const std::string cm = openads::util::client_setting( + "OPENADS_ENGINE_LOCK_CYCLE_MS", "engine_lock_cycle_ms"); + if (!cm.empty()) + d.cycle_ms = static_cast(std::stoul(cm)); + } catch (...) {} + return d; + }(); + return p; +} + +// Sleep for the interval the given policy prescribes before retry `attempt`. +inline void lock_retry_sleep(const LockPolicy& p, std::uint32_t attempt) { auto ms_ = p.sleep_before_attempt(attempt); if (ms_ > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(ms_)); } } +// Sleep for the interval the client-facing policy prescribes. +inline void lock_retry_sleep(std::uint32_t attempt) { + lock_retry_sleep(lock_retry_policy(), attempt); +} + } // namespace openads::abi diff --git a/src/abi/runtime.cpp b/src/abi/runtime.cpp new file mode 100644 index 00000000..9f95e9e2 --- /dev/null +++ b/src/abi/runtime.cpp @@ -0,0 +1,10 @@ +#include "abi/runtime.h" + +namespace openads::abi::detail { + +ProcessState& state() { + static ProcessState process_state; + return process_state; +} + +} // namespace openads::abi::detail diff --git a/src/abi/runtime.h b/src/abi/runtime.h new file mode 100644 index 00000000..4f71f65b --- /dev/null +++ b/src/abi/runtime.h @@ -0,0 +1,27 @@ +#pragma once + +#include "session/connection.h" +#include "session/handle_registry.h" + +#include +#include +#include +#include + +namespace openads::abi::detail { + +// Process-wide state shared by ACE entry points. Keep this as the sole owner +// while ace_exports.cpp is split into domain translation units. +struct ProcessState { + // Recursive because SQL UNION execution deliberately re-enters + // AdsExecuteSQLDirect while the outer entry point holds this lock. + std::recursive_mutex mu; + session::HandleRegistry registry; + std::unordered_map> conns; + std::vector> remote_finds; +}; + +ProcessState& state(); + +} // namespace openads::abi::detail diff --git a/src/drivers/cdx/cdx_index.cpp b/src/drivers/cdx/cdx_index.cpp index 5e58dce3..252522d7 100644 --- a/src/drivers/cdx/cdx_index.cpp +++ b/src/drivers/cdx/cdx_index.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,13 @@ bool want_fsync_() { return v; } +// Opt-in lock diagnostics only. Set OPENADS_LOCK_DIAG to a writable file +// path before starting the process; unset/empty means no diagnostic file. +std::FILE* open_lock_diag_() { + const char* path = std::getenv("OPENADS_LOCK_DIAG"); + return path && path[0] != '\0' ? std::fopen(path, "a") : nullptr; +} + std::mutex g_cdx_alloc_mu; std::unordered_map g_cdx_alloc_tail; @@ -738,7 +746,7 @@ util::Result CdxIndex::ensure_write_lock_() { auto& slot = g_cdx_write_locks[path_]; if (!slot) slot = std::make_shared(); e = slot; - if (auto* f = std::fopen("/tmp/lock_diag.log", "a")) { + if (auto* f = open_lock_diag_()) { std::fprintf(f, "ensure_write_lock ENTER path=%s users=%zu has_os=%d fd=%p\n", path_.c_str(), e->users, (int)(bool)e->os_lock, file_.native_handle()); std::fclose(f); } @@ -765,7 +773,7 @@ util::Result CdxIndex::ensure_write_lock_() { auto l = acquire_cdx_os_lock_(file_, platform::LockKind::Exclusive); if (!l) { - if (auto* f = std::fopen("/tmp/cdx_diag.log", "a")) { + if (auto* f = open_lock_diag_()) { std::fprintf(f, "acquire_cdx_os_lock FAIL path=%s fd=%p code=%d sub=%d msg=%s\n", path_.c_str(), file_.native_handle(), l.error().code, l.error().sub_code, l.error().message.c_str()); std::fclose(f); @@ -841,7 +849,7 @@ CdxIndex::open_named(const std::string& path, const std::string& tag_name) { mode_ = mode; path_ = canonicalize_path(path); - if (auto* f = std::fopen("/tmp/lock_diag.log", "a")) { + if (auto* f = open_lock_diag_()) { std::fprintf(f, "open_named orig=%s canon=%s fd_will_open\n", path.c_str(), path_.c_str()); std::fclose(f); } diff --git a/src/engine/lock_mgr.cpp b/src/engine/lock_mgr.cpp index cdfae05d..b72d7709 100644 --- a/src/engine/lock_mgr.cpp +++ b/src/engine/lock_mgr.cpp @@ -1,3 +1,4 @@ +#include #include "engine/lock_mgr.h" namespace openads::engine { @@ -150,6 +151,18 @@ LockMgr::try_lock_record_excl(platform::File& f, TableTypeForLock t, LockingMode return LockHandle{std::move(bl).value(), off, 1}; } +util::Result LockMgr::probe_record(platform::File& f, + TableTypeForLock t, LockingMode m, + std::uint32_t recno) { + return platform::ByteLock::probe(f, record_lock_offset(t, m, recno), 1); +} + +util::Result LockMgr::probe_file(platform::File& f, TableTypeForLock t, + LockingMode m) { + return platform::ByteLock::probe(f, file_lock_offset(t, m), + file_lock_length(t)); +} + bool LockMgr::unlock_table(platform::File& f, TableTypeForLock t, LockingMode m) { Key k{&f, file_lock_offset(t, m)}; auto it = held_.find(k); diff --git a/src/engine/lock_mgr.h b/src/engine/lock_mgr.h index 50d1606d..51177f1e 100644 --- a/src/engine/lock_mgr.h +++ b/src/engine/lock_mgr.h @@ -66,6 +66,15 @@ class LockMgr { TableTypeForLock t, LockingMode m, std::uint32_t recno); + // Non-destructive "held by any owner" queries (mtfix11): compute the + // same scheme-aware byte offsets as the lock ops and ask the OS. The + // caller's own registrations are invisible to the query - check them + // first (Table::is_record_locked_any does). + util::Result probe_record(platform::File& f, TableTypeForLock t, + LockingMode m, std::uint32_t recno); + util::Result probe_file (platform::File& f, TableTypeForLock t, + LockingMode m); + // Decrement the per-key refcount. Returns true when the refcount reached // zero so the caller should release the OS-level byte lock held in its // LockHandle; false when nested acquires remain and the OS lock must stay. @@ -83,6 +92,17 @@ class LockMgr { static std::uint64_t file_lock_offset(TableTypeForLock t, LockingMode m); static std::uint64_t file_lock_length(TableTypeForLock t); + // True when the scheme's FLock range spans every record-lock byte + // (Cdx/Vfp: FLock covers 0x40000001..0x7FFFFFFE, records live inside + // it). There a record-byte probe already reports another owner's + // FLock, and probing the file-lock range would false-positive on any + // record lock. Ntx/Adt keep the FLock byte outside the record + // region, so they need a separate file-lock probe for FLock + // visibility. + static bool file_lock_covers_records(TableTypeForLock t) noexcept { + return t == TableTypeForLock::Cdx || t == TableTypeForLock::Vfp; + } + // VFP-scheme record locks mirror the record's physical position, so // the manager needs the table geometry. Set right after the driver // opens (Table::open / from_driver); zeroes degrade to a single diff --git a/src/engine/server_fs.cpp b/src/engine/server_fs.cpp index be5d2fe1..999e6353 100644 --- a/src/engine/server_fs.cpp +++ b/src/engine/server_fs.cpp @@ -212,6 +212,11 @@ util::Result> fs_directory(const std::string& base_dir, util::Result fs_dir_exist(const std::string& abs_path) { std::error_code ec; bool d = fs::is_directory(abs_path, ec); + // A missing directory is an ordinary negative probe, not an I/O error. + // std::filesystem::is_directory(path, ec) sets no_such_file_or_directory + // in ec on some standard libraries (notably Linux/libstdc++). + if (ec == std::errc::no_such_file_or_directory) + return false; if (ec) return io_err(static_cast(openads::AE_INTERNAL_ERROR), "dir exist failed", abs_path); diff --git a/src/engine/table.cpp b/src/engine/table.cpp index 78fcb1f5..ed0689ba 100644 --- a/src/engine/table.cpp +++ b/src/engine/table.cpp @@ -1222,16 +1222,19 @@ util::Result Table::append_record() { // erase until that commit lands (see append_pending_recno_ in table.h). append_pending_recno_ = recno_; append_keys_done_.clear(); - // LOCAL ADT exclusive opens already grant unrestricted writes; taking - // an OS byte-range lock for every append and retaining all of them until - // UnlockTable makes the kernel lock tree grow without bound (~1k/s at - // 10k records with no indexes). Keep other table types unchanged. + // Exclusive ADT opens permit writes without a record lock. Avoid + // accumulating an unused append auto-lock on each new row, including + // when this engine Table is used by the server. Shared ADT and other + // table types retain their existing auto-lock behavior. // Do NOT use the blocking kernel acquire: a peer FLock/Browse covers the VFP rec-lock // range and LockFileEx waits forever (B_BIG N=700 convoy). Retry with // the ACE lock budget, then roll the blank row back so a timeout // cannot leave a durable empty record. if (mode_ != OpenMode::Exclusive || type_ != TableType::Adt) { - const auto p = openads::abi::lock_retry_policy(); + // Engine-internal budget, decoupled from the client-facing ACE lock + // policy (single-attempt by default): the app cannot retry this + // internal auto-lock itself, so it keeps its own short wait. + const auto& p = openads::abi::engine_append_lock_policy(); const auto t0 = std::chrono::steady_clock::now(); const auto deadline = t0 + std::chrono::milliseconds( p.budget_ms() == 0 ? 1 : p.budget_ms()); @@ -1245,7 +1248,7 @@ util::Result Table::append_record() { std::chrono::steady_clock::now() >= deadline) { break; } - openads::abi::lock_retry_sleep(i); + openads::abi::lock_retry_sleep(p, i); } if (!locked) { const std::uint32_t failed = recno_; @@ -1976,6 +1979,24 @@ std::vector Table::held_record_locks() const { return out; } +util::Result Table::is_record_locked_any(std::uint32_t recno) { + if (table_lock_.has_value()) return true; + if (recno_locks_.count(recno) != 0) return true; + if (!driver_ || mode_ == OpenMode::Read) return false; + const auto lt = to_lock_type_(); + auto rec = locks_.probe_record(driver_->file(), lt, locking_, recno); + if (!rec) return rec.error(); + if (rec.value()) return true; + // Cdx/Vfp FLock ranges span every record-lock byte, so the record + // probe above already reports another owner's FLock - and a + // file-lock probe here would false-positive on ANY held record + // lock (those bytes live inside the FLock range). Only Ntx/Adt, + // whose FLock byte sits outside the record region, need the + // file-lock probe to see another owner's FLock. + if (LockMgr::file_lock_covers_records(lt)) return false; + return locks_.probe_file(driver_->file(), lt, locking_); +} + util::Result Table::try_lock_table_excl() { if (mode_ == OpenMode::Read) return {}; if (table_lock_) return {}; @@ -2030,6 +2051,8 @@ void Table::set_order(std::unique_ptr idx) { } void Table::clear_order() { + explicit_order_focus_ = false; + created_order_focus_ = false; order_.reset(); } diff --git a/src/engine/table.h b/src/engine/table.h index 4c6c5b17..aa174765 100644 --- a/src/engine/table.h +++ b/src/engine/table.h @@ -341,6 +341,15 @@ class Table { // exposing the LockMgr internals. std::vector held_record_locks() const; + // Locked-by-any-owner query (mtfix11, SAP AdsIsRecordLocked + // semantics): this table's own registrations first, then a + // non-destructive OS probe of the record's lock byte and of the + // table's file-lock byte - a record also reads locked while ANOTHER + // owner holds the file lock (FLock covers every record). Read-only + // tables and tables without a driver file answer from the own-list + // only: they cannot hold locks and (Win32) cannot probe either. + util::Result is_record_locked_any(std::uint32_t recno); + // Recno-sequence cursor (M10.6). When non-empty, goto_top / // goto_bottom / skip walk this list of recnos in order instead of // the natural append order or any active index. Used by SQL @@ -542,6 +551,11 @@ class Table { // Order + scope surface (M3). void set_order(std::unique_ptr idx); void clear_order(); + // Explicit ACE focus permits ordered table-handle navigation. Merely + // navigating an index handle must not change the table-handle contract. + bool explicit_order_focus() const noexcept { return explicit_order_focus_ || created_order_focus_; } + void set_explicit_order_focus(bool focused) noexcept { explicit_order_focus_ = focused; created_order_focus_ = false; } + void set_created_order_focus(bool focused) noexcept { created_order_focus_ = focused; } // Take ownership of the active index back from the Table, leaving // it without an order. Returns nullptr if no order was set. std::unique_ptr take_order(); @@ -667,6 +681,8 @@ class Table { std::unordered_map recno_locks_; std::optional table_lock_; std::optional order_; + bool explicit_order_focus_ = false; + bool created_order_focus_ = false; std::vector extra_index_views_; State state_ = State::Bof; std::uint32_t recno_ = 0; diff --git a/src/network/client.cpp b/src/network/client.cpp index 72de1235..2b81a082 100644 --- a/src/network/client.cpp +++ b/src/network/client.cpp @@ -5,6 +5,8 @@ #include "openads/error.h" #include +#include +#include #include #include #include @@ -152,17 +154,21 @@ std::size_t parse_row_trailer_into(RemoteTable* rt, // Returns the offset where trailer parsing stopped, so callers // can read trailing sections (reposition-bound piggyback). if (rt == nullptr || pos >= pl.size()) { - if (rt) rt->row_valid = false; + if (rt) { rt->row_valid = false; rt->anchor_ok = false; } return pos; } rt->prefetch_queue.clear(); // M12.21 option C — every nav ack re-anchors the server cursor to the // client's logical position, so the lag resets to zero. rt->cursor_lag = 0; + rt->pair_local_position = false; rt->prefetch_dir = 0; std::uint8_t has_row = pl[pos++]; if (has_row == 0) { rt->row_valid = false; + // Landed on no row: the server cursor is at a phantom, which + // certifies no recno anchor for the R2 self-goto check. + rt->anchor_ok = false; // Lookahead count is always 0 when has_row=0, but the 2 bytes // are still on the wire: consume them so the returned offset // points past the trailer (trailing sections key off it). @@ -172,10 +178,19 @@ std::size_t parse_row_trailer_into(RemoteTable* rt, auto end = parse_one_row(pl, pos, rt->current_recno, rt->current_deleted, rt->current_row); if (end == static_cast(-1)) { - rt->row_valid = false; return pos; + rt->row_valid = false; rt->anchor_ok = false; return pos; } pos = end; rt->row_valid = true; + // mtfix12 R2 — every server-shipped landing certifies the cursor + // anchor: the server cursor IS current_recno right now (the ack + // contract resets cursor_lag to 0 above). Serves self-GotoRecord + // and GetRecordNum while the freshness seq holds. + rt->anchor_recno = rt->current_recno; + rt->anchor_ok = true; + rt->anchor_seq = rt->conn != nullptr ? rt->conn->nav_seq() : 0; + rt->anchor_tbl_seq = + rt->conn != nullptr ? rt->conn->tbl_nav_seq(rt->id) : 0; // M12.21 lookahead block. [u16 count] then count rows. if (pos + 2 > pl.size()) return pos; // M12.18 server (no lookahead) — done. std::uint16_t la = read_u16_le(&pl[pos]); pos += 2; @@ -233,8 +248,89 @@ bool parse_tls_uri(const std::string& uri, return parse_scheme_uri(uri, "tls://", host, port, data_dir); } +namespace { +std::atomic g_frame_trace_hook{nullptr}; +} // namespace + +void set_frame_trace_hook(FrameTraceHook hook) noexcept { + g_frame_trace_hook.store(hook, std::memory_order_relaxed); +} + +std::uint64_t RemoteConnection::tbl_nav_seq(std::uint32_t id) { + std::lock_guard lk(mu_); + const auto it = tbl_nav_seqs_.find(id); + return it != tbl_nav_seqs_.end() ? it->second : 0; +} + +std::uint64_t RemoteConnection::tbl_write_gen(std::uint32_t id) { + std::lock_guard lk(mu_); + const auto it = tbl_write_gens_.find(id); + return it != tbl_write_gens_.end() ? it->second : 0; +} + +void RemoteConnection::note_tbl_nav(std::uint32_t id) { + std::lock_guard lk(mu_); + ++tbl_nav_seqs_[id]; +} + +void RemoteConnection::note_all_tables_nav() { + std::lock_guard lk(mu_); + for (auto& [id, v] : tbl_nav_seqs_) ++v; +} + util::Result RemoteConnection::request(const Frame& f) { std::lock_guard lk(mu_); + frame_seq_.fetch_add(1, std::memory_order_relaxed); + switch (f.opcode) { + case Opcode::AppendBlank: case Opcode::SetField: + case Opcode::SetFields: case Opcode::RecallRecord: + case Opcode::ExecuteSQL: case Opcode::Reindex: + case Opcode::PackTable: case Opcode::ZapTable: + data_epoch_.fetch_add(1, std::memory_order_relaxed); + break; + default: + break; + } + // mtfix12 — per-table generations. Every frame funnels through + // here, so classifying by opcode is exhaustive by construction. + // nav: cursor/visibility-affecting (mirrors note_nav_frame per + // table; FlushTable/FlushFileBuffers intentionally excluded — + // they move no cursor, and dropping them is exactly the widening + // the per-table envelope opts into). write: content-affecting + // (row bytes may differ after it lands). Index-keyed ops + // (Seek/SeekLast/SkipUnique/SetScope/ClearScope) and the + // conn-wide ShowDeleted are bumped by their methods, which know + // the binding. + if (f.payload.size() >= 4) { + const std::uint32_t tid = read_u32_le(f.payload.data()); + switch (f.opcode) { + case Opcode::GotoTop: case Opcode::GotoBottom: + case Opcode::GotoRecord: case Opcode::Skip: + case Opcode::AppendBlank: case Opcode::PackTable: + case Opcode::ZapTable: case Opcode::Reindex: + case Opcode::SetAOF: case Opcode::ClearAOFRemote: + case Opcode::CustomizeAOF: case Opcode::SetOrder: + case Opcode::SetOrderByName: + ++tbl_nav_seqs_[tid]; + break; + default: break; + } + switch (f.opcode) { + case Opcode::AppendBlank: case Opcode::SetField: + case Opcode::SetFields: case Opcode::SetRecord: + case Opcode::DeleteRecord: case Opcode::RecallRecord: + case Opcode::PackTable: case Opcode::ZapTable: + case Opcode::Reindex: + ++tbl_write_gens_[tid]; + break; + default: break; + } + } + const FrameTraceHook trace_hook = + g_frame_trace_hook.load(std::memory_order_relaxed); + const auto trace_t0 = trace_hook != nullptr + ? std::chrono::steady_clock::now() + : std::chrono::steady_clock::time_point{}; // Fail fast on a disconnected handle. rddads hands us connection // handles that AdsDisconnect already tore down (its "current // connection" can point at a handle another thread just closed); @@ -265,6 +361,17 @@ util::Result RemoteConnection::request(const Frame& f) { transport_->close(); return rep.error(); } + if (trace_hook != nullptr) { + const long long us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - trace_t0).count()); + const std::uint32_t tid = + f.payload.size() >= 4 ? read_u32_le(f.payload.data()) : 0u; + trace_hook(this, static_cast(f.opcode), tid, + f.payload.size(), + static_cast(rep.value().opcode), + rep.value().payload.size(), us); + } // M12.10 — Error frame payload prefixed with [u32 LE ace_code]. // Parse it back into the util::Error so callers see the real ACE // code (5036, 7077, 5066, ...) instead of a generic 5000. @@ -312,7 +419,8 @@ void connect_pack_payload(std::vector& payload, // to a client that only understands forward ones (see kCapPrefetchBackward). std::uint32_t caps = kCapPrefetchConsume | kCapPrefetchBackward | kCapOpenTableMode | kCapSetFieldsBatch - | kCapFlushInCloseAll | kCapNavOrderFuse; + | kCapFlushInCloseAll | kCapNavOrderFuse + | kCapNavBoundaryPair; for (int i = 0; i < 4; ++i) payload.push_back(static_cast((caps >> (8 * i)) & 0xFFu)); } @@ -447,6 +555,18 @@ void RemoteConnection::parked_store(std::string key, } } +bool RemoteConnection::parked_contains(const std::string& key) const { + std::lock_guard lk(park_mu_); + const auto now = std::chrono::steady_clock::now(); + for (const auto& e : parked_) { + const auto age = + std::chrono::duration_cast(now - e.parked_at); + if (age.count() > static_cast(kParkedTtlSec)) continue; + if (e.key == key && e.table) return true; + } + return false; +} + void RemoteConnection::parked_flush( std::vector>& out) { std::lock_guard lk(park_mu_); @@ -587,6 +707,7 @@ util::Result RemoteConnection::apply_open_row(RemoteTable* rt, return {}; } + util::Result RemoteConnection::close_table(std::uint32_t id) { Frame req; req.opcode = Opcode::CloseTable; @@ -625,6 +746,8 @@ static void apply_bound_trailer(RemoteTable* rt, bool bof, bool eof, rt->recno_bound = recno; rt->recno_bound_ok = true; rt->recno_bound_seq = seq; + rt->bound_at = std::chrono::steady_clock::now(); + rt->bound_data_epoch = rt->conn->data_epoch(); if (has_count) { rt->count_bound = reccount; rt->count_bound_ok = true; @@ -647,6 +770,94 @@ static bool apply_bound_tail(RemoteTable* rt, return true; } +// mtfix12 R1 — pair-requested acks carry an explicit [u8 ack_flags] +// byte right after the row trailer so section offsets are deterministic +// (the plain length-inference in apply_bound_tail cannot tell a 6-byte +// bound followed by a pair section from a 10-byte bound). ack_flags +// bit 0x01: the main bound tail carries reccount (10 bytes). Applies +// the bound exactly like apply_bound_tail and returns the offset just +// past it. Missing/short tail: nothing applied, returns tend (the +// server declined pair entirely; no certification, wire fallback). +static std::size_t pair_ack_bound_end(RemoteTable* rt, + const std::vector& pl, + std::size_t tend) { + if (rt == nullptr || pl.size() < tend + 1) return tend; + const std::uint8_t af = pl[tend]; + const std::size_t blen = (af & 0x01) ? 10 : 6; + if (pl.size() < tend + 1 + blen) return tend; + apply_bound_trailer(rt, pl[tend + 1] != 0, pl[tend + 2] != 0, + read_u32_le(pl.data() + tend + 3), + (af & 0x01) ? read_u32_le(pl.data() + tend + 7) : 0u, + (af & 0x01) != 0); + return tend + 1 + blen; +} + +// mtfix12 R1 — boundary-pair tail (see kCapNavBoundaryPair in wire.h). +// Layout at [off, ...): +// [u8 flags][u32 trailer_len][trailer][bound 6|10][u32 keycount?] +// flags: 0x02 = bound carries reccount (10 bytes), 0x04 = keycount +// present (ordered tables). The trailer is row-trailer format +// (pack_row_trailer bytes at lookahead depth 0) describing the OPPOSITE +// boundary's landing, read by the server in the same atomic visit as +// the nav that carried it. Stored for the ABI layer to apply verbatim +// if the adjacent opposite-boundary call arrives while the conn-wide +// freshness envelope holds (same rule as remote_nav_duplicate) and no +// write touched the table since (write_gen). Absent/malformed tail = +// no certification: pair_valid stays false and the next +// opposite-boundary call goes to the wire exactly as before. +static bool apply_pair_tail(RemoteTable* rt, int opp_which, + std::uint32_t order, + const std::vector& pl, + std::size_t off) { + if (rt == nullptr || pl.size() < off + 5) return false; + const std::uint8_t flags = pl[off]; + const std::uint32_t tlen = read_u32_le(pl.data() + off + 1); + const std::size_t bound_len = (flags & 0x02) ? 10 : 6; + const std::size_t need = + 5 + static_cast(tlen) + bound_len + + ((flags & 0x04) ? 4 : 0); + if (pl.size() < off + need) return false; + std::size_t p = off + 5; + rt->pair_blob.assign(pl.begin() + static_cast(p), + pl.begin() + static_cast(p + tlen)); + p += tlen; + rt->pair_bof = pl[p] != 0; + rt->pair_eof = pl[p + 1] != 0; + rt->pair_recno = read_u32_le(pl.data() + p + 2); + rt->pair_has_count = (flags & 0x02) != 0; + rt->pair_reccount = rt->pair_has_count + ? read_u32_le(pl.data() + p + 6) : 0u; + p += bound_len; + rt->pair_has_keycount = (flags & 0x04) != 0; + rt->pair_keycount = rt->pair_has_keycount + ? read_u32_le(pl.data() + p) : 0u; + rt->pair_which = opp_which; + rt->pair_order = order; + rt->pair_seq = rt->conn != nullptr ? rt->conn->nav_seq() : 0; + rt->pair_write_gen = + rt->conn != nullptr ? rt->conn->tbl_write_gen(rt->id) : 0; + rt->pair_valid = true; + return true; +} + +util::Result RemoteConnection::apply_pair_blob(RemoteTable* rt) { + if (rt == nullptr || !rt->pair_valid) { + return util::Error{5000, 0, "apply_pair_blob: no certification", ""}; + } + // Same byte path a wire ack for the opposite boundary would take: + // the stored trailer parse re-anchors lag and resets prefetch, + // then the certified bound values land through the shared trailer + // application so every bound/recno/count stamp matches. + parse_row_trailer_into(rt, rt->pair_blob, 0); + rt->pair_local_position = true; + apply_bound_trailer(rt, rt->pair_bof, rt->pair_eof, rt->pair_recno, + rt->pair_reccount, rt->pair_has_count); + if (rt->pair_has_keycount) { + rt->key_counts[rt->pair_order] = rt->pair_keycount; + } + return {}; +} + util::Result RemoteConnection::goto_top(std::uint32_t id) { note_nav_frame(); Frame req; @@ -684,6 +895,12 @@ util::Result RemoteConnection::goto_top(RemoteTable* rt) { static_cast(rt->cache_records_hint & 0xFFu)); req.payload.push_back( static_cast((rt->cache_records_hint >> 8) & 0xFFu)); + // mtfix12 R1: ask for the opposite boundary's certification (the + // back-to-back dbGoTop(); dbGoBottom() ritual). Caps-gated: old + // servers never see the byte (flag 0x02, no order section). + const bool want_pair = server_nav_boundary_pair(); + if (want_pair) req.payload.push_back(0x02); + rt->pair_valid = false; auto rep = request(req); if (!rep) return rep.error(); if (rep.value().opcode != Opcode::GotoTopAck) { @@ -693,7 +910,14 @@ util::Result RemoteConnection::goto_top(RemoteTable* rt) { parse_row_trailer_into(rt, rep.value().payload, 0); // Reposition-bound piggyback (see goto_record): trailing // [u8 bof][u8 eof][u32 recno], length-gated. - apply_bound_tail(rt, rep.value().payload, tend); + if (want_pair) { + const std::size_t off = + pair_ack_bound_end(rt, rep.value().payload, tend); + apply_pair_tail(rt, 2, rt->server_order_id, + rep.value().payload, off); + } else { + apply_bound_tail(rt, rep.value().payload, tend); + } return {}; } @@ -1074,6 +1298,10 @@ util::Result RemoteConnection::goto_bottom(RemoteTable* rt) { Frame req; req.opcode = Opcode::GotoBottom; write_u32_le(rt->id, req.payload); + // mtfix12 R1: opposite-boundary certification (see goto_top). + const bool want_pair = server_nav_boundary_pair(); + if (want_pair) req.payload.push_back(0x02); + rt->pair_valid = false; auto rep = request(req); if (!rep) return rep.error(); if (rep.value().opcode != Opcode::GotoBottomAck) { @@ -1083,7 +1311,14 @@ util::Result RemoteConnection::goto_bottom(RemoteTable* rt) { parse_row_trailer_into(rt, rep.value().payload, 0); // Reposition-bound piggyback (see goto_record): trailing // [u8 bof][u8 eof][u32 recno], length-gated. - apply_bound_tail(rt, rep.value().payload, tend); + if (want_pair) { + const std::size_t off = + pair_ack_bound_end(rt, rep.value().payload, tend); + apply_pair_tail(rt, 1, rt->server_order_id, + rep.value().payload, off); + } else { + apply_bound_tail(rt, rep.value().payload, tend); + } return {}; } @@ -1101,8 +1336,11 @@ util::Result RemoteConnection::goto_top_fused(RemoteTable* rt, static_cast(rt->cache_records_hint & 0xFFu)); req.payload.push_back( static_cast((rt->cache_records_hint >> 8) & 0xFFu)); - req.payload.push_back(0x01); + const bool want_pair = server_nav_boundary_pair(); + req.payload.push_back(static_cast( + 0x01 | (want_pair ? 0x02 : 0x00))); write_u32_le(order_id, req.payload); + rt->pair_valid = false; auto rep = request(req); if (!rep) return rep.error(); if (rep.value().opcode != Opcode::GotoTopAck) { @@ -1110,13 +1348,23 @@ util::Result RemoteConnection::goto_top_fused(RemoteTable* rt, } const std::size_t tend = parse_row_trailer_into(rt, rep.value().payload, 0); - // Reposition-bound piggyback (see goto_record): trailing - // [u8 bof][u8 eof][u32 recno](+[u32 reccount]), length-gated. - apply_bound_tail(rt, rep.value().payload, tend); - // Fused switch only: the server certified the new order's key - // count past the bound tail ([u32]). Rotation visits ask it - // next; serve from the per-order map instead of a frame. - { + if (want_pair) { + // Pair-requested fused ack: [rowtrailer][u8 ack_flags][bound] + // [u32 fused key count][pair section]. + const std::size_t off = + pair_ack_bound_end(rt, rep.value().payload, tend); + const auto& pl = rep.value().payload; + if (pl.size() >= off + 4) { + rt->key_counts[order_id] = read_u32_le(pl.data() + off); + } + apply_pair_tail(rt, 2, order_id, pl, off + 4); + } else { + // Reposition-bound piggyback (see goto_record): trailing + // [u8 bof][u8 eof][u32 recno](+[u32 reccount]), length-gated. + apply_bound_tail(rt, rep.value().payload, tend); + // Fused switch only: the server certified the new order's key + // count past the bound tail ([u32]). Rotation visits ask it + // next; serve from the per-order map instead of a frame. const auto& pl = rep.value().payload; if (pl.size() >= tend + 14) { rt->key_counts[order_id] = read_u32_le(pl.data() + tend + 10); @@ -1131,8 +1379,11 @@ util::Result RemoteConnection::goto_bottom_fused(RemoteTable* rt, Frame req; req.opcode = Opcode::GotoBottom; write_u32_le(rt->id, req.payload); - req.payload.push_back(0x01); + const bool want_pair = server_nav_boundary_pair(); + req.payload.push_back(static_cast( + 0x01 | (want_pair ? 0x02 : 0x00))); write_u32_le(order_id, req.payload); + rt->pair_valid = false; auto rep = request(req); if (!rep) return rep.error(); if (rep.value().opcode != Opcode::GotoBottomAck) { @@ -1140,11 +1391,19 @@ util::Result RemoteConnection::goto_bottom_fused(RemoteTable* rt, } const std::size_t tend = parse_row_trailer_into(rt, rep.value().payload, 0); - // Reposition-bound piggyback (see goto_record): trailing - // [u8 bof][u8 eof][u32 recno](+[u32 reccount]), length-gated. - apply_bound_tail(rt, rep.value().payload, tend); - // Fused switch only: certified key count past the bound tail. - { + if (want_pair) { + const std::size_t off = + pair_ack_bound_end(rt, rep.value().payload, tend); + const auto& pl = rep.value().payload; + if (pl.size() >= off + 4) { + rt->key_counts[order_id] = read_u32_le(pl.data() + off); + } + apply_pair_tail(rt, 1, order_id, pl, off + 4); + } else { + // Reposition-bound piggyback (see goto_record): trailing + // [u8 bof][u8 eof][u32 recno](+[u32 reccount]), length-gated. + apply_bound_tail(rt, rep.value().payload, tend); + // Fused switch only: certified key count past the bound tail. const auto& pl = rep.value().payload; if (pl.size() >= tend + 14) { rt->key_counts[order_id] = read_u32_le(pl.data() + tend + 10); @@ -2449,12 +2708,10 @@ std::uint64_t read_u64_le(const std::uint8_t* p) { } // namespace +// File state can change on another connection or directly on the host. +// Every existence probe must reach the server, just like DirExist. util::Result RemoteConnection::file_exists(const std::string& path) { - { - std::lock_guard lk(file_exists_mu_); - if (file_exists_cache_.count(path) != 0) return true; - } Frame req; req.opcode = Opcode::FileExists; push_lp_str(req.payload, path); @@ -2463,17 +2720,7 @@ RemoteConnection::file_exists(const std::string& path) { if (rep.value().opcode != Opcode::FileExistsAck || rep.value().payload.empty()) return fs_wire_err(rep.value(), "FileExists"); - bool exists = rep.value().payload[0] != 0; - if (exists) { - std::lock_guard lk(file_exists_mu_); - file_exists_cache_.insert(path); - } - return exists; -} - -void RemoteConnection::file_exists_invalidate() { - std::lock_guard lk(file_exists_mu_); - file_exists_cache_.clear(); + return rep.value().payload[0] != 0; } util::Result @@ -2485,7 +2732,6 @@ RemoteConnection::file_erase(const std::string& path) { if (!rep) return rep.error(); if (rep.value().opcode != Opcode::FileEraseAck) return fs_wire_err(rep.value(), "FileErase"); - file_exists_invalidate(); return {}; } @@ -2500,7 +2746,6 @@ RemoteConnection::file_rename(const std::string& old_p, if (!rep) return rep.error(); if (rep.value().opcode != Opcode::FileRenameAck) return fs_wire_err(rep.value(), "FileRename"); - file_exists_invalidate(); return {}; } @@ -2709,6 +2954,10 @@ RemoteConnection::zip_list(const std::string& zip) { return o; } +// Directory state may change through other sessions or on the server host. +// Do not cache existence or skip an idempotent mkdir: a stale positive answer +// can make a fresh organization fail its first CreateTable after its directory +// was removed outside this connection. util::Result RemoteConnection::dir_exist(const std::string& path) { Frame req; @@ -2842,6 +3091,7 @@ util::Result RemoteConnection::skip_unique(std::uint32_t index_id, std::int32_t direction) { note_nav_frame(); + note_all_tables_nav(); // index-keyed, no parent resolved here Frame req; req.opcode = Opcode::SkipUnique; write_u32_le(index_id, req.payload); write_u32_le(static_cast(direction), req.payload); @@ -2859,6 +3109,7 @@ RemoteConnection::set_scope(std::uint32_t index_id, const std::string& key, std::uint16_t data_type) { note_nav_frame(); + note_all_tables_nav(); // index-keyed visibility change // Payload: u32 index_id | u16 which | u16 data_type | bytes key. // Key length is the trailing byte count (payload.size() - 8). Frame req; req.opcode = Opcode::SetScope; @@ -2916,6 +3167,7 @@ RemoteConnection::fetch_current_row(RemoteTable* rt) { void RemoteConnection::show_deleted(bool visible) noexcept { note_nav_frame(); + note_all_tables_nav(); // conn-wide visibility change if (!transport_ || !transport_->valid()) return; Frame req; req.opcode = Opcode::ShowDeleted; @@ -2929,6 +3181,7 @@ util::Result RemoteConnection::clear_scope(std::uint32_t index_id, std::uint16_t which) { note_nav_frame(); + note_all_tables_nav(); // index-keyed visibility change Frame req; req.opcode = Opcode::ClearScope; write_u32_le(index_id, req.payload); write_u16_le(which, req.payload); @@ -2947,6 +3200,9 @@ RemoteConnection::seek(std::uint32_t index_id, std::uint8_t last, RemoteTable* parent) { note_nav_frame(); + // Index-keyed op: request() cannot map it to a table, so the + // binding the caller resolved bumps the per-table generation here. + if (parent != nullptr) note_tbl_nav(parent->id); Frame req; req.opcode = last ? Opcode::SeekLast : Opcode::Seek; write_u32_le(index_id, req.payload); diff --git a/src/network/client.h b/src/network/client.h index 82d175fb..6167a117 100644 --- a/src/network/client.h +++ b/src/network/client.h @@ -46,6 +46,19 @@ struct AggregateBatch { struct RemoteTable; +// Diagnostics only: optional per-frame hook, called after every +// completed request/reply round trip (wire_trace). nullptr = off, which +// is the default; the request path then pays one relaxed atomic load. +// Arguments: connection, request opcode, first u32 of the request +// payload (the table id for table-scoped opcodes; meaningless for +// Hello/Connect/OpenTable), request payload bytes, reply opcode, reply +// payload bytes, microseconds spent in send+receive. +using FrameTraceHook = void (*)(const void* conn, std::uint8_t op, + std::uint32_t tid, std::size_t req_bytes, + std::uint8_t rep_op, std::size_t rep_bytes, + long long us); +void set_frame_trace_hook(FrameTraceHook hook) noexcept; + // M12.5 — wire client used by ace64.dll's dual-mode dispatch. // `RemoteConnection` opens a TCP socket to an OpenADS server, // sends a Connect frame for the data_dir, and exposes a small @@ -106,6 +119,20 @@ class RemoteConnection { return (server_caps_ & kCapNavOrderFuse) != 0; } + // Server's FlushTable handler does the full file-buffers flush + // (see kCapFlushTableDurable): a commit needs no trailing + // FlushFileBuffers frame. + bool server_flush_table_durable() const noexcept { + return (server_caps_ & kCapFlushTableDurable) != 0; + } + + // Server certifies the opposite boundary on GotoTop/GotoBottom + // (see kCapNavBoundaryPair): one frame proves both ends, so a + // back-to-back dbGoTop(); dbGoBottom() ritual costs one RTT. + bool server_nav_boundary_pair() const noexcept { + return (server_caps_ & kCapNavBoundaryPair) != 0; + } + // Current cursor-generation sequence (see nav_seq_). Relaxed load // is enough: it only orders the ABI layer's own duplicate // detection, never data. @@ -113,6 +140,24 @@ class RemoteConnection { return nav_seq_.load(std::memory_order_relaxed); } + // mtfix12 — per-table generations (R1 pair guard / R2 per-table + // envelope). Keyed by wire table id, bumped inside request() for + // every cursor/visibility-affecting frame (nav) and + // content-affecting frame (write) — central by construction, so no + // ABI call site can miss one. Index-keyed ops (Seek/SeekLast/ + // SkipUnique/SetScope/ClearScope) and the connection-wide + // ShowDeleted bump through note_tbl_nav/note_all_tables_nav from + // the methods that know the binding. Table-id reuse after a close + // only starts a new table's counter higher — conservative, never + // stale-accepting. + std::uint64_t tbl_nav_seq(std::uint32_t id); + std::uint64_t tbl_write_gen(std::uint32_t id); + // Index-keyed cursor op whose parent table the caller resolved. + void note_tbl_nav(std::uint32_t id); + // Connection-wide visibility change (ShowDeleted) or an + // index-keyed op with no resolved parent: expire every table. + void note_all_tables_nav(); + // Server version from the HelloAck handshake ("openads/1.09.27"; // pre-1.8.14 servers answer the literal "openads/0.3.2"). Empty // when the Hello probe failed — callers treat that as unknown, @@ -355,15 +400,9 @@ class RemoteConnection { util::Result drop_table(const std::string& name, std::uint16_t delete_files); - // Server filesystem (EnableFileFunc on server). - // Positive-only existence cache (rddads probes the same production - // bags every USE): a "true" answer is served locally until any - // file-mutating op on this connection (erase/rename/drop/create, - // via file_exists_invalidate) clears it. Negatives always go to - // the wire — caching "missing" would hide a concurrently created - // table, while a stale "present" degrades to a clean open error. + // Server filesystem (EnableFileFunc on server). External state is never + // cached: other sessions and host operations can change it at any time. util::Result file_exists(const std::string& path); - void file_exists_invalidate(); util::Result file_erase(const std::string& path); util::Result file_rename(const std::string& old_p, const std::string& new_p); @@ -452,6 +491,13 @@ class RemoteConnection { // and skips the round-trip). util::Result apply_open_row(RemoteTable* rt, const std::vector& trailer); + // mtfix12 R1 — apply the certified opposite-boundary landing + // (pair_blob, row-trailer format) exactly as that boundary's wire + // ack would have: same row-trailer parse (row cache, prefetch + // reset, lag re-anchor), then the certified bound values and the + // scoped key count. Caller has already verified freshness + // (pair_seq/pair_write_gen) via remote_nav_pair. + util::Result apply_pair_blob(RemoteTable* rt); // M12.6 — remote write surface. util::Result append_blank(std::uint32_t id); util::Result set_field(std::uint32_t id, @@ -532,6 +578,40 @@ class RemoteConnection { util::Result mutex_unlock(const std::string& name); util::Result mutex_destroy(const std::string& name); + // Count of wire round trips issued on this connection (every + // request(), reads included). Unlike nav_seq() this also moves on + // locks, fetches and counts, so "no frame since X" means nothing at + // all reached the server in between (used by AdsRefreshRecord to + // serve the row a GotoRecord ack just delivered). + std::uint64_t frame_seq() const noexcept { + return frame_seq_.load(std::memory_order_relaxed); + } + + // Count of frames that can ADD rows or change row contents on the + // server (append, field writes, recall, SQL, pack/zap/reindex) sent on + // this connection. The empty-table window (see AdsSeek) closes as soon + // as this moves, so this station never misses its own new record. + std::uint64_t data_epoch() const noexcept { + return data_epoch_.load(std::memory_order_relaxed); + } + + // Per-connection record-length memo for re-opens of the same table. + // Keyed by lowercased table name + the full schema (names, types, + // widths, decimals) from the open ack, so any restructure changes + // the key and misses. Thread-safe. + bool recall_record_length(const std::string& key, + std::uint32_t& out) { + std::lock_guard lk(reclen_mu_); + auto it = reclen_memo_.find(key); + if (it == reclen_memo_.end()) return false; + out = it->second; + return true; + } + void remember_record_length(const std::string& key, std::uint32_t v) { + std::lock_guard lk(reclen_mu_); + if (reclen_memo_.size() < 4096) reclen_memo_[key] = v; + } + private: util::Result request(const Frame& f); @@ -545,6 +625,10 @@ class RemoteConnection { std::unique_ptr transport_; std::mutex mu_; + // mtfix12 per-table generations (see tbl_nav_seq/tbl_write_gen). + // Guarded by mu_ (bumped inside request(), which already holds it). + std::unordered_map tbl_nav_seqs_; + std::unordered_map tbl_write_gens_; // Server caps echoed in ConnectAck (0 when the server predates caps). std::uint32_t server_caps_ = 0; // Monotonic cursor-generation counter. Bumped ONLY by frames that @@ -558,16 +642,16 @@ class RemoteConnection { // staleness is impossible by construction: observing a change // requires a cursor-affecting frame, which is exactly what bumps. std::atomic nav_seq_{0}; + // See frame_seq(): bumped by every request(). + std::atomic frame_seq_{0}; + std::atomic data_epoch_{0}; + std::mutex reclen_mu_; + std::unordered_map reclen_memo_; // Raw HelloAck payload (see above). Written once during // connect_with_transport, read afterwards without mu_ (the // connection is fully established before any other thread // can hold its handle). std::string server_version_; - // Positive-only file-existence cache (see file_exists). Guarded - // by its own mutex: consulted on the read path, cleared by - // file-mutating ops, never held across wire calls. - std::set file_exists_cache_; - std::mutex file_exists_mu_; public: // Deferred disconnect (MT shared connections). AdsDisconnect on a @@ -604,6 +688,8 @@ class RemoteConnection { std::vector>& evicted); // Drain everything (disconnect). Caller really-closes each entry. void parked_flush(std::vector>& out); + // Diagnostics (wire_trace): is an unexpired entry parked under key? + bool parked_contains(const std::string& key) const; private: std::vector parked_; @@ -613,6 +699,11 @@ class RemoteConnection { // Per-handle wrapper for a remote table. Stores back-pointer to // the connection plus the server-side table id. struct RemoteTable { + // Only an explicit SetIndexOrder opts table-handle navigation into an order. + bool explicit_order_focus = false; + bool created_order_focus = false; + // A boundary blob changed the logical cursor without moving the server. + bool pair_local_position = false; RemoteConnection* conn = nullptr; std::uint32_t id = 0; // True when this table was counted in conn->deferred_open_tables at @@ -649,6 +740,20 @@ struct RemoteTable { bool open_exclusive = false; // mapped mode (never pooled) bool ever_locked = false; // AppendBlank/LockRecord/Table bool scope_touched = false; // SetScope (server state unclear) + // Client-side lock ledger (WAN chattiness): mirrors the record and + // table locks this connection is known to hold on the server. Lets + // AdsUnlockTable skip the frame when nothing is held (rddads + // dbUnlock() calls it blindly before every lock/append), lets + // AdsGetAllLocks answer locally, and lets the close path park a + // table whose locks were all released (the blunt ever_locked flag + // alone used to force a real close + 7-frame reopen per save). + // locks_uncertain covers locks the ledger cannot name: the append + // auto-lock (AppendBlankAck carries no recno) and LockRecord(0) + // with no valid cursor. Only a wire UnlockTable (or close) clears + // it. Conservative throughout: doubt always goes to the wire. + std::set held_recs; + bool table_lock_held = false; + bool locks_uncertain = false; // M12.18 — recno + deleted flag arrive together with the row // bytes so AdsGetRecordNum / AdsIsRecordDeleted can serve from // cache instead of a separate RTT each. @@ -708,6 +813,11 @@ struct RemoteTable { std::uint32_t count_bound = 0; bool count_bound_ok = false; std::uint64_t count_bound_seq = 0; + // Wall time of the last bound tail (server certification) and the + // connection data_epoch() at that moment. Drive the short "still + // empty" window for physically empty tables (AdsSeek/AdsGotoRecord). + std::chrono::steady_clock::time_point bound_at{}; + std::uint64_t bound_data_epoch = 0; // Last wire nav op on this table (0 = none/other, 1 = GotoTop, // 2 = GotoBottom), whether it produced a row, and the connection // nav_seq_ at the time. Serves two WAN-chattiness kills with one @@ -726,6 +836,56 @@ struct RemoteTable { // A top in order A says nothing about order B, so the duplicate // check requires the context to match, not just the op. std::uint32_t last_nav_order = 0; + + // mtfix12 R1 — boundary-pair certification. A GotoTop/GotoBottom ack + // on a kCapNavBoundaryPair server carries the OPPOSITE boundary's + // complete landing state, read by the server in the same atomic + // visit: the row blob in row-trailer format (pack_row_trailer + // bytes, lookahead depth 0), that landing's bound values, and the + // scoped key count. While the conn-wide nav_seq holds (identical + // envelope to remote_nav_duplicate) and no write touched this table + // since (write_gen), a back-to-back opposite-boundary call applies + // the blob exactly as the wire ack would have: same parser, same + // bound application, same keyno sync — byte-identical state, one + // RTT saved. pair_which is the boundary this certifies (1 = top, + // 2 = bottom), i.e. the OPPOSITE of the nav that carried it. + bool pair_valid = false; + int pair_which = 0; + std::uint32_t pair_order = 0; + std::uint64_t pair_seq = 0; + std::uint64_t pair_write_gen = 0; + std::vector pair_blob; + bool pair_bof = false; + bool pair_eof = false; + std::uint32_t pair_recno = 0; + bool pair_has_count = false; + std::uint32_t pair_reccount = 0; + bool pair_has_keycount = false; + std::uint32_t pair_keycount = 0; + // Write generation snapshot: RemoteConnection::tbl_write_gen(id) + // at certification time (the conn bumps the live counter inside + // request() for every content-affecting frame, so no call site can + // miss it). The pair blob was read before any such change, so it + // must not overwrite fresher row state — guard on equality at + // serve time. + // mtfix12 R2 — certified server-cursor anchor. Set whenever a wire + // nav ack (or the open warm row) lands this handle's cursor on a + // row: the server cursor IS anchor_recno at that instant (lag is + // 0 by ack contract). Conn-wide envelope: valid while + // anchor_seq == conn nav_seq. Per-table opt-in envelope + // (OPENADS_NAV_SELF_GOTO=table): valid while anchor_tbl_seq == + // tbl_change_seq — server cursors are per handle, so only THIS + // handle's frames can move it. Position-only certification; row + // bytes come from the existing row cache. + std::uint32_t anchor_recno = 0; + bool anchor_ok = false; + std::uint64_t anchor_seq = 0; + std::uint64_t anchor_tbl_seq = 0; + // anchor_tbl_seq snapshots RemoteConnection::tbl_nav_seq(id) — the + // per-table cursor/visibility generation bumped centrally in + // request(). Purely-local visibility mutations (filter/scope/order + // set+clear — the sites that reset last_nav) must also expire the + // anchor: they clear anchor_ok directly. // M12.19 — cached record count. Serves AdsGetRecordCount and // AdsGetRelKeyPos (scrollbar) without an extra RTT. Invalidated // on writes that may change the row count: AppendBlank / @@ -855,6 +1015,15 @@ struct RemoteTable { std::uint16_t cached_table_type = 0; bool record_length_cached = false; std::uint32_t cached_record_length = 0; + // Set by a wire AdsGotoRecord that landed on a row: the connection + // frame_seq() right after the ack and the recno it delivered. While + // no other frame has gone out and the cursor still sits on that row, + // an AdsRefreshRecord would re-read the very row that ack carried + // (the server's GotoRecord already re-read it from disk), so the + // refresh is served from it with no round trip. + bool goto_row_fresh = false; + std::uint64_t goto_row_frame_seq = 0; + std::uint32_t goto_row_recno = 0; // Deferred teardown (WAN chattiness: rddads issues FlushFileBuffers // + CloseAllIndexes before every CloseTable — 2 wasted frames per // USE, since the server close flushes data and purges index @@ -888,6 +1057,15 @@ struct RemoteTable { // adopt-or-emit decision, so a parked snapshot can never reference // dead server ids. bool indexes_parked = false; + // Nav stamp parked alongside the index snapshot: the CloseAll -> + // OpenIndex (unpark) cycle sends no frames, so a stamp taken just + // before the park still describes the live server cursor as long as + // nav_seq is unchanged when the same order is restored. Zeroed + // whenever the parked snapshot dies for real. + int parked_nav_which = 0; + std::uint32_t parked_nav_order = 0; + bool parked_nav_row = false; + std::uint64_t parked_nav_seq = 0; std::vector> parked_by_tag; std::vector parked_handles; std::uint32_t parked_active = 0; diff --git a/src/network/server.cpp b/src/network/server.cpp index ec1e93b5..8895acc4 100644 --- a/src/network/server.cpp +++ b/src/network/server.cpp @@ -396,6 +396,57 @@ void Server::add_session_table(std::uint64_t id, std::int32_t delta, } } +bool Server::try_register_open(std::uint64_t id, + const std::string& canon_path, + bool exclusive) { + std::lock_guard lk(open_reg_mu_); + auto it = open_reg_.find(canon_path); + if (it != open_reg_.end()) { + auto& e = it->second; + // Another session's exclusive hold denies every open mode. + if (e.exclusive_sid != 0 && e.exclusive_sid != id) return false; + if (exclusive) { + // An exclusive request needs the path free of OTHER sessions. + for (const auto& [sid, n] : e.shared) { + if (sid != id && n > 0) return false; + } + if (e.exclusive_sid != 0 && e.exclusive_sid != id) return false; + e.exclusive_sid = id; + ++e.exclusive_opens; + } else { + ++e.shared[id]; + } + return true; + } + auto& e = open_reg_[canon_path]; + if (exclusive) { + e.exclusive_sid = id; + e.exclusive_opens = 1; + } else { + e.shared[id] = 1; + } + return true; +} + +void Server::unregister_open(std::uint64_t id, + const std::string& canon_path, + bool exclusive) { + std::lock_guard lk(open_reg_mu_); + auto it = open_reg_.find(canon_path); + if (it == open_reg_.end()) return; + auto& e = it->second; + if (exclusive) { + if (e.exclusive_sid == id && e.exclusive_opens > 0) { + if (--e.exclusive_opens == 0) e.exclusive_sid = 0; + } + } else { + if (auto sit = e.shared.find(id); sit != e.shared.end()) { + if (--sit->second == 0) e.shared.erase(sit); + } + } + if (e.shared.empty() && e.exclusive_opens == 0) open_reg_.erase(it); +} + void Server::install_session_socket(std::uint64_t id, Socket s) { std::lock_guard lk(info_mu_); sockets_[id] = s; diff --git a/src/network/server.h b/src/network/server.h index 456405d3..bd7d1f41 100644 --- a/src/network/server.h +++ b/src/network/server.h @@ -213,6 +213,17 @@ class Server { // sessions_mu_. std::unordered_map session_threads_; std::vector finished_threads_; + + // mtfix11 - exclusive-open registry (see try_register_open above). + struct OpenRegEntry { + // Non-exclusive opens per session (a session may hold several). + std::unordered_map shared; + // Exclusive holder: owning session + handle multiplicity. + std::uint64_t exclusive_sid = 0; + std::uint32_t exclusive_opens = 0; + }; + std::mutex open_reg_mu_; + std::unordered_map open_reg_; std::atomic thread_seq_{1}; // Enterprise step 3 — when the sharded-reactor pool is enabled // (OPENADS_SERVER_POOL), accept_loop hands each accepted socket to this @@ -272,6 +283,31 @@ class Server { std::int32_t delta, const std::string& table_name = std::string()); + // mtfix11 - SAP exclusive-open enforcement across wire sessions. + // The drivers map DriverOpenMode::Exclusive to a plain open (POSIX + // has no share modes), so without this registry an ADS_EXCLUSIVE + // open was a silent no-op: any other session could open the same + // table, where SAP ADS denies both directions (reindex/pack + // workflows rely on that denial to keep newcomers out). + // + // Keyed by the engine table's resolved absolute path. An open is + // denied when another session holds the path exclusive, or when the + // request is exclusive and another session holds the path open in + // any mode. Same-session reopens always pass (a connection may USE + // the same table in several workareas; the wire dual-handle twin is + // opened through the local ABI connection and never lands here). + // Engine-internal opens (SQL cursors, replication apply, mgmt) + // bypass the registry by construction - they never reach the wire + // OpenTable handler. + // + // try_register_open registers and returns true when allowed, false + // when denied (nothing registered). unregister_open drops one + // previously-registered open. + bool try_register_open(std::uint64_t id, const std::string& canon_path, + bool exclusive); + void unregister_open(std::uint64_t id, const std::string& canon_path, + bool exclusive); + // studio.web.0.11 — drop a single session by id (closes its // socket; the session_loop's next read_frame returns and the // session thread exits cleanly). Returns true if the id was diff --git a/src/network/session.cpp b/src/network/session.cpp index ac0fe3cb..d25d502e 100644 --- a/src/network/session.cpp +++ b/src/network/session.cpp @@ -17,6 +17,8 @@ #include "openads/ace.h" #include "openads/error.h" #include "abi/lock_retry_policy.h" +#include "abi/create_table_diag.h" +#include "abi/last_error.h" #include "engine/server_fs.h" #include "platform/fs_sandbox.h" #include "platform/path.h" @@ -544,7 +546,11 @@ void Session::cleanup() { // else still references the path, and skips LockRegistry cleanup). if (sess_conn_) { for (auto& [id, h] : tbls_) { - (void)id; + if (auto rit = tbl_open_reg_.find(id); + rit != tbl_open_reg_.end()) { + srv_->unregister_open(sid_, rit->second.first, + rit->second.second); + } if (auto* t = sess_conn_->lookup_table(h)) { (void)t->flush(); openads::mgmt::LockRegistry::instance().remove_all_for_table(t); @@ -555,6 +561,7 @@ void Session::cleanup() { } tbls_.clear(); tbl_open_paths_.clear(); + tbl_open_reg_.clear(); // 5) oads_FOpen / AdsFOpen files for this session. files_.clear(); @@ -1143,6 +1150,79 @@ void Session::pack_bound_trailer(Frame& reply, std::uint32_t id) { } } +// mtfix12 R1 (kCapNavBoundaryPair) — see session.h. The pair section: +// [u8 flags][u32 trailer_len][trailer][bound 6|10] +// flags bit 0x02 = pair bound carries reccount. Bit 0x04 (optional key +// count) remains unset: counting all scoped index keys on every GoTop +// made B_BIG's 10-thread job 8.4 s instead of ~220 ms. If an opposite +// GoBottom is actually consumed, the client asks for its key count only +// when its per-order cache is cold, preserving scoped key-number truth. +// The trailer is pack_row_trailer output at lookahead depth 0 for the +// OPPOSITE boundary, read inside this same visit, so +// the client's adjacent opposite-boundary call applies it verbatim. +void Session::pack_boundary_pair(Frame& reply, std::uint32_t id, + int which, ADSHANDLE hord, + openads::engine::Table* tbl) { + const bool to_bottom = (which == 1); // certify the opposite end + std::uint8_t flags = 0; + Frame blob; + blob.opcode = reply.opcode; + Frame bnd; + bnd.opcode = reply.opcode; + bool granted = false; + if (hord != 0) { + // Ordered table: navigate the ABI twin (it carries the order + // and scope), restore the requested boundary exactly after. + UNSIGNED32 save_rec = 0; + UNSIGNED16 sb = 0, se = 0; + (void)AdsGetRecordNum(hord, 0, &save_rec); + (void)AdsAtBOF(hord, &sb); + (void)AdsAtEOF(hord, &se); + const bool empty_landing = (sb != 0 && se != 0); + const UNSIGNED32 grc = + to_bottom ? AdsGotoBottom(hord) : AdsGotoTop(hord); + if (grc == 0) { + pack_row_trailer(blob, id, 0); + pack_bound_trailer(bnd, id); + if (bnd.payload.size() >= 10) flags |= 0x02; + granted = true; + } + if (empty_landing) { + (void)(which == 1 ? AdsGotoTop(hord) : AdsGotoBottom(hord)); + } else { + (void)AdsGotoRecord(hord, save_rec); + } + } else if (tbl != nullptr) { + const std::uint32_t save_rec = tbl->recno(); + const bool empty_landing = tbl->bof() && tbl->eof(); + auto gr = to_bottom ? tbl->goto_bottom() : tbl->goto_top(); + if (gr) { + pack_row_trailer(blob, id, 0); + pack_bound_trailer(bnd, id); + if (bnd.payload.size() >= 10) flags |= 0x02; + granted = true; + } + if (empty_landing) { + if (which == 1) (void)tbl->goto_top(); + else (void)tbl->goto_bottom(); + } else { + (void)tbl->goto_record(save_rec); + } + } + if (!granted) return; // client falls back to a plain second frame + reply.payload.push_back(flags); + const std::uint32_t tlen = + static_cast(blob.payload.size()); + reply.payload.push_back(static_cast( tlen & 0xFFu)); + reply.payload.push_back(static_cast((tlen >> 8) & 0xFFu)); + reply.payload.push_back(static_cast((tlen >> 16) & 0xFFu)); + reply.payload.push_back(static_cast((tlen >> 24) & 0xFFu)); + reply.payload.insert(reply.payload.end(), + blob.payload.begin(), blob.payload.end()); + reply.payload.insert(reply.payload.end(), + bnd.payload.begin(), bnd.payload.end()); +} + // M12.22/M12.23 — read-ahead depth for one forward Skip. // // `hint` is what the client asked for via AdsCacheRecords, or @@ -1721,7 +1801,9 @@ DispatchResult Session::dispatch(const Frame& f) { const std::uint32_t scaps = openads::network::kCapSetFieldsBatch | openads::network::kCapFlushInCloseAll | - openads::network::kCapNavOrderFuse; + openads::network::kCapNavOrderFuse | + openads::network::kCapFlushTableDurable | + openads::network::kCapNavBoundaryPair; reply.payload.push_back( static_cast( scaps & 0xFFu)); reply.payload.push_back( @@ -1877,7 +1959,7 @@ DispatchResult Session::dispatch(const Frame& f) { std::string rel; auto open_mode = openads::engine::OpenMode::Shared; if (client_open_table_mode_ok_ && f.payload.size() >= 2) { - // M12.x extended payload: [u16 LE mode][table_name_bytes] + // M12.x extended payload: [u16 mode][table_name_bytes] std::uint16_t mode_u16 = static_cast( static_cast(f.payload[0]) | (static_cast(f.payload[1]) << 8)); @@ -1899,10 +1981,14 @@ DispatchResult Session::dispatch(const Frame& f) { if (!stripped.empty()) rel = std::move(stripped); } } + openads::abi::create_diag::Scope open_diag(rel, + std::to_string(sid_) + "." + openads::abi::create_diag::new_id()); auto th = sess_conn_->open_table(rel, openads::engine::TableType::Cdx, open_mode); if (!th) { + openads::abi::create_diag::log("server-open-fail", th.error().code, + "table-open", th.error().sub_code); std::fprintf(stderr, "[srv] OpenTable FAILED rel='%s' code=%d msg='%s'\n", rel.c_str(), th.error().code, th.error().message.c_str()); @@ -1911,8 +1997,32 @@ DispatchResult Session::dispatch(const Frame& f) { break; } std::uint32_t id = next_id_++; + // mtfix11 - enforce ADS_EXCLUSIVE across wire sessions (SAP + // semantics; the drivers alone treat Exclusive as a plain + // open, so a reindex/pack exclusive hold never kept a second + // instance's opens out). Open-then-register keeps the + // registry key canonical (the engine's resolved path); a + // denied open is closed again and answered with 7040 + // AE_FILE_IN_USE - the same code this codebase maps Win32 + // sharing violations to. + if (auto* tbl = sess_conn_->lookup_table(th.value())) { + const std::string& canon = tbl->path(); + const bool excl = + (open_mode == openads::engine::OpenMode::Exclusive); + if (!canon.empty()) { + if (!srv_->try_register_open(sid_, canon, excl)) { + sess_conn_->close_table(th.value()); + openads::abi::create_diag::log("server-open-exclusive-fail", 7040); + reply = err("OpenTable: table in use exclusively", + 7040); + break; + } + tbl_open_reg_.emplace(id, std::make_pair(canon, excl)); + } + } tbls_.emplace(id, th.value()); tbl_open_paths_.emplace(id, rel); + openads::abi::create_diag::log("server-open-ok"); srv_->add_session_table(sid_, +1, rel); reply.opcode = Opcode::OpenTableAck; write_u32_le(id, reply.payload); @@ -2000,6 +2110,12 @@ DispatchResult Session::dispatch(const Frame& f) { if (it != tbls_.end()) { sess_conn_->close_table(it->second); tbls_.erase(it); + if (auto rit = tbl_open_reg_.find(id); + rit != tbl_open_reg_.end()) { + srv_->unregister_open(sid_, rit->second.first, + rit->second.second); + tbl_open_reg_.erase(rit); + } std::string tname; if (auto pit = tbl_open_paths_.find(id); pit != tbl_open_paths_.end()) tname = pit->second; @@ -2048,12 +2164,21 @@ DispatchResult Session::dispatch(const Frame& f) { // navigating, collapsing SetOrder+GotoTop into one frame. // Length-gated (old clients stop at byte 6); cursor tables // above ignore it (their orders are query-fixed). + // mtfix12 R1: byte 6 is a flags byte on new clients — + // bit 0x01 fused order section (kCapNavOrderFuse, same + // wire shape as before: old clients send exactly 0x01), + // bit 0x02 boundary-pair request (kCapNavBoundaryPair). bool fused_order = false; - if (f.payload.size() >= 11 && f.payload[6] == 0x01) { - std::uint32_t oiid = read_u32_le(f.payload.data() + 7); - UNSIGNED32 oorc = install_table_order(id, oiid); - if (oorc != 0) { reply = err("SetOrder", oorc); break; } - fused_order = true; + bool want_pair = false; + if (f.payload.size() >= 7) { + const std::uint8_t fl = f.payload[6]; + want_pair = (fl & 0x02) != 0; + if ((fl & 0x01) != 0 && f.payload.size() >= 11) { + std::uint32_t oiid = read_u32_le(f.payload.data() + 7); + UNSIGNED32 oorc = install_table_order(id, oiid); + if (oorc != 0) { reply = err("SetOrder", oorc); break; } + fused_order = true; + } } auto it = tbls_.find(id); if (it == tbls_.end() || !sess_conn_) { @@ -2101,7 +2226,21 @@ DispatchResult Session::dispatch(const Frame& f) { } pack_row_trailer(reply, id, next_lookahead(id, gt_hint)); // Reposition truth rides after the trailer (see GotoRecord). - pack_bound_trailer(reply, id); + // mtfix12 R1: pair-requested acks carry an explicit + // [u8 ack_flags] first (bit 0x01 = bound has reccount) so + // the client parses section offsets deterministically. + if (want_pair) { + Frame bnd; + bnd.opcode = reply.opcode; + pack_bound_trailer(bnd, id); + reply.payload.push_back( + bnd.payload.size() >= 10 ? 1 : 0); + reply.payload.insert(reply.payload.end(), + bnd.payload.begin(), + bnd.payload.end()); + } else { + pack_bound_trailer(reply, id); + } // Fused switch only: the app almost always asks this // order's key count next (scrollbar setup), so certify it // now ([u32] after the bound tail) instead of paying a @@ -2118,6 +2257,11 @@ DispatchResult Session::dispatch(const Frame& f) { reply.payload.push_back(static_cast((fkc >> 16) & 0xFFu)); reply.payload.push_back(static_cast((fkc >> 24) & 0xFFu)); } + // mtfix12 R1: the opposite boundary's landing state, + // read and packed inside this same visit. + if (want_pair) { + pack_boundary_pair(reply, id, 1, hord, tbl); + } // Sync AFTER packing — pack_row_trailer walks the ABI cursor // through the block and restores it, so the engine cursor has to be // anchored to where the ABI cursor finally lands (same reason as @@ -2583,12 +2727,20 @@ DispatchResult Session::dispatch(const Frame& f) { // Fused nav+order: trailing [u8 0x01][u32 order_id]. // (GotoBottom carries no depth hint, so the section starts // at byte 4.) + // mtfix12 R1: byte 4 is a flags byte on new clients (see + // GotoTop): bit 0x01 fused order section, bit 0x02 + // boundary-pair request. bool fused_order = false; - if (f.payload.size() >= 9 && f.payload[4] == 0x01) { - std::uint32_t oiid = read_u32_le(f.payload.data() + 5); - UNSIGNED32 oorc = install_table_order(id, oiid); - if (oorc != 0) { reply = err("SetOrder", oorc); break; } - fused_order = true; + bool want_pair = false; + if (f.payload.size() >= 5) { + const std::uint8_t fl = f.payload[4]; + want_pair = (fl & 0x02) != 0; + if ((fl & 0x01) != 0 && f.payload.size() >= 9) { + std::uint32_t oiid = read_u32_le(f.payload.data() + 5); + UNSIGNED32 oorc = install_table_order(id, oiid); + if (oorc != 0) { reply = err("SetOrder", oorc); break; } + fused_order = true; + } } auto it = tbls_.find(id); if (it == tbls_.end() || !sess_conn_) { @@ -2610,7 +2762,18 @@ DispatchResult Session::dispatch(const Frame& f) { } reply.opcode = Opcode::GotoBottomAck; pack_row_trailer(reply, id); - pack_bound_trailer(reply, id); + if (want_pair) { + Frame bnd; + bnd.opcode = reply.opcode; + pack_bound_trailer(bnd, id); + reply.payload.push_back( + bnd.payload.size() >= 10 ? 1 : 0); + reply.payload.insert(reply.payload.end(), + bnd.payload.begin(), + bnd.payload.end()); + } else { + pack_bound_trailer(reply, id); + } // Fused switch only: certify the new order's key count // (see GotoTop). if (fused_order) { @@ -2625,6 +2788,9 @@ DispatchResult Session::dispatch(const Frame& f) { reply.payload.push_back(static_cast((fkc >> 16) & 0xFFu)); reply.payload.push_back(static_cast((fkc >> 24) & 0xFFu)); } + if (want_pair) { + pack_boundary_pair(reply, id, 2, hord, tbl); + } break; } // M12.15 — info / lock / maintenance / AOF. @@ -3523,16 +3689,26 @@ DispatchResult Session::dispatch(const Frame& f) { auto nb = to_cbuf(name); auto fb = to_cbuf(fields); ADSHANDLE hTable = 0; + openads::abi::create_diag::Scope cd_scope(name, + std::to_string(sid_) + "." + openads::abi::create_diag::new_id()); + openads::abi::create_diag::log("server-wire-begin"); UNSIGNED32 rrc = AdsCreateTable( abi_conn_, nb.data(), nullptr, table_type, char_type, 0, 0, memo_bs, fb.data(), &hTable); if (rrc != 0) { + const auto captured = openads::abi::last_error_code(); + openads::abi::create_diag::log("server-abi-fail", rrc, + captured == static_cast(rrc) ? "last-error-matches" : "last-error-differs"); reply = err("CreateTable", rrc); break; } + openads::abi::create_diag::log("server-abi-ok"); // Files are on disk under the data dir; release the local // handle so the client's subsequent OpenTable can take Shared. - if (hTable != 0) (void)AdsCloseTable(hTable); + if (hTable != 0) { + const auto close_rc = AdsCloseTable(hTable); + openads::abi::create_diag::log(close_rc ? "server-close-fail" : "server-close-ok", close_rc); + } reply.opcode = Opcode::CreateTableAck; break; } @@ -4223,6 +4399,10 @@ DispatchResult Session::dispatch(const Frame& f) { } auto* tbl = sess_conn_->lookup_table(it->second); if (!tbl) { reply = err("AppendBlank: lookup failed"); break; } + auto diag_name = tbl_open_paths_.find(id); + openads::abi::create_diag::Scope append_diag( + diag_name == tbl_open_paths_.end() ? std::string() : diag_name->second, + std::to_string(sid_) + ".append." + std::to_string(id)); // M12.16 dual-handle: CreateIndex/OpenIndex bind bags on the // parallel ABI Table (tbls_h_), not on the engine Table used by // the historical write path. Writing only through the engine @@ -4231,7 +4411,12 @@ DispatchResult Session::dispatch(const Frame& f) { // exists so sync_all_indexes_ updates every bound tag. if (auto hit = tbls_h_.find(id); hit != tbls_h_.end()) { UNSIGNED32 rrc = AdsAppendRecord(hit->second); - if (rrc != 0) { reply = err("AppendBlank", rrc); break; } + if (rrc != 0) { + const auto captured = openads::abi::last_error_code(); + openads::abi::create_diag::log("server-append-abi-fail", rrc, + captured == static_cast(rrc) ? "last-error-matches" : "last-error-differs"); + reply = err("AppendBlank", rrc); break; + } // Storm fix — the client commits with DbCommit→FlushTable // (which flushes this same handle); a per-append flush here // multiplied CDX page writes ~9x under the 700-storm and @@ -4241,9 +4426,17 @@ DispatchResult Session::dispatch(const Frame& f) { tbl->set_pending_append(true); } else { auto r = tbl->append_record(); - if (!r) { reply = err("AppendBlank: append_record failed"); break; } + if (!r) { + openads::abi::create_diag::log("server-append-engine-fail", r.error().code, + "append-record", r.error().sub_code); + reply = err("AppendBlank: append_record failed"); break; + } tbl->set_pending_append(true); } + if (openads::abi::create_diag::enabled( + diag_name == tbl_open_paths_.end() ? std::string() : diag_name->second) && + diag_first_append_.insert(id).second) + openads::abi::create_diag::log("server-first-append-ok"); reply.opcode = Opcode::AppendBlankAck; break; } @@ -4973,6 +5166,20 @@ DispatchResult Session::dispatch(const Frame& f) { if (!tbl) { reply = err("Reindex: lookup failed"); break; } auto r = tbl->reindex(); if (!r) { reply = err("Reindex: reindex failed"); break; } + // The table's bags (production .cdx/.z01 and any OpenIndex + // bag) are bound on the ABI twin (tbls_h_), not on the engine + // table, so the engine reindex above finds no index and is a + // no-op. Rebuild the twin's bags too, as Pack/Zap already do. + // Flush first so the rebuild reads every committed row; + // AdsGotoTop refreshes the twin's cached record count. + if (auto hit = tbls_h_.find(id); hit != tbls_h_.end()) { + if (auto fl = tbl->flush(); !fl) { + reply = err("Reindex: flush failed"); break; + } + (void)AdsGotoTop(hit->second); + UNSIGNED32 rrc = AdsReindex(hit->second); + if (rrc != 0) { reply = err("Reindex", rrc); break; } + } reply.opcode = Opcode::ReindexAck; break; } diff --git a/src/network/session.h b/src/network/session.h index c283b533..81c420a5 100644 --- a/src/network/session.h +++ b/src/network/session.h @@ -87,6 +87,13 @@ class Session { // Original OpenTable payload (DD alias or relative path). ensure_abi_handle // must reopen the same physical file — basename-only breaks subdir tables. std::unordered_map tbl_open_paths_; + // Diagnostic-only first append marker; no change to table operations. + std::unordered_set diag_first_append_; + // mtfix11 - Server::try_register_open bookkeeping per wire table id: + // (engine-resolved canonical path, exclusive flag) as registered at + // OpenTable; consumed at CloseTable / teardown for unregister_open. + std::unordered_map> + tbl_open_reg_; std::unordered_map cursor_tbls_; // M12.16 — lazy-promoted ABI handle parallel to tbls_. std::unordered_map tbls_h_; @@ -226,6 +233,17 @@ class Session { // what must stay a pure read. A freshly repositioned twin is not // in limbo; both-true genuinely means an empty cursor. void pack_bound_trailer(Frame& reply, std::uint32_t id); + // mtfix12 R1 (kCapNavBoundaryPair): append the OPPOSITE boundary's + // landing state (row blob at lookahead depth 0, bound values, + // scoped key count for ordered tables) after the requested + // boundary's own sections. `which` is the boundary just navigated + // (1 = top, 2 = bottom); the pair certifies the other end. The + // cursor is restored exactly before returning; on any failure + // nothing is appended (the client length-gates and falls back to + // a plain second frame). Read-only wrt caller-visible state. + void pack_boundary_pair(Frame& reply, std::uint32_t id, + int which, ADSHANDLE hord, + openads::engine::Table* tbl); // Warm OpenTableAck sections (USE latency): schema + first-row TLVs // appended after the bag field (see wire.h OpenTableAckSections). // The row section positions the engine cursor exactly like an diff --git a/src/network/wire.h b/src/network/wire.h index 3fbe051b..b9935788 100644 --- a/src/network/wire.h +++ b/src/network/wire.h @@ -648,6 +648,37 @@ inline constexpr std::uint32_t kCapFlushInCloseAll = 0x00000020u; // never emit it. inline constexpr std::uint32_t kCapNavOrderFuse = 0x00000040u; +// Durable FlushTable: the server's FlushTable handler already performs +// the full file-buffers flush (ABI twin via AdsFlushFileBuffers, then +// the engine table), i.e. exactly the work of a standalone +// FlushFileBuffers frame. A client that sees this bit clears its dirty +// flag when its own FlushTable lands and skips the trailing +// FlushFileBuffers -- one frame per commit instead of two. Same +// two-way gating as kCapFlushInCloseAll; old servers never echo it, so +// the client keeps sending both frames there. +inline constexpr std::uint32_t kCapFlushTableDurable = 0x00000080u; + +// Boundary-pair certification (mtfix12 R1): a GotoTop/GotoBottom request +// may carry one more trailing flag bit (see the flag bytes below) asking +// the server to read the OPPOSITE boundary in the same atomic visit and +// append its full landing state to the ack (row blob + bound values + +// scoped key count). The client stamps that certification and serves a +// back-to-back opposite-boundary call (the dbGoTop(); dbGoBottom() +// ritual) with zero frames between proof and serve -- the same +// conn-wide freshness envelope the shipped duplicate-suppression uses. +// Same two-way gating as kCapNavOrderFuse: the client only sets the +// flag when the ConnectAck echo carries this bit, so old servers never +// see it and old clients never emit it. +// +// Request layouts (new server parses bits, old layouts stay valid): +// GotoTop: [u32 id][u16 depth][u8 flags]([u32 order]) +// GotoBottom: [u32 id][u8 flags]([u32 order]) +// flags bit 0x01 = fused order section present (kCapNavOrderFuse) +// flags bit 0x02 = boundary-pair certification requested (this bit) +// Pre-mtfix12 clients send flags == 0x01 exactly when fusing, which the +// new server reads as "fused, no pair" -- bit-exact with the old parse. +inline constexpr std::uint32_t kCapNavBoundaryPair = 0x00000100u; + // Warm OpenTableAck sections (USE latency). After the fixed // `[u32 id][u16 bag_len][bag]` prefix, the ack carries // `[u8 section_count]` then that many TLVs: diff --git a/src/platform/lock.h b/src/platform/lock.h index 85db7c19..ae093e1d 100644 --- a/src/platform/lock.h +++ b/src/platform/lock.h @@ -25,6 +25,15 @@ class ByteLock { std::uint64_t length, LockKind kind); + // Non-destructive conflict query (mtfix11): true when any byte of the + // range is locked by ANOTHER owner (another handle/OFD/process). + // Never takes or removes a lock. Callers must consult their own lock + // registrations first: POSIX OFD queries cannot see the querying fd's + // own locks, and a Win32 same-handle overlap also reports a conflict - + // neither layer can distinguish "mine". + static util::Result probe(File& f, std::uint64_t offset, + std::uint64_t length); + util::Result release(); // Internal: construct from a native handle and the locked range. diff --git a/src/platform/lock_posix.cpp b/src/platform/lock_posix.cpp index 579070e0..ca640a27 100644 --- a/src/platform/lock_posix.cpp +++ b/src/platform/lock_posix.cpp @@ -10,6 +10,17 @@ #include #include +// OFD (open-file-description) locks are used only where they are proven +// to work. The macOS SDK defines F_OFD_SETLK, but there they made the +// engine wait on its own handles (a fresh table refused its first +// append lock and CI sat on it), so macOS keeps the process-scoped +// F_SETLK/F_SETLKW locks the engine was written for. +#if defined(F_OFD_SETLK) && !defined(__APPLE__) +#define OPENADS_USE_OFD_LOCKS 1 +#else +#define OPENADS_USE_OFD_LOCKS 0 +#endif + namespace openads::platform { namespace { @@ -28,7 +39,7 @@ util::Error os_error(const char* op) { // should still contend (Win32 LockFile is fd-scoped). OFD locks // are tied to the open file description, matching the Win32 // semantics used by the engine. -#ifdef F_OFD_SETLK +#if OPENADS_USE_OFD_LOCKS constexpr int kSetLk = F_OFD_SETLK; [[maybe_unused]] constexpr int kSetLkW = F_OFD_SETLKW; #else @@ -103,14 +114,13 @@ void ByteLock::release_() noexcept { util::Result ByteLock::acquire(File& f, std::uint64_t offset, std::uint64_t length, LockKind kind) { -#if defined(F_OFD_SETLK) && !defined(__APPLE__) +#if OPENADS_USE_OFD_LOCKS return do_lock(f, offset, length, kind, kSetLkW); #else - // macOS: its SDK defines the OFD commands, but a blocking F_OFD_SETLKW - // waits forever when a leaked holder lives in this same process (CI - // test runs sat on it until the job timeout). Poll the non-blocking - // command and give up after a bounded wait, so a stall becomes a lock - // error instead of a hang. + // Process-scoped locks (macOS) can still wait forever on a stale + // holder in another process. Poll the non-blocking command and give + // up after a bounded wait, so a stall becomes a lock error instead of + // a hang. const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); for (;;) { @@ -128,21 +138,20 @@ util::Result ByteLock::acquire(File& f, std::uint64_t offset, int fd = static_cast( reinterpret_cast(f.native_handle()) - 1); long holder = -1; -#ifdef F_OFD_GETLK - q.l_pid = 0; - const int kGetLk = F_OFD_GETLK; -#else - const int kGetLk = F_GETLK; -#endif - if (::fcntl(fd, kGetLk, &q) == 0 && q.l_type != F_UNLCK) { + const int get_rc = ::fcntl(fd, F_GETLK, &q); + const int get_errn = (get_rc == -1) ? errno : 0; + if (get_rc == 0 && q.l_type != F_UNLCK) { holder = static_cast(q.l_pid); } std::fprintf(stderr, "openads: byte lock wait timed out (offset=%llu len=%llu " - "holder_pid=%ld self_pid=%ld)\n", + "set_errno=%d get_rc=%d get_errno=%d get_type=%d " + "holder_pid=%ld self_pid=%ld fd=%d)\n", static_cast(offset), - static_cast(length), holder, - static_cast(::getpid())); + static_cast(length), + r.error().sub_code, get_rc, get_errn, + static_cast(q.l_type), holder, + static_cast(::getpid()), fd); return r; } std::this_thread::sleep_for(std::chrono::milliseconds(5)); @@ -156,6 +165,29 @@ util::Result ByteLock::try_acquire(File& f, std::uint64_t offset, return do_lock(f, offset, length, kind, kSetLk); } +util::Result ByteLock::probe(File& f, std::uint64_t offset, + std::uint64_t length) { + struct flock fl{}; + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = static_cast(fold_lock_offset(offset)); + fl.l_len = static_cast(length); + fl.l_pid = 0; + // native_handle() stores (fd + 1) to avoid the nullptr/fd-0 collision. + int fd = static_cast(reinterpret_cast(f.native_handle()) - 1); +#if OPENADS_USE_OFD_LOCKS + // F_OFD_GETLK reports conflicts against OTHER open file descriptions - + // exactly "another handle/session holds this byte". The querying fd's + // own locks are invisible to it; callers check their own list first. + if (::fcntl(fd, F_OFD_GETLK, &fl) == -1) return os_error("fcntl(F_OFD_GETLK)"); +#else + // Pre-3.15 fallback: process-scoped GETLK reports other PROCESSES only; + // same-process conflicts stay invisible (degraded, never wrong-safe). + if (::fcntl(fd, F_GETLK, &fl) == -1) return os_error("fcntl(F_GETLK)"); +#endif + return fl.l_type != F_UNLCK; +} + util::Result ByteLock::release() { release_(); return {}; diff --git a/src/platform/lock_win32.cpp b/src/platform/lock_win32.cpp index 40d3d64c..7f55bf4a 100644 --- a/src/platform/lock_win32.cpp +++ b/src/platform/lock_win32.cpp @@ -77,6 +77,27 @@ util::Result ByteLock::try_acquire(File& f, std::uint64_t offset, return do_lock(f, offset, length, kind, LOCKFILE_FAIL_IMMEDIATELY); } +util::Result ByteLock::probe(File& f, std::uint64_t offset, + std::uint64_t length) { + // Win32 has no query API: a non-blocking take-and-release is the only + // conflict probe. A same-handle overlap also fails with + // ERROR_LOCK_VIOLATION, so callers must exclude their own + // registrations before probing. + OVERLAPPED ov{}; + ov.Offset = static_cast(offset & 0xFFFFFFFFu); + ov.OffsetHigh = static_cast(offset >> 32); + DWORD lo = static_cast(length & 0xFFFFFFFFu); + DWORD hi = static_cast(length >> 32); + HANDLE h = reinterpret_cast(f.native_handle()); + if (::LockFileEx(h, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, + 0, lo, hi, &ov)) { + ::UnlockFileEx(h, 0, lo, hi, &ov); + return false; + } + if (::GetLastError() == ERROR_LOCK_VIOLATION) return true; + return os_error("LockFileEx probe"); +} + util::Result ByteLock::release() { release_(); return {}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 821ebec0..220e8130 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -209,6 +209,7 @@ add_executable(openads_unit_tests unit/abi_remote_index_reopen_test.cpp unit/abi_remote_zombie_lock_test.cpp unit/network_skip_depth_test.cpp + unit/abi_index_focus_contract_test.cpp unit/abi_remote_index_nav_test.cpp unit/abi_remote_date_index_scope_test.cpp unit/abi_remote_prodcdx_test.cpp @@ -268,6 +269,7 @@ add_executable(openads_unit_tests unit/abi_adt_smoke_test.cpp unit/abi_adt_create_test.cpp unit/abi_adt_append_exclusive_test.cpp + unit/abi_adt_append_exclusive_upstream_test.cpp unit/abi_adt_wide_numeric_test.cpp unit/abi_adi_create_test.cpp unit/abi_adi_separate_bag_test.cpp @@ -333,12 +335,19 @@ add_executable(openads_unit_tests unit/network_version_test.cpp unit/network_nav_batch_test.cpp unit/network_use_budget_test.cpp + unit/network_local_answers_test.cpp + unit/network_empty_window_test.cpp + unit/network_remote_reindex_test.cpp + unit/network_frame_trace_test.cpp + unit/network_boundary_pair_test.cpp + unit/network_commit_slimming_test.cpp unit/network_teardown_batch_test.cpp unit/network_pool_mt_test.cpp unit/network_mutex_release_test.cpp unit/network_lock_exclusion_test.cpp unit/network_login_gate_stress_test.cpp unit/abi_remote_lock_introspection_test.cpp + unit/network_exclusive_open_test.cpp unit/backend_tier1_test.cpp unit/abi_oads_file_funcs_test.cpp unit/abi_create_index_path_test.cpp @@ -490,6 +499,13 @@ endif() add_test(NAME openads_unit_tests COMMAND openads_unit_tests -tce=*[slow]*,*flaky*) add_test(NAME openads_unit_tests_slow COMMAND openads_unit_tests -tc=*[slow]* -tce=*flaky*) +# Session-pool MT stress: the pool test in 30 fresh processes (each run +# already repeats its 4-thread phase 25x). Catches rare races that one +# in-suite pass misses. Every patch must keep this green. +if(UNIX) + add_test(NAME network_pool_mt_repeat COMMAND sh -c "i=0; while [ $i -lt 30 ]; do \"$0\" -tc=\"Session pool: MT*\" >/dev/null 2>&1 || { echo \"pool MT test failed on run $i\"; \"$0\" -tc=\"Session pool: MT*\"; exit 1; }; i=$((i+1)); done; echo \"pool MT test: 30 clean runs\"" $) +endif() + # SQL URI backend smoke (sqlite://): runs a focused doctest subset in CI. if(OPENADS_WITH_POSTGRESQL) add_test(NAME pg_live_smoke COMMAND openads_unit_tests diff --git a/tests/unit/abi_adi_separate_bag_test.cpp b/tests/unit/abi_adi_separate_bag_test.cpp index 7446834a..4f19e85b 100644 --- a/tests/unit/abi_adi_separate_bag_test.cpp +++ b/tests/unit/abi_adi_separate_bag_test.cpp @@ -133,6 +133,7 @@ TEST_CASE("ADI separate (non-structural) bag on ADT: reopen and navigate") { ADSHANDLE ah[8] = {0}; UNSIGNED16 n = 8; REQUIRE(AdsOpenIndex(hTable2, idxfile, ah, &n) == AE_SUCCESS); + REQUIRE(AdsSetIndexOrderByHandle(hTable2, ah[0]) == AE_SUCCESS); REQUIRE(AdsGotoTop(hTable2) == AE_SUCCESS); std::vector seen; diff --git a/tests/unit/abi_adt_append_exclusive_test.cpp b/tests/unit/abi_adt_append_exclusive_test.cpp index c7f3eafb..bbafd1a0 100644 --- a/tests/unit/abi_adt_append_exclusive_test.cpp +++ b/tests/unit/abi_adt_append_exclusive_test.cpp @@ -1,59 +1,30 @@ #include "doctest.h" #include "openads/ace.h" - #include #include #include - -namespace fs = std::filesystem; - -TEST_CASE("LOCAL ADT exclusive bulk append does not accumulate record locks") { - const fs::path dir = fs::temp_directory_path() / - "openads_adt_append_exclusive_regression"; - std::error_code ec; - fs::remove_all(dir, ec); - fs::create_directories(dir, ec); - UNSIGNED8 srv[260]{}; - const std::string root = dir.string(); - REQUIRE(root.size() < sizeof(srv)); - std::memcpy(srv, root.c_str(), root.size()); - ADSHANDLE conn = 0, table = 0; - REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, - &conn) == AE_SUCCESS); - UNSIGNED8 name[] = "bulk.adt"; - UNSIGNED8 fields[] = "NAME,Character,30;AGE,Numeric,5"; - REQUIRE(AdsCreateTable(conn, name, nullptr, ADS_ADT, ADS_ANSI, - 0, 0, 0, fields, &table) == AE_SUCCESS); - REQUIRE(AdsCloseTable(table) == AE_SUCCESS); - REQUIRE(AdsOpenTable(conn, name, nullptr, ADS_ADT, ADS_ANSI, - ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, - ADS_EXCLUSIVE, &table) == AE_SUCCESS); - REQUIRE(AdsSetDeferredFlush(table, 1) == AE_SUCCESS); - UNSIGNED8 field[] = "NAME"; - UNSIGNED8 value[] = "written"; - for (int i = 1; i <= 10000; ++i) { - REQUIRE(AdsAppendRecord(table) == AE_SUCCESS); - REQUIRE(AdsSetString(table, field, value, 7) == AE_SUCCESS); - REQUIRE(AdsWriteRecord(table) == AE_SUCCESS); - if (i == 1 || i == 5000 || i == 10000) { - UNSIGNED16 nlocks = 999; - REQUIRE(AdsGetNumLocks(table, &nlocks) == AE_SUCCESS); - CHECK(nlocks == 0); - } - } - REQUIRE(AdsCloseTable(table) == AE_SUCCESS); - REQUIRE(AdsOpenTable(conn, name, nullptr, ADS_ADT, ADS_ANSI, - ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, - ADS_READONLY, &table) == AE_SUCCESS); - UNSIGNED32 count = 0; - REQUIRE(AdsGetRecordCount(table, ADS_RESPECTFILTERS, &count) == AE_SUCCESS); - CHECK(count == 10000u); - REQUIRE(AdsGotoRecord(table, 10000) == AE_SUCCESS); - UNSIGNED8 out[64]{}; - UNSIGNED32 len = sizeof(out); - REQUIRE(AdsGetString(table, field, out, &len, 0) == AE_SUCCESS); - CHECK(std::string(reinterpret_cast(out), len) == "written"); - REQUIRE(AdsCloseTable(table) == AE_SUCCESS); - REQUIRE(AdsDisconnect(conn) == AE_SUCCESS); - fs::remove_all(dir, ec); +TEST_CASE("mtfix21 ADT exclusive append no lock pile and Shared guard preserved") { + namespace fs=std::filesystem; + auto dir=fs::temp_directory_path()/"mtfix21_adt_regression"; + std::error_code ec;fs::remove_all(dir,ec);fs::create_directories(dir); + auto path=dir.string();ADSHANDLE conn=0,t=0; + REQUIRE(AdsConnect60(reinterpret_cast(path.data()),ADS_LOCAL_SERVER,nullptr,nullptr,0,&conn)==0); + UNSIGNED8 name[]="bulk.adt",fields[]="NAME,Character,30;AGE,Numeric,5",fld[]="NAME"; + REQUIRE(AdsCreateTable(conn,name,nullptr,ADS_ADT,ADS_ANSI,0,0,0,fields,&t)==0); + REQUIRE(AdsCloseTable(t)==0); + REQUIRE(AdsOpenTable(conn,name,nullptr,ADS_ADT,ADS_ANSI,ADS_COMPATIBLE_LOCKING,ADS_IGNORERIGHTS,ADS_EXCLUSIVE,&t)==0); + for(int i=1;i<=10000;++i){ + REQUIRE(AdsAppendRecord(t)==0);auto value=std::to_string(i); + REQUIRE(AdsSetString(t,fld,reinterpret_cast(value.data()),static_cast(value.size()))==0); + if(i==1||i==5000||i==10000){UNSIGNED16 locks=999;REQUIRE(AdsGetNumLocks(t,&locks)==0);CHECK(locks==0);} + } + REQUIRE(AdsCloseTable(t)==0); + REQUIRE(AdsOpenTable(conn,name,nullptr,ADS_ADT,ADS_ANSI,ADS_COMPATIBLE_LOCKING,ADS_IGNORERIGHTS,ADS_SHARED,&t)==0); + UNSIGNED32 count=0;REQUIRE(AdsGetRecordCount(t,ADS_RESPECTFILTERS,&count)==0);CHECK(count==10000); + for(UNSIGNED32 i:{1u,5000u,10000u}){REQUIRE(AdsGotoRecord(t,i)==0);UNSIGNED8 out[64]{};UNSIGNED32 len=sizeof(out);REQUIRE(AdsGetString(t,fld,out,&len,0)==0);CHECK(std::string(reinterpret_cast(out),len)==std::to_string(i));} + UNSIGNED8 v[]="changed";CHECK(AdsSetString(t,fld,v,7)==5035); + REQUIRE(AdsAppendRecord(t)==0);UNSIGNED16 locks=0;REQUIRE(AdsGetNumLocks(t,&locks)==0);CHECK(locks==1); + REQUIRE(AdsSetString(t,fld,v,7)==0);REQUIRE(AdsWriteRecord(t)==0);REQUIRE(AdsUnlockRecord(t,0)==0); + REQUIRE(AdsGetNumLocks(t,&locks)==0);CHECK(locks==0); + REQUIRE(AdsCloseTable(t)==0);REQUIRE(AdsDisconnect(conn)==0);fs::remove_all(dir,ec); } diff --git a/tests/unit/abi_adt_append_exclusive_upstream_test.cpp b/tests/unit/abi_adt_append_exclusive_upstream_test.cpp new file mode 100644 index 00000000..c7f3eafb --- /dev/null +++ b/tests/unit/abi_adt_append_exclusive_upstream_test.cpp @@ -0,0 +1,59 @@ +#include "doctest.h" +#include "openads/ace.h" + +#include +#include +#include + +namespace fs = std::filesystem; + +TEST_CASE("LOCAL ADT exclusive bulk append does not accumulate record locks") { + const fs::path dir = fs::temp_directory_path() / + "openads_adt_append_exclusive_regression"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir, ec); + UNSIGNED8 srv[260]{}; + const std::string root = dir.string(); + REQUIRE(root.size() < sizeof(srv)); + std::memcpy(srv, root.c_str(), root.size()); + ADSHANDLE conn = 0, table = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, + &conn) == AE_SUCCESS); + UNSIGNED8 name[] = "bulk.adt"; + UNSIGNED8 fields[] = "NAME,Character,30;AGE,Numeric,5"; + REQUIRE(AdsCreateTable(conn, name, nullptr, ADS_ADT, ADS_ANSI, + 0, 0, 0, fields, &table) == AE_SUCCESS); + REQUIRE(AdsCloseTable(table) == AE_SUCCESS); + REQUIRE(AdsOpenTable(conn, name, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_EXCLUSIVE, &table) == AE_SUCCESS); + REQUIRE(AdsSetDeferredFlush(table, 1) == AE_SUCCESS); + UNSIGNED8 field[] = "NAME"; + UNSIGNED8 value[] = "written"; + for (int i = 1; i <= 10000; ++i) { + REQUIRE(AdsAppendRecord(table) == AE_SUCCESS); + REQUIRE(AdsSetString(table, field, value, 7) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(table) == AE_SUCCESS); + if (i == 1 || i == 5000 || i == 10000) { + UNSIGNED16 nlocks = 999; + REQUIRE(AdsGetNumLocks(table, &nlocks) == AE_SUCCESS); + CHECK(nlocks == 0); + } + } + REQUIRE(AdsCloseTable(table) == AE_SUCCESS); + REQUIRE(AdsOpenTable(conn, name, nullptr, ADS_ADT, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_READONLY, &table) == AE_SUCCESS); + UNSIGNED32 count = 0; + REQUIRE(AdsGetRecordCount(table, ADS_RESPECTFILTERS, &count) == AE_SUCCESS); + CHECK(count == 10000u); + REQUIRE(AdsGotoRecord(table, 10000) == AE_SUCCESS); + UNSIGNED8 out[64]{}; + UNSIGNED32 len = sizeof(out); + REQUIRE(AdsGetString(table, field, out, &len, 0) == AE_SUCCESS); + CHECK(std::string(reinterpret_cast(out), len) == "written"); + REQUIRE(AdsCloseTable(table) == AE_SUCCESS); + REQUIRE(AdsDisconnect(conn) == AE_SUCCESS); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/abi_alternating_append_test.cpp b/tests/unit/abi_alternating_append_test.cpp index a9f35ef3..8bd2347a 100644 --- a/tests/unit/abi_alternating_append_test.cpp +++ b/tests/unit/abi_alternating_append_test.cpp @@ -383,6 +383,9 @@ TEST_CASE("Editing an indexed field moves the key and keeps order with duplicate UNSIGNED16 nidx = 8; REQUIRE(AdsOpenIndex(hTbl, (UNSIGNED8*)"big.cdx", idxs, &nidx) == 0); + // This test walks via the table handle after an index seek; opt in + // explicitly. Stock rddads uses the index handle until focus is zero. + REQUIRE(AdsSetIndexOrderByHandle(hTbl, idxs[0]) == 0); // "Edward" now has 3 keys; "Zzztop" exactly 1, at the end of the order. UNSIGNED16 found = 0; REQUIRE(AdsSeek(idxs[0], (UNSIGNED8*)"Edward", 6, ADS_STRINGKEY, diff --git a/tests/unit/abi_index_focus_contract_test.cpp b/tests/unit/abi_index_focus_contract_test.cpp new file mode 100644 index 00000000..f846adf2 --- /dev/null +++ b/tests/unit/abi_index_focus_contract_test.cpp @@ -0,0 +1,131 @@ +// Harbour rddads focus contract. Bug report relayed by Pritpal Bedi. +// rddads uppercases name lookup; focus 0/"" only changes its hOrdCurrent. +#include "doctest.h" +#include "openads/ace.h" +#include "network/server.h" +#include +#include +#include +#include +namespace { +UNSIGNED8* focus_bytes(std::string& s) { return reinterpret_cast(s.data()); } +void focus_expect_rec(ADSHANDLE t, UNSIGNED32 n) { + UNSIGNED32 got = 0; + REQUIRE(AdsGetRecordNum(t, ADS_IGNOREFILTERS, &got) == 0); + INFO("expected rec=" << n << " table=" << t); + CHECK(got == n); +} +void focus_fixture(const std::filesystem::path& dir, UNSIGNED16 type) { + std::filesystem::create_directories(dir); + std::string path = dir.string(); ADSHANDLE c = 0, t = 0; + REQUIRE(AdsConnect60(focus_bytes(path), ADS_LOCAL_SERVER, nullptr, nullptr, 0, &c) == 0); + std::string name = type == ADS_ADT ? "focus.adt" : "focus.dbf"; + std::string defs = "lower,Character,8;UPPER,Character,8"; + REQUIRE(AdsCreateTable(c, focus_bytes(name), nullptr, type, ADS_ANSI, + ADS_PROPRIETARY_LOCKING, 0, 0, focus_bytes(defs), &t) == 0); + for (const char* value : {"D", "B", "A", "C"}) { + REQUIRE(AdsAppendRecord(t) == 0); + std::string k = "lower", k2 = "UPPER", v = value; + REQUIRE(AdsSetString(t, focus_bytes(k), focus_bytes(v), 1) == 0); + REQUIRE(AdsSetString(t, focus_bytes(k2), focus_bytes(v), 1) == 0); + REQUIRE(AdsWriteRecord(t) == 0); + } + std::string bag = (dir / (type == ADS_ADT ? "focus.adi" : "focus.cdx")).string(); + for (const char* tag : {"lower", "UPPER"}) { + std::string tagname = tag, expr = tag; ADSHANDLE h = 0; + REQUIRE(AdsCreateIndex61(t, focus_bytes(bag), focus_bytes(tagname), + focus_bytes(expr), nullptr, nullptr, ADS_COMPOUND, 512, &h) == 0); + } + REQUIRE(AdsCloseTable(t) == 0); + REQUIRE(AdsDisconnect(c) == 0); +} +void focus_check(const std::filesystem::path& dir, UNSIGNED16 type, bool remote) { + INFO("type=" << type << " remote=" << remote); + openads::network::Server srv; + std::string path = dir.string(); + if (remote) { + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + path = "tcp://127.0.0.1:" + std::to_string(srv.port()) + "/" + path; + } + ADSHANDLE c = 0, t = 0; + REQUIRE(AdsConnect60(focus_bytes(path), remote ? ADS_REMOTE_SERVER : ADS_LOCAL_SERVER, + nullptr, nullptr, 0, &c) == 0); + std::string name = type == ADS_ADT ? "focus.adt" : "focus.dbf"; + REQUIRE(AdsOpenTable(c, focus_bytes(name), nullptr, type, ADS_ANSI, + ADS_PROPRIETARY_LOCKING, 0, ADS_SHARED, &t) == 0); + REQUIRE(AdsGotoTop(t) == 0); focus_expect_rec(t, 1); + ADSHANDLE first = 0, second = 0; + REQUIRE(AdsGetIndexHandleByOrder(t, 1, &first) == 0); + REQUIRE(AdsGetIndexHandleByOrder(t, 2, &second) == 0); + for (const char* tag : {"lower", "LOWER", "LoWeR ", "upper", "UPPER"}) { + std::string wanted = tag; ADSHANDLE h = 0; + REQUIRE(AdsGetIndexHandle(t, focus_bytes(wanted), &h) == 0); + CHECK(h == (wanted[0] == 'u' || wanted[0] == 'U' ? second : first)); + } + // Exactly stock rddads' zero and empty-name sequence: no reset API call. + for (int repeat = 0; repeat < 2; ++repeat) { + { INFO("index top second"); REQUIRE(AdsGotoTop(second) == 0); focus_expect_rec(t, 3); } + REQUIRE(AdsSkip(second, 1) == 0); focus_expect_rec(t, 2); + REQUIRE(AdsGotoTop(t) == 0); focus_expect_rec(t, 1); + REQUIRE(AdsSkip(t, 1) == 0); focus_expect_rec(t, 2); + REQUIRE(AdsGotoBottom(t) == 0); focus_expect_rec(t, 4); + { INFO("natural backwards skip"); REQUIRE(AdsSkip(t, -1) == 0); focus_expect_rec(t, 3); } + // Direct Skip must restore natural order without a preceding GoTop. + { INFO("index top first"); REQUIRE(AdsGotoTop(first) == 0); focus_expect_rec(t, 3); } + { INFO("direct natural skip"); REQUIRE(AdsSkip(t, 1) == 0); focus_expect_rec(t, 4); } + } + // Retained scopes must not keep natural focus indexed. + std::string scope = "B"; + REQUIRE(AdsSetScope(first, ADS_TOP, focus_bytes(scope), 1, ADS_STRINGKEY) == 0); + REQUIRE(AdsGotoTop(first) == 0); focus_expect_rec(t, 2); + REQUIRE(AdsGotoTop(t) == 0); focus_expect_rec(t, 1); + REQUIRE(AdsGotoTop(first) == 0); focus_expect_rec(t, 2); + REQUIRE(AdsClearScope(first, ADS_TOP) == 0); + // Explicit focus APIs must still opt table-handle calls into index order. + REQUIRE(AdsSetIndexOrderByHandle(t, second) == 0); + REQUIRE(AdsGotoTop(t) == 0); focus_expect_rec(t, 3); + REQUIRE(AdsSkip(t, 1) == 0); focus_expect_rec(t, 2); + for (int repeat = 0; repeat < 2; ++repeat) { + REQUIRE(AdsSetIndexOrderByHandle(t, 0) == 0); + REQUIRE(AdsGotoTop(t) == 0); focus_expect_rec(t, 1); + } + std::string tag = "LOWER", empty; + REQUIRE(AdsSetIndexOrder(t, focus_bytes(tag)) == 0); + REQUIRE(AdsGotoBottom(t) == 0); focus_expect_rec(t, 1); + REQUIRE(AdsSetIndexOrder(t, focus_bytes(empty)) == 0); + REQUIRE(AdsGotoBottom(t) == 0); focus_expect_rec(t, 4); + // A newly created tag supports historical table-handle navigation, + // then stock rddads index navigation makes table calls natural again. + std::string extra = "EXTRA", expr = "lower"; ADSHANDLE fresh = 0; + std::string bag = (dir / (type == ADS_ADT ? "new.adi" : "new.cdx")).string(); + REQUIRE(AdsCreateIndex61(t, focus_bytes(bag), focus_bytes(extra), + focus_bytes(expr), nullptr, nullptr, ADS_COMPOUND, 512, &fresh) == 0); + REQUIRE(AdsGotoTop(t) == 0); focus_expect_rec(t, 3); + REQUIRE(AdsGotoTop(fresh) == 0); focus_expect_rec(t, 3); + REQUIRE(AdsGotoTop(t) == 0); focus_expect_rec(t, 1); + // Natural focus must leave every tag open, and all keys maintained. + UNSIGNED16 count = 0; REQUIRE(AdsGetNumIndexes(t, &count) == 0); CHECK(count == 3); + REQUIRE(AdsAppendRecord(t) == 0); + std::string key = "lower", key2 = "UPPER", v = "0"; + REQUIRE(AdsSetString(t, focus_bytes(key), focus_bytes(v), 1) == 0); + REQUIRE(AdsSetString(t, focus_bytes(key2), focus_bytes(v), 1) == 0); + REQUIRE(AdsWriteRecord(t) == 0); + REQUIRE(AdsGotoTop(first) == 0); focus_expect_rec(t, 5); + REQUIRE(AdsGotoTop(second) == 0); focus_expect_rec(t, 5); + REQUIRE(AdsCloseTable(t) == 0); REQUIRE(AdsDisconnect(c) == 0); + if (remote) srv.stop(); +} +} +TEST_CASE("RDD focus: case-insensitive tags and implicit index to natural navigation") { + // Default native ADI and DBF/CDX. Rerouted ADI is exercised by the + // companion invocation with OPENADS_ADT_CDX_INDEX=1 (cached configuration). + for (UNSIGNED16 type : {static_cast(ADS_ADT), static_cast(ADS_CDX)}) { + for (bool remote : {false, true}) { + auto dir = std::filesystem::temp_directory_path() / + ("openads_focus_contract_" + std::to_string(type) + (remote ? "_remote" : "_local")); + std::error_code ec; std::filesystem::remove_all(dir, ec); + focus_fixture(dir, type); focus_check(dir, type, remote); + std::filesystem::remove_all(dir, ec); + } + } +} diff --git a/tests/unit/abi_index_smoke_test.cpp b/tests/unit/abi_index_smoke_test.cpp index d3c4d43e..5d55923e 100644 --- a/tests/unit/abi_index_smoke_test.cpp +++ b/tests/unit/abi_index_smoke_test.cpp @@ -97,6 +97,7 @@ TEST_CASE("ABI index smoke: create NTX, seek, walk in order, scope") { REQUIRE(AdsGetRecordNum(hTable, 0, &recno) == 0); CHECK(recno == 3); + REQUIRE(AdsSetIndexOrderByHandle(hTable, hIndex) == 0); // Scope: top = BBBB, bottom = CCCC -> only recno 3 then recno 1. UNSIGNED8 stop[8] = "BBBB"; UNSIGNED8 sbot[8] = "CCCC"; diff --git a/tests/unit/abi_lock_retry_test.cpp b/tests/unit/abi_lock_retry_test.cpp index 5110abda..d5eed0bf 100644 --- a/tests/unit/abi_lock_retry_test.cpp +++ b/tests/unit/abi_lock_retry_test.cpp @@ -143,9 +143,59 @@ TEST_CASE("M9.18 retry loop sleeps cycle_ms * retry_count when contended") { REQUIRE(AdsUnlockTable(hTableA) == 0); - // Restore defaults so other tests aren't affected. + // Restore the default single-attempt policy so other tests aren't + // affected (the process default is retry_count 0 since 1.09.68-mtfix6). REQUIRE(AdsSetLockCycle(0, 100) == 0); - REQUIRE(AdsSetLockRetryCount(0, 10) == 0); + REQUIRE(AdsSetLockRetryCount(0, 0) == 0); + + REQUIRE(AdsCloseTable(hTableA) == 0); + REQUIRE(AdsDisconnect(hConnA) == 0); + REQUIRE(AdsCloseTable(hTableB) == 0); + REQUIRE(AdsDisconnect(hConnB) == 0); + fs::remove_all(dir, ec); +} + +TEST_CASE("M9.18 single-attempt policy never sleeps when contended") { + // mtfix6 default: a failed lock returns immediately - the application + // owns any retry loop (xBase RLOCK()/FLOCK() semantics). With + // retry_count 0 the call must never wait, pass or fail. + const auto dir = fs::temp_directory_path() / "openads_m9_18_lock_fast"; + std::error_code ec; + fs::remove_all(dir, ec); + stage_dbf(dir); + + UNSIGNED8 srv[256]; + std::memcpy(srv, dir.string().c_str(), dir.string().size() + 1); + UNSIGNED8 leaf[16] = "data"; + + ADSHANDLE hConnA = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, + nullptr, nullptr, 0, &hConnA) == 0); + ADSHANDLE hTableA = 0; + REQUIRE(AdsOpenTable(hConnA, leaf, leaf, ADS_CDX, + 1, 1, 0, 1, &hTableA) == 0); + + ADSHANDLE hConnB = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, + nullptr, nullptr, 0, &hConnB) == 0); + ADSHANDLE hTableB = 0; + REQUIRE(AdsOpenTable(hConnB, leaf, leaf, ADS_CDX, + 1, 1, 0, 1, &hTableB) == 0); + + // Single attempt, 100 ms cycle: the cycle must never be slept. + REQUIRE(AdsSetLockCycle(0, 100) == 0); + REQUIRE(AdsSetLockRetryCount(0, 0) == 0); + + REQUIRE(AdsLockTable(hTableA) == 0); + + auto t0 = std::chrono::steady_clock::now(); + AdsLockTable(hTableB); // result irrelevant - timing is the assertion + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + + CHECK(elapsed < 50); // single attempt on an in-process server: no sleep + + REQUIRE(AdsUnlockTable(hTableA) == 0); REQUIRE(AdsCloseTable(hTableA) == 0); REQUIRE(AdsDisconnect(hConnA) == 0); diff --git a/tests/unit/abi_lock_unlock_full_test.cpp b/tests/unit/abi_lock_unlock_full_test.cpp index 578e4480..7e214f0b 100644 --- a/tests/unit/abi_lock_unlock_full_test.cpp +++ b/tests/unit/abi_lock_unlock_full_test.cpp @@ -69,7 +69,14 @@ ADSHANDLE open_shared(ADSHANDLE hConn, const char* name) { UNSIGNED8 tname[64] = {}; std::memcpy(tname, name, std::strlen(name) + 1); // NOLINT ADSHANDLE hT = 0; - REQUIRE(AdsOpenTable(hConn, tname, tname, ADS_CDX, 1, 1, 0, 1, &hT) == 0); + // usMode ADS_SHARED: this helper always opened exclusive (usMode=1), + // which only "worked" while exclusive opens were a silent no-op. With + // mtfix11's SAP-faithful enforcement a second connection's exclusive + // open of the same table is correctly denied, so open what the name + // says. (Argument order: usLockType, usCheckRights, usMode.) + REQUIRE(AdsOpenTable(hConn, tname, tname, ADS_CDX, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, + ADS_SHARED, &hT) == 0); return hT; } @@ -373,7 +380,14 @@ TEST_CASE("lockfull remote: threads on a shared connection don't leak locks") { ADSHANDLE hT = handles[t]; for (int c = 0; c < kCycles; ++c) { const UNSIGNED32 rec = 1 + (c % 5); - UNSIGNED32 rc = AdsLockRecord(hT, rec); + // mtfix6: locks are single-attempt; the application owns + // the retry loop under contention. + UNSIGNED32 rc = 1; + for (int a = 0; a < 500 && rc != 0; ++a) { + rc = AdsLockRecord(hT, rec); + if (rc != 0) + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } if (rc != 0) { fprintf(stderr, "[lockfull] lock rc=%u\n", rc); ++failures; break; } UNSIGNED8 f[] = "VAL"; rc = AdsSetLong(hT, f, c); diff --git a/tests/unit/abi_mt_contention_test.cpp b/tests/unit/abi_mt_contention_test.cpp index 510d5851..47ea2d84 100644 --- a/tests/unit/abi_mt_contention_test.cpp +++ b/tests/unit/abi_mt_contention_test.cpp @@ -283,7 +283,16 @@ TEST_CASE("MT: record lock contention across threads honours the byte lock") { std::atomic b_rc{0xFFFFFFFFu}; std::thread tb([&] { AdsGotoRecord(hTB, 7); - b_rc = AdsLockRecord(hTB, 0); + // mtfix6: locks are single-attempt now - a contended lock fails + // immediately and the application owns the retry loop. Poll like + // an app would until A releases. + UNSIGNED32 rc = 1; + for (int a = 0; a < 400 && rc != 0; ++a) { + rc = AdsLockRecord(hTB, 0); + if (rc != 0) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + b_rc = rc; b_done = 1; }); std::this_thread::sleep_for(std::chrono::milliseconds(500)); @@ -344,8 +353,13 @@ TEST_CASE("MT: readers always see a consistent walk while writers append [flaky] ADS_CHECKRIGHTS, ADS_SHARED, &hTbl) != 0) { continue; } + UNSIGNED16 capacity = 1; if (AdsOpenIndex(hTbl, (UNSIGNED8*)"mt.cdx", &hIdx, - nullptr) != 0) { + &capacity) != 0) { + AdsCloseTable(hTbl); + continue; + } + if (AdsSetIndexOrderByHandle(hTbl, hIdx) != 0) { AdsCloseTable(hTbl); continue; } diff --git a/tests/unit/abi_navigation_test.cpp b/tests/unit/abi_navigation_test.cpp index 9d18b558..37df2597 100644 --- a/tests/unit/abi_navigation_test.cpp +++ b/tests/unit/abi_navigation_test.cpp @@ -262,13 +262,15 @@ TEST_CASE("Nav structural CDX auto-open keeps natural order") { REQUIRE(AdsCloseTable(hT) == 0); } -TEST_CASE("Nav explicit AdsOpenIndex activates the first tag") { +TEST_CASE("Nav explicit focus after AdsOpenIndex walks the first tag") { NavConn c("openads_nav_explidx"); make_zulu5(c.hConn, "nave.dbf", "nave.cdx"); ADSHANDLE hT = open_table(c.hConn, "nave.dbf"); ADSHANDLE hI = 0; - REQUIRE(AdsOpenIndex(hT, (UNSIGNED8*)"nave.cdx", &hI, nullptr) == 0); + UNSIGNED16 capacity = 1; + REQUIRE(AdsOpenIndex(hT, (UNSIGNED8*)"nave.cdx", &hI, &capacity) == 0); + REQUIRE(AdsSetIndexOrderByHandle(hT, hI) == 0); REQUIRE(AdsGotoTop(hT) == 0); check_pos(hT, 2, 0, 0, "gotop (alpha, first key)"); REQUIRE(AdsGotoBottom(hT) == 0); diff --git a/tests/unit/abi_ntx_multitag_test.cpp b/tests/unit/abi_ntx_multitag_test.cpp index 56803fb5..ee33212a 100644 --- a/tests/unit/abi_ntx_multitag_test.cpp +++ b/tests/unit/abi_ntx_multitag_test.cpp @@ -123,7 +123,8 @@ TEST_CASE("M9.14 NTX multi-tag binding: two .ntx files coexist on one table") { REQUIRE(AdsGetRecordNum(hTable, 0, &recno) == 0); CHECK(recno == 2); - // After SEQ activated, GotoTop now walks SEQ order → "01"(rec2). + REQUIRE(AdsSetIndexOrderByHandle(hTable, h_seq) == 0); + // After explicit SEQ focus, GotoTop walks SEQ order → "01"(rec2). REQUIRE(AdsGotoTop(hTable) == 0); REQUIRE(AdsGetRecordNum(hTable, 0, &recno) == 0); CHECK(recno == 2); // SEQ "01" is rec2 @@ -150,7 +151,8 @@ TEST_CASE("M9.14 NTX multi-tag binding: two .ntx files coexist on one table") { REQUIRE(AdsGetRecordNum(hTable, 0, &recno) == 0); CHECK(recno == 4); - // GotoTop on TAGORD → AAAA(rec2) again. + REQUIRE(AdsSetIndexOrderByHandle(hTable, h_first) == 0); + // GotoTop on TAGORD -> AAAA(rec2) again. REQUIRE(AdsGotoTop(hTable) == 0); REQUIRE(AdsGetRecordNum(hTable, 0, &recno) == 0); CHECK(recno == 2); @@ -216,7 +218,8 @@ TEST_CASE("M9.14 NTX multi-tag: AdsOpenIndex re-binds two pre-existing files") { REQUIRE(AdsGetNumIndexes(hTable, &nidx) == 0); CHECK(nidx == 2); - // First-opened binding is the active order: TAG(AAAA) is rec2. + REQUIRE(AdsSetIndexOrderByHandle(hTable, arr1[0]) == 0); + // Explicit focus on first binding: TAG(AAAA) is rec2. REQUIRE(AdsGotoTop(hTable) == 0); UNSIGNED32 recno = 0; REQUIRE(AdsGetRecordNum(hTable, 0, &recno) == 0); diff --git a/tests/unit/abi_pritpal_lock_test.cpp b/tests/unit/abi_pritpal_lock_test.cpp index 82efc65e..dc043579 100644 --- a/tests/unit/abi_pritpal_lock_test.cpp +++ b/tests/unit/abi_pritpal_lock_test.cpp @@ -134,9 +134,9 @@ TEST_CASE("Remote: record lock contention — B's lock fails, does not hang") { // Must have returned promptly (~30ms max), NOT hung forever CHECK(elapsed < 2000); - // Restore defaults + // Restore the default single-attempt policy (mtfix6). REQUIRE(AdsSetLockCycle(0, 100) == 0); - REQUIRE(AdsSetLockRetryCount(0, 10) == 0); + REQUIRE(AdsSetLockRetryCount(0, 0) == 0); // Cleanup REQUIRE(AdsUnlockRecord(hTblA, 1) == 0); @@ -255,9 +255,9 @@ TEST_CASE("Remote: FLock contention — B's FLock fails after retries") { CHECK(rc != 0); // Must NOT succeed CHECK(elapsed < 2000); // Must return promptly - // Restore defaults + // Restore the default single-attempt policy (mtfix6). REQUIRE(AdsSetLockCycle(0, 100) == 0); - REQUIRE(AdsSetLockRetryCount(0, 10) == 0); + REQUIRE(AdsSetLockRetryCount(0, 0) == 0); // Cleanup REQUIRE(AdsUnlockTable(hTblA) == 0); diff --git a/tests/unit/abi_remote_create_table_test.cpp b/tests/unit/abi_remote_create_table_test.cpp index 93461661..7fc69ff6 100644 --- a/tests/unit/abi_remote_create_table_test.cpp +++ b/tests/unit/abi_remote_create_table_test.cpp @@ -8,6 +8,10 @@ #include "network/server.h" #include +#include +#include +#include +#include #include #include #include @@ -134,3 +138,121 @@ TEST_CASE("remote AdsCreateTable with drive-rooted name still under data dir") { REQUIRE(AdsDisconnect(hConn) == 0); fs::remove_all(data, ec); } + +TEST_CASE("CreateTable diagnostic isolates post-write reopen failure") { + using openads::network::Server; + auto data = fs::temp_directory_path() / "openads_create_diag_post_open"; + auto diag = data / "create-diag.log"; + std::error_code ec; + fs::remove_all(data, ec); + fs::create_directories(data); +#if defined(_WIN32) + // Windows MAX_PATH cannot express this long-path reopen fixture. Instead + // force failure at companion memo creation, still after the DBF write. + const std::string rel = "probe.dbf"; + fs::create_directories(data / "probe.fpt"); + // Keep the directory non-empty so the server's pre-create remove() + // cannot erase it; the memo create must then fail on Windows. + std::ofstream(data / "probe.fpt" / "occupy") << "x"; +#else + // The server's ABI create writes the requested relative path, but its + // fixed 260-byte post-create name buffer truncates a longer path. This + // deterministically fails the reopen after the DBF has been written. + std::string rel; + for (int i = 0; i < 55; ++i) rel += "nest/"; + rel += "probe.dbf"; + fs::create_directories(data / fs::path(rel).parent_path()); +#endif +#if defined(_WIN32) + _putenv_s("OPENADS_CREATE_TABLE_DIAG_FILE", diag.string().c_str()); + _putenv_s("OPENADS_CREATE_TABLE_DIAG_FILTER", "probe.dbf"); +#else + setenv("OPENADS_CREATE_TABLE_DIAG_FILE", diag.string().c_str(), 1); + setenv("OPENADS_CREATE_TABLE_DIAG_FILTER", "probe.dbf", 1); +#endif + Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + ADSHANDLE conn = remote_connect(data, srv.port()); + std::vector name(rel.begin(), rel.end()); + name.push_back(0); +#if defined(_WIN32) + UNSIGNED8 fields[] = "ID,Numeric,4,0;NOTE,Memo"; +#else + UNSIGNED8 fields[] = "ID,Numeric,4,0"; +#endif + ADSHANDLE table = 0; + const UNSIGNED32 rc = AdsCreateTable(conn, name.data(), nullptr, ADS_CDX, ADS_ANSI, + 0, 0, 0, fields, &table); + const UNSIGNED32 reported = rc; + UNSIGNED32 last = 0; + UNSIGNED8 message[512]{}; + UNSIGNED16 message_len = sizeof(message); + REQUIRE(AdsGetLastError(&last, message, &message_len) == 0); + CHECK(rc != 0); + CHECK(last == reported); + CHECK(fs::exists(data / rel)); + std::ifstream log(diag); + std::string content((std::istreambuf_iterator(log)), + std::istreambuf_iterator()); + CHECK(content.find("stage=dbf-write-ok") != std::string::npos); +#if defined(_WIN32) + CHECK(content.find("stage=memo-create-fail code=" + + std::to_string(reported)) != std::string::npos); +#else + CHECK(content.find("stage=server-abi-reopen-fail code=" + + std::to_string(reported)) != std::string::npos); +#endif + CHECK(content.find("stage=client-wire-fail code=" + + std::to_string(reported)) != std::string::npos); + CHECK(content.find("NOTE") == std::string::npos); + (void)AdsDisconnect(conn); +#if defined(_WIN32) + _putenv_s("OPENADS_CREATE_TABLE_DIAG_FILE", ""); + _putenv_s("OPENADS_CREATE_TABLE_DIAG_FILTER", ""); +#else + unsetenv("OPENADS_CREATE_TABLE_DIAG_FILE"); + unsetenv("OPENADS_CREATE_TABLE_DIAG_FILTER"); +#endif + fs::remove_all(data, ec); +} + +TEST_CASE("CreateTable broad diagnostic records first-user open and append") { + using openads::network::Server; + auto data = fs::temp_directory_path() / "openads_create_diag_first_user"; + auto diag = data / "create-diag.log"; + std::error_code ec; + fs::remove_all(data, ec); + fs::create_directories(data); +#if defined(_WIN32) + _putenv_s("OPENADS_CREATE_TABLE_DIAG_FILE", diag.string().c_str()); + _putenv_s("OPENADS_CREATE_TABLE_DIAG_FILTER", ""); +#else + setenv("OPENADS_CREATE_TABLE_DIAG_FILE", diag.string().c_str(), 1); + unsetenv("OPENADS_CREATE_TABLE_DIAG_FILTER"); +#endif + Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + ADSHANDLE conn = remote_connect(data, srv.port()); + UNSIGNED8 name[] = "USERCFG.dbf"; + UNSIGNED8 fields[] = "ID,Numeric,4,0"; + ADSHANDLE table = 0; + REQUIRE(AdsCreateTable(conn, name, nullptr, ADS_CDX, ADS_ANSI, + 0, 0, 0, fields, &table) == 0); + REQUIRE(AdsAppendRecord(table) == 0); + REQUIRE(AdsCloseTable(table) == 0); + REQUIRE(AdsDisconnect(conn) == 0); + std::ifstream log(diag); + std::string content((std::istreambuf_iterator(log)), + std::istreambuf_iterator()); + CHECK(content.find("table=USERCFG.dbf stage=dbf-write-ok") != std::string::npos); + CHECK(content.find("table=USERCFG.dbf stage=server-open-ok") != std::string::npos); + CHECK(content.find("table=USERCFG.dbf stage=client-append-enter") != std::string::npos); + CHECK(content.find("table=USERCFG.dbf stage=client-append-ok") != std::string::npos); + CHECK(content.find("table=USERCFG.dbf stage=server-first-append-ok") != std::string::npos); +#if defined(_WIN32) + _putenv_s("OPENADS_CREATE_TABLE_DIAG_FILE", ""); +#else + unsetenv("OPENADS_CREATE_TABLE_DIAG_FILE"); +#endif + fs::remove_all(data, ec); +} diff --git a/tests/unit/abi_remote_index_nav_test.cpp b/tests/unit/abi_remote_index_nav_test.cpp index c7ced1c6..44bee286 100644 --- a/tests/unit/abi_remote_index_nav_test.cpp +++ b/tests/unit/abi_remote_index_nav_test.cpp @@ -914,6 +914,10 @@ TEST_CASE("remote AdsSetScope constrains GotoTop/Skip walk") { }; CHECK(visible_count(hOrd) == 3); + // A table handle after implicit index focus is natural, even while + // that index retains scopes. Explicit focus opts table navigation in. + CHECK(visible_count(hRT) == 4); + REQUIRE(AdsSetIndexOrderByHandle(hRT, hOrd) == 0); CHECK(visible_count(hRT) == 3); REQUIRE(AdsClearScope(hOrd, ADS_TOP) == 0); diff --git a/tests/unit/abi_remote_lock_introspection_test.cpp b/tests/unit/abi_remote_lock_introspection_test.cpp index efaa3b8d..f2835ca3 100644 --- a/tests/unit/abi_remote_lock_introspection_test.cpp +++ b/tests/unit/abi_remote_lock_introspection_test.cpp @@ -5,10 +5,9 @@ // remote handles (ace_exports.cpp returned early for get_remote_table()), // so Harbour dbRecordInfo(DBRI_LOCKED) and dbRLockList() under ADSCDX // always answered .F./{} — Vouch's IsLogged() thread bailed out. -// AdsIsRecordLocked (0x13) answers own-handle state server-side: the -// engine Table's table lock plus the session ABI twin's record locks, -// which include append auto-locks. Whether ANOTHER handle owns a lock is -// deliberately out of scope (that would be a separate global-state API). +// AdsIsRecordLocked answers from the client lock ledger when complete; +// GetAllLocks (0x15) falls back to the server's own-handle list for append +// auto-locks that the client cannot name. The global OS probe stays internal. #include "doctest.h" #include "openads/ace.h" #include "openads/error.h" @@ -81,8 +80,9 @@ TEST_CASE("M12.36 remote AdsIsRecordLocked reflects this connection's locks") { UNSIGNED8 tname[] = "li.dbf"; ADSHANDLE hTable = 0; - REQUIRE(AdsOpenTable(hConn, tname, nullptr, ADS_CDX, ADS_ANSI, ADS_SHARED, - ADS_COMPATIBLE_LOCKING, ADS_DEFAULT, &hTable) + REQUIRE(AdsOpenTable(hConn, tname, nullptr, ADS_CDX, ADS_ANSI, +ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, +ADS_SHARED, &hTable) == AE_SUCCESS); // Nothing locked yet. @@ -128,8 +128,9 @@ TEST_CASE("M12.36 remote AdsGetAllLocks enumerates held record locks") { UNSIGNED8 tname[] = "li.dbf"; ADSHANDLE hTable = 0; - REQUIRE(AdsOpenTable(hConn, tname, nullptr, ADS_CDX, ADS_ANSI, ADS_SHARED, - ADS_COMPATIBLE_LOCKING, ADS_DEFAULT, &hTable) + REQUIRE(AdsOpenTable(hConn, tname, nullptr, ADS_CDX, ADS_ANSI, +ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, +ADS_SHARED, &hTable) == AE_SUCCESS); REQUIRE(AdsLockRecord(hTable, 2) == AE_SUCCESS); diff --git a/tests/unit/abi_setorder_zero_test.cpp b/tests/unit/abi_setorder_zero_test.cpp index 16812e06..40f37ed1 100644 --- a/tests/unit/abi_setorder_zero_test.cpp +++ b/tests/unit/abi_setorder_zero_test.cpp @@ -88,7 +88,9 @@ void exercise_setorder_zero(ADSHANDLE hConn) { REQUIRE(AdsGetIndexHandleByOrder(hT, 1, &hOrd) == 0); REQUIRE(hOrd != 0); - // Activate the order via index-handle navigation (the rddads pattern). + // Explicit focus opts table-handle navigation into indexed order. + // Stock rddads implicit focus is covered by abi_index_focus_contract_test. + REQUIRE(AdsSetIndexOrderByHandle(hT, hOrd) == 0); REQUIRE(AdsGotoTop(hOrd) == 0); CHECK(walk_recnos(hT) == std::vector( std::begin(kIndexed), std::end(kIndexed))); diff --git a/tests/unit/network_boundary_pair_test.cpp b/tests/unit/network_boundary_pair_test.cpp new file mode 100644 index 00000000..a596d479 --- /dev/null +++ b/tests/unit/network_boundary_pair_test.cpp @@ -0,0 +1,272 @@ +// tests/unit/network_boundary_pair_test.cpp +// mtfix12 R1/R2/R3 — boundary-pair certification, self-GotoRecord +// suppression, and GetRecordNum anchoring, end to end over a loopback +// server: per-opcode frame counts prove which calls hit the wire, and +// reads prove the locally-served state is what the wire would have +// returned. +#include "doctest.h" +#include "network/client.h" +#include "network/server.h" +#include "openads/ace.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +std::atomic g_top{0}, g_bottom{0}, g_goto{0}, g_recnum{0}, g_keycount{0}; + +void count_nav(const void*, std::uint8_t op, std::uint32_t, std::size_t, + std::uint8_t, std::size_t, long long) { + if (op == 0x40) g_top.fetch_add(1); // GotoTop + if (op == 0x64) g_bottom.fetch_add(1); // GotoBottom + if (op == 0x58) g_goto.fetch_add(1); // GotoRecord + if (op == 0x4E) g_recnum.fetch_add(1); + if (op == 0xB0) g_keycount.fetch_add(1); // GetKeyCount +} + +void reset_counts() { + g_top = 0; g_bottom = 0; g_goto = 0; g_recnum = 0; g_keycount = 0; +} + +std::uint32_t recno_of(ADSHANDLE h) { + UNSIGNED32 n = 0; + REQUIRE(AdsGetRecordNum(h, 0, &n) == AE_SUCCESS); + return n; +} + +std::string id_field(ADSHANDLE h) { + char buf[64] = {}; + UNSIGNED32 len = sizeof(buf) - 1; + UNSIGNED8 fld[] = "ID"; + REQUIRE(AdsGetField(h, fld, reinterpret_cast(buf), &len, + ADS_NONE) == AE_SUCCESS); + return std::string(buf, len); +} + +} // namespace + +TEST_CASE("boundary pair, self-goto, and recno anchor over loopback") { + auto dir = fs::temp_directory_path() / "openads_bpair"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + + // Seed two 5-row tables locally. + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC0 = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hC0) + == AE_SUCCESS); + UNSIGNED8 def[] = "ID,N,8,0"; + for (const char* nm : {"BP1.DBF", "BP2.DBF"}) { + UNSIGNED8 tn[64]{}; + std::memcpy(tn, nm, std::strlen(nm) + 1); + ADSHANDLE h = 0; + REQUIRE(AdsCreateTable(hC0, tn, nullptr, ADS_CDX, 0, 0, 0, 0, def, + &h) == AE_SUCCESS); + for (int i = 1; i <= 5; ++i) { + REQUIRE(AdsAppendRecord(h) == AE_SUCCESS); + char v[16]; + std::snprintf(v, sizeof(v), "%d", i); + UNSIGNED8 fld[] = "ID"; + REQUIRE(AdsSetField(h, fld, + reinterpret_cast(v), + static_cast(std::strlen(v))) + == AE_SUCCESS); + } + REQUIRE(AdsCloseTable(h) == AE_SUCCESS); + } + REQUIRE(AdsDisconnect(hC0) == AE_SUCCESS); + + openads::network::Server s; + REQUIRE(s.start("127.0.0.1", 0).has_value()); + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(s.port()), dir.string().c_str()); + UNSIGNED8 sb[512]{}; + std::memcpy(sb, uri, std::strlen(uri) + 1); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(sb, ADS_REMOTE_SERVER, nullptr, nullptr, 0, &hC) + == AE_SUCCESS); + + UNSIGNED8 t1[] = "BP1.DBF"; + UNSIGNED8 t2[] = "BP2.DBF"; + ADSHANDLE hT = 0, hU = 0; + REQUIRE(AdsOpenTable(hC, t1, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + REQUIRE(AdsOpenTable(hC, t2, nullptr, ADS_CDX, 0, 0, 0, 0, &hU) + == AE_SUCCESS); + + openads::network::set_frame_trace_hook(&count_nav); + + // Settle: position somewhere known; whatever it costs is not counted. + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hU) == AE_SUCCESS); + reset_counts(); + + // R2 — self-GotoRecord: first GO wires and certifies the anchor, + // the repeat is served locally. + REQUIRE(AdsGotoRecord(hT, 3) == AE_SUCCESS); + CHECK(g_goto.load() == 1u); + CHECK(recno_of(hT) == 3u); + CHECK(g_recnum.load() == 0u); // R3: from the anchor + REQUIRE(AdsGotoRecord(hT, 3) == AE_SUCCESS); + CHECK(g_goto.load() == 1u); // no new frame + CHECK(recno_of(hT) == 3u); + + // R1 — the GotoTop ack certified the bottom in the same visit. + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(g_top.load() == 1u); + CHECK(recno_of(hT) == 1u); + REQUIRE(AdsGotoBottom(hT) == AE_SUCCESS); + CHECK(g_bottom.load() == 0u); // served from the pair blob + CHECK(recno_of(hT) == 5u); // and it IS the bottom row + CHECK(id_field(hT) == "5"); + UNSIGNED16 atEof = 1; + REQUIRE(AdsAtEOF(hT, &atEof) == AE_SUCCESS); + CHECK(atEof == 0); + + // The pair serve stamped bottom; a consecutive bottom is the old + // duplicate path, and the top that follows re-wires (no fresh + // certification for top was made by the LOCAL bottom serve... the + // certification from the earlier GotoBottom... none happened, so + // this top must wire and re-certify bottom). + REQUIRE(AdsGotoBottom(hT) == AE_SUCCESS); + CHECK(g_bottom.load() == 0u); // duplicate of the pair serve + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(g_top.load() == 2u); // wire: re-certifies bottom + REQUIRE(AdsGotoBottom(hT) == AE_SUCCESS); + CHECK(g_bottom.load() == 0u); // pair again + CHECK(recno_of(hT) == 5u); + + // A write on this table kills the certification (the blob's row + // bytes predate it) and the frame expires the conn-wide seq anyway. + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(g_top.load() == 3u); // wire: certify bottom again + REQUIRE(AdsLockRecord(hT, 0) == AE_SUCCESS); + UNSIGNED8 fld[] = "ID"; + UNSIGNED8 seven[] = "7"; + REQUIRE(AdsSetField(hT, fld, seven, 1) == AE_SUCCESS); + REQUIRE(AdsUnlockRecord(hT, 0) == AE_SUCCESS); + REQUIRE(AdsGotoBottom(hT) == AE_SUCCESS); + CHECK(g_bottom.load() == 1u); // back on the wire + CHECK(recno_of(hT) == 5u); + CHECK(id_field(hT) == "5"); // row 5 untouched by the write + + // The wire GotoBottom also certified the TOP: this GotoTop is + // itself pair-served (symmetric certification). + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + CHECK(g_top.load() == 3u); // pair-served from the bottom + CHECK(recno_of(hT) == 1u); + // Conn-wide envelope: nav on ANOTHER table expires the pair. + REQUIRE(AdsGotoRecord(hU, 2) == AE_SUCCESS); // other table's nav + REQUIRE(AdsGotoBottom(hT) == AE_SUCCESS); + CHECK(g_bottom.load() == 2u); // wire: conn seq moved + + openads::network::set_frame_trace_hook(nullptr); + + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hU) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + s.stop(); + fs::remove_all(dir, ec); +} + +TEST_CASE("boundary pair: scoped ordered bottom counts on demand") { + auto dir = fs::temp_directory_path() / "openads_bpair_scoped"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + UNSIGNED8 path[512]{}; + std::memcpy(path, dir.string().c_str(), dir.string().size()); + ADSHANDLE local = 0; + REQUIRE(AdsConnect60(path, ADS_LOCAL_SERVER, nullptr, nullptr, 0, + &local) == AE_SUCCESS); + UNSIGNED8 tn[] = "SCOPE.DBF"; + UNSIGNED8 def[] = "GRP,C,1,0;ID,C,2,0"; + ADSHANDLE tbl = 0; + REQUIRE(AdsCreateTable(local, tn, nullptr, ADS_CDX, 0, 0, 0, 0, + def, &tbl) == AE_SUCCESS); + UNSIGNED8 grp[] = "GRP", id[] = "ID"; + for (const char* value : {"A1", "A2", "A3", "B4"}) { + REQUIRE(AdsAppendRecord(tbl) == AE_SUCCESS); + REQUIRE(AdsSetField(tbl, grp, (UNSIGNED8*)value, 1) == AE_SUCCESS); + REQUIRE(AdsSetField(tbl, id, (UNSIGNED8*)(value + 1), 1) + == AE_SUCCESS); + REQUIRE(AdsWriteRecord(tbl) == AE_SUCCESS); + } + // One deleted A row makes physical count (4), scoped count (3) + // and scoped SET DELETED count (2) distinct. + REQUIRE(AdsGotoRecord(tbl, 2) == AE_SUCCESS); + REQUIRE(AdsDeleteRecord(tbl) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(tbl) == AE_SUCCESS); + UNSIGNED8 bag[] = "SCOPE.CDX", tag[] = "BYGRP", expr[] = "GRP+ID"; + ADSHANDLE idx = 0; + REQUIRE(AdsCreateIndex61(tbl, bag, tag, expr, nullptr, nullptr, + ADS_COMPOUND, 512, &idx) == AE_SUCCESS); + REQUIRE(AdsCloseTable(tbl) == AE_SUCCESS); + REQUIRE(AdsDisconnect(local) == AE_SUCCESS); + + openads::network::Server server; + REQUIRE(server.start("127.0.0.1", 0).has_value()); + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(server.port()), dir.string().c_str()); + UNSIGNED8 sb[512]{}; + std::memcpy(sb, uri, std::strlen(uri) + 1); + ADSHANDLE conn = 0; + REQUIRE(AdsConnect60(sb, ADS_REMOTE_SERVER, nullptr, nullptr, 0, + &conn) == AE_SUCCESS); + REQUIRE(AdsOpenTable(conn, tn, nullptr, ADS_CDX, 0, 0, 0, 0, + &tbl) == AE_SUCCESS); + REQUIRE(AdsGetIndexHandleByOrder(tbl, 1, &idx) == AE_SUCCESS); + REQUIRE(idx != 0); + UNSIGNED8 a[] = "A"; + REQUIRE(AdsSetScope(idx, ADS_TOP, a, 1, ADS_STRINGKEY) == AE_SUCCESS); + REQUIRE(AdsSetScope(idx, ADS_BOTTOM, a, 1, ADS_STRINGKEY) == AE_SUCCESS); + REQUIRE(AdsShowDeleted(0) == AE_SUCCESS); + openads::network::set_frame_trace_hook(&count_nav); + reset_counts(); + REQUIRE(AdsGotoTop(idx) == AE_SUCCESS); + CHECK(recno_of(tbl) == 1u); + CHECK(g_keycount.load() == 0u); // no eager count from a plain GoTop + REQUIRE(AdsGotoBottom(idx) == AE_SUCCESS); + CHECK(g_bottom.load() == 0u); // opposite landing served from pair + CHECK(recno_of(tbl) == 3u); // deleted row 2 skipped + // A cold scoped order count is fetched only when Bottom is consumed. + CHECK(g_keycount.load() == 1u); + UNSIGNED32 kc = 0; + REQUIRE(AdsGetKeyCount(idx, 0, &kc) == AE_SUCCESS); + CHECK(kc == 2u); // not physical 4 or scoped 3 + CHECK(g_keycount.load() == 1u); // cached after bottom + REQUIRE(AdsGotoTop(idx) == AE_SUCCESS); + CHECK(recno_of(tbl) == 1u); + + // Scope to no rows; top and bottom agree on BOF+EOF without + // treating a physical row or deleted key as visible. + UNSIGNED8 z[] = "Z"; + REQUIRE(AdsSetScope(idx, ADS_TOP, z, 1, ADS_STRINGKEY) == AE_SUCCESS); + REQUIRE(AdsSetScope(idx, ADS_BOTTOM, z, 1, ADS_STRINGKEY) == AE_SUCCESS); + REQUIRE(AdsGotoTop(idx) == AE_SUCCESS); + UNSIGNED16 b = 0, e = 0; + REQUIRE(AdsAtBOF(tbl, &b) == AE_SUCCESS); + REQUIRE(AdsAtEOF(tbl, &e) == AE_SUCCESS); + CHECK(b == 1); CHECK(e == 1); + REQUIRE(AdsGotoBottom(idx) == AE_SUCCESS); + REQUIRE(AdsAtBOF(tbl, &b) == AE_SUCCESS); + REQUIRE(AdsAtEOF(tbl, &e) == AE_SUCCESS); + CHECK(b == 1); CHECK(e == 1); + REQUIRE(AdsGetKeyCount(idx, 0, &kc) == AE_SUCCESS); + CHECK(kc == 0u); + openads::network::set_frame_trace_hook(nullptr); + REQUIRE(AdsShowDeleted(1) == AE_SUCCESS); + REQUIRE(AdsCloseTable(tbl) == AE_SUCCESS); + REQUIRE(AdsDisconnect(conn) == AE_SUCCESS); + server.stop(); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/network_commit_slimming_test.cpp b/tests/unit/network_commit_slimming_test.cpp new file mode 100644 index 00000000..dc912753 --- /dev/null +++ b/tests/unit/network_commit_slimming_test.cpp @@ -0,0 +1,359 @@ +// tests/unit/network_commit_slimming_test.cpp +// Commit-path slimming for the Vouch WAN voucher save (mtfix8): +// a) AdsUnlockTable with an empty client lock ledger sends no frame +// (rddads dbUnlock() calls it blindly before every lock/append); +// b) AdsGetAllLocks answers from the ledger while it is provably +// complete (no append auto-lock outstanding, no table lock); +// c) AdsWriteRecord on a clean table sends no FlushTable frame +// (DBCOMMITALL touches every workarea), and on a +// kCapFlushTableDurable server the trailing AdsFlushFileBuffers +// owes nothing either -- one frame per commit; +// d) a table whose locks were all released parks at close instead +// of real-closing (the blunt ever_locked flag used to force a +// 7-frame reopen per save); +// e) File and directory existence always re-probe the server; external +// sessions and host operations may change them. +// mtfix10: +// f) AdsGetNumLocks shares the GetAllLocks ledger fast path (rddads +// polls it after every commit/unlock -- it used to ride the wire +// op even when the ledger was provably complete); +// g) FileExists no longer uses an open-table shortcut: it checks the +// exact path even when a table or index with that stem is open. +#include "doctest.h" +#include "mgmt/mg_stats.h" +#include "network/server.h" +#include "openads/ace.h" + +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +constexpr std::uint8_t kOpOpenTable = 0x20; +constexpr std::uint8_t kOpCloseTable = 0x22; +constexpr std::uint8_t kOpGetAllLocks = 0x15; +constexpr std::uint8_t kOpFlushTable = 0x5A; +constexpr std::uint8_t kOpUnlockTable = 0x78; +constexpr std::uint8_t kOpFlushFile = 0x7E; +constexpr std::uint8_t kOpFileExists = 0xE0; +constexpr std::uint8_t kOpDirExist = 0xEC; +constexpr std::uint8_t kOpDirMake = 0xEE; + +fs::path cs_tmp_dir() { + return fs::temp_directory_path() / "openads_commit_slim_test"; +} + +void cs_wipe() { + std::error_code ec; + fs::remove_all(cs_tmp_dir(), ec); + fs::create_directories(cs_tmp_dir(), ec); +} + +void cs_seed(const fs::path& dir) { + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + UNSIGNED8 def[] = "ID,N,8,0"; + UNSIGNED8 tname[] = "CS.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hConn, tname, nullptr, ADS_CDX, 0, 0, 0, 0, def, + &hT) == AE_SUCCESS); + UNSIGNED8 fld[] = "ID"; + for (int i = 1; i <= 5; ++i) { + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + REQUIRE(AdsSetDouble(hT, fld, i * 10) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); + } + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); +} + +std::uint64_t cs_op(std::uint8_t op) { + return openads::mgmt::process_mg_stats() + .op_timing[op].count.load(std::memory_order_relaxed); +} + +ADSHANDLE cs_connect_remote(const fs::path& dir, std::uint16_t port) { + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(port), dir.string().c_str()); + UNSIGNED8 srvbuf[512]{}; + std::memcpy(srvbuf, uri, std::strlen(uri) + 1); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srvbuf, ADS_REMOTE_SERVER, nullptr, nullptr, 0, + &hConn) == AE_SUCCESS); + return hConn; +} + +ADSHANDLE cs_open(ADSHANDLE hConn) { + UNSIGNED8 tname[] = "CS.DBF"; + ADSHANDLE hTable = 0; + REQUIRE(AdsOpenTable(hConn, tname, nullptr, ADS_CDX, ADS_ANSI, ADS_SHARED, + ADS_COMPATIBLE_LOCKING, ADS_DEFAULT, &hTable) + == AE_SUCCESS); + return hTable; +} + +} // namespace + +TEST_CASE("Commit slimming: blind UnlockTable and GetAllLocks stay local") { + cs_wipe(); + auto dir = cs_tmp_dir(); + cs_seed(dir); + + openads::network::Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + ADSHANDLE hConn = cs_connect_remote(dir, srv.port()); + ADSHANDLE hTable = cs_open(hConn); + + // Fresh table, nothing held: dbUnlock() costs nothing. + std::uint64_t un0 = cs_op(kOpUnlockTable); + REQUIRE(AdsUnlockTable(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpUnlockTable) == un0); + + // A held lock forces the real frame; then the ledger is clear and + // the next blind unlock is free again. + REQUIRE(AdsLockRecord(hTable, 1) == AE_SUCCESS); + REQUIRE(AdsUnlockTable(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpUnlockTable) == un0 + 1); + REQUIRE(AdsUnlockTable(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpUnlockTable) == un0 + 1); + + // GetAllLocks answers from the ledger with the right recnos. + REQUIRE(AdsLockRecord(hTable, 1) == AE_SUCCESS); + REQUIRE(AdsLockRecord(hTable, 3) == AE_SUCCESS); + const std::uint64_t gal0 = cs_op(kOpGetAllLocks); + UNSIGNED32 recs[8]{}; + UNSIGNED16 n = 8; + REQUIRE(AdsGetAllLocks(hTable, recs, &n) == AE_SUCCESS); + CHECK(cs_op(kOpGetAllLocks) == gal0); + CHECK(n == 2u); + + // GetNumLocks shares the ledger fast path (mtfix10). + UNSIGNED16 nl = 0; + REQUIRE(AdsGetNumLocks(hTable, &nl) == AE_SUCCESS); + CHECK(cs_op(kOpGetAllLocks) == gal0); + CHECK(nl == 2u); + bool saw1 = false, saw3 = false; + for (UNSIGNED16 i = 0; i < n && i < 8; ++i) { + saw1 = saw1 || recs[i] == 1u; + saw3 = saw3 || recs[i] == 3u; + } + CHECK(saw1); + CHECK(saw3); + + // An append auto-locks server-side with no recno in the ack: the + // ledger goes uncertain, GetAllLocks returns to the wire, and the + // following UnlockTable really goes out (and releases it). + REQUIRE(AdsAppendRecord(hTable) == AE_SUCCESS); + n = 8; + REQUIRE(AdsGetAllLocks(hTable, recs, &n) == AE_SUCCESS); + CHECK(cs_op(kOpGetAllLocks) == gal0 + 1); + CHECK(n >= 3u); // 1, 3 and the auto-locked blank + nl = 0; + REQUIRE(AdsGetNumLocks(hTable, &nl) == AE_SUCCESS); + CHECK(cs_op(kOpGetAllLocks) == gal0 + 2); // uncertain: back to wire + CHECK(nl >= 3u); + un0 = cs_op(kOpUnlockTable); + REQUIRE(AdsUnlockTable(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpUnlockTable) == un0 + 1); + + // Ledger provably empty again: local answer, zero locks. + n = 8; + REQUIRE(AdsGetAllLocks(hTable, recs, &n) == AE_SUCCESS); + CHECK(cs_op(kOpGetAllLocks) == gal0 + 2); + CHECK(n == 0u); + nl = 99; + REQUIRE(AdsGetNumLocks(hTable, &nl) == AE_SUCCESS); + CHECK(cs_op(kOpGetAllLocks) == gal0 + 2); + CHECK(nl == 0u); + + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + srv.stop(); +} + +TEST_CASE("Commit slimming: clean commit and durable FlushTable") { + cs_wipe(); + auto dir = cs_tmp_dir(); + cs_seed(dir); + + openads::network::Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + ADSHANDLE hConn = cs_connect_remote(dir, srv.port()); + ADSHANDLE hTable = cs_open(hConn); + + // Nothing written since open: COMMIT sends no FlushTable. + const std::uint64_t ft0 = cs_op(kOpFlushTable); + const std::uint64_t ff0 = cs_op(kOpFlushFile); + REQUIRE(AdsWriteRecord(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpFlushTable) == ft0); + + // A real write: FlushTable goes out once, and the durable server + // makes the trailing FlushFileBuffers a no-op. + UNSIGNED8 fld[] = "ID"; + REQUIRE(AdsLockRecord(hTable, 2) == AE_SUCCESS); + REQUIRE(AdsSetDouble(hTable, fld, 99.0) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpFlushTable) == ft0 + 1); + REQUIRE(AdsFlushFileBuffers(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpFlushFile) == ff0); + REQUIRE(AdsUnlockTable(hTable) == AE_SUCCESS); + + // The write really landed: a fresh open reads 99 back. + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + ADSHANDLE hT2 = cs_open(hConn); + REQUIRE(AdsGotoRecord(hT2, 2) == AE_SUCCESS); + double val = 0.0; + REQUIRE(AdsGetDouble(hT2, fld, &val) == AE_SUCCESS); + CHECK(val == 99.0); + REQUIRE(AdsCloseTable(hT2) == AE_SUCCESS); + + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + srv.stop(); +} + +TEST_CASE("Commit slimming: released-lock tables park at close") { + cs_wipe(); + auto dir = cs_tmp_dir(); + cs_seed(dir); + + openads::network::Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + ADSHANDLE hConn = cs_connect_remote(dir, srv.port()); + ADSHANDLE hTable = cs_open(hConn); + + // Lock, unlock, close: nothing held at close, so the table parks + // (no CloseTable frame) and the reopen is a park hit. + REQUIRE(AdsLockRecord(hTable, 4) == AE_SUCCESS); + REQUIRE(AdsUnlockTable(hTable) == AE_SUCCESS); + const std::uint64_t cl0 = cs_op(kOpCloseTable); + const std::uint64_t op0 = cs_op(kOpOpenTable); + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + CHECK(cs_op(kOpCloseTable) == cl0); + + ADSHANDLE hT2 = cs_open(hConn); + CHECK(cs_op(kOpOpenTable) == op0); + REQUIRE(AdsGotoRecord(hT2, 4) == AE_SUCCESS); + UNSIGNED8 fld[] = "ID"; + double val = 0.0; + REQUIRE(AdsGetDouble(hT2, fld, &val) == AE_SUCCESS); + CHECK(val == 40.0); + REQUIRE(AdsCloseTable(hT2) == AE_SUCCESS); + + // Append + write + unlock + close: the auto-lock is released, so + // this parks too -- the old ever_locked rule real-closed here. + ADSHANDLE hT3 = cs_open(hConn); + REQUIRE(AdsAppendRecord(hT3) == AE_SUCCESS); + REQUIRE(AdsSetDouble(hT3, fld, 60.0) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hT3) == AE_SUCCESS); + REQUIRE(AdsUnlockTable(hT3) == AE_SUCCESS); + const std::uint64_t cl1 = cs_op(kOpCloseTable); + REQUIRE(AdsCloseTable(hT3) == AE_SUCCESS); + CHECK(cs_op(kOpCloseTable) == cl1); + + // A lock STILL held at close keeps the real close (no park leak). + ADSHANDLE hT4 = cs_open(hConn); + REQUIRE(AdsLockRecord(hT4, 1) == AE_SUCCESS); + const std::uint64_t cl2 = cs_op(kOpCloseTable); + REQUIRE(AdsCloseTable(hT4) == AE_SUCCESS); + CHECK(cs_op(kOpCloseTable) == cl2 + 1); + + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + srv.stop(); +} + +TEST_CASE("Commit slimming: directory state always reflects server changes") { + cs_wipe(); + auto dir = cs_tmp_dir(); + cs_seed(dir); + + openads::network::Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + srv.set_enable_file_func(true); + ADSHANDLE hConn = cs_connect_remote(dir, srv.port()); + + const std::uint64_t de0 = cs_op(kOpDirExist); + const std::uint64_t dm0 = cs_op(kOpDirMake); + UNSIGNED16 ex = 0; + REQUIRE(AdsDirExist(hConn, (UNSIGNED8*)".", &ex) == AE_SUCCESS); + CHECK(ex == 1u); + REQUIRE(AdsDirExist(hConn, (UNSIGNED8*)".", &ex) == AE_SUCCESS); + CHECK(ex == 1u); + CHECK(cs_op(kOpDirExist) == de0 + 2); + REQUIRE(AdsDirMake(hConn, (UNSIGNED8*)".") == AE_SUCCESS); + CHECK(cs_op(kOpDirMake) == dm0 + 1); + + REQUIRE(AdsDirMake(hConn, (UNSIGNED8*)"cs_sub") == AE_SUCCESS); + CHECK(cs_op(kOpDirMake) == dm0 + 2); + REQUIRE(fs::is_directory(dir / "cs_sub")); + REQUIRE(AdsDirExist(hConn, (UNSIGNED8*)"cs_sub", &ex) == AE_SUCCESS); + CHECK(ex == 1u); + CHECK(cs_op(kOpDirExist) == de0 + 3); + + // Simulate host maintenance or a second client: deletion bypasses this + // connection's DirRemove, so its next probe must reach the server. + std::error_code ec; + REQUIRE(fs::remove(dir / "cs_sub", ec)); + REQUIRE_FALSE(ec); + REQUIRE(AdsDirExist(hConn, (UNSIGNED8*)"cs_sub", &ex) == AE_SUCCESS); + CHECK(ex == 0u); + CHECK(cs_op(kOpDirExist) == de0 + 4); + REQUIRE(AdsDirMake(hConn, (UNSIGNED8*)"cs_sub") == AE_SUCCESS); + CHECK(cs_op(kOpDirMake) == dm0 + 3); + CHECK(fs::is_directory(dir / "cs_sub")); + REQUIRE(AdsDirExist(hConn, (UNSIGNED8*)"cs_sub", &ex) == AE_SUCCESS); + CHECK(ex == 1u); + CHECK(cs_op(kOpDirExist) == de0 + 5); + + // A repeat mkdir must still reach the server (and remain idempotent), + // not claim success from an obsolete positive answer. + REQUIRE(fs::remove(dir / "cs_sub", ec)); + REQUIRE_FALSE(ec); + REQUIRE(AdsDirMake(hConn, (UNSIGNED8*)"cs_sub") == AE_SUCCESS); + CHECK(cs_op(kOpDirMake) == dm0 + 4); + CHECK(fs::is_directory(dir / "cs_sub")); + REQUIRE(AdsDirRemove(hConn, (UNSIGNED8*)"cs_sub") == AE_SUCCESS); + + // FileExists is also a live probe. A positive answer for an open table + // must not bypass the server, nor may a missing bag be cached. + ADSHANDLE hTable = cs_open(hConn); + const std::uint64_t fe0 = cs_op(kOpFileExists); + REQUIRE(AdsCheckExistence(hConn, (UNSIGNED8*)"CS.DBF", &ex) + == AE_SUCCESS); + CHECK(ex == 1u); + CHECK(cs_op(kOpFileExists) == fe0 + 1); + REQUIRE(AdsCheckExistence(hConn, (UNSIGNED8*)"CS.Z01", &ex) + == AE_SUCCESS); + CHECK(ex == 0u); + REQUIRE(AdsCheckExistence(hConn, (UNSIGNED8*)"CS.Z01", &ex) + == AE_SUCCESS); + CHECK(ex == 0u); + CHECK(cs_op(kOpFileExists) == fe0 + 3); + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + + // Positive-to-missing and missing-to-positive after external changes + // on the server, without a mutation through this connection. + REQUIRE(AdsCheckExistence(hConn, (UNSIGNED8*)"cs_sub", &ex) + == AE_SUCCESS); + CHECK(ex == 0u); + REQUIRE(fs::create_directory(dir / "cs_sub", ec)); + REQUIRE_FALSE(ec); + REQUIRE(AdsCheckExistence(hConn, (UNSIGNED8*)"cs_sub", &ex) + == AE_SUCCESS); + CHECK(ex == 1u); + REQUIRE(fs::remove(dir / "cs_sub", ec)); + REQUIRE_FALSE(ec); + REQUIRE(AdsCheckExistence(hConn, (UNSIGNED8*)"cs_sub", &ex) + == AE_SUCCESS); + CHECK(ex == 0u); + CHECK(cs_op(kOpFileExists) == fe0 + 6); + + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + srv.stop(); +} diff --git a/tests/unit/network_empty_window_test.cpp b/tests/unit/network_empty_window_test.cpp new file mode 100644 index 00000000..2b0f2c8f --- /dev/null +++ b/tests/unit/network_empty_window_test.cpp @@ -0,0 +1,119 @@ +// tests/unit/network_empty_window_test.cpp +// Short "still empty" window (patch 2): after the server certifies a +// table physically empty, Seek / GO n are answered locally for up to +// OPENADS_EMPTY_TTL_MS (default 1500 ms). Own writes close the window +// immediately; after the window the wire is used again. +#include "doctest.h" +#include "mgmt/mg_stats.h" +#include "network/server.h" +#include "openads/ace.h" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +std::uint64_t ew_op(std::uint8_t op) { + return openads::mgmt::process_mg_stats() + .op_timing[op] + .count.load(std::memory_order_relaxed); +} + +} // namespace + +TEST_CASE("Remote empty window: local not-found, closes on own write/TTL") { + auto dir = fs::temp_directory_path() / "openads_emptywin"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC0 = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hC0) + == AE_SUCCESS); + UNSIGNED8 def[] = "ID,N,8,0"; + UNSIGNED8 tn0[] = "EW.DBF"; + ADSHANDLE hT0 = 0; + REQUIRE(AdsCreateTable(hC0, tn0, nullptr, ADS_CDX, 0, 0, 0, 0, def, &hT0) + == AE_SUCCESS); + ADSHANDLE hI0 = 0; + UNSIGNED8 bag[] = "EW.CDX"; + UNSIGNED8 tag[] = "BYID"; + UNSIGNED8 exp[] = "STR(ID,8)"; + REQUIRE(AdsCreateIndex61(hT0, bag, tag, exp, nullptr, nullptr, + ADS_COMPOUND, 512, &hI0) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT0) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC0) == AE_SUCCESS); + + openads::network::Server s; + REQUIRE(s.start("127.0.0.1", 0).has_value()); + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(s.port()), dir.string().c_str()); + UNSIGNED8 sb[512]{}; + std::memcpy(sb, uri, std::strlen(uri) + 1); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(sb, ADS_REMOTE_SERVER, nullptr, nullptr, 0, &hC) + == AE_SUCCESS); + + UNSIGNED8 tn[] = "EW.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + ADSHANDLE hOrd = 0; + UNSIGNED8 want[] = "BYID"; + REQUIRE(AdsGetIndexHandle(hT, want, &hOrd) == AE_SUCCESS); + REQUIRE(AdsSetIndexOrderByHandle(hT, hOrd) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hOrd) == AE_SUCCESS); // server certifies empty + + UNSIGNED8 key[] = " 7"; + const auto sk0 = ew_op(0x90); + const auto gr0 = ew_op(0x58); + UNSIGNED16 found = 9; + REQUIRE(AdsSeek(hOrd, key, 8, ADS_STRINGKEY, ADS_HARDSEEK, &found) + == AE_SUCCESS); + CHECK(found == 0); + REQUIRE(AdsGotoRecord(hT, 1) == AE_SUCCESS); + CHECK(ew_op(0x90) - sk0 == 0u); + CHECK(ew_op(0x58) - gr0 == 0u); + UNSIGNED16 b = 0, e = 0; + REQUIRE(AdsAtBOF(hT, &b) == AE_SUCCESS); + REQUIRE(AdsAtEOF(hT, &e) == AE_SUCCESS); + CHECK(b == 1); + CHECK(e == 1); + + SUBCASE("own append closes the window") { + UNSIGNED8 f1[] = "ID"; + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + REQUIRE(AdsSetDouble(hT, f1, 7) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); + const auto sk1 = ew_op(0x90); + found = 0; + REQUIRE(AdsSeek(hOrd, key, 8, ADS_STRINGKEY, ADS_HARDSEEK, &found) + == AE_SUCCESS); + CHECK(ew_op(0x90) - sk1 == 1u); + CHECK(found == 1); + } + + SUBCASE("window ends after the TTL") { + std::this_thread::sleep_for(std::chrono::milliseconds(2100)); + const auto sk1 = ew_op(0x90); + found = 9; + REQUIRE(AdsSeek(hOrd, key, 8, ADS_STRINGKEY, ADS_HARDSEEK, &found) + == AE_SUCCESS); + CHECK(ew_op(0x90) - sk1 == 1u); + CHECK(found == 0); + } + + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + s.stop(); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/network_exclusive_open_test.cpp b/tests/unit/network_exclusive_open_test.cpp new file mode 100644 index 00000000..6563b5f6 --- /dev/null +++ b/tests/unit/network_exclusive_open_test.cpp @@ -0,0 +1,194 @@ +// network_exclusive_open_test.cpp -- mtfix11: ADS_EXCLUSIVE open +// enforcement across wire sessions. +// +// Root cause pinned here: the drivers map DriverOpenMode::Exclusive to a +// plain open (POSIX has no share modes), so an exclusive hold was a +// silent no-op - a second instance's opens sailed through a reindex/pack +// exclusive hold that SAP ADS would have denied, and the app's +// "only one user" startup abort never fired. The Server now keeps an +// open registry keyed by the engine's resolved path and denies +// conflicting opens with 7040 AE_FILE_IN_USE. +// +// Client park-pool interaction: our AdsCloseTable PARKS poolable shared +// tables (the server handle stays open for fast reuse), so a shared +// close does NOT release the server-side registration - the table IS +// still open on that session, and the registry correctly keeps blocking +// a peer's exclusive request. Exclusive tables are never parked, and a +// disconnect sweeps every registration the session still holds; those +// are the two release paths this test drives. +#include "doctest.h" +#include "openads/ace.h" +#include "openads/error.h" +#include "network/server.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using openads::AE_SUCCESS; + +namespace { + +fs::path excl_tmp_dir() { + return fs::temp_directory_path() / "openads_remote_excl_open"; +} + +// Recreates /ex.dbf with 3 rows, NAME C(8). +void seed_excl_fixture(const fs::path& dir) { + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + + UNSIGNED8 srv[512]{}; + const auto d = dir.string(); + std::memcpy(srv, d.c_str(), d.size()); + ADSHANDLE hConn = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hConn) + == AE_SUCCESS); + + UNSIGNED8 tname[] = "ex.dbf"; + UNSIGNED8 def[] = "NAME,C,8,0"; + ADSHANDLE hTable = 0; + REQUIRE(AdsCreateTable(hConn, tname, nullptr, ADS_CDX, ADS_ANSI, + 0, 0, 0, def, &hTable) == AE_SUCCESS); + UNSIGNED8 fld[] = "NAME"; + for (int i = 0; i < 3; ++i) { + char v[9]; + std::snprintf(v, sizeof(v), "R%07d", i); + REQUIRE(AdsAppendRecord(hTable) == AE_SUCCESS); + REQUIRE(AdsSetString(hTable, fld, reinterpret_cast(v), + 8) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hTable) == AE_SUCCESS); + } + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); +} + +UNSIGNED32 open_mode(ADSHANDLE hConn, const char* name, UNSIGNED16 mode, + ADSHANDLE* ph) { + return AdsOpenTable(hConn, + reinterpret_cast(const_cast(name)), + nullptr, ADS_CDX, ADS_ANSI, + ADS_COMPATIBLE_LOCKING, ADS_IGNORERIGHTS, mode, ph); +} + +// Disconnect teardown is asynchronous with respect to another session's +// next request: the client returns from AdsDisconnect once its socket is +// down, while the server still has to observe the hangup and sweep that +// session's registrations. Poll briefly for the acquisition the sweep +// must unblock instead of assuming the sweep has already run. +UNSIGNED32 open_eventually(ADSHANDLE hConn, const char* name, + UNSIGNED16 mode, ADSHANDLE* ph) { + UNSIGNED32 rc = AE_SUCCESS; + for (int i = 0; i < 100; ++i) { + rc = open_mode(hConn, name, mode, ph); + if (rc == AE_SUCCESS) return rc; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + return rc; +} + +ADSHANDLE connect_remote(const UNSIGNED8* srvbuf) { + ADSHANDLE h = 0; + REQUIRE(AdsConnect60(const_cast(srvbuf), ADS_REMOTE_SERVER, + nullptr, nullptr, 0, &h) == AE_SUCCESS); + return h; +} + +} // namespace + +TEST_CASE("mtfix11 exclusive open denies other sessions both directions") { + auto dir = excl_tmp_dir(); + seed_excl_fixture(dir); + + openads::network::Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + + char uri[512]; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(srv.port()), dir.string().c_str()); + UNSIGNED8 srvbuf[512]{}; + std::memcpy(srvbuf, uri, std::strlen(uri) + 1); + + const char* tname = "ex.dbf"; + ADSHANDLE hA = connect_remote(srvbuf); + ADSHANDLE hB = connect_remote(srvbuf); + ADSHANDLE tA = 0, tA2 = 0, tB = 0; + + // 1. An exclusive holder blocks other sessions in both modes. + REQUIRE(open_mode(hA, tname, ADS_EXCLUSIVE, &tA) == AE_SUCCESS); + CHECK(open_mode(hB, tname, ADS_SHARED, &tB) != AE_SUCCESS); + CHECK(tB == 0); + CHECK(open_mode(hB, tname, ADS_EXCLUSIVE, &tB) != AE_SUCCESS); + CHECK(tB == 0); + + // 2. The holding session itself may open the table again (several + // workareas on one connection are legitimate). + REQUIRE(open_mode(hA, tname, ADS_SHARED, &tA2) == AE_SUCCESS); + + // 3. Closing the exclusive handle releases that hold (exclusive + // tables are never parked, so this close reaches the wire) - but + // the session's remaining shared open still blocks B's exclusive. + REQUIRE(AdsCloseTable(tA) == AE_SUCCESS); + CHECK(open_mode(hB, tname, ADS_EXCLUSIVE, &tB) != AE_SUCCESS); + CHECK(tB == 0); + + // 4. Closing the shared handle parks it client-side; the following + // disconnect drains the park pool (a live open table would DEFER + // the disconnect instead) and the wire close frees the registry. + REQUIRE(AdsCloseTable(tA2) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hA) == AE_SUCCESS); + REQUIRE(open_eventually(hB, tname, ADS_EXCLUSIVE, &tB) == AE_SUCCESS); + + // 5. Now B holds it exclusive: A's opens are denied. + hA = connect_remote(srvbuf); + tA = 0; + CHECK(open_mode(hA, tname, ADS_SHARED, &tA) != AE_SUCCESS); + CHECK(tA == 0); + tA = 0; + CHECK(open_mode(hA, tname, ADS_EXCLUSIVE, &tA) != AE_SUCCESS); + CHECK(tA == 0); + + // 6. B's exclusive close is a real close: A gets in shared. + REQUIRE(AdsCloseTable(tB) == AE_SUCCESS); + REQUIRE(open_mode(hA, tname, ADS_SHARED, &tA) == AE_SUCCESS); + + // 7. Two shared holders coexist; a third session's exclusive fails. + ADSHANDLE hC = connect_remote(srvbuf); + ADSHANDLE tC = 0; + REQUIRE(open_mode(hC, tname, ADS_SHARED, &tC) == AE_SUCCESS); + ADSHANDLE hD = connect_remote(srvbuf); + ADSHANDLE tD = 0; + CHECK(open_mode(hD, tname, ADS_EXCLUSIVE, &tD) != AE_SUCCESS); + CHECK(tD == 0); + + // 8. An upgrade to exclusive requires the table free of OTHER + // sessions: A still holds it shared, so C's exclusive is denied. + ADSHANDLE tC2 = 0; + CHECK(open_mode(hC, tname, ADS_EXCLUSIVE, &tC2) != AE_SUCCESS); + CHECK(tC2 == 0); + + // 9. Once A lets go (parked close + disconnect drain), C's upgrade + // succeeds and D stays locked out. + REQUIRE(AdsCloseTable(tA) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hA) == AE_SUCCESS); + REQUIRE(open_eventually(hC, tname, ADS_EXCLUSIVE, &tC2) == AE_SUCCESS); + CHECK(open_mode(hD, tname, ADS_SHARED, &tD) != AE_SUCCESS); + CHECK(tD == 0); + + // 10. C's exclusive close releases the exclusive hold; D may now + // share alongside C's still-open shared handle. + REQUIRE(AdsCloseTable(tC2) == AE_SUCCESS); + REQUIRE(open_mode(hD, tname, ADS_SHARED, &tD) == AE_SUCCESS); + + REQUIRE(AdsCloseTable(tC) == AE_SUCCESS); + REQUIRE(AdsCloseTable(tD) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hB) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hD) == AE_SUCCESS); +} diff --git a/tests/unit/network_frame_trace_test.cpp b/tests/unit/network_frame_trace_test.cpp new file mode 100644 index 00000000..9a15742e --- /dev/null +++ b/tests/unit/network_frame_trace_test.cpp @@ -0,0 +1,88 @@ +// tests/unit/network_frame_trace_test.cpp +// wire_trace diagnostics: the optional per-frame hook sees every +// completed round trip while installed, and nothing once removed. +// Default (no hook) is the production path. +#include "doctest.h" +#include "network/client.h" +#include "network/server.h" +#include "openads/ace.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +std::atomic g_frames{0}; +std::atomic g_open_frames{0}; +std::atomic g_open_acks{0}; + +void count_frame(const void* conn, std::uint8_t op, std::uint32_t /*tid*/, + std::size_t /*req_bytes*/, std::uint8_t rep_op, + std::size_t /*rep_bytes*/, long long us) { + CHECK(conn != nullptr); + CHECK(us >= 0); + g_frames.fetch_add(1); + if (op == 0x20) { + g_open_frames.fetch_add(1); + if (rep_op == 0x21) g_open_acks.fetch_add(1); + } +} + +} // namespace + +TEST_CASE("wire trace: frame hook counts round trips only while installed") { + auto dir = fs::temp_directory_path() / "openads_frametrace"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + + // Seed one table locally. + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC0 = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hC0) + == AE_SUCCESS); + UNSIGNED8 def[] = "ID,N,8,0"; + UNSIGNED8 tn0[] = "FT.DBF"; + ADSHANDLE hT0 = 0; + REQUIRE(AdsCreateTable(hC0, tn0, nullptr, ADS_CDX, 0, 0, 0, 0, def, &hT0) + == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT0) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC0) == AE_SUCCESS); + + openads::network::Server s; + REQUIRE(s.start("127.0.0.1", 0).has_value()); + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(s.port()), dir.string().c_str()); + UNSIGNED8 sb[512]{}; + std::memcpy(sb, uri, std::strlen(uri) + 1); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(sb, ADS_REMOTE_SERVER, nullptr, nullptr, 0, &hC) + == AE_SUCCESS); + + openads::network::set_frame_trace_hook(&count_frame); + UNSIGNED8 tn[] = "FT.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + openads::network::set_frame_trace_hook(nullptr); + + CHECK(g_frames.load() >= 1u); + CHECK(g_open_frames.load() == 1u); + CHECK(g_open_acks.load() == 1u); + + // Removed: further traffic is not reported. + const unsigned seen = g_frames.load(); + REQUIRE(AdsGotoBottom(hT) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + CHECK(g_frames.load() == seen); + + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + s.stop(); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/network_local_answers_test.cpp b/tests/unit/network_local_answers_test.cpp new file mode 100644 index 00000000..0730194e --- /dev/null +++ b/tests/unit/network_local_answers_test.cpp @@ -0,0 +1,214 @@ +// tests/unit/network_local_answers_test.cpp +// Client-only frame cuts for the Vouch WAN startup (patch 1): +// a) AdsGetTableType answered from the opened name's extension, and +// AdsGetRecordLength remembered per connection for re-opens; +// b) no GetKeyCount after an ordered GotoBottom the server certified +// empty; +// c) AdsGotoRecord(0) on a certified-empty table served locally; +// d) AdsRefreshRecord right after a GotoRecord (nothing sent between) +// served from the GotoRecord ack. +// Each case also checks the answers match what the wire would say, and +// that the wire is still used when the proof does not hold. +#include "doctest.h" + +#include +#include +#include "mgmt/mg_stats.h" +#include "network/server.h" +#include "openads/ace.h" + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +std::uint64_t la_op(std::uint8_t op) { + return openads::mgmt::process_mg_stats() + .op_timing[op] + .count.load(std::memory_order_relaxed); +} + +void la_make_table(ADSHANDLE hC, const char* name, const char* bagname, + int rows) { + UNSIGNED8 def[] = "ID,N,8,0;NM,C,10,0"; + UNSIGNED8 tn[64]{}; + std::memcpy(tn, name, std::strlen(name)); + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, def, &hT) + == AE_SUCCESS); + UNSIGNED8 f1[] = "ID"; + for (int i = 1; i <= rows; ++i) { + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + REQUIRE(AdsSetDouble(hT, f1, i * 10) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); + } + ADSHANDLE hI = 0; + UNSIGNED8 bag[64]{}; + std::memcpy(bag, bagname, std::strlen(bagname)); + UNSIGNED8 tag[] = "BYID"; + UNSIGNED8 exp[] = "ID"; + REQUIRE(AdsCreateIndex61(hT, bag, tag, exp, nullptr, nullptr, + ADS_COMPOUND, 512, &hI) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); +} + +} // namespace + +TEST_CASE("Remote local answers: empty tables, table type, refresh") { + auto dir = fs::temp_directory_path() / "openads_localans"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC0 = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hC0) + == AE_SUCCESS); + la_make_table(hC0, "LAEMPTY.DBF", "LAEMPTY.CDX", 0); + la_make_table(hC0, "LAFULL.DBF", "LAFULL.CDX", 5); + REQUIRE(AdsDisconnect(hC0) == AE_SUCCESS); + + openads::network::Server s; + REQUIRE(s.start("127.0.0.1", 0).has_value()); + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(s.port()), dir.string().c_str()); + UNSIGNED8 sb[512]{}; + std::memcpy(sb, uri, std::strlen(uri) + 1); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(sb, ADS_REMOTE_SERVER, nullptr, nullptr, 0, &hC) + == AE_SUCCESS); + + UNSIGNED8 want[] = "BYID"; + + SUBCASE("empty ordered table: no GetKeyCount, local GotoRecord") { + UNSIGNED8 tn[] = "LAEMPTY.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + ADSHANDLE hOrd = 0; + REQUIRE(AdsGetIndexHandle(hT, want, &hOrd) == AE_SUCCESS); + REQUIRE(AdsSetIndexOrderByHandle(hT, hOrd) == AE_SUCCESS); + + // rddads order: top first (installs the order), then bottom. + REQUIRE(AdsGotoTop(hOrd) == AE_SUCCESS); + const auto kc0 = la_op(0xB0); + const auto gr0 = la_op(0x58); + const auto tt0 = la_op(0x6A); + REQUIRE(AdsGotoBottom(hOrd) == AE_SUCCESS); + CHECK(la_op(0xB0) - kc0 == 0u); // (b) no key count frame + + UNSIGNED16 b = 0, e = 0; + REQUIRE(AdsAtBOF(hT, &b) == AE_SUCCESS); + REQUIRE(AdsAtEOF(hT, &e) == AE_SUCCESS); + CHECK(b == 1); + CHECK(e == 1); + + REQUIRE(AdsGotoRecord(hT, 0) == AE_SUCCESS); // (c) + CHECK(la_op(0x58) - gr0 == 0u); + b = 0; e = 0; + REQUIRE(AdsAtBOF(hT, &b) == AE_SUCCESS); + REQUIRE(AdsAtEOF(hT, &e) == AE_SUCCESS); + CHECK(b == 1); + CHECK(e == 1); + + UNSIGNED16 tt = 0; + REQUIRE(AdsGetTableType(hT, &tt) == AE_SUCCESS); // (a) + CHECK(tt == ADS_CDX); + CHECK(la_op(0x6A) - tt0 == 0u); + + // GO n>0 is not answered by these local answers (a peer may have + // appended). The separate still-empty window (OPENADS_EMPTY_TTL_MS, + // at most 2s) may answer it right after an empty certification, so + // let that window lapse first. + std::this_thread::sleep_for(std::chrono::milliseconds(2100)); + REQUIRE(AdsGotoRecord(hT, 1) == AE_SUCCESS); + CHECK(la_op(0x58) - gr0 == 1u); + + // The wire still agrees: the explicit key count is 0. + UNSIGNED32 kc = 99; + REQUIRE(AdsGetKeyCount(hOrd, 0, &kc) == AE_SUCCESS); + CHECK(kc == 0u); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + } + + SUBCASE("non-empty table: GO 0 and keycount still use the wire") { + UNSIGNED8 tn[] = "LAFULL.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + ADSHANDLE hOrd = 0; + REQUIRE(AdsGetIndexHandle(hT, want, &hOrd) == AE_SUCCESS); + REQUIRE(AdsSetIndexOrderByHandle(hT, hOrd) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hOrd) == AE_SUCCESS); + REQUIRE(AdsGotoBottom(hOrd) == AE_SUCCESS); + // Non-empty order: the key count is the real one (5), however + // it was obtained (wire or an existing piggyback). + UNSIGNED32 kcf = 0; + REQUIRE(AdsGetKeyCount(hOrd, 0, &kcf) == AE_SUCCESS); + CHECK(kcf == 5u); + const auto gr0 = la_op(0x58); + REQUIRE(AdsGotoRecord(hT, 0) == AE_SUCCESS); + CHECK(la_op(0x58) - gr0 == 1u); + UNSIGNED16 e = 0; + REQUIRE(AdsAtEOF(hT, &e) == AE_SUCCESS); + CHECK(e == 1); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + } + + SUBCASE("refresh after goto is local; after a lock it is not") { + UNSIGNED8 tn[] = "LAFULL.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + const auto rf0 = la_op(0x68); + REQUIRE(AdsGotoRecord(hT, 3) == AE_SUCCESS); + REQUIRE(AdsRefreshRecord(hT) == AE_SUCCESS); // (d) + CHECK(la_op(0x68) - rf0 == 0u); + UNSIGNED32 rn = 0; + REQUIRE(AdsGetRecordNum(hT, ADS_IGNOREFILTERS, &rn) == AE_SUCCESS); + CHECK(rn == 3u); + double v = 0; + UNSIGNED8 f1[] = "ID"; + REQUIRE(AdsGetDouble(hT, f1, &v) == AE_SUCCESS); + CHECK(v == doctest::Approx(30.0)); + + // A second refresh has no fresh GotoRecord behind it: wire. + REQUIRE(AdsRefreshRecord(hT) == AE_SUCCESS); + CHECK(la_op(0x68) - rf0 == 1u); + + // GotoRecord, then a lock frame, then refresh: must re-read. + REQUIRE(AdsGotoRecord(hT, 4) == AE_SUCCESS); + REQUIRE(AdsLockRecord(hT, 4) == AE_SUCCESS); + REQUIRE(AdsRefreshRecord(hT) == AE_SUCCESS); + CHECK(la_op(0x68) - rf0 == 2u); + REQUIRE(AdsUnlockRecord(hT, 4) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + } + + SUBCASE("record length remembered across re-opens") { + UNSIGNED8 tn[] = "LAFULL.DBF"; + const auto rl0 = la_op(0x6C); + for (int i = 0; i < 2; ++i) { + ADSHANDLE hT = 0; + // Exclusive: never parked, so the second open is a real + // re-open on a new handle. + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, + ADS_EXCLUSIVE, &hT) == AE_SUCCESS); + UNSIGNED32 len = 0; + REQUIRE(AdsGetRecordLength(hT, &len) == AE_SUCCESS); + CHECK(len == 19u); // 1 (delete flag) + 8 + 10 + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + } + CHECK(la_op(0x6C) - rl0 <= 1u); + } + + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + s.stop(); + fs::remove_all(dir, ec); +} diff --git a/tests/unit/network_lock_exclusion_test.cpp b/tests/unit/network_lock_exclusion_test.cpp index 5ec9cd1c..1c36b852 100644 --- a/tests/unit/network_lock_exclusion_test.cpp +++ b/tests/unit/network_lock_exclusion_test.cpp @@ -3,9 +3,10 @@ // Field: process A locks its username record for life; occasionally a // second process B locks the SAME record and proceeds (never on // DBFCDX). The gate is LockRecord -> server try_lock_record_excl -> -// OS byte lock; IsRecordLocked is own-table-only by design (see -// Session::IsRecordLocked + AdsIsRecordLocked) and can never serve as -// the cross-process check. +// OS byte lock. IsRecordLocked was own-table-only at birth (the +// single-login guard that QUERIED instead of attempting stayed blind); +// mtfix11 makes it cross-owner aware via the OS probe, matching SAP - +// but the enforcement gate below still rests on LockRecord attempts. // // Matrix: A holds rec 3 (natural vs ordered/twin path on each side), // B must fail the same recno. Then close/reopen handover: A closes diff --git a/tests/unit/network_nav_batch_test.cpp b/tests/unit/network_nav_batch_test.cpp index 12f3607b..5b6a9e69 100644 --- a/tests/unit/network_nav_batch_test.cpp +++ b/tests/unit/network_nav_batch_test.cpp @@ -172,12 +172,17 @@ TEST_CASE("Nav batching: duplicate GotoTop/GotoBottom skip their frame") { REQUIRE(AdsGotoTop(hTable) == AE_SUCCESS); CHECK(nb_op(kOpGotoTop) == top0); - // Bottom twice: one frame, then suppressed. + // The warm open did not certify a pair, so the first bottom wires; + // the duplicate stays local. REQUIRE(AdsGotoBottom(hTable) == AE_SUCCESS); REQUIRE(AdsGotoBottom(hTable) == AE_SUCCESS); CHECK(nb_op(kOpGotoBottom) == bot0 + 1); - // A wire nav in between invalidates: top goes out again. + // Restored pair certification: bottom's ack certifies top, so this + // call consumes the pair without another wire frame. + REQUIRE(AdsGotoTop(hTable) == AE_SUCCESS); + CHECK(nb_op(kOpGotoTop) == top0); + REQUIRE(AdsGotoRecord(hTable, 2) == AE_SUCCESS); REQUIRE(AdsGotoTop(hTable) == AE_SUCCESS); CHECK(nb_op(kOpGotoTop) == top0 + 1); @@ -292,6 +297,8 @@ TEST_CASE("Nav batching: order context defeats duplicate suppression") { nb_w REQUIRE(AdsGotoTop(hOrd) == AE_SUCCESS); CHECK(nb_op(kOpGotoTop) == top0 + 1); + // Explicit focus keeps table-handle navigation indexed. + REQUIRE(AdsSetIndexOrderByHandle(hTable, hOrd) == AE_SUCCESS); // Table-handle top with the order still bound: same position, // suppressed as well. REQUIRE(AdsGotoTop(hTable) == AE_SUCCESS); @@ -454,11 +461,19 @@ TEST_CASE("Nav batching: deferred SetOrder fuses into GotoTop") { nb_wipe(); REQUIRE(AdsGetRecordNum(hTable, 0, &rec) == AE_SUCCESS); CHECK(rec == 2u); - // Same pair to the bottom: fused the same way. + // Same pattern to the bottom: fused the same way. mtfix12 R1 + // notes: the fused GotoTop above certified the bottom in the same + // visit, so a plain ordered GotoBottom here would be pair-served + // and exercise nothing. A pending order blocks the pair serve (the + // certified order no longer provably matches), so defer a natural + // switch first — the bottom on the TABLE handle fuses it into one + // frame exactly like leg one. (ByHandle(hOrd) can't be the second + // switch: with the acked binding still hOrd it reads as a + // same-order no-op and would leave the natural switch pending.) + REQUIRE(AdsSetIndexOrderByHandle(hTable, 0) == AE_SUCCESS); const std::uint64_t bot0 = nb_op(kOpGotoBottom); - REQUIRE(AdsSetIndexOrderByHandle(hTable, hOrd) == AE_SUCCESS); CHECK(nb_op(kOpSetOrder) == so0); - REQUIRE(AdsGotoBottom(hOrd) == AE_SUCCESS); + REQUIRE(AdsGotoBottom(hTable) == AE_SUCCESS); CHECK(nb_op(kOpSetOrder) == so0); CHECK(nb_op(kOpGotoBottom) == bot0 + 1); diff --git a/tests/unit/network_open_warm_test.cpp b/tests/unit/network_open_warm_test.cpp index 221fc0a2..9d67d97d 100644 --- a/tests/unit/network_open_warm_test.cpp +++ b/tests/unit/network_open_warm_test.cpp @@ -168,7 +168,8 @@ TEST_CASE("Pooled re-USE: close/reopen skips OpenTable+CloseTable frames") { srv.stop(); } -TEST_CASE("Pooled re-USE skipped after a lock (real close)") { ow_wipe(); +TEST_CASE("Pooled re-USE after a released lock still parks") { + ow_wipe(); auto dir = ow_tmp_dir(); seed_ow_fixture(dir); @@ -194,12 +195,14 @@ TEST_CASE("Pooled re-USE skipped after a lock (real close)") { ow_wipe(); REQUIRE(AdsUnlockRecord(hTable, 0) == AE_SUCCESS); REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); - // Locked once: pool-ineligible, so the reopen is a full wire open. + // Locked then fully released: still poolable (mtfix8) -- the close + // parks the handle and the reopen is a park hit, no wire open. Only + // outstanding locks (locks_held) force a real close now. const std::uint64_t open_before = op_count(0x20); REQUIRE(AdsOpenTable(hConn, tname, nullptr, ADS_CDX, ADS_ANSI, ADS_SHARED, ADS_COMPATIBLE_LOCKING, ADS_DEFAULT, &hTable) == AE_SUCCESS); - CHECK(op_count(0x20) == open_before + 1); + CHECK(op_count(0x20) == open_before); CHECK(ow_get(hTable, "NM") == "alpha"); REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); diff --git a/tests/unit/network_pool_mt_test.cpp b/tests/unit/network_pool_mt_test.cpp index a4f493c3..6225273c 100644 --- a/tests/unit/network_pool_mt_test.cpp +++ b/tests/unit/network_pool_mt_test.cpp @@ -134,9 +134,17 @@ TEST_CASE("Session pool: MT threads open/read/close on parallel lanes") { if (AdsCloseTable(hT) != AE_SUCCESS) failures.fetch_add(1); } }; - std::vector th; - for (int w = 0; w < 4; ++w) th.emplace_back(worker, w); - for (auto& t : th) t.join(); + // Repeat the whole 4-thread phase: the old data race on the shared + // remote-table store (unlocked park-path erase) hit ~1 run in 30, so + // one pass per test run was not enough to catch it. Rounds after the + // first also exercise park/adopt across threads (fresh thread ids + // deal onto lanes again). + constexpr int kRounds = 25; + for (int round = 0; round < kRounds; ++round) { + std::vector th; + for (int w = 0; w < 4; ++w) th.emplace_back(worker, w); + for (auto& t : th) t.join(); + } CHECK(failures.load() == 0); // Pool + deferred accounting intact: reopen on the same handle works, diff --git a/tests/unit/network_remote_reindex_test.cpp b/tests/unit/network_remote_reindex_test.cpp new file mode 100644 index 00000000..a939c29c --- /dev/null +++ b/tests/unit/network_remote_reindex_test.cpp @@ -0,0 +1,279 @@ +// tests/unit/network_remote_reindex_test.cpp +// Remote AdsReindex must rebuild the table's bags. On the server the +// bags are bound on the ABI twin handle, not on the engine table, so a +// reindex of the engine table alone rebuilt nothing and still reported +// success (rddads ordListRebuild ignores the return code anyway). +#include "doctest.h" +#include "network/server.h" +#include "openads/ace.h" + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace { + +UNSIGNED32 rr_local_key_count(const fs::path& dir) { + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hC) + == AE_SUCCESS); + UNSIGNED8 tn[] = "RR.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + UNSIGNED8 tag[] = "BYID"; + ADSHANDLE hI = 0; + REQUIRE(AdsGetIndexHandle(hT, tag, &hI) == AE_SUCCESS); + UNSIGNED32 n = 0; + REQUIRE(AdsGetKeyCount(hI, ADS_IGNOREFILTERS, &n) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + return n; +} + +void rr_append(ADSHANDLE hT, double id) { + UNSIGNED8 fld[] = "ID"; + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + REQUIRE(AdsSetDouble(hT, fld, id) == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); +} + +} // namespace + +TEST_CASE("Remote AdsReindex rebuilds the bag bound on the server twin") { + auto dir = fs::temp_directory_path() / "openads_remote_reindex"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + + // Stage a stale production bag locally: build it over 2 rows, save a + // copy, add 2 more rows, then put the 2-key copy back. + { + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, + &hC) == AE_SUCCESS); + UNSIGNED8 def[] = "ID,N,8,0"; + UNSIGNED8 tn[] = "RR.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, def, + &hT) == AE_SUCCESS); + UNSIGNED8 bag[] = "RR.CDX"; + UNSIGNED8 tag[] = "BYID"; + UNSIGNED8 exp[] = "STR(ID,8)"; + ADSHANDLE hI = 0; + REQUIRE(AdsCreateIndex61(hT, bag, tag, exp, nullptr, nullptr, + ADS_COMPOUND, 512, &hI) == AE_SUCCESS); + rr_append(hT, 2); + rr_append(hT, 1); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + } + REQUIRE(rr_local_key_count(dir) == 2); + fs::copy_file(dir / "RR.CDX", dir / "RR.OLD", + fs::copy_options::overwrite_existing); + { + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, + &hC) == AE_SUCCESS); + UNSIGNED8 tn[] = "RR.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + rr_append(hT, 4); + rr_append(hT, 3); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + } + fs::copy_file(dir / "RR.OLD", dir / "RR.CDX", + fs::copy_options::overwrite_existing); + REQUIRE(rr_local_key_count(dir) == 2); // stale: 4 rows, 2 keys + + // Remote REINDEX, the way rddads ordListRebuild issues it. + { + openads::network::Server s; + REQUIRE(s.start("127.0.0.1", 0).has_value()); + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(s.port()), + dir.string().c_str()); + UNSIGNED8 sb[512]{}; + std::memcpy(sb, uri, std::strlen(uri)); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(sb, ADS_REMOTE_SERVER, nullptr, nullptr, 0, + &hC) == AE_SUCCESS); + UNSIGNED8 tn[] = "RR.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + UNSIGNED8 tag[] = "BYID"; + ADSHANDLE hI = 0; + REQUIRE(AdsGetIndexHandle(hT, tag, &hI) == AE_SUCCESS); + REQUIRE(AdsSetIndexOrderByHandle(hT, hI) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hT) == AE_SUCCESS); + REQUIRE(AdsReindex(hT) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + s.stop(); + } + + CHECK(rr_local_key_count(dir) == 4); + fs::remove_all(dir, ec); +} + +// Vouch-shaped remote REINDEX: CDX-format production bag named .Z01 +// with several tags, table opened remotely (shared and exclusive) with no +// order set, then AdsReindex. Every tag must be rebuilt and the bag file +// rewritten, for both an empty table and a stale non-empty one. +namespace { + +const char* const kRzTags[] = {"TA", "TB", "TC"}; + +void rz_counts(const fs::path& dir, UNSIGNED32 out[3]) { + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, &hC) + == AE_SUCCESS); + UNSIGNED8 tn[] = "RZ.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + UNSIGNED8 bag[] = "RZ.Z01"; + ADSHANDLE arr[16]{}; + UNSIGNED16 alen = 16; + (void)AdsOpenIndex(hT, bag, arr, &alen); + for (int i = 0; i < 3; ++i) { + UNSIGNED8 tg[8]{}; + std::memcpy(tg, kRzTags[i], std::strlen(kRzTags[i])); + ADSHANDLE hI = 0; + REQUIRE(AdsGetIndexHandle(hT, tg, &hI) == AE_SUCCESS); + out[i] = 0; + REQUIRE(AdsGetKeyCount(hI, ADS_IGNOREFILTERS, &out[i]) == AE_SUCCESS); + } + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); +} + +void rz_append(ADSHANDLE hT, double id) { + UNSIGNED8 fid[] = "ID"; + UNSIGNED8 fnm[] = "NM"; + char nm[16]; + std::snprintf(nm, sizeof(nm), "N%05d", static_cast(id)); + REQUIRE(AdsAppendRecord(hT) == AE_SUCCESS); + REQUIRE(AdsSetDouble(hT, fid, id) == AE_SUCCESS); + REQUIRE(AdsSetString(hT, fnm, reinterpret_cast(nm), + static_cast(std::strlen(nm))) + == AE_SUCCESS); + REQUIRE(AdsWriteRecord(hT) == AE_SUCCESS); +} + +void rz_run(bool empty, UNSIGNED16 mode) { + CAPTURE(empty); + CAPTURE(mode); + auto dir = fs::temp_directory_path() / "openads_remote_reindex_z01"; + std::error_code ec; + fs::remove_all(dir, ec); + fs::create_directories(dir); + const UNSIGNED32 want = empty ? 0 : 4; + { + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, + &hC) == AE_SUCCESS); + UNSIGNED8 def[] = "ID,N,8,0;NM,C,10,0"; + UNSIGNED8 tn[] = "RZ.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsCreateTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, def, + &hT) == AE_SUCCESS); + const char* exprs[] = {"STR(ID,8)", "NM", "NM+STR(ID,8)"}; + for (int i = 0; i < 3; ++i) { + UNSIGNED8 bag[] = "RZ.Z01"; + UNSIGNED8 tg[8]{}; + std::memcpy(tg, kRzTags[i], std::strlen(kRzTags[i])); + UNSIGNED8 ex[32]{}; + std::memcpy(ex, exprs[i], std::strlen(exprs[i])); + ADSHANDLE hI = 0; + REQUIRE(AdsCreateIndex61(hT, bag, tg, ex, nullptr, nullptr, + ADS_COMPOUND, 512, &hI) == AE_SUCCESS); + } + if (!empty) { rz_append(hT, 2); rz_append(hT, 1); } + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + } + if (!empty) { + fs::copy_file(dir / "RZ.Z01", dir / "RZ.OLD", + fs::copy_options::overwrite_existing); + UNSIGNED8 srv[512]{}; + std::memcpy(srv, dir.string().c_str(), dir.string().size()); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(srv, ADS_LOCAL_SERVER, nullptr, nullptr, 0, + &hC) == AE_SUCCESS); + UNSIGNED8 tn[] = "RZ.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, 0, &hT) + == AE_SUCCESS); + UNSIGNED8 bag[] = "RZ.Z01"; + ADSHANDLE arr[16]{}; + UNSIGNED16 alen = 16; + (void)AdsOpenIndex(hT, bag, arr, &alen); + rz_append(hT, 4); + rz_append(hT, 3); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + fs::copy_file(dir / "RZ.OLD", dir / "RZ.Z01", + fs::copy_options::overwrite_existing); + UNSIGNED32 c[3]{}; + rz_counts(dir, c); + CHECK(c[0] == 2); // stale before the remote reindex + } + const auto old_t = fs::file_time_type::clock::now() - std::chrono::hours(24); + fs::last_write_time(dir / "RZ.Z01", old_t); + { + openads::network::Server s; + REQUIRE(s.start("127.0.0.1", 0).has_value()); + char uri[512]{}; + std::snprintf(uri, sizeof(uri), "tcp://127.0.0.1:%u/%s", + static_cast(s.port()), + dir.string().c_str()); + UNSIGNED8 sb[512]{}; + std::memcpy(sb, uri, std::strlen(uri)); + ADSHANDLE hC = 0; + REQUIRE(AdsConnect60(sb, ADS_REMOTE_SERVER, nullptr, nullptr, 0, + &hC) == AE_SUCCESS); + UNSIGNED8 tn[] = "RZ.DBF"; + ADSHANDLE hT = 0; + REQUIRE(AdsOpenTable(hC, tn, nullptr, ADS_CDX, 0, 0, 0, mode, &hT) + == AE_SUCCESS); + CHECK(AdsReindex(hT) == AE_SUCCESS); + REQUIRE(AdsCloseTable(hT) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hC) == AE_SUCCESS); + s.stop(); + } + CHECK(fs::last_write_time(dir / "RZ.Z01") > old_t + std::chrono::hours(1)); + UNSIGNED32 c[3]{}; + rz_counts(dir, c); + CHECK(c[0] == want); + CHECK(c[1] == want); + CHECK(c[2] == want); + fs::remove_all(dir, ec); +} + +} // namespace + +TEST_CASE("Remote AdsReindex, Vouch shape: multi-tag .Z01, no order") { + rz_run(true, ADS_SHARED); + rz_run(true, ADS_EXCLUSIVE); + rz_run(false, ADS_SHARED); + rz_run(false, ADS_EXCLUSIVE); +} diff --git a/tests/unit/network_teardown_batch_test.cpp b/tests/unit/network_teardown_batch_test.cpp index 5db28aa2..cc0c2144 100644 --- a/tests/unit/network_teardown_batch_test.cpp +++ b/tests/unit/network_teardown_batch_test.cpp @@ -6,8 +6,8 @@ // - Flush/CloseAll defer to the close that absorbs them (server close // flushes via its shadow handle and purges index bindings); any // other intervening wire op flushes them first, in order. -// - CheckExistence serves positive answers locally until a -// file-mutating op clears the cache (negatives always go out). +// - CheckExistence always probes the server; external file/dir changes +// can occur outside this connection. // - SetOrder to the ack-confirmed binding skips its frame (SetOrder // never moves the cursor). // - Order-handle key counts ride the parent's order cache. @@ -234,7 +234,7 @@ TEST_CASE("Teardown batching: merged flush survives for unparked closes") { srv.stop(); } -TEST_CASE("Teardown batching: existence positives cache until mutation") { +TEST_CASE("Teardown batching: existence probes reflect external changes") { tb_wipe(); auto dir = tb_tmp_dir(); tb_seed(dir); @@ -253,14 +253,13 @@ TEST_CASE("Teardown batching: existence positives cache until mutation") { CHECK(ex == 1u); REQUIRE(AdsCheckExistence(hConn, fn, &ex) == AE_SUCCESS); CHECK(ex == 1u); - CHECK(tb_op(kOpFileExists) == fe0 + 1); + CHECK(tb_op(kOpFileExists) == fe0 + 3); - // A file-mutating op clears the cache: the next check goes out and - // reports the drop. + // A file-mutating op also changes the next live answer. REQUIRE(AdsDropTable(hConn, fn, 1) == AE_SUCCESS); REQUIRE(AdsCheckExistence(hConn, fn, &ex) == AE_SUCCESS); CHECK(ex == 0u); - CHECK(tb_op(kOpFileExists) == fe0 + 2); + CHECK(tb_op(kOpFileExists) == fe0 + 4); REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); srv.stop(); @@ -359,7 +358,9 @@ TEST_CASE("Teardown batching: immutable metadata caches per handle") { } CHECK(tt == 2u); // ADS_CDX CHECK(rl > 0u); - CHECK(tb_op(0x6A) == tt0 + 1); + // Table type: answered from the opened name's extension (0 frames) + // when it has one; at most one frame otherwise. + CHECK(tb_op(0x6A) <= tt0 + 1); CHECK(tb_op(0x6C) == rl0 + 1); REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); @@ -586,7 +587,7 @@ TEST_CASE("Index park: CreateIndex invalidates the snapshot") { srv.stop(); } -TEST_CASE("Teardown batching: open bags answer existence locally") { +TEST_CASE("Teardown batching: open bags do not hide filesystem changes") { tb_wipe(); auto dir = tb_tmp_dir(); tb_seed(dir); @@ -597,28 +598,20 @@ TEST_CASE("Teardown batching: open bags answer existence locally") { ADSHANDLE hConn = tb_connect_remote(dir, srv.port()); ADSHANDLE hTable = tb_open(hConn); - // The production bag is bound by this open table: existence is - // trivially true (any spelling that stems the same, including - // the .z01 production alias) with zero frames. + // A live table or bag cannot establish existence for a differently + // spelled path. The exact spelling must be checked on the server. const std::uint64_t fe0 = tb_op(kOpFileExists); UNSIGNED16 ex = 0; - UNSIGNED8 bag1[] = "TB.CDX"; - REQUIRE(AdsCheckExistence(hConn, bag1, &ex) == AE_SUCCESS); - CHECK(ex == 1u); - UNSIGNED8 bag2[] = "tb.cdx"; - REQUIRE(AdsCheckExistence(hConn, bag2, &ex) == AE_SUCCESS); - CHECK(ex == 1u); - UNSIGNED8 bag3[] = "TB.z01"; - REQUIRE(AdsCheckExistence(hConn, bag3, &ex) == AE_SUCCESS); + UNSIGNED8 bag[] = "TB.CDX"; + REQUIRE(AdsCheckExistence(hConn, bag, &ex) == AE_SUCCESS); CHECK(ex == 1u); - CHECK(tb_op(kOpFileExists) == fe0); - - // A genuinely missing file still goes to the wire (negatives are - // never cached: the file may appear at any time). + UNSIGNED8 alias[] = "TB.z01"; + REQUIRE(AdsCheckExistence(hConn, alias, &ex) == AE_SUCCESS); + CHECK(ex == 0u); UNSIGNED8 missing[] = "NOPE.CDX"; REQUIRE(AdsCheckExistence(hConn, missing, &ex) == AE_SUCCESS); CHECK(ex == 0u); - CHECK(tb_op(kOpFileExists) == fe0 + 1); + CHECK(tb_op(kOpFileExists) == fe0 + 3); REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); @@ -670,3 +663,74 @@ TEST_CASE("KeyCount: rotation revisits ride the per-order map") { REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); srv.stop(); } + +TEST_CASE("Parked nav stamp survives CloseAll/OpenIndex unpark") { + tb_wipe(); + auto dir = tb_tmp_dir(); + tb_seed(dir); + + openads::network::Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + ADSHANDLE hConn = tb_connect_remote(dir, srv.port()); + ADSHANDLE hTable = tb_open(hConn); + + // Bind the bag and take the order to its top: one wire GotoTop that + // stamps (top, order) on the client. + UNSIGNED8 bag[] = "TB.CDX"; + ADSHANDLE hIdx = 0; + UNSIGNED16 nidx = 1; + REQUIRE(AdsOpenIndex(hTable, bag, &hIdx, &nidx) == AE_SUCCESS); + REQUIRE(nidx >= 1); + REQUIRE(AdsGotoTop(hIdx) == AE_SUCCESS); + + // The rddads rotation pattern: OrdListClear (parked, no frame) then + // the same bag reopened (unpark hit, no frame). + const std::uint64_t gt_before = tb_op(kOpGotoTop); + REQUIRE(AdsCloseAllIndexes(hTable) == AE_SUCCESS); + ADSHANDLE hIdx2 = 0; + nidx = 1; + REQUIRE(AdsOpenIndex(hTable, bag, &hIdx2, &nidx) == AE_SUCCESS); + // The post-unpark GotoTop(idx) must dedupe against the parked stamp: + // the server never moved, so no refresh frame is owed. + REQUIRE(AdsGotoTop(hIdx2) == AE_SUCCESS); + CHECK(tb_op(kOpGotoTop) == gt_before); + UNSIGNED32 rec = 0; + REQUIRE(AdsGetRecordNum(hTable, ADS_IGNOREFILTERS, &rec) == AE_SUCCESS); + CHECK(rec == 1u); + + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + srv.stop(); +} + +TEST_CASE("Parked nav stamp does not survive an intervening wire nav") { + tb_wipe(); + auto dir = tb_tmp_dir(); + tb_seed(dir); + + openads::network::Server srv; + REQUIRE(srv.start("127.0.0.1", 0).has_value()); + ADSHANDLE hConn = tb_connect_remote(dir, srv.port()); + ADSHANDLE hTable = tb_open(hConn); + + UNSIGNED8 bag[] = "TB.CDX"; + ADSHANDLE hIdx = 0; + UNSIGNED16 nidx = 1; + REQUIRE(AdsOpenIndex(hTable, bag, &hIdx, &nidx) == AE_SUCCESS); + REQUIRE(AdsGotoTop(hIdx) == AE_SUCCESS); + + REQUIRE(AdsCloseAllIndexes(hTable) == AE_SUCCESS); + // A wire nav inside the window bumps the cursor seq: the parked + // stamp no longer proves anything and must be ignored. + REQUIRE(AdsGotoBottom(hTable) == AE_SUCCESS); + ADSHANDLE hIdx2 = 0; + nidx = 1; + REQUIRE(AdsOpenIndex(hTable, bag, &hIdx2, &nidx) == AE_SUCCESS); + const std::uint64_t gt_before = tb_op(kOpGotoTop); + REQUIRE(AdsGotoTop(hIdx2) == AE_SUCCESS); + CHECK(tb_op(kOpGotoTop) == gt_before + 1); + + REQUIRE(AdsCloseTable(hTable) == AE_SUCCESS); + REQUIRE(AdsDisconnect(hConn) == AE_SUCCESS); + srv.stop(); +} diff --git a/tests/unit/network_use_budget_test.cpp b/tests/unit/network_use_budget_test.cpp index e11a1fb0..b3eca3e4 100644 --- a/tests/unit/network_use_budget_test.cpp +++ b/tests/unit/network_use_budget_test.cpp @@ -111,7 +111,8 @@ TEST_CASE("Nav batching: full USE cycle stays within wire budget") { // boundary probes and duplicate navs contribute zero frames. CHECK(total <= 14u); CHECK(delta(0x40) == 1u); // GotoTop: second suppressed - CHECK(delta(0x64) == 1u); // GotoBottom: second suppressed + CHECK(delta(0x64) == 0u); // mtfix15 restores opposite-boundary pair; + // bottom and duplicate are served locally CHECK(delta(0x48) == 0u); // AtEOF: served locally CHECK(delta(0x4C) == 0u); // AtBOF: served locally