Fix Go to Definition for cached BIG archives

crates/server/src/backend.rs

@@ -14,7 +14,6 @@ use std::time::{Duration, Instant};
 
 use base64::Engine;
 use dashmap::DashMap;
-use percent_encoding::percent_decode_str;
 use ropey::Rope;
 use serde::Deserialize;
 use tower_lsp::lsp_types::*;
@@ -39,6 +38,7 @@ use crate::scan::{
 };
 #[cfg(test)]
 use crate::scan::{parse_w3d_models, scan_big, scan_roots};
+use crate::uri::canonical_uri;
 
 const CLEAR_INDEX_CACHE_COMMAND: &str = "zerosyntax.clearIndexCache";
 const REBUILD_INDEX_CACHE_COMMAND: &str = "zerosyntax.rebuildIndexCache";
@@ -459,40 +459,6 @@ pub struct VirtualFileParams {
     uri: String,
 }
 
-/// Normalise a client-supplied URI to the form [`Url::from_file_path`] produces.
-/// On Windows, VS Code sends `file:///c%3A/…` (percent-encoded colon, lowercase
-/// drive letter) while `from_file_path` produces `file:///C:/…`. The mismatch
-/// makes the same file land in the `WorkspaceIndex` under two different keys,
-/// so every definition appears duplicated. Round-tripping through the file-path
-/// canonicalises both percent-encoding and drive-letter casing. Non-`file:`
-/// schemes other than `big:` are returned unchanged.
-fn canonical_uri(uri: Url) -> Url {
-    if uri.scheme() == "file" {
-        if let Ok(path) = uri.to_file_path() {
-            if let Ok(canonical) = Url::from_file_path(path) {
-                return canonical;
-            }
-        }
-    } else if uri.scheme() == "big" {
-        let Ok(mut path) = percent_decode_str(uri.path())
-            .decode_utf8()
-            .map(|path| path.into_owned())
-        else {
-            return uri;
-        };
-        if path.as_bytes().get(1).is_some_and(u8::is_ascii_lowercase)
-            && path.as_bytes().get(2) == Some(&b':')
-        {
-            let drive = char::from(path.as_bytes()[1].to_ascii_uppercase()).to_string();
-            path.replace_range(1..2, &drive);
-        }
-        let mut canonical = Url::parse("big:///").expect("static BIG URI is valid");
-        canonical.set_path(&path);
-        return canonical;
-    }
-    uri
-}
-
 fn is_map_layer_file(file: &str) -> bool {
     file.rsplit(['/', '\\']).next().is_some_and(|name| {
         name.eq_ignore_ascii_case("map.ini") || name.eq_ignore_ascii_case("solo.ini")

crates/server/src/cache.rs

@@ -17,7 +17,7 @@ const STORE_FILE: &str = "index-v1.sqlite3";
 const STORE_VERSION: i64 = 1;
 /// Bump when `CachedEntry` serialization or any extractor feeding it changes.
 /// SQL layout changes instead bump `STORE_VERSION` and the store filename.
-const PRODUCER_ABI: &[u8] = b"zerosyntax-physical-input-v3";
+const PRODUCER_ABI: &[u8] = b"zerosyntax-physical-input-v4";
 const BUSY_TIMEOUT: Duration = Duration::from_secs(2);
 const TOUCH_INTERVAL_SECS: i64 = 24 * 60 * 60;
 const MAX_AGE_SECS: i64 = 30 * 24 * 60 * 60;

crates/server/src/main.rs

@@ -7,6 +7,7 @@ mod cli;
 mod convert;
 mod progress;
 mod scan;
+mod uri;
 
 use backend::Backend;
 use tower_lsp::{LspService, Server};

crates/server/src/scan.rs

@@ -19,6 +19,7 @@ use zerosyntax_analysis::Analyzer;
 use zerosyntax_w3d::W3dFile;
 
 use crate::cache::{self, Fingerprint, InputCache};
+use crate::uri::canonical_uri;
 
 pub(crate) type ScanEntry = (
     String,
@@ -339,7 +340,7 @@ fn big_uri(path: &Path, entry: &str) -> String {
     let archive = path.to_string_lossy().replace('\\', "/");
     let mut uri = Url::parse("big:///").expect("static BIG URI is valid");
     uri.set_path(&format!("{archive}!/{entry}"));
-    uri.to_string()
+    canonical_uri(uri).to_string()
 }
 
 fn file_stem_str(path: &str) -> String {
@@ -1014,6 +1015,17 @@ mod tests {
         assert_eq!(postcard::from_bytes::<Vec<u8>>(&payload).unwrap(), value);
     }
 
+    #[test]
+    fn big_uri_normalizes_configured_drive_letter() {
+        assert_eq!(
+            big_uri(
+                Path::new("c:/Game Folder/Base #.big"),
+                "Data/INI/Object.ini"
+            ),
+            "big:///C:/Game%20Folder/Base%20%23.big!/Data/INI/Object.ini"
+        );
+    }
+
     #[test]
     fn big_archive_is_one_cached_input_with_many_virtual_entries() {
         let root = unique_temp_dir("big-cache");

crates/server/src/uri.rs

@@ -0,0 +1,34 @@
+//! Shared URI identity for scanned inputs and client requests.
+
+use percent_encoding::percent_decode_str;
+use tower_lsp::lsp_types::Url;
+
+/// Normalize file and BIG URIs before using them as index or source-text keys.
+/// VS Code percent-encodes drive colons and lowercases drive letters; scanners
+/// can also receive either drive spelling through configured base roots.
+pub(crate) fn canonical_uri(uri: Url) -> Url {
+    if uri.scheme() == "file" {
+        if let Ok(path) = uri.to_file_path() {
+            if let Ok(canonical) = Url::from_file_path(path) {
+                return canonical;
+            }
+        }
+    } else if uri.scheme() == "big" {
+        let Ok(mut path) = percent_decode_str(uri.path())
+            .decode_utf8()
+            .map(|path| path.into_owned())
+        else {
+            return uri;
+        };
+        if path.as_bytes().get(1).is_some_and(u8::is_ascii_lowercase)
+            && path.as_bytes().get(2) == Some(&b':')
+        {
+            let drive = char::from(path.as_bytes()[1].to_ascii_uppercase()).to_string();
+            path.replace_range(1..2, &drive);
+        }
+        let mut canonical = Url::parse("big:///").expect("static BIG URI is valid");
+        canonical.set_path(&path);
+        return canonical;
+    }
+    uri
+}

crates/server/tests/e2e.py

@@ -1241,6 +1241,9 @@ def wait_for2(pred, what, timeout=15.0):
     except Exception:
         proc2.kill()
 
+    from e2e_big_cache import check_big_cache
+    check_big_cache(exe, frame, reader)
+
     print("\nALL E2E CHECKS PASSED")
     return 0
 

crates/server/tests/e2e_big_cache.py

@@ -0,0 +1,145 @@
+"""Regression for #66: navigate into and within BIG files after a cache restart."""
+import os
+from pathlib import Path
+import queue
+import struct
+import subprocess
+import tempfile
+import threading
+import time
+import urllib.parse
+
+
+def check_big_cache(exe, frame, reader):
+    exe = str(Path(exe).resolve())
+    with tempfile.TemporaryDirectory(prefix="zerosyntax-big-navigation-") as temp:
+        root = Path(temp)
+        archive = root / "Base Cache #.big"
+        text = (
+            "; Archived implementation\r\n"
+            "CommandButton CachedButton\r\n  Command = UNIT_BUILD\r\nEnd\r\n"
+            "CommandSet CachedSet\r\n  1 = CachedButton\r\nEnd\r\n"
+        )
+        name = b"Data\\INI\\Cached.ini\0"
+        content = text.encode("utf-8")
+        offset = 16 + 8 + len(name)
+        archive.write_bytes(
+            b"BIGF" + struct.pack(">III", offset + len(content), 1, 0)
+            + struct.pack(">II", offset, len(content)) + name + content
+        )
+        configured_path = str(archive)
+        if os.name == "nt":
+            # A common spelling in baseIniRoots. Clients canonicalize the drive
+            # differently; the index and source-text cache must still agree.
+            configured_path = configured_path[0].lower() + configured_path[1:]
+        env = {**os.environ, "LOCALAPPDATA": str(root / "cache"),
+               "XDG_CACHE_HOME": str(root / "cache")}
+        source_uri = (root / "Reference.ini").as_uri()
+
+        for warm in (False, True):
+            proc = subprocess.Popen(
+                [exe], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL, bufsize=0, env=env,
+            )
+            messages = queue.Queue()
+            thread = threading.Thread(target=reader, args=(proc.stdout, messages), daemon=True)
+            thread.start()
+
+            def send(method, params, request_id=None):
+                message = {"jsonrpc": "2.0", "method": method}
+                if params is not None:
+                    message["params"] = params
+                if request_id is not None:
+                    message["id"] = request_id
+                proc.stdin.write(frame(message))
+                proc.stdin.flush()
+
+            def wait_for(predicate):
+                deadline = time.monotonic() + 15
+                while time.monotonic() < deadline:
+                    try:
+                        message = messages.get(timeout=max(0.001, deadline - time.monotonic()))
+                    except queue.Empty:
+                        break
+                    assert message is not None, "BIG navigation server exited early"
+                    assert "_parse_error" not in message, message
+                    if predicate(message):
+                        return message
+                raise AssertionError("timed out waiting for BIG navigation server")
+
+            def request(method, params, request_id):
+                send(method, params, request_id)
+                response = wait_for(lambda m: m.get("id") == request_id)
+                assert "error" not in response, response
+                return response["result"]
+
+            def open_doc(uri, document_text):
+                send("textDocument/didOpen", {"textDocument": {
+                    "uri": uri, "languageId": "generals-ini", "version": 1,
+                    "text": document_text,
+                }})
+                diagnostics = wait_for(
+                    lambda m: m.get("method") == "textDocument/publishDiagnostics"
+                )
+                assert not diagnostics["params"]["diagnostics"], diagnostics
+
+            def definition(uri, line, character, request_id):
+                return request("textDocument/definition", {
+                    "textDocument": {"uri": uri},
+                    "position": {"line": line, "character": character},
+                }, request_id)
+
+            try:
+                request("initialize", {
+                    "capabilities": {}, "rootUri": None,
+                    "initializationOptions": {"baseIniRoots": [configured_path]},
+                }, 1)
+                send("initialized", {})
+                indexed = wait_for(lambda m: m.get("method") == "window/logMessage"
+                                   and "indexing completed" in m["params"]["message"])
+                expected = "1 cached, 0 reparsed" if warm else "0 cached, 1 reparsed"
+                assert expected in indexed["params"]["message"], indexed
+
+                open_doc(source_uri, "Object CacheUser\n  CommandSet = CachedSet\nEnd\n")
+                locations = definition(source_uri, 1, 19, 2)
+                assert locations and len(locations) == 1, (warm, locations)
+                target = locations[0]
+                assert target["uri"].startswith("big:"), target
+                assert target["range"] == {
+                    "start": {"line": 4, "character": 11},
+                    "end": {"line": 4, "character": 20},
+                }, target
+                assert request("zerosyntax/readVirtualFile", {"uri": target["uri"]}, 3) == text
+
+                # Reproduce VS Code's URI serialization, then navigate from
+                # the virtual document to another definition in the archive.
+                path = urllib.parse.unquote(urllib.parse.urlsplit(target["uri"]).path)
+                if len(path) > 2 and path[2] == ":":
+                    path = path[:1] + path[1].lower() + path[2:]
+                client_uri = "big:" + urllib.parse.quote(path, safe="/")
+                assert request("zerosyntax/readVirtualFile", {"uri": client_uri}, 4) == text
+                open_doc(client_uri, text)
+                nested = definition(client_uri, 5, 10, 5)
+                assert nested == [{"uri": target["uri"], "range": {
+                    "start": {"line": 1, "character": 14},
+                    "end": {"line": 1, "character": 26},
+                }}], nested
+                request("shutdown", None, 6)
+                send("exit", None)
+                proc.stdin.close()
+                assert proc.wait(timeout=5) == 0
+            finally:
+                if proc.poll() is None:
+                    proc.kill()
+                    proc.wait(timeout=5)
+                thread.join(timeout=2)
+                proc.stdin.close()
+                proc.stdout.close()
+            print(f"OK: BIG definition navigation with {'warm' if warm else 'cold'} cache")
+
+
+if __name__ == "__main__":
+    import sys
+    from e2e import frame, reader
+
+    check_big_cache(sys.argv[1], frame, reader)