diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bd4d275..c313a5d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,18 +9,18 @@ jobs: fail-fast: false matrix: os: ["windows-latest", "ubuntu-latest", "macOS-latest"] - go: ["1.24.x"] + go: ["1.26.x"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 with: fetch-depth: 1 - - uses: WillAbides/setup-go-faster@v1.8.0 + - uses: actions/setup-go@v6 with: go-version: ${{ matrix.go }} - run: "go test -race ./..." - - uses: dominikh/staticcheck-action@v1.3.1 + - uses: dominikh/staticcheck-action@v1.4.0 with: - version: "2025.1" + version: "2025.1.1" install-go: false cache-key: ${{ matrix.go }} diff --git a/.gitignore b/.gitignore index 8e70cc6b..290ea923 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,10 @@ ctrld-* # generated file cmd/cli/rsrc_*.syso +ctrld +ctrld.exe + +# Local planning artifacts - never commit +/SPEC.md +/tasks/ +/SPEC-*.md diff --git a/README.md b/README.md index 5ff7445b..c9251987 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ A highly configurable DNS forwarding proxy with support for: - Multiple upstreams with fallbacks - Multiple network policy driven DNS query steering (via network cidr, MAC address or FQDN) - Policy driven domain based "split horizon" DNS with wildcard support -- Integrations with common router vendors and firmware - LAN client discovery via DHCP, mDNS, ARP, NDP, hosts file parsing - Prometheus metrics exporter @@ -25,50 +24,32 @@ All DNS protocols are supported, including: - `DNS-over-HTTP/3` (DOH3) - `DNS-over-QUIC` -# Use Cases -1. Use secure DNS protocols on networks and devices that don't natively support them (legacy routers, legacy OSes, TVs, smart toasters). +## Use Cases +1. Use secure DNS protocols on networks and devices that don't natively support them (legacy OSes, TVs, smart toasters). 2. Create source IP based DNS routing policies with variable secure DNS upstreams. Subnet 1 (admin) uses upstream resolver A, while Subnet 2 (employee) uses upstream resolver B. 3. Create destination IP based DNS routing policies with variable secure DNS upstreams. Listener 1 uses upstream resolver C, while Listener 2 uses upstream resolver D. 4. Create domain level "split horizon" DNS routing policies to send internal domains (*.company.int) to a local DNS server, while everything else goes to another upstream. -5. Deploy on a router and create LAN client specific DNS routing policies from a web GUI (When using ControlD.com). ## OS Support -- Windows (386, amd64, arm) -- Windows Server (386, amd64) +- Windows Desktop (386, amd64, arm64) - MacOS (amd64, arm64) -- Linux (386, amd64, arm, mips) -- FreeBSD (386, amd64, arm) -- Common routers (See below) - - -### Supported Routers -You can run `ctrld` on any supported router. The list of supported routers and firmware includes: -- Asus Merlin -- DD-WRT -- Firewalla -- FreshTomato -- GL.iNet -- OpenWRT -- pfSense / OPNsense -- Synology -- Ubiquiti (UniFi, EdgeOS) - -`ctrld` will attempt to interface with dnsmasq (or Windows Server) whenever possible and set itself as the upstream, while running on port 5354. On FreeBSD based OSes, `ctrld` will terminate dnsmasq and unbound in order to be able to listen on port 53 directly. - -# Install +- Linux (386, amd64, arm, arm64, mips, mipsle, mips64) +- FreeBSD (386, amd64, arm, arm64) + +## Install There are several ways to download and install `ctrld`. ## Quick Install The simplest way to download and install `ctrld` is to use the following installer command on any UNIX-like platform: ```shell -sh -c 'sh -c "$(curl -sL https://api.controld.com/dl)"' +sh -c 'sh -c "$(curl -sL https://api.controld.com/dl?version=client)"' ``` Windows user and prefer Powershell (who doesn't)? No problem, execute this command instead in administrative PowerShell: ```shell -(Invoke-WebRequest -Uri 'https://api.controld.com/dl/ps1' -UseBasicParsing).Content | Set-Content "$env:TEMPctrld_install.ps1"; Invoke-Expression "& '$env:TEMPctrld_install.ps1'" +(Invoke-WebRequest -Uri 'https://api.controld.com/dl/ps1?version=client' -UseBasicParsing).Content | Set-Content "$env:TEMP\ctrld_install.ps1"; Invoke-Expression "& '$env:TEMP\ctrld_install.ps1'" ``` Or you can pull and run a Docker container from [Docker Hub](https://hub.docker.com/r/controldns/ctrld) @@ -80,7 +61,7 @@ docker run -d --name=ctrld -p 127.0.0.1:53:53/tcp -p 127.0.0.1:53:53/udp control Alternatively, if you know what you're doing you can download pre-compiled binaries from the [Releases](https://github.com/Control-D-Inc/ctrld/releases) section for the appropriate platform. ## Build -Lastly, you can build `ctrld` from source which requires `go1.21+`: +Lastly, you can build `ctrld` from source which requires `go1.24+`: ```shell go build ./cmd/ctrld @@ -99,7 +80,7 @@ docker build -t controldns/ctrld . -f docker/Dockerfile ``` -# Usage +## Usage The cli is self documenting, so feel free to run `--help` on any sub-command to get specific usages. ## Arguments @@ -112,7 +93,7 @@ _/ ___\ __\_ __ \ | / __ | \/ dns forwarding proxy \/ Usage: - ctrld [command] + ctrld-client [command] Available Commands: run Run the DNS proxy server @@ -128,12 +109,12 @@ Available Commands: log Manage runtime debug logs Flags: - -h, --help help for ctrld + -h, --help help for ctrld-client -s, --silent do not write any log output - -v, --verbose count verbose log output, "-v" basic logging, "-vv" debug level logging - --version version for ctrld + -v, --verbose count verbose log output, "-v" basic logging, "-vv" debug logging + --version version for ctrld-client -Use "ctrld [command] --help" for more information about a command. +Use "ctrld-client [command] --help" for more information about a command. ``` ## Basic Run Mode @@ -143,12 +124,12 @@ This is the most basic way to run `ctrld`, in foreground mode. Unless you alread Windows (Admin Shell) ```shell - ctrld.exe run + ctrld-client.exe run ``` Linux or Macos ```shell - sudo ctrld run + sudo ctrld-client run ``` You can then run a test query using a DNS client, for example, `dig`: @@ -161,25 +142,23 @@ You can then run a test query using a DNS client, for example, `dig`: If `verify.controld.com` resolves, you're successfully using the default Control D upstream. From here, you can start editing the config file that was generated. To enforce a new config, restart the server. ## Service Mode -This mode will run the application as a background system service on any Windows, MacOS, Linux, FreeBSD distribution or supported router. This will create a generic `ctrld.toml` file in the **C:\ControlD** directory (on Windows) or `/etc/controld/` (almost everywhere else), start the system service, and **configure the listener on all physical network interface**. Service will start on OS boot. - -When Control D upstreams are used on a router type device, `ctrld` will [relay your network topology](https://docs.controld.com/docs/device-clients) to Control D (LAN IPs, MAC addresses, and hostnames), and you will be able to see your LAN devices in the web panel, view analytics and apply unique profiles to them. +This mode will run the application as a background system service on any Windows, MacOS, Linux or FreeBSD distribution. This will create a generic `ctrld.toml` file in the **C:\ControlD** directory (on Windows) or `/etc/controld/` (almost everywhere else), start the system service, and **configure the listener on all physical network interface**. Service will start on OS boot. ### Command Windows (Admin Shell) ```shell - ctrld.exe start + ctrld-client.exe start ``` Linux or Macos ``` - sudo ctrld start + sudo ctrld-client start ``` -If `ctrld` is not in your system path (you installed it manually), you will need to run the above commands from the directory where you installed `ctrld`. +If `ctrld-client` is not in your system path (you installed it manually), you will need to run the above commands from the directory where you installed it. -In order to stop the service, and restore your DNS to original state, simply run `ctrld stop`. If you wish to stop and uninstall the service permanently, run `ctrld uninstall`. +In order to stop the service, and restore your DNS to original state, simply run `ctrld-client stop`. If you wish to stop and uninstall the service permanently, run `ctrld-client uninstall`. ## Unmanaged Service Mode This mode functions similarly to the "Service Mode" above except it will simply start a system service and the config defined listeners, but **will not make any changes to any network interfaces**. You can then set the `ctrld` listener(s) IP on the desired network interfaces manually. @@ -188,36 +167,36 @@ This mode functions similarly to the "Service Mode" above except it will simply Windows (Admin Shell) ```shell - ctrld.exe service start + ctrld-client.exe service start ``` Linux or Macos ```shell - sudo ctrld service start + sudo ctrld-client service start ``` -# Configuration +## Configuration `ctrld` can be configured in variety of different ways, which include: API, local config file or via cli launch args. ## API Based Auto Configuration -Application can be started with a specific Control D resolver config, instead of the default one. Simply supply your Resolver ID with a `--cd` flag, when using the `start` (service) mode. In this mode, the application will automatically choose a non-conflicting IP and/or port and configure itself as the upstream to whatever process is running on port 53 (like dnsmasq or Windows DNS Server). This mode is used when the 1 liner installer command from the Control D onboarding guide is executed. +Application can be started with a specific Control D resolver config, instead of the default one. Simply supply your Resolver ID with a `--cd` flag, when using the `start` (service) mode. This mode is used when the 1 liner installer command from the Control D onboarding guide is executed. The following command will use your own personal Control D Device resolver, and start the application in service mode. Your resolver ID is displayed on the "Show Resolvers" screen for the relevant Control D Endpoint. Windows (Admin Shell) ```shell -ctrld.exe start --cd abcd1234 +ctrld-client.exe start --cd abcd1234 ``` Linux or Macos ```shell -sudo ctrld start --cd abcd1234 +sudo ctrld-client start --cd abcd1234 ``` Once you run the above command, the following things will happen: - You resolver configuration will be fetched from the API, and config file templated with the resolver data - Application will start as a service, and keep running (even after reboot) until you run the `stop` or `uninstall` sub-commands -- All physical network interface will be updated to use the listener started by the service or dnsmasq upstream will be switched to `ctrld` +- All physical network interface will be updated to use the listener started by the service - All DNS queries will be sent to the listener ## Manual Configuration @@ -257,7 +236,7 @@ If you're unable to use a config file, `ctrld` can be be supplied with basic con ### Example ``` -ctrld run --listen=127.0.0.1:53 --primary_upstream=https://freedns.controld.com/p2 --secondary_upstream=10.0.10.1:53 --domains=*.company.int,very-secure.local --log /path/to/log.log +ctrld-client run --listen=127.0.0.1:53 --primary_upstream=https://freedns.controld.com/p2 --secondary_upstream=10.0.10.1:53 --domains=*.company.int,very-secure.local --log /path/to/log.log ``` The above will start a foreground process and: @@ -266,5 +245,69 @@ The above will start a foreground process and: - Excluding `*.company.int` and `very-secure.local` matching queries, that are forwarded to `10.0.10.1:53` - Write a debug log to `/path/to/log.log` +## DNS Intercept Mode +When running `ctrld` alongside VPN software, DNS conflicts can cause intermittent failures, bypassed filtering, or configuration loops. DNS Intercept Mode prevents these issues by transparently capturing all DNS traffic on the system and routing it through `ctrld`, without modifying network adapter DNS settings. + +DNS Intercept Mode (`dns`) is the default for new installations on macOS and on Windows desktop editions when no mode is selected with `--intercept-mode` or `service.intercept_mode`; existing Standard installations keep interface DNS mode across upgrades. Windows Server and other platforms keep interface DNS mode unless intercept is selected explicitly. See [Intercept Mode Default](docs/intercept-mode-default.md). + +### When to Use +Enable DNS Intercept Mode if you: +- Use corporate VPN software (F5, Cisco AnyConnect, Palo Alto GlobalProtect, Zscaler) +- Run overlay networks like Tailscale or WireGuard +- Experience random DNS failures when VPN connects/disconnects +- See gaps in your Control D analytics when VPN is active +- Have endpoint security software that also manages DNS + +### Command + +Windows (Admin Shell) +```shell +ctrld-client.exe start --intercept-mode dns --cd RESOLVER_ID_HERE +``` + +macOS +```shell +sudo ctrld-client start --intercept-mode dns --cd RESOLVER_ID_HERE +``` + +`--intercept-mode dns` automatically detects VPN internal domains and routes them to the VPN's DNS server, while Control D handles everything else. On macOS and Windows desktop editions the flag can be left out: `dns` is the default. On Windows Server it must be given. + +To use interface DNS mode instead, on a new or an existing service: + +Windows (Admin Shell) +```shell +ctrld-client.exe start --intercept-mode off +``` + +macOS +```shell +sudo ctrld-client start --intercept-mode off +``` + +This removes the intercept rules and reverts to standard interface-based DNS configuration. The selection is persisted (`intercept_mode = "off"` in `ctrld.toml`), so it survives restarts and upgrades. + +### Platform Support +| Platform | Supported | Mechanism | +|----------|-----------|-----------| +| Windows | ✅ | NRPT (Name Resolution Policy Table) | +| macOS | ✅ | pf (packet filter) redirect | +| Linux | ❌ | Not currently supported | + +### Features +- **VPN split routing** — VPN-specific domains are automatically detected and forwarded to the VPN's DNS server +- **Captive portal recovery** — Wi-Fi login pages (hotels, airports, coffee shops) work automatically +- **No network adapter changes** — DNS settings stay untouched, eliminating conflicts entirely +- **Automatic port 53 conflict resolution** — if another process (e.g., `mDNSResponder` on macOS) is already using port 53, `ctrld` automatically listens on a different port. OS-level packet interception redirects all DNS traffic to `ctrld` transparently, so no manual configuration is needed. This only applies to intercept mode. + +### Tested VPN Software +- F5 BIG-IP APM +- Cisco AnyConnect +- Palo Alto GlobalProtect +- Tailscale (including Exit Nodes) +- Windscribe +- WireGuard + +For more details, see the [DNS Intercept Mode documentation](https://docs.controld.com/docs/dns-intercept). + ## Contributing See [Contribution Guideline](./docs/contributing.md) diff --git a/client_info_darwin.go b/client_info_darwin.go deleted file mode 100644 index 4c3d10b2..00000000 --- a/client_info_darwin.go +++ /dev/null @@ -1,4 +0,0 @@ -package ctrld - -// SelfDiscover reports whether ctrld should only do self discover. -func SelfDiscover() bool { return true } diff --git a/client_info_others.go b/client_info_others.go deleted file mode 100644 index d728913a..00000000 --- a/client_info_others.go +++ /dev/null @@ -1,6 +0,0 @@ -//go:build !windows && !darwin - -package ctrld - -// SelfDiscover reports whether ctrld should only do self discover. -func SelfDiscover() bool { return false } diff --git a/client_info_windows.go b/client_info_windows.go deleted file mode 100644 index f20bca78..00000000 --- a/client_info_windows.go +++ /dev/null @@ -1,18 +0,0 @@ -package ctrld - -import ( - "golang.org/x/sys/windows" -) - -// isWindowsWorkStation reports whether ctrld was run on a Windows workstation machine. -func isWindowsWorkStation() bool { - // From https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa - const VER_NT_WORKSTATION = 0x0000001 - osvi := windows.RtlGetVersion() - return osvi.ProductType == VER_NT_WORKSTATION -} - -// SelfDiscover reports whether ctrld should only do self discover. -func SelfDiscover() bool { - return isWindowsWorkStation() -} diff --git a/cmd/cli/ad_domain.go b/cmd/cli/ad_domain.go new file mode 100644 index 00000000..3654b0d2 --- /dev/null +++ b/cmd/cli/ad_domain.go @@ -0,0 +1,36 @@ +package cli + +import ( + "strings" + "sync/atomic" +) + +// activeDirectoryDomain is the machine's Active Directory domain, recorded when +// addExtraSplitDnsRule detects it (Windows only; empty elsewhere). +// +// Names in it are resolved by the domain controller through upstream.os. The +// OS resolver races the LAN servers against Control D's public resolver unless +// the query is marked as a LAN query, and a split-rule match returns before +// that marking, so without this AD names (DC hostnames, DC-locator SRV +// records carrying the machine name, WPAD) were sent to 76.76.2.0 in plaintext. +var activeDirectoryDomain atomic.Pointer[string] + +func setActiveDirectoryDomain(domain string) { + d := strings.TrimSuffix(strings.ToLower(strings.TrimSpace(domain)), ".") + activeDirectoryDomain.Store(&d) +} + +// inActiveDirectoryDomain reports whether name is the AD domain or below it. +func inActiveDirectoryDomain(name string) bool { + d := activeDirectoryDomain.Load() + if d == nil || *d == "" { + return false + } + name = strings.TrimSuffix(strings.ToLower(name), ".") + return name == *d || strings.HasSuffix(name, "."+*d) +} + +// osUpstreamOnly reports whether upstreams is exactly the OS resolver. +func osUpstreamOnly(upstreams []string) bool { + return len(upstreams) == 1 && upstreams[0] == upstreamOS +} diff --git a/cmd/cli/ad_domain_test.go b/cmd/cli/ad_domain_test.go new file mode 100644 index 00000000..6db180fd --- /dev/null +++ b/cmd/cli/ad_domain_test.go @@ -0,0 +1,73 @@ +package cli + +import ( + "context" + "testing" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" +) + +func withActiveDirectoryDomain(t *testing.T, domain string) { + t.Helper() + prev := activeDirectoryDomain.Load() + setActiveDirectoryDomain(domain) + t.Cleanup(func() { activeDirectoryDomain.Store(prev) }) +} + +func Test_inActiveDirectoryDomain(t *testing.T) { + withActiveDirectoryDomain(t, "Corp.Lab.") + for name, want := range map[string]bool{ + "corp.lab.": true, + "DC01.corp.lab.": true, + "_ldap._tcp.dc._msdcs.corp.lab.": true, + "wpad.corp.lab": true, + "notcorp.lab.": false, + "corp.lab.example.com.": false, + "13gmuaedixa.verify.controld.com.": false, + } { + if got := inActiveDirectoryDomain(name); got != want { + t.Errorf("inActiveDirectoryDomain(%q) = %v, want %v", name, got, want) + } + } + activeDirectoryDomain.Store(nil) + if inActiveDirectoryDomain("dc01.corp.lab.") { + t.Error("no AD domain recorded must match nothing") + } +} + +// A split-rule match returns before the LAN-query marking; AD names routed to +// the OS resolver must still be marked, so osResolver drops 76.76.2.0. +func Test_handleSpecialQueryTypes_ADRuleIsLanQuery(t *testing.T) { + withActiveDirectoryDomain(t, "corp.lab") + p := newTestProg(t) + for _, tc := range []struct { + name string + qname string + qtype uint16 + upstreams []string + wantLan bool + }{ + {"AD host via OS", "DC01.corp.lab.", dns.TypeA, []string{upstreamOS}, true}, + {"DC locator SRV via OS", "_ldap._tcp.dc._msdcs.corp.lab.", dns.TypeSRV, []string{upstreamOS}, true}, + {"AD name, explicit upstream", "dc01.corp.lab.", dns.TypeA, []string{upstreamPrefix + "0"}, false}, + {"bypass rule, public name", "example.com.", dns.TypeA, []string{upstreamOS}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + m := new(dns.Msg) + m.SetQuestion(tc.qname, tc.qtype) + req := &proxyRequest{msg: m, ufr: &upstreamForResult{matched: true, matchedPolicy: "My Policy", matchedNetwork: "no network", matchedRule: "*.corp.lab"}} + ctx := context.Background() + upstreams := tc.upstreams + var configs []*ctrld.UpstreamConfig + if res := p.handleSpecialQueryTypes(&ctx, req, &upstreams, &configs); res != nil { + t.Fatalf("unexpected local answer: %v", res) + } + lan, _ := ctx.Value(ctrld.LanQueryCtxKey{}).(bool) + if lan != tc.wantLan { + t.Errorf("LAN query = %v, want %v", lan, tc.wantLan) + } + }) + } +} diff --git a/cmd/cli/ad_os_lookup_test.go b/cmd/cli/ad_os_lookup_test.go new file mode 100644 index 00000000..0834a94c --- /dev/null +++ b/cmd/cli/ad_os_lookup_test.go @@ -0,0 +1,155 @@ +package cli + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" +) + +// adLookupResolver records whether each resolve carried the LAN mark, which is +// what keeps osResolver off Control D's public resolver. Only the OS resolver +// answers, so an explicit upstream fails over to it. +type adLookupResolver struct { + name string + seen *[]adLookupResolve +} + +type adLookupResolve struct { + resolver string + lan bool +} + +func (r adLookupResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + lan, _ := ctx.Value(ctrld.LanQueryCtxKey{}).(bool) + *r.seen = append(*r.seen, adLookupResolve{r.name, lan}) + if r.name != osUpstreamConfig.Name { + return nil, errors.New("upstream unreachable") + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, nil +} + +func newAdLookupProg(t *testing.T, leak bool) (*prog, *[]adLookupResolve) { + t.Helper() + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{ + "0": {Name: "explicit resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: "192.0.2.53:53", Timeout: 100}, + }} + cfg.Service.LeakOnUpstreamFailure = &leak + withActiveDirectoryDomain(t, "corp.lab") + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + p.recoveryCancel = func() {} + p.um.after = func(time.Duration, func()) {} + p.lanLoopGuard = newLoopGuard() + p.ptrLoopGuard = newLoopGuard() + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + var seen []adLookupResolve + origNewResolver := newResolverFn + t.Cleanup(func() { newResolverFn = origNewResolver }) + newResolverFn = func(_ context.Context, uc *ctrld.UpstreamConfig) (ctrld.Resolver, error) { + return adLookupResolver{name: uc.Name, seen: &seen}, nil + } + return p, &seen +} + +func adLookupQuery(p *prog, name string, ufr *upstreamForResult) *proxyResponse { + msg := new(dns.Msg) + qtype := dns.TypeA + if strings.HasPrefix(name, "_") { + qtype = dns.TypeSRV + } + msg.SetQuestion(name, qtype) + ufr.srcAddr = "192.168.0.1:1234" + return p.proxy(context.Background(), &proxyRequest{msg: msg, ufr: ufr}) +} + +func wantAdLookupResolves(t *testing.T, got []adLookupResolve, want ...adLookupResolve) { + t.Helper() + if !slices.Equal(got, want) { + t.Fatalf("resolves = %+v, want %+v", got, want) + } +} + +// Drive proxy() itself, so the test fails if a path stops marking the query. +func Test_proxyKeepsAdLookupsOffThePublicResolver(t *testing.T) { + captureDebugMainLog(t) + osName := osUpstreamConfig.Name + for _, tc := range []struct { + name string + qname string + ufr *upstreamForResult + leak bool + want []adLookupResolve + }{ + { + // The auto-added AD rule has no targets, so proxy() routes it to the OS resolver. + name: "auto-added AD rule", + qname: "dc01.corp.lab.", + ufr: &upstreamForResult{matched: true}, + want: []adLookupResolve{{osName, true}}, + }, + { + name: "DC locator SRV through an AD rule", + qname: "_ldap._tcp.dc._msdcs.corp.lab.", + ufr: &upstreamForResult{matched: true}, + want: []adLookupResolve{{osName, true}}, + }, + { + name: "OS rule for a public name keeps the public fallback", + qname: "example.com.", + ufr: &upstreamForResult{matched: true}, + want: []adLookupResolve{{osName, false}}, + }, + { + name: "explicit rule for an AD name falls back to the OS resolver", + qname: "dc01.corp.lab.", + ufr: &upstreamForResult{matched: true, upstreams: []string{upstreamPrefix + "0"}}, + leak: true, + want: []adLookupResolve{{"explicit resolver", false}, {osName, true}}, + }, + { + name: "explicit rule for a public name falls back with the public resolver", + qname: "example.com.", + ufr: &upstreamForResult{matched: true, upstreams: []string{upstreamPrefix + "0"}}, + leak: true, + want: []adLookupResolve{{"explicit resolver", false}, {osName, false}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + p, seen := newAdLookupProg(t, tc.leak) + if res := adLookupQuery(p, tc.qname, tc.ufr); res == nil || res.answer == nil { + t.Fatal("the proxy returned no answer") + } + wantAdLookupResolves(t, *seen, tc.want...) + }) + } +} + +// The DNS-intercept recovery bypass forwards queries before any rule handling. +func Test_proxyRecoveryBypassKeepsAdLookupsOffThePublicResolver(t *testing.T) { + captureDebugMainLog(t) + useDNSIntercept(t) + osName := osUpstreamConfig.Name + for _, tc := range []struct { + qname string + lan bool + }{{"dc01.corp.lab.", true}, {"example.com.", false}} { + t.Run(tc.qname, func(t *testing.T) { + p, seen := newAdLookupProg(t, false) + p.recoveryBypass.Store(true) + if res := adLookupQuery(p, tc.qname, &upstreamForResult{upstreams: []string{upstreamPrefix + "0"}}); res == nil || res.answer == nil { + t.Fatal("the proxy returned no answer") + } + wantAdLookupResolves(t, *seen, adLookupResolve{osName, tc.lan}) + }) + } +} diff --git a/cmd/cli/ad_others.go b/cmd/cli/ad_others.go index b23476fe..6a7417fb 100644 --- a/cmd/cli/ad_others.go +++ b/cmd/cli/ad_others.go @@ -8,8 +8,3 @@ import ( // addExtraSplitDnsRule adds split DNS rule if present. func addExtraSplitDnsRule(_ *ctrld.Config) bool { return false } - -// getActiveDirectoryDomain returns AD domain name of this computer. -func getActiveDirectoryDomain() (string, error) { - return "", nil -} diff --git a/cmd/cli/ad_windows.go b/cmd/cli/ad_windows.go index 66180a90..af1ff572 100644 --- a/cmd/cli/ad_windows.go +++ b/cmd/cli/ad_windows.go @@ -1,31 +1,27 @@ package cli import ( - "io" - "log" - "os" "strings" - "github.com/microsoft/wmi/pkg/base/host" - hh "github.com/microsoft/wmi/pkg/hardware/host" - "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/system" ) // addExtraSplitDnsRule adds split DNS rule for domain if it's part of active directory. func addExtraSplitDnsRule(cfg *ctrld.Config) bool { - domain, err := getActiveDirectoryDomain() + domain, err := system.GetActiveDirectoryDomain() if err != nil { - mainLog.Load().Debug().Msgf("unable to get active directory domain: %v", err) + mainLog.Load().Debug().Msgf("Unable to get active directory domain: %v", err) return false } if domain == "" { - mainLog.Load().Debug().Msg("no active directory domain found") + mainLog.Load().Debug().Msg("No active directory domain found") return false } // Network rules are lowercase during toml config marshaling, // lowercase the domain here too for consistency. domain = strings.ToLower(domain) + setActiveDirectoryDomain(domain) domainRuleAdded := addSplitDnsRule(cfg, domain) wildcardDomainRuleRuleAdded := addSplitDnsRule(cfg, "*."+strings.TrimPrefix(domain, ".")) return domainRuleAdded || wildcardDomainRuleRuleAdded @@ -40,34 +36,12 @@ func addSplitDnsRule(cfg *ctrld.Config, domain string) bool { } for _, rule := range lc.Policy.Rules { if _, ok := rule[domain]; ok { - mainLog.Load().Debug().Msgf("split-rule %q already existed for listener.%s", domain, n) + mainLog.Load().Debug().Msgf("Split-rule %q already existed for listener.%s", domain, n) return false } } - mainLog.Load().Debug().Msgf("adding split-rule %q for listener.%s", domain, n) + mainLog.Load().Debug().Msgf("Adding split-rule %q for listener.%s", domain, n) lc.Policy.Rules = append(lc.Policy.Rules, ctrld.Rule{domain: []string{}}) } return true } - -// getActiveDirectoryDomain returns AD domain name of this computer. -func getActiveDirectoryDomain() (string, error) { - log.SetOutput(io.Discard) - defer log.SetOutput(os.Stderr) - whost := host.NewWmiLocalHost() - cs, err := hh.GetComputerSystem(whost) - if cs != nil { - defer cs.Close() - } - if err != nil { - return "", err - } - pod, err := cs.GetPropertyPartOfDomain() - if err != nil { - return "", err - } - if pod { - return cs.GetPropertyDomain() - } - return "", nil -} diff --git a/cmd/cli/ad_windows_test.go b/cmd/cli/ad_windows_test.go index 6abd25f9..c987fe13 100644 --- a/cmd/cli/ad_windows_test.go +++ b/cmd/cli/ad_windows_test.go @@ -5,14 +5,16 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/system" "github.com/Control-D-Inc/ctrld/testhelper" - "github.com/stretchr/testify/assert" ) func Test_getActiveDirectoryDomain(t *testing.T) { start := time.Now() - domain, err := getActiveDirectoryDomain() + domain, err := system.GetActiveDirectoryDomain() if err != nil { t.Fatal(err) } diff --git a/cmd/cli/api_maintenance.go b/cmd/cli/api_maintenance.go new file mode 100644 index 00000000..b84244cf --- /dev/null +++ b/cmd/cli/api_maintenance.go @@ -0,0 +1,251 @@ +package cli + +import ( + "crypto/sha256" + "encoding/hex" + "math/rand" + "os" + "strings" + "time" + + "github.com/go-playground/validator/v10" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// configReadFromDisk records that startup read a configuration file this host +// already had, instead of generating a default one because none existed. +// +// The distinction only matters while the API is in maintenance. A generated +// default carries the public freedns upstreams, so falling back to it would +// answer queries outside the Control D profile the operator provisioned — +// quietly serving a policy nobody chose, which is worse than not starting. +// Only a file the host already had can stand in for the configuration the API +// would have returned. +var configReadFromDisk bool + +// maintenanceNoConfigDetail is what ctrld reports when maintenance stops a +// start that has nothing to fall back on. It names the operation that cannot +// happen rather than the fetch that failed, because "could not fetch resolver +// config" reads as a network fault and sends the reader after connectivity +// that is not broken. +const maintenanceNoConfigDetail = "cannot provision during Control D API maintenance: " + + "this host has no previously working configuration to fall back on. Retry once maintenance ends." + +// managedConfigMarkerFile records that the configuration file next to it was +// written from an API answer, and for which identity. +// +// File presence cannot carry this on its own. A start that has no +// configuration writes the seeded default before the API is ever reached, so a +// provisioning that failed leaves a file behind that the next start reads +// successfully. That file holds the public freedns upstreams, so trusting it +// would run the endpoint outside the Control D profile it was provisioned for +// — the exact outcome the fallback exists to avoid. +const managedConfigMarkerFile = ".managed_config" + +// managedConfigMarkerPath is a var so tests can place the marker in a temp dir. +var managedConfigMarkerPath = func() string { return absHomeDir(managedConfigMarkerFile) } + +// managedConfigMarker fingerprints both the identity and the configuration file +// the marker is recorded for. +// +// The identity alone cannot carry it. A marker that names only the identity +// outlives the file it vouched for: delete or move that file and the next start +// regenerates the seeded default, which the stale marker then certifies. The +// same happens when --config points somewhere else under the same home. Binding +// to the contents means any file the API did not produce fails the check, +// without having to detect how it got there. +// +// The raw identity is kept off disk: it is redacted from retained logs, and a +// marker only ever has to answer whether this is the same identity. +func managedConfigMarker(uid, path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + identity := sha256.Sum256([]byte(uid)) + contents := sha256.Sum256(data) + return hex.EncodeToString(identity[:]) + ":" + hex.EncodeToString(contents[:]), nil +} + +// recordManagedConfig notes that the configuration on disk came from the API +// for uid. The caller must have persisted that configuration first, or the +// marker would vouch for a file that does not match it. +func recordManagedConfig(uid string) error { + if uid == "" { + return nil + } + marker, err := managedConfigMarker(uid, defaultConfigFile) + if err != nil { + return err + } + return os.WriteFile(managedConfigMarkerPath(), []byte(marker), 0600) +} + +// managedConfigRecordedFor reports whether the configuration file in use was +// written from an API answer for uid. +func managedConfigRecordedFor(uid string) bool { + if uid == "" { + return false + } + recorded, err := os.ReadFile(managedConfigMarkerPath()) + if err != nil { + return false + } + want, err := managedConfigMarker(uid, defaultConfigFile) + if err != nil { + return false + } + return strings.TrimSpace(string(recorded)) == want +} + +// lastKnownConfigUsable reports whether cfg can carry this start by itself. +// +// It is deliberately quiet: this is a probe, and the caller reports either +// outcome. Startup validates the configuration again further on and reports +// its own errors there, so a failure here must not also print them. +func lastKnownConfigUsable(cfg *ctrld.Config) bool { + if !configReadFromDisk || cfg == nil { + return false + } + // A file this host already had is necessary but not sufficient: it must + // also be one the API produced, for the identity being asked for now. + if !managedConfigRecordedFor(cdUID) { + return false + } + return ctrld.ValidateConfig(validator.New(), cfg) == nil +} + +// useLastKnownConfigDuringMaintenance reports whether startup may continue on +// the configuration already on disk because the API is in maintenance. +// +// Maintenance says nothing about this host's configuration — the same request +// succeeds once the window ends — so stopping DNS for its duration turns a +// temporary API outage into an endpoint outage. Continuing is only honest when +// the configuration on disk is one the API itself produced on an earlier run: +// configReadFromDisk establishes that this host already had the file, and the +// managed-config marker that the API produced it for this identity. The marker +// is the load-bearing half - a file alone can be a generated default. +// +// The resolver config stays nil: this run has none, and the reload path +// already treats a nil one as "no previous response to compare against", so +// the first successful reload after maintenance adopts the fresh answer. +func useLastKnownConfigDuringMaintenance(err error, cfg *ctrld.Config) bool { + if !controld.IsMaintenance(err) { + return false + } + logger := mainLog.Load().With().Str("mode", "cd") + if !lastKnownConfigUsable(cfg) { + journal(logger.Error()).Str("reason", "api_maintenance"). + Msg("Control D API is in maintenance and no previously working configuration is on disk") + return false + } + journal(logger.Warn()).Str("reason", "api_maintenance"). + Msg("Control D API is in maintenance; starting with the last known working configuration on disk") + return true +} + +// logResolverConfigFetchFailure reports a resolver-config fetch that failed on +// a path which keeps serving DNS afterwards: the periodic reload and the +// self-uninstall probe. +// +// Maintenance gets its own line, and it is journaled. "could not fetch +// resolver config" reads as a fault on this host and sends the reader after +// connectivity that is not broken, which is how the reported incident was +// first triaged. It also leaves a later "why is my configuration stale" +// question unanswerable once the debug stream has rotated. The reload runs +// hourly by default, so one retained line per window is bounded. +func logResolverConfigFetchFailure(logger *ctrld.Logger, err error) { + if controld.IsMaintenance(err) { + journal(logger.Warn()).Err(err).Str("reason", "api_maintenance"). + Msg("Control D API is in maintenance; keeping the current configuration until it returns") + return + } + logger.Warn().Err(err).Msg("Could not fetch resolver config") +} + +const ( + // maintenanceReloadInterval is how soon a run that started on the + // configuration on disk asks the API again. It is deliberately far below + // refetch_time: such a run holds no answer from the API at all, so its + // exclude list, Internal Domains, deactivation pin and self-upgrade check + // are all still those of the previous run until one arrives. + maintenanceReloadInterval = 5 * time.Minute + // maintenanceReloadJitter spreads those retries across the hosts that + // started during the same window. + // + // The hourly ticker needs no jitter because each host starts it at its own + // boot time, which spreads the fleet on its own. A maintenance window + // removes that spread: every host that falls back does so within the same + // few minutes, so a fixed retry would put them all in lockstep and land as + // one burst on the API exactly as it comes back. + maintenanceReloadJitter = 2 * time.Minute +) + +// maintenanceReloadDelay returns the wait before the next retry of a run that +// started on the configuration on disk. It is a var so a test can drive the +// reload loop without waiting out a real interval. +var maintenanceReloadDelay = func() time.Duration { + return maintenanceReloadInterval + time.Duration(rand.Int63n(int64(maintenanceReloadJitter))) +} + +// maintenanceConfigApplied records that a run which started on the +// configuration on disk has handed an answer from the API to the reload path. +// +// It is deliberately not called when the fetch succeeds, nor when the response +// is handed to the reload. The reload receiver reads the configuration and +// fetches again, and either step can fail; until it does not, the endpoint is +// still serving what it started with. Clearing the flag earlier would stop the +// faster retry and report a recovery that had not happened. +func (p *prog) maintenanceConfigApplied(logger *ctrld.Logger) { + if p.startedInAPIMaintenance.CompareAndSwap(true, false) { + journal(logger.Info()).Str("reason", "api_maintenance"). + Msg("Control D API answered again; this run applied the configuration it returned") + } +} + +// persistManagedConfig writes cfg and, on a run the API manages, records that +// the file it just wrote came from there. +// +// The two belong together. The marker is bound to the file's contents, and +// every API-driven change rewrites that file, so a write that does not refresh +// the marker leaves the host ineligible for the very fallback its own +// configuration is meant to provide - including recovery from a maintenance +// start. A failed write certifies nothing, and a run with no resolver UID +// writes nothing the API produced: generated defaults and NextDNS +// configurations must never be certified. +func persistManagedConfig(cfg *ctrld.Config, logger *ctrld.Logger) { + if err := writeConfigFile(cfg); err != nil { + logger.Error().Err(err).Msg("Could not write new config") + return + } + if cdUID == "" { + return + } + if err := recordManagedConfig(cdUID); err != nil { + logger.Warn().Err(err).Msg("Could not record that this configuration came from the API") + } +} + +// recordManagedConfigForStart refreshes the marker once a start has finished +// writing its configuration. +// +// fetchedThisRun is whether the API answered this run. wasManagedBeforeWrite is +// whether the file already carried a marker for this identity before anything +// this start rewrote it. Either is enough: a maintenance fallback fetches +// nothing, yet still rewrites the file through the listener and intercept-mode +// updates, and the marker is bound to the file's contents - so without the +// second the next start in the same window loses the fallback this one used. +// Neither means the file is a generated default or otherwise unmanaged, which +// must never be certified. +func recordManagedConfigForStart(fetchedThisRun, wasManagedBeforeWrite bool) { + if !fetchedThisRun && !wasManagedBeforeWrite { + return + } + if err := recordManagedConfig(cdUID); err != nil { + // Not fatal: this run is fine. It only costs the next start its + // fallback if the API is in maintenance then. + mainLog.Load().Warn().Err(err).Msg("Could not record that this configuration came from the API") + } +} diff --git a/cmd/cli/api_maintenance_firewall_test.go b/cmd/cli/api_maintenance_firewall_test.go new file mode 100644 index 00000000..7aebc24e --- /dev/null +++ b/cmd/cli/api_maintenance_firewall_test.go @@ -0,0 +1,96 @@ +package cli + +import ( + "context" + "net/netip" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// TestMaintenanceFallbackFirewallAllowedDestinations covers Firewall Mode on a +// maintenance fallback start. The organization's Allowed Destinations exist +// only in the API answer, which such a run does not have, so the exception set +// is empty until the API returns. That is a deliberate fail-closed state: it +// must be named in the retained journal, and the first answer after recovery +// must re-apply the list even though nothing else is compared as changed. +func TestMaintenanceFallbackFirewallAllowedDestinations(t *testing.T) { + allowed := netip.MustParseAddr("203.0.113.10") + + t.Run("fallback start journals that allowed destinations are not enforced", func(t *testing.T) { + logs := captureDebugMainLog(t) + p := progWithAllowList() + p.logger.Store(mainLog.Load()) + p.startedInAPIMaintenance.Store(true) + + p.syncAllowedDestinations() + + events := jsonLogEvents(t, logs, "Firewall: organization allowed destinations are not enforced until the Control D API answers; started on the configuration on disk during maintenance") + if len(events) != 1 { + t.Fatalf("fail-closed warnings: got %d, want 1", len(events)) + } + if events[0]["level"] != "warn" { + t.Errorf("level = %v, want warn", events[0]["level"]) + } + if events[0]["reason"] != "api_maintenance" { + t.Errorf("reason = %v, want api_maintenance", events[0]["reason"]) + } + if events[0]["journal"] != true { + t.Error("the fail-closed warning is not retained in the journal") + } + if p.allowList.Contains(allowed) { + t.Fatalf("%s allowed with no resolver config", allowed) + } + }) + + t.Run("a normal start with no resolver config stays quiet", func(t *testing.T) { + logs := captureDebugMainLog(t) + p := progWithAllowList() + p.logger.Store(mainLog.Load()) + + p.syncAllowedDestinations() + + if got := len(jsonLogEvents(t, logs, "Firewall: organization allowed destinations are not enforced until the Control D API answers; started on the configuration on disk during maintenance")); got != 0 { + t.Fatalf("fail-closed warnings outside maintenance: got %d, want 0", got) + } + }) + + t.Run("firewall mode off stays quiet", func(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + p.startedInAPIMaintenance.Store(true) + + p.syncAllowedDestinations() + + if got := len(jsonLogEvents(t, logs, "Firewall: organization allowed destinations are not enforced until the Control D API answers; started on the configuration on disk during maintenance")); got != 0 { + t.Fatalf("fail-closed warnings with Firewall Mode off: got %d, want 0", got) + } + }) + + t.Run("pending recovery re-applies the list", func(t *testing.T) { + p := progWithAllowList() + p.apiReloadCh = make(chan *ctrld.Config, 1) + p.startedInAPIMaintenance.Store(true) + // The fallback start applied an empty set. + p.syncAllowedDestinations() + if p.allowList.Contains(allowed) { + t.Fatalf("%s allowed before recovery", allowed) + } + + p.applyFetchedResolverConfig(context.Background(), discardLogger(), &controld.ResolverConfig{ + DestinationIPs: []string{"203.0.113.10"}, + }, false, time.Now().Unix()) + + if !p.allowList.Contains(allowed) { + t.Fatalf("recovery did not re-apply the organization list: %s blocked", allowed) + } + select { + case <-p.apiReloadCh: + default: + t.Fatal("pending recovery did not deliver the answer to the apply path") + } + }) +} diff --git a/cmd/cli/api_maintenance_review_test.go b/cmd/cli/api_maintenance_review_test.go new file mode 100644 index 00000000..f3b19595 --- /dev/null +++ b/cmd/cli/api_maintenance_review_test.go @@ -0,0 +1,544 @@ +package cli + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/spf13/viper" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// TestMaintenanceFallbackRejectsGeneratedDefaultAcrossStarts covers the gap a +// boolean set by hand cannot: a first start with no configuration writes the +// seeded default before the API is ever reached, so a provisioning that failed +// leaves a file the next start reads successfully. Without the managed-config +// marker that file is accepted as a "last known configuration" and the +// endpoint runs on the public freedns upstreams instead of its profile. +func TestMaintenanceFallbackRejectsGeneratedDefaultAcrossStarts(t *testing.T) { + dir := t.TempDir() + oldV, oldFile, oldPath := v, defaultConfigFile, configPath + oldRead, oldUID, oldMarker := configReadFromDisk, cdUID, managedConfigMarkerPath + t.Cleanup(func() { + v, defaultConfigFile, configPath = oldV, oldFile, oldPath + configReadFromDisk, cdUID, managedConfigMarkerPath = oldRead, oldUID, oldMarker + }) + configPath = "" + cdUID = "provisioned-uid" + managedConfigMarkerPath = func() string { return filepath.Join(dir, managedConfigMarkerFile) } + cfgPath := filepath.Join(dir, "ctrld.toml") + + // One process start, using the real config reader and default generation. + start := func(t *testing.T) (*ctrld.Config, bool) { + t.Helper() + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "ctrld") + ctrld.SetConfigNameWithPath(v, "ctrld", dir) + defaultConfigFile = cfgPath + configReadFromDisk = false + // writeDefaultConfig is false: the file exists by now, and default + // generation is covered above. + readConfigFile(false, false) + started := &ctrld.Config{} + if err := v.Unmarshal(started); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return started, lastKnownConfigUsable(started) + } + + // The failed first start: no configuration existed, so ctrld generated the + // seeded default and wrote it. Reproduced by writing exactly what + // readConfigFile writes there, minus its listener probe, which binds + // privileged ports and cannot run in a test. What matters for the guard is + // the file's content and that no read set configReadFromDisk. + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "ctrld") + defaultConfigFile = cfgPath + configReadFromDisk = false + generated := &ctrld.Config{} + if err := v.Unmarshal(generated); err != nil { + t.Fatalf("unmarshal defaults: %v", err) + } + if err := writeConfigFile(generated); err != nil { + t.Fatalf("write generated default: %v", err) + } + if configReadFromDisk { + t.Fatal("generating a default must not count as reading one") + } + if lastKnownConfigUsable(generated) { + t.Fatal("the first start accepted a configuration it had just generated") + } + // Prove the generated default is the thing that must never be trusted. + var endpoints []string + for _, uc := range generated.Upstream { + endpoints = append(endpoints, uc.Endpoint) + } + if !strings.Contains(strings.Join(endpoints, " "), "freedns.controld.com") { + t.Fatalf("expected the seeded freedns default, got %v", endpoints) + } + + // Second start: the leftover file now reads back through the real reader, + // but no API answer ever produced it. + second, ok := start(t) + if !configReadFromDisk { + t.Fatal("the leftover file was not read, so the boundary is not being exercised") + } + if ok { + t.Fatal("a generated default was accepted as a last known configuration on a later start") + } + + // A genuine provisioning records the marker; the fallback then applies. + if err := recordManagedConfig(cdUID); err != nil { + t.Fatalf("recordManagedConfig: %v", err) + } + if _, ok := start(t); !ok { + t.Fatal("a configuration the API produced was rejected") + } + + // The marker belongs to one identity and must not carry to another. + cdUID = "a-different-uid" + if _, ok := start(t); ok { + t.Fatal("a marker for another identity authorized this one") + } + _ = second +} + +// TestMaintenanceRecoveryDeliversReload covers what recovery has to mean. The +// first answer after a maintenance start is the only one that differs from the +// nil configuration the run holds; if it is compared away, every later answer +// equals the stored one and the excludes and Internal Domains of the previous +// run stay in place for good. +func TestMaintenanceRecoveryDeliversReload(t *testing.T) { + oldFetch, oldDelay, oldUID := fetchResolverConfigFn, maintenanceReloadDelay, cdUID + t.Cleanup(func() { + fetchResolverConfigFn, maintenanceReloadDelay, cdUID = oldFetch, oldDelay, oldUID + }) + cdUID = "test-uid" + maintenanceReloadDelay = func() time.Duration { return time.Millisecond } + + var calls atomic.Int64 + fetchResolverConfigFn = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + calls.Add(1) + // A plain answer: no custom config, and excludes the run has never seen. + return &controld.ResolverConfig{Exclude: []string{"excluded.example"}}, nil + } + + p := &prog{ + cfg: &ctrld.Config{Service: ctrld.ServiceConfig{RefetchTime: ptrTo(3600)}}, + stopCh: make(chan struct{}), + runAbortCh: make(chan struct{}), + apiForceReloadCh: make(chan struct{}), + apiReloadCh: make(chan *ctrld.Config), + } + p.logger.Store(discardLogger()) + p.startedInAPIMaintenance.Store(true) + + done := make(chan struct{}) + go func() { defer close(done); p.apiConfigReload() }() + t.Cleanup(func() { + close(p.stopCh) + <-done + }) + + select { + case <-p.apiReloadCh: + // Stand in for the reload receiver applying it. + p.maintenanceConfigApplied(discardLogger()) + case <-time.After(10 * time.Second): + t.Fatal("recovery never delivered a reload, so the endpoint kept the configuration it started with") + } + + deadline := time.Now().Add(5 * time.Second) + for p.startedInAPIMaintenance.Load() { + if time.Now().After(deadline) { + t.Fatal("the run stayed in maintenance after applying the configuration") + } + time.Sleep(time.Millisecond) + } + if calls.Load() == 0 { + t.Fatal("no fetch happened") + } +} + +// TestDeactivationPinUnknownDuringMaintenance covers the PIN state a +// maintenance start leaves behind. cdDeactivationPin's default means "the API +// says there is none"; a run that never asked must not present that as +// permission to deactivate. +func TestDeactivationPinUnknownDuringMaintenance(t *testing.T) { + oldUID, oldPin, oldKnown := cdUID, cdDeactivationPin.Load(), deactivationPinKnown.Load() + t.Cleanup(func() { + cdUID = oldUID + cdDeactivationPin.Store(oldPin) + deactivationPinKnown.Store(oldKnown) + deactivationLockedUntil.Store(0) + }) + cdUID = "test-uid" + deactivationLockedUntil.Store(0) + + // The handler re-fetches before it checks the PIN. Without this the test + // reaches the production API with a made-up uid, and the statuses below + // would depend on that call failing rather than on the maintenance answer + // the case is about. + oldFetch := fetchResolverConfig + t.Cleanup(func() { fetchResolverConfig = oldFetch }) + var fetches atomic.Int64 + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + fetches.Add(1) + return nil, maintenanceAPIErr(http.StatusServiceUnavailable) + } + + post := func(t *testing.T) int { + t.Helper() + p := &prog{cs: &controlServer{mux: http.NewServeMux()}, pinCodeValidCh: make(chan struct{}, 1)} + p.logger.Store(discardLogger()) + p.registerControlServerHandler() + body, _ := json.Marshal(&deactivationRequest{Pin: defaultDeactivationPin}) + req := httptest.NewRequest(http.MethodPost, deactivationPath, strings.NewReader(string(body))) + w := httptest.NewRecorder() + p.cs.mux.ServeHTTP(w, req) + return w.Code + } + + t.Run("a run that never asked refuses", func(t *testing.T) { + cdDeactivationPin.Store(defaultDeactivationPin) + deactivationPinKnown.Store(false) + if got := post(t); got != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d: an unknown PIN authorized deactivation", got, http.StatusServiceUnavailable) + } + }) + + t.Run("a confirmed no-PIN configuration still allows it", func(t *testing.T) { + cdDeactivationPin.Store(defaultDeactivationPin) + deactivationPinKnown.Store(true) + if got := post(t); got != http.StatusOK { + t.Fatalf("status = %d, want 200: a device with no PIN can no longer be deactivated", got) + } + }) + + t.Run("an answering API settles the question", func(t *testing.T) { + // The re-fetch is what makes the PIN knowable again mid-run, so the + // stub has to be able to answer, not only fail. + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return &controld.ResolverConfig{}, nil + } + cdDeactivationPin.Store(defaultDeactivationPin) + deactivationPinKnown.Store(false) + if got := post(t); got != http.StatusOK { + t.Fatalf("status = %d, want 200: an API answer did not settle the PIN", got) + } + if !deactivationPinKnown.Load() { + t.Fatal("an API answer left the PIN unknown") + } + }) + + if fetches.Load() == 0 { + t.Fatal("the handler never reached the fetch seam, so the stub proved nothing") + } +} + +// The client must not report an unverifiable PIN as a wrong one, and must +// still refuse the operation. +func TestUnverifiableDeactivationPinIsARefusal(t *testing.T) { + if !isCheckDeactivationPinErr(errUnverifiableDeactivationPin) { + t.Fatal("an unverifiable PIN did not stop stop/uninstall") + } +} + +// TestManagedMarkerDoesNotOutliveItsConfig covers the provenance residual: a +// marker naming only the identity survives the file it vouched for, so a +// previously managed host whose config goes missing regenerates the seeded +// default and the stale marker certifies it. +func TestManagedMarkerDoesNotOutliveItsConfig(t *testing.T) { + dir := t.TempDir() + oldV, oldFile, oldPath := v, defaultConfigFile, configPath + oldRead, oldUID, oldMarker := configReadFromDisk, cdUID, managedConfigMarkerPath + t.Cleanup(func() { + v, defaultConfigFile, configPath = oldV, oldFile, oldPath + configReadFromDisk, cdUID, managedConfigMarkerPath = oldRead, oldUID, oldMarker + }) + configPath = "" + cdUID = "provisioned-uid" + managedConfigMarkerPath = func() string { return filepath.Join(dir, managedConfigMarkerFile) } + cfgPath := filepath.Join(dir, "ctrld.toml") + + start := func(t *testing.T) bool { + t.Helper() + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "ctrld") + ctrld.SetConfigNameWithPath(v, "ctrld", dir) + defaultConfigFile = cfgPath + configReadFromDisk = false + readConfigFile(false, false) + started := &ctrld.Config{} + if err := v.Unmarshal(started); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return lastKnownConfigUsable(started) + } + + // A managed configuration and its marker, written the way a successful run + // writes them. + managed := provisionedConfig() + defaultConfigFile = cfgPath + if err := writeConfigFile(managed); err != nil { + t.Fatalf("write managed config: %v", err) + } + if err := recordManagedConfig(cdUID); err != nil { + t.Fatalf("recordManagedConfig: %v", err) + } + if !start(t) { + t.Fatal("positive control: a managed configuration was rejected") + } + + // The file goes missing and the next start regenerates the seeded default, + // while the marker for the same identity stays behind. + if err := os.Remove(cfgPath); err != nil { + t.Fatal(err) + } + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "ctrld") + defaultConfigFile = cfgPath + regenerated := &ctrld.Config{} + if err := v.Unmarshal(regenerated); err != nil { + t.Fatalf("unmarshal defaults: %v", err) + } + if err := writeConfigFile(regenerated); err != nil { + t.Fatalf("write regenerated default: %v", err) + } + var endpoints []string + for _, uc := range regenerated.Upstream { + endpoints = append(endpoints, uc.Endpoint) + } + if !strings.Contains(strings.Join(endpoints, " "), "freedns.controld.com") { + t.Fatalf("expected the seeded freedns default, got %v", endpoints) + } + + if start(t) { + t.Fatal("a stale marker certified a regenerated default") + } +} + +// TestMaintenanceRecoveryOutlivesAFailedApply covers the other residual: the +// reload receiver fetches again and can fail. Acknowledging recovery when the +// trigger is sent would clear the flag while the endpoint still serves what it +// started with, and every later identical answer would then compare equal to +// the already-stored response and never be applied again. +func TestMaintenanceRecoveryOutlivesAFailedApply(t *testing.T) { + oldFetch, oldDelay, oldUID := fetchResolverConfigFn, maintenanceReloadDelay, cdUID + t.Cleanup(func() { + fetchResolverConfigFn, maintenanceReloadDelay, cdUID = oldFetch, oldDelay, oldUID + }) + cdUID = "test-uid" + maintenanceReloadDelay = func() time.Duration { return time.Millisecond } + + // The same answer every time, as a recovered API would give. + fetchResolverConfigFn = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return &controld.ResolverConfig{Exclude: []string{"excluded.example"}}, nil + } + + p := &prog{ + cfg: &ctrld.Config{Service: ctrld.ServiceConfig{RefetchTime: ptrTo(3600)}}, + stopCh: make(chan struct{}), + runAbortCh: make(chan struct{}), + apiForceReloadCh: make(chan struct{}), + apiReloadCh: make(chan *ctrld.Config), + } + p.logger.Store(discardLogger()) + p.startedInAPIMaintenance.Store(true) + + // A receiver that takes the trigger and then fails to apply, the way the + // real one does when its own fetch cannot reach the API. + var delivered atomic.Int64 + go func() { + for { + select { + case <-p.apiReloadCh: + delivered.Add(1) + case <-p.stopCh: + return + } + } + }() + + done := make(chan struct{}) + go func() { defer close(done); p.apiConfigReload() }() + t.Cleanup(func() { + close(p.stopCh) + <-done + }) + + // A failed apply must leave the run pending and keep redelivering, even + // though every answer after the first is identical. + deadline := time.Now().Add(10 * time.Second) + for delivered.Load() < 3 { + if time.Now().After(deadline) { + t.Fatalf("redelivery stopped after %d triggers; an identical answer was compared away", delivered.Load()) + } + time.Sleep(time.Millisecond) + } + if !p.startedInAPIMaintenance.Load() { + t.Fatal("recovery was acknowledged without the configuration being applied") + } +} + +// TestManagedReloadKeepsFallbackEligibility covers the provenance residual on +// the reload path: the marker is bound to the file's contents, and every +// API-driven change rewrites that file. A reload that persists a new managed +// configuration without refreshing the marker leaves the host unable to fall +// back on the configuration it just successfully applied. +func TestManagedReloadKeepsFallbackEligibility(t *testing.T) { + dir := t.TempDir() + oldV, oldFile, oldPath := v, defaultConfigFile, configPath + oldRead, oldUID, oldMarker := configReadFromDisk, cdUID, managedConfigMarkerPath + t.Cleanup(func() { + v, defaultConfigFile, configPath = oldV, oldFile, oldPath + configReadFromDisk, cdUID, managedConfigMarkerPath = oldRead, oldUID, oldMarker + }) + configPath = "" + cdUID = "provisioned-uid" + managedConfigMarkerPath = func() string { return filepath.Join(dir, managedConfigMarkerFile) } + cfgPath := filepath.Join(dir, "ctrld.toml") + + // A restart that reads whatever is on disk now, as a maintenance start does. + restart := func(t *testing.T) bool { + t.Helper() + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "ctrld") + ctrld.SetConfigNameWithPath(v, "ctrld", dir) + defaultConfigFile = cfgPath + configReadFromDisk = false + readConfigFile(false, false) + started := &ctrld.Config{} + if err := v.Unmarshal(started); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return lastKnownConfigUsable(started) + } + + // Managed startup. + defaultConfigFile = cfgPath + if err := writeConfigFile(provisionedConfig()); err != nil { + t.Fatal(err) + } + if err := recordManagedConfig(cdUID); err != nil { + t.Fatal(err) + } + if !restart(t) { + t.Fatal("a managed startup was not eligible to fall back") + } + + // An API-driven reload persists a changed managed configuration, through + // the same call the reload receiver makes. + changed := provisionedConfig() + changed.Upstream["0"].Endpoint = "https://dns.controld.com/changed5678" + logger := mainLog.Load() + defaultConfigFile = cfgPath + persistManagedConfig(changed, logger) + + onDisk, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(onDisk), "changed5678") { + t.Fatal("the reload did not persist the changed configuration") + } + if !restart(t) { + t.Fatal("a successfully applied managed configuration was not eligible to fall back") + } + + // The guards must survive: a write with no resolver UID certifies nothing. + cdUID = "" + stillChanged := provisionedConfig() + stillChanged.Upstream["0"].Endpoint = "https://dns.controld.com/unmanaged9999" + persistManagedConfig(stillChanged, logger) + cdUID = "provisioned-uid" + if restart(t) { + t.Fatal("a write outside managed mode refreshed the marker") + } +} + +// TestFallbackStartRewriteKeepsEligibility covers the startup half of the +// content-bound marker. A maintenance fallback fetches nothing, so it records +// no new marker, but it can still rewrite the file through the listener or +// intercept-mode updates. If the marker is left on the pre-write bytes, the +// next start in the same window is denied the fallback this one just used. +func TestFallbackStartRewriteKeepsEligibility(t *testing.T) { + dir := t.TempDir() + oldV, oldFile, oldPath := v, defaultConfigFile, configPath + oldRead, oldUID, oldMarker := configReadFromDisk, cdUID, managedConfigMarkerPath + t.Cleanup(func() { + v, defaultConfigFile, configPath = oldV, oldFile, oldPath + configReadFromDisk, cdUID, managedConfigMarkerPath = oldRead, oldUID, oldMarker + }) + configPath = "" + cdUID = "provisioned-uid" + managedConfigMarkerPath = func() string { return filepath.Join(dir, managedConfigMarkerFile) } + cfgPath := filepath.Join(dir, "ctrld.toml") + + restart := func(t *testing.T) bool { + t.Helper() + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "ctrld") + ctrld.SetConfigNameWithPath(v, "ctrld", dir) + defaultConfigFile = cfgPath + configReadFromDisk = false + readConfigFile(false, false) + started := &ctrld.Config{} + if err := v.Unmarshal(started); err != nil { + t.Fatalf("unmarshal: %v", err) + } + return lastKnownConfigUsable(started) + } + + // An earlier successful run left a managed configuration and its marker. + defaultConfigFile = cfgPath + if err := writeConfigFile(provisionedConfig()); err != nil { + t.Fatal(err) + } + if err := recordManagedConfig(cdUID); err != nil { + t.Fatal(err) + } + if !restart(t) { + t.Fatal("a managed configuration was not eligible to fall back") + } + + // A maintenance start: no fetch happened, so nothing new is recorded, but + // the listener/intercept updates still rewrite the file. This mirrors the + // startup ordering in run(): remember provenance, rewrite, then re-record. + managedConfigFetched := false + wasManagedConfig := managedConfigRecordedFor(cdUID) + if !wasManagedConfig { + t.Fatal("the pre-write file was not recognized as managed") + } + rewritten := provisionedConfig() + rewritten.Listener["0"].Port = 5354 // e.g. the macOS intercept port fallback + if err := writeConfigFile(rewritten); err != nil { + t.Fatal(err) + } + recordManagedConfigForStart(managedConfigFetched, wasManagedConfig) + + if !restart(t) { + t.Fatal("a rewrite during a maintenance fallback denied the next start its fallback") + } + + // The guard must survive: a rewrite with no prior managed provenance and no + // fetch still certifies nothing. + if err := os.Remove(managedConfigMarkerPath()); err != nil { + t.Fatal(err) + } + if managedConfigRecordedFor(cdUID) { + t.Fatal("the marker survived its own removal") + } + if restart(t) { + t.Fatal("a file with no managed provenance was accepted") + } +} diff --git a/cmd/cli/api_maintenance_test.go b/cmd/cli/api_maintenance_test.go new file mode 100644 index 00000000..bb84e2cf --- /dev/null +++ b/cmd/cli/api_maintenance_test.go @@ -0,0 +1,376 @@ +package cli + +import ( + "context" + "net/http" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// maintenanceAPIErr builds the API's hard maintenance answer: the dedicated +// code with the maintenance message. +func maintenanceAPIErr(status int) error { + e := &controld.ErrorResponse{StatusCode: status} + e.ErrorField.Code = controld.MaintenanceCode + e.ErrorField.Message = controld.MaintenanceMessage + " Please try again later." + return e +} + +// legacyMaintenanceAPIErr is the answer of a deployment that predates the +// dedicated code: the maintenance message alone. +func legacyMaintenanceAPIErr(status int) error { + e := &controld.ErrorResponse{StatusCode: status} + e.ErrorField.Message = controld.MaintenanceMessage + " Please try again later." + return e +} + +// deviceInvalidErr models the API refusing a device that no longer exists. +func deviceInvalidErr() error { + e := &controld.ErrorResponse{StatusCode: http.StatusNotFound} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = "device does not exist" + return e +} + +// provisionedConfig models the configuration an earlier successful run wrote. +func provisionedConfig() *ctrld.Config { + return &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: "127.0.0.1", Port: 53}, + }, + Network: map[string]*ctrld.NetworkConfig{ + "0": {Name: "Network 0", Cidrs: []string{"0.0.0.0/0"}}, + }, + Upstream: map[string]*ctrld.UpstreamConfig{ + "0": {Name: "Control D", Type: ctrld.ResolverTypeDOH, Endpoint: "https://dns.controld.com/abcd1234", Timeout: 5000}, + }, + } +} + +// withConfigOnDisk sets up what startup would have left behind: whether a +// configuration file was read, and whether a previous run recorded that the +// API produced it. Both are required for a fallback, and the marker lives in a +// temp dir so a real one on this host cannot influence the result. +func withConfigOnDisk(t *testing.T, read, managed bool) { + t.Helper() + oldRead, oldMarker, oldUID := configReadFromDisk, managedConfigMarkerPath, cdUID + t.Cleanup(func() { + configReadFromDisk, managedConfigMarkerPath, cdUID = oldRead, oldMarker, oldUID + }) + dir := t.TempDir() + managedConfigMarkerPath = func() string { return filepath.Join(dir, managedConfigMarkerFile) } + oldFile := defaultConfigFile + t.Cleanup(func() { defaultConfigFile = oldFile }) + defaultConfigFile = filepath.Join(dir, "ctrld.toml") + cdUID = "test-uid" + configReadFromDisk = read + if managed { + // The marker is bound to the file it vouches for, so one has to exist. + if err := writeConfigFile(provisionedConfig()); err != nil { + t.Fatalf("write config: %v", err) + } + if err := recordManagedConfig(cdUID); err != nil { + t.Fatalf("recordManagedConfig: %v", err) + } + } +} + +// Test_apiFailureCodeClassifiesMaintenance covers the classification the whole +// fallback rests on. Maintenance must not be reported as unreachability, which +// is what the incident showed (API_UNREACHABLE after a successful connection), +// and must never reach the device-invalid path that self-uninstalls. +func Test_apiFailureCodeClassifiesMaintenance(t *testing.T) { + for _, answer := range []func(int) error{maintenanceAPIErr, legacyMaintenanceAPIErr} { + for _, status := range []int{http.StatusServiceUnavailable, http.StatusBadGateway, http.StatusBadRequest, http.StatusNotFound} { + code, ok := apiFailureCode(answer(status)) + if !ok || code != provisionCodeAPIMaintenance { + t.Fatalf("status %d: code = %q ok = %v, want API_MAINTENANCE", status, code, ok) + } + } + } +} + +// Test_maintenanceIsNotAPermanentRejection covers the status trap: a +// maintenance answer carrying a client-error status would otherwise be read as +// "this request will be refused again" and stop the service for good. +func Test_maintenanceIsNotAPermanentRejection(t *testing.T) { + for _, answer := range []func(int) error{maintenanceAPIErr, legacyMaintenanceAPIErr} { + for _, status := range []int{http.StatusBadRequest, http.StatusNotFound, http.StatusForbidden} { + if _, permanent := permanentAPIRejection(answer(status)); permanent { + t.Fatalf("status %d classified as a permanent rejection", status) + } + } + } +} + +func Test_useLastKnownConfigDuringMaintenance(t *testing.T) { + t.Run("a previously working config carries the start", func(t *testing.T) { + withConfigOnDisk(t, true, true) + if !useLastKnownConfigDuringMaintenance(maintenanceAPIErr(http.StatusServiceUnavailable), provisionedConfig()) { + t.Fatal("maintenance did not fall back to the config on disk") + } + }) + + t.Run("a generated default is not a fallback", func(t *testing.T) { + // A fresh install: no file existed, so startup wrote the freedns + // default. Serving it would answer outside the operator's profile. + withConfigOnDisk(t, false, false) + if useLastKnownConfigDuringMaintenance(maintenanceAPIErr(http.StatusServiceUnavailable), provisionedConfig()) { + t.Fatal("a generated default was accepted as a last known configuration") + } + }) + + t.Run("an unusable config on disk is not a fallback", func(t *testing.T) { + withConfigOnDisk(t, true, true) + cfg := provisionedConfig() + cfg.Upstream = nil + if useLastKnownConfigDuringMaintenance(maintenanceAPIErr(http.StatusServiceUnavailable), cfg) { + t.Fatal("an invalid config was accepted as a last known configuration") + } + }) + + t.Run("a file with no record of an API answer is not a fallback", func(t *testing.T) { + withConfigOnDisk(t, true, false) + if useLastKnownConfigDuringMaintenance(maintenanceAPIErr(http.StatusServiceUnavailable), provisionedConfig()) { + t.Fatal("a configuration no API answer produced was accepted") + } + }) + + t.Run("other failures never take this path", func(t *testing.T) { + withConfigOnDisk(t, true, true) + for _, err := range []error{retryableNetworkErr(), deviceInvalidErr()} { + if useLastKnownConfigDuringMaintenance(err, provisionedConfig()) { + t.Fatalf("%v was treated as maintenance", err) + } + } + }) +} + +// Test_maintenanceWithoutFallbackFails covers the other half of the request: +// when there is nothing to fall back on, ctrld still stops, but says what +// cannot be done rather than reporting a fetch or connectivity failure. +func Test_maintenanceWithoutFallbackFails(t *testing.T) { + exitCode, notified := stubProvisionGlobals(t) + uninstalled := false + uninstallInvalidCdUIDFn = func(*prog, *ctrld.Logger, bool) bool { uninstalled = true; return true } + + handleAPIPreflightFailure(&prog{}, maintenanceAPIErr(http.StatusServiceUnavailable), func() { *notified = true }) + + if want := provisionExitCodeForCode[provisionCodeAPIMaintenance]; *exitCode != want { + t.Errorf("exit = %d, want %d", *exitCode, want) + } + if uninstalled { + t.Error("maintenance reached the self-uninstall path") + } + if !*notified { + t.Error("notify not called") + } + r, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if r.Code != string(provisionCodeAPIMaintenance) { + t.Errorf("code = %q, want API_MAINTENANCE", r.Code) + } + if r.Message != maintenanceNoConfigDetail { + t.Errorf("message = %q, want the maintenance guidance", r.Message) + } +} + +// maintenanceWithDeviceCode is the answer that makes the guard load-bearing: +// maintenance carried alongside the device-invalid code. Without the guard in +// selfUninstallCheck, this uninstalls a working install over a temporary API +// outage. +func maintenanceWithDeviceCode() error { + e := &controld.ErrorResponse{StatusCode: http.StatusServiceUnavailable} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = controld.MaintenanceMessage + " Please try again later." + return e +} + +func stubSelfUninstall(t *testing.T) *bool { + t.Helper() + old := selfUninstallFn + t.Cleanup(func() { selfUninstallFn = old }) + called := false + selfUninstallFn = func(*prog, *ctrld.Logger) { called = true } + return &called +} + +func newSelfUninstallTestProg() *prog { + return &prog{dnsWatcherStopCh: make(chan struct{})} +} + +func Test_selfUninstallCheckSkipsMaintenance(t *testing.T) { + t.Run("maintenance never uninstalls", func(t *testing.T) { + called := stubSelfUninstall(t) + p := newSelfUninstallTestProg() + selfUninstallCheck(maintenanceWithDeviceCode(), p, discardLogger()) + if *called { + t.Fatal("maintenance triggered a self-uninstall") + } + select { + case <-p.dnsWatcherStopCh: + t.Fatal("maintenance stopped the DNS watchers") + default: + } + }) + + t.Run("a real device rejection still uninstalls", func(t *testing.T) { + called := stubSelfUninstall(t) + selfUninstallCheck(deviceInvalidErr(), newSelfUninstallTestProg(), discardLogger()) + if !*called { + t.Fatal("a deleted device no longer self-uninstalls") + } + }) + + t.Run("an unrelated failure does nothing", func(t *testing.T) { + called := stubSelfUninstall(t) + selfUninstallCheck(retryableNetworkErr(), newSelfUninstallTestProg(), discardLogger()) + if *called { + t.Fatal("a network failure triggered a self-uninstall") + } + }) +} + +// Test_logResolverConfigFetchFailure covers what the reload paths say. The +// incident was first triaged as a connectivity fault because the only line was +// "could not fetch resolver config" after a successful connection. +func Test_logResolverConfigFetchFailure(t *testing.T) { + t.Run("maintenance is named and retained", func(t *testing.T) { + logs := captureDebugMainLog(t) + logger := mainLog.Load().With().Str("mode", "api-reload") + logResolverConfigFetchFailure(logger, maintenanceAPIErr(http.StatusServiceUnavailable)) + + events := jsonLogEvents(t, logs, "Control D API is in maintenance; keeping the current configuration until it returns") + if len(events) != 1 { + t.Fatalf("maintenance events: got %d, want 1", len(events)) + } + if events[0]["reason"] != "api_maintenance" { + t.Errorf("reason = %v, want api_maintenance", events[0]["reason"]) + } + if events[0]["journal"] != true { + t.Error("the maintenance line is not retained in the journal") + } + }) + + t.Run("other failures keep the existing line", func(t *testing.T) { + logs := captureDebugMainLog(t) + logger := mainLog.Load().With().Str("mode", "api-reload") + logResolverConfigFetchFailure(logger, retryableNetworkErr()) + + if got := len(jsonLogEvents(t, logs, "Could not fetch resolver config")); got != 1 { + t.Fatalf("generic fetch-failure events: got %d, want 1", got) + } + }) +} + +// Test_maintenanceReloadDelay covers the two properties the retry depends on: +// it is far below refetch_time so a degraded run recovers promptly, and it is +// spread so hosts that fell back together do not return as one burst. +func Test_maintenanceReloadDelay(t *testing.T) { + const refetchDefault = time.Hour + seen := make(map[time.Duration]bool) + for range 200 { + d := maintenanceReloadDelay() + if d < maintenanceReloadInterval || d >= maintenanceReloadInterval+maintenanceReloadJitter { + t.Fatalf("delay %s outside [%s, %s)", d, maintenanceReloadInterval, maintenanceReloadInterval+maintenanceReloadJitter) + } + if d >= refetchDefault { + t.Fatalf("delay %s is no faster than the normal refetch interval", d) + } + seen[d] = true + } + if len(seen) < 2 { + t.Fatal("every delay was identical; the fleet would retry in lockstep") + } +} + +// Test_maintenanceRetryRecoversBeforeRefetchInterval drives the real reload +// loop: a run that started on the configuration on disk must ask the API again +// without waiting out refetch_time, and must stop asking once it gets an +// answer. +func Test_maintenanceRetryRecoversBeforeRefetchInterval(t *testing.T) { + oldFetch, oldDelay, oldUID := fetchResolverConfigFn, maintenanceReloadDelay, cdUID + t.Cleanup(func() { + fetchResolverConfigFn, maintenanceReloadDelay, cdUID = oldFetch, oldDelay, oldUID + }) + cdUID = "test-uid" + maintenanceReloadDelay = func() time.Duration { return time.Millisecond } + + var calls atomic.Int64 + recovered := make(chan struct{}) + var once sync.Once + fetchResolverConfigFn = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + // Stay in maintenance for the first few retries, then answer. + if calls.Add(1) < 3 { + return nil, maintenanceAPIErr(http.StatusServiceUnavailable) + } + once.Do(func() { close(recovered) }) + return &controld.ResolverConfig{}, nil + } + + p := &prog{ + cfg: &ctrld.Config{ + // An hour: the test must pass because of the maintenance retry, + // never because the normal ticker happened to fire. + Service: ctrld.ServiceConfig{RefetchTime: ptrTo(3600)}, + }, + stopCh: make(chan struct{}), + runAbortCh: make(chan struct{}), + apiForceReloadCh: make(chan struct{}), + apiReloadCh: make(chan *ctrld.Config), + } + p.logger.Store(discardLogger()) + p.startedInAPIMaintenance.Store(true) + + // Recovery now means the configuration reached the reload, so something + // has to receive it. + go func() { + for { + select { + case <-p.apiReloadCh: + // Stand in for the reload receiver applying it. + p.maintenanceConfigApplied(discardLogger()) + case <-p.stopCh: + return + } + } + }() + + done := make(chan struct{}) + go func() { defer close(done); p.apiConfigReload() }() + t.Cleanup(func() { + close(p.stopCh) + <-done + }) + + select { + case <-recovered: + case <-time.After(10 * time.Second): + t.Fatal("the maintenance retry never reached the API") + } + + // The flag clears on the successful fetch, which is what stops the retry. + deadline := time.Now().Add(5 * time.Second) + for p.startedInAPIMaintenance.Load() { + if time.Now().After(deadline) { + t.Fatal("the run stayed in maintenance after the API answered") + } + time.Sleep(time.Millisecond) + } + + settled := calls.Load() + time.Sleep(50 * time.Millisecond) + if extra := calls.Load() - settled; extra > 1 { + t.Fatalf("the faster retry kept running after recovery: %d more fetches", extra) + } +} + +func ptrTo[T any](v T) *T { return &v } diff --git a/cmd/cli/api_reload_shutdown_test.go b/cmd/cli/api_reload_shutdown_test.go new file mode 100644 index 00000000..9864463b --- /dev/null +++ b/cmd/cli/api_reload_shutdown_test.go @@ -0,0 +1,160 @@ +package cli + +import ( + "context" + "encoding/base64" + "net" + "os" + "os/exec" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func TestAPIConfigReloadShutdown(t *testing.T) { + const env = "CTRLD_TEST_API_RELOAD_SHUTDOWN" + mode := os.Getenv(env) + if mode == "" { + for _, mode := range []string{"stop-exclude", "abort-exclude", "stop-custom", "abort-custom", "stop-internal", "abort-internal", "stop-fetch", "abort-fetch", "live-exclude", "live-custom", "live-internal"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestAPIConfigReloadShutdown$", "-test.v") + cmd.Env = append(os.Environ(), env+"="+mode) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", out) + }) + } + return + } + cdUID = "testuid" + version = "dev" + p := &prog{ + cfg: &ctrld.Config{}, rc: &controld.ResolverConfig{}, + stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), + apiForceReloadCh: make(chan struct{}), apiReloadCh: make(chan *ctrld.Config), + } + p.logger.Store(mainLog.Load()) + rc := &controld.ResolverConfig{Exclude: []string{"internal.test"}} + if strings.HasSuffix(mode, "internal") { + rc.Exclude = nil + rc.SplitDNS = []controld.SplitDNS{{Domain: "internal.test", Mode: controld.SplitDNSModeOS}} + } + if strings.HasSuffix(mode, "custom") { + rc.Ctrld.CustomConfig = base64.StdEncoding.EncodeToString([]byte(`[listener.0] +ip = "127.0.0.1" +port = 5354 +[network.0] +cidrs = ["127.0.0.0/8"] +[upstream.0] +type = "legacy" +endpoint = "127.0.0.1" +`)) + } + entered, canceled := make(chan struct{}), make(chan struct{}) + fetchResolverConfigFn = func(ctx context.Context, _ *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + close(entered) + if strings.HasSuffix(mode, "fetch") { + <-ctx.Done() + close(canceled) + // A late successful response must not schedule a reload after stop. + return rc, nil + } + return rc, nil + } + done := make(chan struct{}) + go func() { p.apiConfigReload(); close(done) }() + p.apiForceReloadCh <- struct{}{} + awaitShutdown(t, entered) + if strings.HasPrefix(mode, "live-") { + select { + case got := <-p.apiReloadCh: + if strings.HasSuffix(mode, "custom") { + require.NotNil(t, got) + require.Equal(t, "127.0.0.1", got.Listener["0"].IP) + } else { + require.Nil(t, got) + } + case <-time.After(2 * time.Second): + t.Fatal("running API loop did not deliver its reload") + } + } else if !strings.HasSuffix(mode, "fetch") { + marker := "Exclude list changes detected, reloading" + if strings.HasSuffix(mode, "custom") { + marker = "Custom config changes detected, reloading" + } else if strings.HasSuffix(mode, "internal") { + marker = "Internal domain changes detected, reloading" + } + require.Eventually(t, func() bool { return strings.Contains(logOutput.String(), marker) }, time.Second, time.Millisecond) + } + if strings.HasPrefix(mode, "abort-") { + close(p.runAbortCh) + } else { + close(p.stopCh) + } + awaitShutdown(t, done) + if strings.HasSuffix(mode, "fetch") { + awaitShutdown(t, canceled) + require.Empty(t, p.rc.Exclude, "late fetch must not update run state") + } +} + +func TestForceFetchingAPIStopsWithoutReceiver(t *testing.T) { + old := cdUID + cdUID = "testuid" + t.Cleanup(func() { cdUID = old }) + for _, abort := range []bool{false, true} { + for _, delivered := range []bool{false, true} { + p := &prog{cfg: &ctrld.Config{}, stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), apiForceReloadCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + p.forceFetchingAPI("testuid.verify.controld.com") + done := p.apiForceReloadGroup.DoChan("force_sync_api", func() (interface{}, error) { t.Error("lost active force-reload worker"); return nil, nil }) + if delivered { + select { + case <-p.apiForceReloadCh: + case <-time.After(time.Second): + t.Fatal("no force-reload signal") + } + } + if abort { + close(p.runAbortCh) + } else { + close(p.stopCh) + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("force-reload send/cooldown survived stop") + } + } + } +} + +type closeCountingLogConn struct { + net.Conn + closes atomic.Int32 +} + +func (c *closeCountingLogConn) Close() error { c.closes.Add(1); return nil } +func (*closeCountingLogConn) Write(b []byte) (int, error) { return len(b), nil } + +func TestStartupLogConnectionCloseOwnership(t *testing.T) { + c := &closeCountingLogConn{} + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{}), logConn: c} + p.logger.Store(mainLog.Load()) + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { defer wg.Done(); p.notifyExitToLogServer(); p.closeLogConn() }() + } + wg.Wait() + require.NoError(t, p.shutdown()) + require.EqualValues(t, 1, c.closes.Load()) + require.Nil(t, p.logConn) +} diff --git a/cmd/cli/api_reload_test.go b/cmd/cli/api_reload_test.go new file mode 100644 index 00000000..74e16475 --- /dev/null +++ b/cmd/cli/api_reload_test.go @@ -0,0 +1,333 @@ +package cli + +import ( + "context" + "net/netip" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// progForRefresh builds a prog that can run a configuration refresh: Firewall +// Mode on, and a buffered reload channel so a refresh that decides ctrld must +// reload does not block on a listener that does not exist in a test. +func progForRefresh() *prog { + p := progWithAllowList() + p.rc = &controld.ResolverConfig{} + p.apiReloadCh = make(chan *ctrld.Config, 1) + return p +} + +// refresh runs one configuration refresh through the handler apiConfigReload +// uses, which is the point: a test that called applyAllowedDestinations directly +// would still pass if the refresh path stopped calling it. +func refresh(t *testing.T, p *prog, forced bool, rc *controld.ResolverConfig) { + t.Helper() + p.applyFetchedResolverConfig(context.Background(), discardLogger(), rc, forced, time.Now().Unix()) +} + +// TestRefreshInternalDomains covers the master-specific refresh handler after +// apiConfigReload delegates to it, including changes with unchanged excludes. +func TestRefreshInternalDomains(t *testing.T) { + for _, forced := range []bool{false, true} { + name := "scheduled" + if forced { + name = "forced" + } + t.Run(name, func(t *testing.T) { + // No platform firewall or service is installed by this fixture. + p := &prog{rc: &controld.ResolverConfig{}, apiReloadCh: make(chan *ctrld.Config, 1)} + p.logger.Store(discardLogger()) + for _, step := range []struct { + name string + entries []controld.SplitDNS + reload bool + }{ + {"absent", nil, false}, + {"add", []controld.SplitDNS{{Domain: "corp.example", Mode: "os"}}, true}, + {"unchanged", []controld.SplitDNS{{Domain: "CORP.EXAMPLE.", Mode: "os"}}, false}, + {"mode", []controld.SplitDNS{{Domain: "corp.example", Mode: "resolvers", Resolvers: []string{"127.0.0.2"}}}, true}, + {"resolver", []controld.SplitDNS{{Domain: "corp.example", Mode: "resolvers", Resolvers: []string{"127.0.0.3"}}}, true}, + {"domain", []controld.SplitDNS{{Domain: "office.example", Mode: "resolvers", Resolvers: []string{"127.0.0.3"}}}, true}, + {"remove", nil, true}, + {"empty unchanged", []controld.SplitDNS{}, false}, + } { + t.Run(step.name, func(t *testing.T) { + rc := &controld.ResolverConfig{SplitDNS: step.entries} + refresh(t, p, forced, rc) + gotReload := false + select { + case cfg := <-p.apiReloadCh: + gotReload = true + if cfg != nil { + t.Fatal("managed change must request config regeneration") + } + default: + } + if gotReload != step.reload { + t.Fatalf("reload = %v, want %v", gotReload, step.reload) + } + if p.rc != rc { + t.Fatal("refresh did not retain the authoritative response") + } + }) + } + // Changes to Internal Domains must not override a custom config. + rc := &controld.ResolverConfig{SplitDNS: []controld.SplitDNS{{Domain: "corp.example", Mode: "os"}}} + rc.Ctrld.CustomConfig = "custom-config-is-authoritative" + refresh(t, p, false, rc) + select { + case <-p.apiReloadCh: + t.Fatal("Internal Domains overrode unchanged custom config") + default: + } + }) + } +} + +// TestRefreshAppliesAllowedDestinations drives the real refresh handler for both +// the scheduled and the forced path, in the case where nothing else about the +// configuration changed - no custom config, unchanged exclusions - so the +// refresh takes its early return. That is where an allowed-destination update is +// easiest to lose, because the refresh has no other work to do. +func TestRefreshAppliesAllowedDestinations(t *testing.T) { + direct := netip.MustParseAddr("203.0.113.10") + inRange := netip.MustParseAddr("198.51.100.7") + + for _, forced := range []bool{false, true} { + name := "scheduled refresh" + if forced { + name = "forced refresh" + } + t.Run(name, func(t *testing.T) { + p := progForRefresh() + + refresh(t, p, forced, &controld.ResolverConfig{ + DestinationIPs: []string{"203.0.113.10", "198.51.100.0/24"}, + }) + if !p.allowList.Contains(direct) || !p.allowList.Contains(inRange) { + t.Fatalf("refresh did not apply the organization list: %s=%v %s=%v", + direct, p.allowList.Contains(direct), inRange, p.allowList.Contains(inRange)) + } + select { + case cfg := <-p.apiReloadCh: + t.Fatalf("unchanged configuration signaled a reload (%v)", cfg) + default: + } + + // A later refresh withdraws one entry and keeps the other. + refresh(t, p, forced, &controld.ResolverConfig{ + DestinationIPs: []string{"203.0.113.10"}, + }) + if p.allowList.Contains(inRange) { + t.Fatalf("%s still allowed after the refresh that withdrew it", inRange) + } + if !p.allowList.Contains(direct) { + t.Fatalf("%s should still be allowed", direct) + } + + // And one clears the list entirely. + refresh(t, p, forced, &controld.ResolverConfig{}) + if p.allowList.Contains(direct) { + t.Fatalf("%s still allowed after the refresh that cleared the list", direct) + } + }) + } +} + +// TestRefreshAppliesAllowedDestinationsWhenReloading covers the other branch of +// the same handler: a refresh that also changes the exclusion list, and so +// signals a ctrld reload, must still apply the destinations - and they must +// survive the allowlist flush that the reload performs. +func TestRefreshAppliesAllowedDestinationsWhenReloading(t *testing.T) { + p := progForRefresh() + p.rc = &controld.ResolverConfig{Exclude: []string{"example.com"}} + direct := netip.MustParseAddr("203.0.113.10") + + refresh(t, p, false, &controld.ResolverConfig{ + Exclude: []string{"example.com", "example.net"}, + DestinationIPs: []string{"203.0.113.10"}, + }) + + select { + case <-p.apiReloadCh: + default: + t.Fatal("exclusion list change did not signal a reload") + } + if !p.allowList.Contains(direct) { + t.Fatalf("%s not allowed after a refresh that reloaded ctrld", direct) + } +} + +// TestRefreshKeepsAllowedDestinationsPendingUntilMirrored pins that the refresh +// path reports honestly: while platform enforcement is rejecting the change, the +// refresh leaves it pending and retries it, instead of recording it as applied. +func TestRefreshKeepsAllowedDestinationsPendingUntilMirrored(t *testing.T) { + p := progForRefresh() + var calls []mirrorCall + failing := true + stubMirror(t, &calls, &failing) + + rc := &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}} + refresh(t, p, false, rc) + if got := p.pendingDestinations(p.allowList); got != 1 { + t.Fatalf("pendingDestinations = %d after a rejected mirror, want 1", got) + } + + refresh(t, p, false, rc) + if len(calls) != 2 { + t.Fatalf("refresh did not retry the rejected change: calls = %v", calls) + } + + failing = false + refresh(t, p, false, rc) + if got := p.pendingDestinations(p.allowList); got != 0 { + t.Fatalf("pendingDestinations = %d after the mirror succeeded, want 0", got) + } +} + +// stubResolverConfigFetch replaces the refresh loop's API call for the duration +// of a test, and points cdUID at a device so apiConfigReload does not return +// immediately. Each fetch returns the config the test currently wants and +// reports on fetched, which is how a test knows a refresh cycle has run. +func stubResolverConfigFetch(t *testing.T, config func() *controld.ResolverConfig, fetched chan<- struct{}) { + t.Helper() + origFetch, origUID := fetchResolverConfigFn, cdUID + t.Cleanup(func() { fetchResolverConfigFn, cdUID = origFetch, origUID }) + + cdUID = "test-uid" + fetchResolverConfigFn = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + rc := config() + select { + case fetched <- struct{}{}: + default: + } + return rc, nil + } +} + +// startRefreshLoop runs apiConfigReload in the background and joins it before the +// test finishes. The loop must not outlive the test: it calls through the same +// package-level stubs the next test replaces, so a leaked one would both race +// those globals and act on another test's prog. +func startRefreshLoop(t *testing.T, p *prog) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + p.apiConfigReload() + }() + t.Cleanup(func() { + close(p.stopCh) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("apiConfigReload did not stop") + } + }) +} + +// waitForCondition polls until cond holds, failing the test if it never does. The +// refresh loop runs in its own goroutine, so its effects land asynchronously. +func waitForCondition(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func TestApiConfigReloadUsesRuntimeMetadata(t *testing.T) { + originalFetch, originalUID := fetchResolverConfigFn, cdUID + t.Cleanup(func() { + fetchResolverConfigFn = originalFetch + cdUID = originalUID + }) + cdUID = "test-uid" + + captured := make(chan *controld.ResolverConfigRequest, 1) + fetchResolverConfigFn = func(_ context.Context, req *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + captured <- req + return &controld.ResolverConfig{}, nil + } + + p := progForRefresh() + p.cfg = &ctrld.Config{} + p.stopCh = make(chan struct{}) + p.apiForceReloadCh = make(chan struct{}) + startRefreshLoop(t, p) + p.apiForceReloadCh <- struct{}{} + + select { + case req := <-captured: + if _, ok := req.Metadata["username"]; ok { + t.Error("periodic/forced API refresh included username") + } + if req.IncludeHostnameHints { + t.Error("periodic/forced API refresh requested hostname hints") + } + case <-time.After(5 * time.Second): + t.Fatal("periodic/forced API refresh did not fetch resolver config") + } +} + +// TestApiConfigReloadAppliesAllowedDestinations drives apiConfigReload itself - +// the loop that owns the refresh ticker and the forced-reload channel - rather +// than the handler it calls, so the wiring between them is covered too: removing +// the handler call from the loop must fail a test, not just removing the work +// inside the handler. +func TestApiConfigReloadAppliesAllowedDestinations(t *testing.T) { + direct := netip.MustParseAddr("203.0.113.10") + + for _, forced := range []bool{true, false} { + name := "forced reload" + if !forced { + name = "refresh ticker" + } + t.Run(name, func(t *testing.T) { + p := progForRefresh() + p.cfg = &ctrld.Config{} + refetch := 1 // seconds; only the ticker path waits for it + p.cfg.Service.RefetchTime = &refetch + p.stopCh = make(chan struct{}) + p.apiForceReloadCh = make(chan struct{}) + + var mu sync.Mutex + destinations := []string{"203.0.113.10"} + fetched := make(chan struct{}, 1) + stubResolverConfigFetch(t, func() *controld.ResolverConfig { + mu.Lock() + defer mu.Unlock() + return &controld.ResolverConfig{DestinationIPs: append([]string(nil), destinations...)} + }, fetched) + + startRefreshLoop(t, p) + + if forced { + p.apiForceReloadCh <- struct{}{} + } + waitForCondition(t, "the destination to be applied", func() bool { + return p.allowList.Contains(direct) + }) + + // The organization withdraws it; the next cycle must take it away. + mu.Lock() + destinations = nil + mu.Unlock() + + if forced { + p.apiForceReloadCh <- struct{}{} + } + waitForCondition(t, "the withdrawn destination to stop being allowed", func() bool { + return !p.allowList.Contains(direct) + }) + }) + } +} diff --git a/cmd/cli/change_logger.go b/cmd/cli/change_logger.go new file mode 100644 index 00000000..52601d8c --- /dev/null +++ b/cmd/cli/change_logger.go @@ -0,0 +1,205 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "tailscale.com/net/netmon" + "time" +) + +// noiseSummaryInterval bounds a storm of noise deltas to one journal line, +// because AirDrop and virtual adapters can churn for hours. +const noiseSummaryInterval = 10 * time.Minute + +// wakeReportInterval merges the reports of the wake sources. netmon polls for +// a time jump every 15 s, so one wake can reach ctrld over half a minute. +const wakeReportInterval = 30 * time.Second + +// repeatLogger holds the last value of each named line, so a line that +// repeats every few seconds reaches the log on change only. +type repeatLogger struct { + mu sync.Mutex + last map[string]string + repeats map[string]uint64 +} + +// changed reports whether the value of a key differs from the value before it. +// The repeat count tells how many lines the caller held back, so the next line +// it writes can carry them. +func (r *repeatLogger) changed(key, value string) (changed bool, repeats uint64) { + r.mu.Lock() + defer r.mu.Unlock() + if r.last == nil { + r.last = make(map[string]string) + r.repeats = make(map[string]uint64) + } + if previous, seen := r.last[key]; seen && previous == value { + r.repeats[key]++ + return false, r.repeats[key] + } + repeats = r.repeats[key] + r.last[key] = value + r.repeats[key] = 0 + return true, repeats +} + +// wakeHoldInterval is the time that a wake report without a gap waits for the +// detector. The detector ticks every two seconds, so it reports well inside +// this hold when it runs. +const wakeHoldInterval = 8 * time.Second + +// wakeReport is one wake as a source saw it. +type wakeReport struct { + source string + gap time.Duration + state *netmon.State +} + +// wakeReporter keeps one report per wake. netmon measures no gap, so its +// report waits for the detector, which measures one. When no detector report +// comes, the held report goes out as it is. +type wakeReporter struct { + mu sync.Mutex + last time.Time + held *wakeReport + // after arms the hold timer. nil means time.AfterFunc, and a test fires + // the hold itself. + after func(time.Duration, func()) +} + +// note takes one report of a wake and hands the report that the journal gets +// to emit. The first source of a window reports. A report with a gap replaces +// a held report without one. Every other report of the window is the same +// wake and drops. +func (w *wakeReporter) note(now time.Time, report wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + gapKnown := report.gap > 0 + if !w.last.IsZero() && now.Sub(w.last) < wakeReportInterval { + if w.held != nil && gapKnown { + w.held = nil + emit(report) + } + return + } + w.last = now + if gapKnown { + w.held = nil + emit(report) + return + } + held := &wakeReport{source: report.source, gap: report.gap, state: report.state} + w.held = held + w.afterFunc()(wakeHoldInterval, func() { w.release(held, emit) }) +} + +// release emits a held report when no source with a gap replaced it. +func (w *wakeReporter) release(held *wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + if w.held != held { + return + } + w.held = nil + emit(*held) +} + +func (w *wakeReporter) afterFunc() func(time.Duration, func()) { + if w.after != nil { + return w.after + } + return func(d time.Duration, fn func()) { time.AfterFunc(d, fn) } +} + +// noiseSummary describes the noise deltas that one journal line covers. +type noiseSummary struct { + Count int + First time.Time + Last time.Time + Interfaces []string +} + +// noiseCoalescer counts the noise deltas of a storm and reports when the +// journal needs a line for them. The count, the first time, and the names +// cover the open window; last covers the storm, so a quiet period ends it. +type noiseCoalescer struct { + mu sync.Mutex + count int + first time.Time + last time.Time + emitted time.Time + names map[string]struct{} +} + +// add records one noise delta and reports whether to log a summary now. The +// summary covers the deltas since the last one, so a storm of any length +// keeps to one line per interval. +func (c *noiseCoalescer) add(interfaces []string, now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + newRun := c.stormEnded(now) + if newRun { + c.clearWindow() + } + c.record(interfaces, now) + if !newRun && now.Sub(c.emitted) < noiseSummaryInterval { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// flush reports the open window and closes it. A storm ends with a delta that +// the daemon acts on, so the last window of the storm reaches the journal +// there instead of waiting for the next storm. +func (c *noiseCoalescer) flush(now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.count == 0 { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// stormEnded is true when the deltas stopped for longer than the interval, so +// a new storm does not inherit the count of the storm before it. +func (c *noiseCoalescer) stormEnded(now time.Time) bool { + return c.last.IsZero() || now.Sub(c.last) > noiseSummaryInterval +} + +func (c *noiseCoalescer) record(interfaces []string, now time.Time) { + if c.count == 0 { + c.first = now + } + c.count++ + c.last = now + if c.names == nil { + c.names = make(map[string]struct{}) + } + for _, name := range interfaces { + c.names[name] = struct{}{} + } +} + +// summary sorts the names, because a log line that keeps its order reads as +// one value across a capture. +func (c *noiseCoalescer) summary() noiseSummary { + return noiseSummary{ + Count: c.count, + First: c.first, + Last: c.last, + Interfaces: slices.Sorted(maps.Keys(c.names)), + } +} + +func (c *noiseCoalescer) clearWindow() { + c.count = 0 + c.first = time.Time{} + c.names = nil +} diff --git a/cmd/cli/change_logger_test.go b/cmd/cli/change_logger_test.go new file mode 100644 index 00000000..4292c4c9 --- /dev/null +++ b/cmd/cli/change_logger_test.go @@ -0,0 +1,198 @@ +package cli + +import ( + "slices" + "testing" + "time" +) + +func Test_repeatLogger(t *testing.T) { + var repeats repeatLogger + for _, step := range []struct { + name string + key string + value string + wantChanged bool + wantRepeats uint64 + }{ + {"the first value of a key is a change", "pf anchor", "intact", true, 0}, + {"the same value repeats", "pf anchor", "intact", false, 1}, + {"the same value repeats again", "pf anchor", "intact", false, 2}, + {"a new value reports the run before it", "pf anchor", "missing", true, 2}, + {"the new value repeats", "pf anchor", "missing", false, 1}, + {"the value before is a change again", "pf anchor", "intact", true, 1}, + {"a second key starts on its own", "tunnels", "utun0", true, 0}, + {"the second key repeats", "tunnels", "utun0", false, 1}, + {"the first key keeps its own count", "pf anchor", "intact", false, 1}, + } { + t.Run(step.name, func(t *testing.T) { + changed, count := repeats.changed(step.key, step.value) + if changed != step.wantChanged || count != step.wantRepeats { + t.Errorf("changed(%q, %q) = %v, %d, want %v, %d", + step.key, step.value, changed, count, step.wantChanged, step.wantRepeats) + } + }) + } +} + +func Test_wakeReporter(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + var wake wakeReporter + var armed []func() + wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + var emitted []wakeReport + emit := func(report wakeReport) { emitted = append(emitted, report) } + fireHeld := func() { + for _, fn := range armed { + fn() + } + armed = nil + } + + // netmon reports first and knows no gap, so its report waits. + wake.note(base, wakeReport{source: "netmon"}, emit) + if len(emitted) != 0 || len(armed) != 1 { + t.Fatalf("a report without a gap must wait: emitted %d, timers %d", len(emitted), len(armed)) + } + // The detector reports inside the hold with the gap, so it replaces the + // held report. + wake.note(base.Add(2*time.Second), wakeReport{source: "detector", gap: 90 * time.Second}, emit) + fireHeld() + if len(emitted) != 1 || emitted[0].source != "detector" { + t.Fatalf("emitted %+v, want the detector report alone", emitted) + } + // Every other report of this wake is the same wake. + wake.note(base.Add(10*time.Second), wakeReport{source: "netmon"}, emit) + wake.note(base.Add(12*time.Second), wakeReport{source: "detector", gap: 91 * time.Second}, emit) + if len(emitted) != 1 { + t.Fatalf("emitted %d reports inside the window, want 1", len(emitted)) + } + + // A wake that only netmon sees reports without a gap when the hold ends. + wake.note(base.Add(40*time.Second), wakeReport{source: "netmon"}, emit) + if len(emitted) != 1 || len(armed) != 1 { + t.Fatalf("a new wake must arm one hold: emitted %d, timers %d", len(emitted), len(armed)) + } + fireHeld() + if len(emitted) != 2 || emitted[1].source != "netmon" || emitted[1].gap != 0 { + t.Fatalf("emitted %+v, want the netmon report after the hold", emitted) + } + // A gap that arrives after the hold is too late: one event per wake. + wake.note(base.Add(50*time.Second), wakeReport{source: "detector", gap: 92 * time.Second}, emit) + if len(emitted) != 2 { + t.Fatalf("emitted %d reports, want no second report of one wake", len(emitted)) + } + + // The detector first reports at once, and netmon adds nothing. + wake.note(base.Add(80*time.Second), wakeReport{source: "detector", gap: 93 * time.Second}, emit) + wake.note(base.Add(82*time.Second), wakeReport{source: "netmon"}, emit) + fireHeld() + if len(emitted) != 3 || emitted[2].source != "detector" || len(armed) != 0 { + t.Fatalf("emitted %+v with %d timers, want one detector report and no hold", emitted, len(armed)) + } +} + +func Test_noiseCoalescer(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + + t.Run("a storm reports once every ten minutes", func(t *testing.T) { + const deltas = 100 + const cadence = 15 * time.Second + var noise noiseCoalescer + var got []noiseSummary + for i := 0; i < deltas; i++ { + interfaces := []string{"awdl0"} + if i%2 == 1 { + interfaces = []string{"llw0", "awdl0"} + } + summary, emit := noise.add(interfaces, base.Add(time.Duration(i)*cadence)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 40, + First: base.Add(cadence), + Last: base.Add(10 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }, { + Count: 40, + First: base.Add(10*time.Minute + cadence), + Last: base.Add(20 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }} + assertSummaries(t, got, want) + }) + + t.Run("a flush reports the open window", func(t *testing.T) { + var noise noiseCoalescer + if _, emit := noise.flush(base); emit { + t.Fatal("an empty coalescer reported a summary") + } + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute} { + noise.add([]string{"awdl0"}, base.Add(offset)) + } + summary, emit := noise.flush(base.Add(3 * time.Minute)) + if !emit { + t.Fatal("the flush reported no summary for an open window") + } + assertSummaries(t, []noiseSummary{summary}, []noiseSummary{{ + Count: 2, + First: base.Add(time.Minute), + Last: base.Add(2 * time.Minute), + Interfaces: []string{"awdl0"}, + }}) + if _, emit = noise.flush(base.Add(4 * time.Minute)); emit { + t.Fatal("the flush reported the same window twice") + } + }) + + t.Run("a gap longer than the interval starts a new run", func(t *testing.T) { + var noise noiseCoalescer + var got []noiseSummary + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute, 17 * time.Minute} { + summary, emit := noise.add([]string{"awdl0"}, base.Add(offset)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 1, + First: base.Add(17 * time.Minute), + Last: base.Add(17 * time.Minute), + Interfaces: []string{"awdl0"}, + }} + assertSummaries(t, got, want) + }) +} + +func assertSummaries(t *testing.T, got, want []noiseSummary) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("got %d summaries, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if !sameSummary(got[i], want[i]) { + t.Errorf("summary %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func sameSummary(got, want noiseSummary) bool { + return got.Count == want.Count && + got.First.Equal(want.First) && + got.Last.Equal(want.Last) && + slices.Equal(got.Interfaces, want.Interfaces) +} diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 1984f702..9d90f618 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -24,6 +24,7 @@ import ( "strings" "sync/atomic" "time" + "unicode" "github.com/Masterminds/semver/v3" "github.com/cuonglm/osinfo" @@ -31,7 +32,6 @@ import ( "github.com/kardianos/service" "github.com/miekg/dns" "github.com/pelletier/go-toml/v2" - "github.com/rs/zerolog" "github.com/spf13/cobra" "github.com/spf13/viper" "tailscale.com/logtail/backoff" @@ -40,7 +40,6 @@ import ( "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/controld" ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" - "github.com/Control-D-Inc/ctrld/internal/router" ) // selfCheckInternalTestDomain is used for testing ctrld self response to clients. @@ -48,8 +47,8 @@ const selfCheckInternalTestDomain = "ctrld" + loopTestDomain const ( windowsForwardersFilename = ".forwarders.txt" oldBinSuffix = "_previous" - oldLogSuffix = ".1" msgExit = "$$EXIT$$" + shutdownTimeout = 10 * time.Second ) var ( @@ -62,10 +61,13 @@ var ( defaultConfigFile = "ctrld.toml" rootCertPool *x509.CertPool errSelfCheckNoAnswer = errors.New("no response from ctrld listener. You can try to re-launch with flag --skip_self_checks") + // Store version once during init to avoid repeated calls to curVersion() + appVersion = curVersion() ) var basicModeFlags = []string{"listen", "primary_upstream", "secondary_upstream", "domains"} +// isNoConfigStart checks if the command is using no-config start mode func isNoConfigStart(cmd *cobra.Command) bool { for _, flagName := range basicModeFlags { if cmd.Flags().Lookup(flagName).Changed { @@ -75,6 +77,16 @@ func isNoConfigStart(cmd *cobra.Command) bool { return false } +// cliName is this client's identity: the command name Cobra prints in help and +// in "--version" output, the file the release pipeline publishes +// (scripts/build.sh executable_name), and therefore the file the self-upgrader +// downloads. +// +// Referenced rather than repeated because "--version" output is parsed as well as +// printed: binaryVersion builds its expected prefix from this, and a literal on +// either side would let a rename break rollback silently. +const cliName = "ctrld-client" + const rootShortDesc = ` __ .__ .___ _____/ |________| | __| _/ @@ -84,34 +96,63 @@ _/ ___\ __\_ __ \ | / __ | \/ dns forwarding proxy \/ ` -var rootCmd = &cobra.Command{ - Use: "ctrld", - Short: strings.TrimLeft(rootShortDesc, "\n"), - Version: curVersion(), - PersistentPreRun: func(cmd *cobra.Command, args []string) { - initConsoleLogging() - }, -} - +// curVersion returns the current version string func curVersion() string { + // Ensure version has proper "v" prefix for semantic versioning + // This is needed because some build systems may provide version without the "v" prefix if version != "dev" && !strings.HasPrefix(version, "v") { version = "v" + version } + // Return version directly if it's not empty and not a dev build + // This avoids unnecessary commit hash concatenation for release versions if version != "" && version != "dev" { - return version + return displayVersion(version) } + // Truncate commit hash to 7 characters for readability + // Git commit hashes are typically 40 characters, but 7 is sufficient for identification if len(commit) > 7 { commit = commit[:7] } return fmt.Sprintf("%s-%s", version, commit) } -func initCLI() { +// displayVersion converts the build tag into the version the client reports. +// +// master carries v2.x.x tags so its releases can be tracked alongside the v1.x.x +// line still cut from the v1.0 branch, but the client presents itself as v1.x.x: +// a v2.0.0 tag reads as v1.0.0, v2.3.1 as v1.3.1. Only master ever carries a +// major >= 2, so the split tag spaces scope this to master by themselves and the +// binary needs no build-time signal for which branch produced it - the v1.0 +// branch's tags have major 1 and pass through untouched, as does anything that is +// not a semantic version ("dev", commit-suffixed builds). +// +// Minor, patch, prerelease and build metadata are preserved, so a v2.1.0-rc1 tag +// stays a release candidate at v1.1.0-rc1 and isStableVersion still classifies it +// the same way. +func displayVersion(v string) string { + sv, err := semver.NewVersion(v) + if err != nil || sv.Major() < 2 { + return v + } + shifted := semver.New(sv.Major()-1, sv.Minor(), sv.Patch(), sv.Prerelease(), sv.Metadata()) + return "v" + shifted.String() +} + +func initCLI() *cobra.Command { // Enable opening via explorer.exe on Windows. // See: https://github.com/spf13/cobra/issues/844. cobra.MousetrapHelpText = "" cobra.EnableCommandSorting = false + rootCmd := &cobra.Command{ + Use: cliName, + Short: strings.TrimLeft(rootShortDesc, "\n"), + Version: appVersion, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + initConsoleLogging() + }, + } + rootCmd.PersistentFlags().CountVarP( &verbose, "verbose", @@ -128,25 +169,45 @@ func initCLI() { rootCmd.SetHelpCommand(&cobra.Command{Hidden: true}) rootCmd.CompletionOptions.HiddenDefaultCmd = true - initRunCmd() - startCmd := initStartCmd() - stopCmd := initStopCmd() - restartCmd := initRestartCmd() - reloadCmd := initReloadCmd(restartCmd) - statusCmd := initStatusCmd() - uninstallCmd := initUninstallCmd() - interfacesCmd := initInterfacesCmd() - initServicesCmd(startCmd, stopCmd, restartCmd, reloadCmd, statusCmd, uninstallCmd, interfacesCmd) - initClientsCmd() - initUpgradeCmd() - initLogCmd() + InitRunCmd(rootCmd) + InitServiceCmd(rootCmd) + InitClientsCmd(rootCmd) + InitUpgradeCmd(rootCmd) + InitLogCmd(rootCmd) + InitDiagCmd(rootCmd) + + return rootCmd } // isMobile reports whether the current OS is a mobile platform. -func isMobile() bool { +var isMobile = func() bool { return runtime.GOOS == "android" || runtime.GOOS == "ios" } +var ( + networkUp = ctrldnet.Up + cleanupStaleDNSInterceptStateFn = cleanupStaleDNSInterceptState +) + +// updateConfigInterceptMode persists an explicit --intercept-mode selection in +// service.intercept_mode. "off" is written as "off", not cleared: with an unset +// value meaning the platform default ("dns" on macOS and Windows desktop), a cleared +// value would turn interception back on at the next start without the flag. +func updateConfigInterceptMode(cfg *ctrld.Config, mode string) bool { + var desired string + switch mode { + case "dns", "hard", "off": + desired = mode + default: + return false + } + if cfg.Service.InterceptMode == desired { + return false + } + cfg.Service.InterceptMode = desired + return true +} + // isAndroid reports whether the current OS is Android. func isAndroid() bool { return runtime.GOOS == "android" @@ -164,8 +225,13 @@ func isStableVersion(vs string) bool { // RunCobraCommand runs ctrld cli. func RunCobraCommand(cmd *cobra.Command) { noConfigStart = isNoConfigStart(cmd) - checkStrFlagEmpty(cmd, cdUidFlagName) - checkStrFlagEmpty(cmd, cdOrgFlagName) + firewallModeFlagChanged = cmd.Flags().Changed("firewall-mode") + if !checkStrFlagEmpty(cmd, cdUidFlagName) { + return + } + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + return + } run(nil, make(chan struct{})) } @@ -204,7 +270,8 @@ func CheckDeactivationPin(pin int64, stopCh chan struct{}) int { // run runs ctrld cli with given app callback and stop channel. func run(appCallback *AppCallback, stopCh chan struct{}) { if stopCh == nil { - mainLog.Load().Fatal().Msg("stopCh is nil") + failRunUnclassified(mainLog.Load().Error(), "run() called with a nil stop channel", nil) + return } waitCh := make(chan struct{}) p := &prog{ @@ -216,9 +283,12 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { dnsWatcherStopCh: make(chan struct{}), apiReloadCh: make(chan *ctrld.Config), apiForceReloadCh: make(chan struct{}), + runDone: make(chan struct{}), + runAbortCh: make(chan struct{}), cfg: &cfg, appCallback: appCallback, } + p.logger.Store(mainLog.Load()) if homedir == "" { if dir, err := userHomeDir(); err == nil { homedir = dir @@ -229,39 +299,45 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { sockDir = d } sockPath := filepath.Join(sockDir, ctrldLogUnixSock) - if addr, err := net.ResolveUnixAddr("unix", sockPath); err == nil { - if conn, err := net.Dial(addr.Network(), addr.String()); err == nil { - lc := &logConn{conn: conn} - consoleWriter.Out = io.MultiWriter(os.Stdout, lc) - p.logConn = lc - } else { - if !errors.Is(err, os.ErrNotExist) { - mainLog.Load().Warn().Err(err).Msg("unable to create log ipc connection") - } + hlc := newHTTPLogClient(sockPath) + + // Test if HTTP log server is available + if err := hlc.Ping(); err != nil { + if !errLogServerUnavailable(err) { + p.Warn().Err(err).Msg("Unable to ping log server") } } else { - mainLog.Load().Warn().Err(err).Msgf("unable to resolve socket address: %s", sockPath) - } - notifyExitToLogServer := func() { - if p.logConn != nil { - _, _ = p.logConn.Write([]byte(msgExit)) - } + // Server is available, use HTTP log client + consoleWriter = newHumanReadableZapCore(io.MultiWriter(os.Stdout, hlc), consoleWriterLevel) + p.logConn = hlc } if daemon && runtime.GOOS == "windows" { - mainLog.Load().Fatal().Msg("Cannot run in daemon mode. Please install a Windows service.") + failRunUnclassified(p.Error(), "cannot run in daemon mode; please install a Windows service", p.notifyExitToLogServer) + return } - if !daemon { + switch { + case isMobile(): + defer p.finishRun() + // There is no OS service manager here, and s.Run parks a goroutine on a + // signal that never arrives, leaking one per start/stop cycle. + if err := p.Start(nil); err != nil { + mainLog.Load().Fatal().Err(err).Msg("failed to start ctrld") + } + case !daemon: // We need to call s.Run() as soon as possible to response to the OS manager, so it // can see ctrld is running and don't mark ctrld as failed service. go func() { - s, err := newService(p, svcConfig) + svcCmd := NewServiceCommand() + svcConfig := svcCmd.createServiceConfig() + s, err := svcCmd.newService(p, svcConfig) if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed create new service") + failRunUnclassified(p.Error().Err(err), fmt.Sprintf("failed to create new service: %v", err), p.notifyExitToLogServer) + return } if err := s.Run(); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to start service") + p.Error().Err(err).Msg("Failed to start service") } }() } @@ -269,133 +345,198 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { tryReadingConfig(writeDefaultConfig) if err := readBase64Config(configBase64); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to read base64 config") + failRunUnclassified(p.Error().Err(err), fmt.Sprintf("failed to read base64 config: %v", err), p.notifyExitToLogServer) + return } processNoConfigFlags(noConfigStart) // After s.Run() was called, if ctrld is going to be terminated for any reason, // write msgExit to p.logConn so others (like "ctrld start") won't have to wait for timeout. p.mu.Lock() - if err := v.Unmarshal(&cfg); err != nil { - notifyExitToLogServer() - mainLog.Load().Fatal().Msgf("failed to unmarshal config: %v", err) - } + unmarshalErr := v.Unmarshal(&cfg) p.mu.Unlock() + if unmarshalErr != nil { + failRunUnclassified(p.Error(), fmt.Sprintf("failed to unmarshal config: %v", unmarshalErr), p.notifyExitToLogServer) + return + } - processLogAndCacheFlags() + processLogAndCacheFlags(v, &cfg) // Log config do not have thing to validate, so it's safe to init log here, // so it's able to log information in processCDFlags. p.initLogging(true) - mainLog.Load().Info().Msgf("starting ctrld %s", curVersion()) - mainLog.Load().Info().Msgf("os: %s", osVersion()) + p.Info().Msgf("Starting ctrld %s", curVersion()) + p.Info().Msgf("OS: %s", osVersion()) + + // Drop enforcement left behind by a previous ctrld process before doing anything + // that needs the network. A previous run that died without cleaning up can leave + // machine-wide block filters installed (Firewall Mode blocks all non-allowlisted + // outbound traffic), which would deny this process's own API bootstrap below and + // leave it retrying forever - never reaching the cleanup that lives inside + // intercept startup. No-op when no stale state exists. + cleanupStaleDNSInterceptStateFn() // Wait for network up. - if !ctrldnet.Up() { - notifyExitToLogServer() - mainLog.Load().Fatal().Msg("network is not up yet") + if !networkUp() { + failRunUnclassified(p.Error(), "network is not up yet", p.notifyExitToLogServer) + return } - p.router = router.New(&cfg, cdUID != "") cs, err := newControlServer(filepath.Join(sockDir, ControlSocketName())) if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not create control server") + p.Warn().Err(err).Msg("Could not create control server") } p.cs = cs - // Processing --cd flag require connecting to ControlD API, which needs valid - // time for validating server certificate. Some routers need NTP synchronization - // to set the current time, so this check must happen before processCDFlags. - if err := p.router.PreRun(); err != nil { - notifyExitToLogServer() - mainLog.Load().Fatal().Err(err).Msg("failed to perform router pre-run check") - } - oldLogPath := cfg.Service.LogPath + // Whether this run got its configuration from the API, which is what the + // managed-config marker may claim once the configuration is on disk. + managedConfigFetched := false if uid := cdUIDFromProvToken(); uid != "" { cdUID = uid + p.initLoggingAfterProvisioning() } if cdUID != "" { - validateCdUpstreamProtocol() - if rc, err := processCDFlags(&cfg); err != nil { + if !validateCdUpstreamProtocol(p.notifyExitToLogServer) { + return + } + // Bound API preflight by the service lifetime. Without this, a stop request + // arriving while the API is unreachable leaves this retry/backoff loop running + // after "Service stopped" was logged, so the process keeps working on behalf of + // a service the OS considers stopped. + pf := runAPIPreflight(p.stopCh, &cfg) + switch { + case pf.stopRequested: + // Stop requested during preflight, whether or not the fetch itself + // succeeded. A successful fetch does not entitle startup to continue: the + // operator asked for a stop, and carrying on would set up listeners and + // interception for a service the OS already considers stopping. + // + // Exit the way a normal stop does: no Fatal, so the OS service manager does + // not see a failed start and apply its restart policy to a service the + // operator just asked to stop. + p.Notice().Msg("Stop requested while fetching resolver config — shutting down") + p.notifyExitToLogServer() + return + case pf.err != nil: if isMobile() { - appCallback.Exit(err.Error()) + appCallback.Exit(pf.err.Error()) return } - cdLogger := mainLog.Load().With().Str("mode", "cd").Logger() - // Performs self-uninstallation if the ControlD device does not exist. - var uer *controld.ErrorResponse - if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { - _ = uninstallInvalidCdUID(p, cdLogger, false) + // Maintenance is temporary and says nothing about this host's + // configuration, so a start that already has a working one carries + // on with it rather than taking DNS down for the window. + if !useLastKnownConfigDuringMaintenance(pf.err, &cfg) { + handleAPIPreflightFailure(p, pf.err, p.notifyExitToLogServer) + return } - notifyExitToLogServer() - cdLogger.Fatal().Err(err).Msg("failed to fetch resolver config") - } else { + // This run holds no answer from the API, so the reload loop asks + // again well before refetch_time would. + p.startedInAPIMaintenance.Store(true) + default: p.mu.Lock() - p.rc = rc + p.rc = pf.rc p.mu.Unlock() + managedConfigFetched = true } } - updated := updateListenerConfig(&cfg, notifyExitToLogServer) + updated := updateListenerConfig(&cfg, p.notifyExitToLogServer) + // Whether the file on disk, before anything below rewrites it, is already + // one the API produced for this identity. + wasManagedConfig := managedConfigRecordedFor(cdUID) + // The marker must never vouch for a configuration that was not written. + // Without this, a first managed run whose listener needed no change would + // record the marker over the seeded default still on disk. + if managedConfigFetched && !wasManagedConfig { + updated = true + } + + // Bootstrap and listener binding both succeeded, so an earlier run's + // recorded failure no longer describes this install. + clearProvisionResult() if cdUID != "" { - processLogAndCacheFlags() + processLogAndCacheFlags(v, &cfg) + } + + // Keep config and the explicit CLI/service mode in sync. In particular, "off" + // must replace a previously persisted dns/hard value, and must be kept as + // "off": an unset value means the platform default, which is "dns" on + // macOS and Windows desktop editions. + if updateConfigInterceptMode(&cfg, interceptMode) { + updated = true + p.Info().Msgf("writing intercept_mode = %q to config (requested %q)", cfg.Service.InterceptMode, interceptMode) + } + + // Persist firewall_mode to config only when provided via CLI flag. + // The flag defaults to "off" for help output, so use Changed() to avoid + // clobbering a config-file value of firewall_mode = "on" on normal starts. + if firewallModeFlagChanged { + if !validateFirewallModeFlag(true, firewallMode, p.notifyExitToLogServer) { + return + } + if cfg.Service.FirewallMode != firewallMode { + cfg.Service.FirewallMode = firewallMode + updated = true + p.Info().Msgf("writing firewall_mode = %q to config", firewallMode) + } + } else if cfg.Service.FirewallMode != "" { + // If firewall_mode is set in config but not via flag, use config value. + firewallMode = cfg.Service.FirewallMode + } else { + firewallMode = "off" + } + + // The API answer is the source of truth for a managed run. Without this, + // a custom config added (or changed) after install is applied in memory + // but never reaches the file, which file readers such as the macOS + // delegated log-send gate still use. + if managedConfigFetched && !updated && configFileDiffers(&cfg) { + updated = true + p.Info().Msg("Config from Control D API differs from the config file") } if updated { if err := writeConfigFile(&cfg); err != nil { - notifyExitToLogServer() - mainLog.Load().Fatal().Err(err).Msg("failed to write config file") - } else { - mainLog.Load().Info().Msg("writing config file to: " + defaultConfigFile) + failRunUnclassified(p.Error().Err(err), fmt.Sprintf("failed to write config file: %v", err), p.notifyExitToLogServer) + return } + p.Info().Msg("Writing config file to: " + defaultConfigFile) } + recordManagedConfigForStart(managedConfigFetched, wasManagedConfig) if newLogPath := cfg.Service.LogPath; newLogPath != "" && oldLogPath != newLogPath { - // After processCDFlags, log config may change, so reset mainLog and re-init logging. - l := zerolog.New(io.Discard) - mainLog.Store(&l) - - // Copy logs written so far to new log file if possible. - if buf, err := os.ReadFile(oldLogPath); err == nil { - if err := os.WriteFile(newLogPath, buf, os.FileMode(0o600)); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not copy old log file") - } - } - initLoggingWithBackup(false) + p.switchLogPath(oldLogPath, newLogPath) } if err := validateConfig(&cfg); err != nil { - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(p.Error().Err(err), fmt.Sprintf("invalid config: %v", err), p.notifyExitToLogServer) + return } initCache() if daemon { exe, err := os.Executable() if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to find the binary") - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(p.Error().Err(err), "failed to find the binary", p.notifyExitToLogServer) + return } curDir, err := os.Getwd() if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to get current working directory") - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(p.Error().Err(err), "failed to get current working directory", p.notifyExitToLogServer) + return } // If running as daemon, re-run the command in background, with daemon off. cmd := exec.Command(exe, append(os.Args[1:], "-d=false")...) cmd.Dir = curDir if err := cmd.Start(); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to start process as daemon") - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(p.Error().Err(err), "failed to start process as daemon", p.notifyExitToLogServer) + return } - mainLog.Load().Info().Int("pid", cmd.Process.Pid).Msg("DNS proxy started") + p.Info().Int("pid", cmd.Process.Pid).Msg("DNS proxy started") os.Exit(0) } @@ -403,7 +544,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { for _, lc := range p.cfg.Listener { if shouldAllocateLoopbackIP(lc.IP) { if err := allocateIP(lc.IP); err != nil { - mainLog.Load().Error().Err(err).Msgf("could not allocate IP: %s", lc.IP) + p.Error().Err(err).Msgf("Could not allocate ip: %s", lc.IP) } } } @@ -414,74 +555,92 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { for _, lc := range p.cfg.Listener { if shouldAllocateLoopbackIP(lc.IP) { if err := deAllocateIP(lc.IP); err != nil { - mainLog.Load().Error().Err(err).Msgf("could not de-allocate IP: %s", lc.IP) + p.Error().Err(err).Msgf("Could not de-allocate ip: %s", lc.IP) } } } }) - if platform := router.Name(); platform != "" { - if cp := router.CertPool(); cp != nil { - rootCertPool = cp - } - if iface != "" { - p.onStarted = append(p.onStarted, func() { - mainLog.Load().Debug().Msg("router setup on start") - if err := p.router.Setup(); err != nil { - mainLog.Load().Error().Err(err).Msg("could not configure router") - } - }) - p.onStopped = append(p.onStopped, func() { - mainLog.Load().Debug().Msg("router cleanup on stop") - if err := p.router.Cleanup(); err != nil { - mainLog.Load().Error().Err(err).Msg("could not cleanup router") - } - }) - } - } p.onStopped = append(p.onStopped, func() { // restore static DNS settings or DHCP p.resetDNS(false, true) - // Iterate over all physical interfaces and restore static DNS if a saved static config exists. - withEachPhysicalInterfaces("", "restore static DNS", func(i *net.Interface) error { - file := savedStaticDnsSettingsFilePath(i) - if _, err := os.Stat(file); err == nil { - if err := restoreDNS(i); err != nil { - mainLog.Load().Error().Err(err).Msgf("Could not restore static DNS on interface %s", i.Name) - } else { - mainLog.Load().Debug().Msgf("Restored static DNS on interface %s successfully", i.Name) - } - } - return nil - }) + restoreSavedStaticDNS("", false) }) close(waitCh) <-stopCh + if !isMobile() { + p.finishRun() + } } -func writeConfigFile(cfg *ctrld.Config) error { +// configFileForWrite returns the config file that writeConfigFile writes: +// viper's file when it read one, then --config, then defaultConfigFile. A +// Control D custom config is read from memory, so viper names no file then. +func configFileForWrite() string { if cfu := v.ConfigFileUsed(); cfu != "" { - defaultConfigFile = cfu - } else if configPath != "" { - defaultConfigFile = configPath + return cfu + } + if configPath != "" { + return configPath + } + return defaultConfigFile +} + +// renderConfigFile returns the exact bytes writeConfigFile writes for cfg. +func renderConfigFile(cfg *ctrld.Config) ([]byte, error) { + var buf bytes.Buffer + if cdUID != "" { + buf.WriteString("# AUTO-GENERATED VIA CD FLAG - DO NOT MODIFY\n\n") } + enc := toml.NewEncoder(&buf).SetIndentTables(true) + if err := enc.Encode(&cfg); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// configFileDiffers reports whether the config file on disk is not what +// writeConfigFile would write for cfg. A missing or unreadable file differs. +func configFileDiffers(cfg *ctrld.Config) bool { + want, err := renderConfigFile(cfg) + if err != nil { + return true + } + have, err := os.ReadFile(configFileForWrite()) + if err != nil { + return true + } + return !bytes.Equal(have, want) +} + +// writeConfigFile writes the configuration to a file +func writeConfigFile(cfg *ctrld.Config) error { + mainLog.Load().Debug().Msg("Writing configuration file") + + defaultConfigFile = configFileForWrite() + data, err := renderConfigFile(cfg) + if err != nil { + return err + } + + mainLog.Load().Debug().Str("config_file", defaultConfigFile).Msg("Opening configuration file for writing") + f, err := os.OpenFile(defaultConfigFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0o644)) if err != nil { + mainLog.Load().Error().Err(err).Str("config_file", defaultConfigFile).Msg("Failed to open configuration file") return err } defer f.Close() - if cdUID != "" { - if _, err := f.WriteString("# AUTO-GENERATED VIA CD FLAG - DO NOT MODIFY\n\n"); err != nil { - return err - } - } - enc := toml.NewEncoder(f).SetIndentTables(true) - if err := enc.Encode(&cfg); err != nil { + if _, err := f.Write(data); err != nil { + mainLog.Load().Error().Err(err).Str("config_file", defaultConfigFile).Msg("Failed to write configuration file") return err } if err := f.Close(); err != nil { + mainLog.Load().Error().Err(err).Str("config_file", defaultConfigFile).Msg("Failed to close configuration file") return err } + + mainLog.Load().Debug().Str("config_file", defaultConfigFile).Msg("Configuration file written successfully") return nil } @@ -493,10 +652,13 @@ func readConfigFile(writeDefaultConfig, notice bool) bool { // If err == nil, there's a config supplied via `--config`, no default config written. err := v.ReadInConfig() if err == nil { + // Only a file the host already had can stand in for the API answer + // during maintenance; the default written below cannot. + configReadFromDisk = true if notice { mainLog.Load().Notice().Msg("Reading config: " + v.ConfigFileUsed()) } - mainLog.Load().Info().Msg("loading config file from: " + v.ConfigFileUsed()) + mainLog.Load().Info().Msg("Loading config file from: " + v.ConfigFileUsed()) defaultConfigFile = v.ConfigFileUsed() return true } @@ -508,22 +670,21 @@ func readConfigFile(writeDefaultConfig, notice bool) bool { // If error is viper.ConfigFileNotFoundError, write default config. if errors.As(err, &viper.ConfigFileNotFoundError{}) { if err := v.Unmarshal(&cfg); err != nil { - mainLog.Load().Fatal().Msgf("failed to unmarshal default config: %v", err) + mainLog.Load().Fatal().Msgf("Failed to unmarshal default config: %v", err) } - nop := zerolog.Nop() - _, _ = tryUpdateListenerConfig(&cfg, &nop, func() {}, true) + _, _ = tryUpdateListenerConfig(&cfg, func() {}, true) addExtraSplitDnsRule(&cfg) if err := writeConfigFile(&cfg); err != nil { - mainLog.Load().Fatal().Msgf("failed to write default config file: %v", err) + mainLog.Load().Fatal().Msgf("Failed to write default config file: %v", err) } else { fp, err := filepath.Abs(defaultConfigFile) if err != nil { - mainLog.Load().Fatal().Msgf("failed to get default config file path: %v", err) + mainLog.Load().Fatal().Msgf("Failed to get default config file path: %v", err) } if cdUID == "" && nextdns == "" { mainLog.Load().Notice().Msg("Generating controld default config: " + fp) } - mainLog.Load().Info().Msg("writing default config file to: " + fp) + mainLog.Load().Info().Msg("Writing default config file to: " + fp) } return false } @@ -532,12 +693,12 @@ func readConfigFile(writeDefaultConfig, notice bool) bool { if errors.As(err, &viper.ConfigParseError{}) { if de := decoderErrorFromTomlFile(v.ConfigFileUsed()); de != nil { row, col := de.Position() - mainLog.Load().Fatal().Msgf("failed to decode config file at line: %d, column: %d, error: %v", row, col, err) + mainLog.Load().Fatal().Msgf("Failed to decode config file at line: %d, column: %d, error: %v", row, col, err) } } // Otherwise, report fatal error and exit. - mainLog.Load().Fatal().Msgf("failed to decode config file: %v", err) + mainLog.Load().Fatal().Msgf("Failed to decode config file: %v", err) return false } @@ -559,11 +720,17 @@ func readBase64Config(configBase64 string) error { if configBase64 == "" { return nil } + + mainLog.Load().Debug().Msg("Reading base64 encoded configuration") + configStr, err := base64.StdEncoding.DecodeString(configBase64) if err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to decode base64 configuration") return fmt.Errorf("invalid base64 config: %w", err) } + mainLog.Load().Debug().Int("config_length", len(configStr)).Msg("Base64 configuration decoded successfully") + // readBase64Config is called when: // // - "--base64_config" flag set. @@ -572,24 +739,39 @@ func readBase64Config(configBase64 string) error { // So we need to re-create viper instance to discard old one. v = viper.NewWithOptions(viper.KeyDelimiter("::")) v.SetConfigType("toml") - return v.ReadConfig(bytes.NewReader(configStr)) + + mainLog.Load().Debug().Msg("Parsing base64 configuration as TOML") + + if err := v.ReadConfig(bytes.NewReader(configStr)); err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to parse base64 configuration as TOML") + return err + } + + mainLog.Load().Debug().Msg("Base64 configuration processed successfully") + return nil } +// processNoConfigFlags processes flags for no-config mode func processNoConfigFlags(noConfigStart bool) { if !noConfigStart { return } + + mainLog.Load().Debug().Msg("Processing no-config mode flags") + if listenAddress == "" || primaryUpstream == "" { mainLog.Load().Fatal().Msg(`"listen" and "primary_upstream" flags must be set in no config mode`) } processListenFlag() endpointAndTyp := func(endpoint string) (string, string) { + mainLog.Load().Debug().Str("endpoint", endpoint).Msg("Processing endpoint for resolver type") typ := ctrld.ResolverTypeFromEndpoint(endpoint) endpoint = strings.TrimPrefix(endpoint, "quic://") if after, found := strings.CutPrefix(endpoint, "h3://"); found { endpoint = "https://" + after } + mainLog.Load().Debug().Str("endpoint", endpoint).Str("type", typ).Msg("Endpoint processed") return endpoint, typ } pEndpoint, pType := endpointAndTyp(primaryUpstream) @@ -599,7 +781,8 @@ func processNoConfigFlags(noConfigStart bool) { Type: pType, Timeout: 5000, } - puc.Init() + loggerCtx := ctrld.LoggerCtx(context.Background(), mainLog.Load()) + puc.Init(loggerCtx) upstream := map[string]*ctrld.UpstreamConfig{"0": puc} if secondaryUpstream != "" { sEndpoint, sType := endpointAndTyp(secondaryUpstream) @@ -609,7 +792,7 @@ func processNoConfigFlags(noConfigStart bool) { Type: sType, Timeout: 5000, } - suc.Init() + suc.Init(loggerCtx) upstream["1"] = suc rules := make([]ctrld.Rule, 0, len(domains)) for _, domain := range domains { @@ -625,9 +808,31 @@ func processNoConfigFlags(noConfigStart bool) { // If cdDeactivationPin equals to this default, it means the pin code is not set from Control D API. const defaultDeactivationPin = -1 +// deactivationPinKnown reports whether this run ever had an answer from the API +// about the deactivation PIN. +// +// cdDeactivationPin cannot carry this: its default means "the API said there is +// no PIN", and a run that started on the configuration on disk during +// maintenance never asked. Without the distinction such a run would present an +// unknown PIN as no PIN and hand out deactivation to anyone who asked. +var deactivationPinKnown atomic.Bool + // cdDeactivationPin is used in cd mode to decide whether stop and uninstall commands can be run. var cdDeactivationPin atomic.Int64 +// Brute-force protection for the deactivation PIN endpoint on the control socket. +// After deactivationMaxFailedAttempts consecutive wrong PINs, further attempts are +// rejected for deactivationLockoutSeconds. Counter resets on a correct PIN. +const ( + deactivationMaxFailedAttempts = 5 + deactivationLockoutSeconds = 60 +) + +var ( + deactivationFailedAttempts atomic.Int64 + deactivationLockedUntil atomic.Int64 +) + func init() { cdDeactivationPin.Store(defaultDeactivationPin) } @@ -637,18 +842,336 @@ func deactivationPinSet() bool { return cdDeactivationPin.Load() != defaultDeactivationPin } -func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { - logger := mainLog.Load().With().Str("mode", "cd").Logger() - logger.Info().Msgf("fetching Controld D configuration from API: %s", cdUID) +// fetchResolverConfig is a test seam for the ControlD resolver-config API call. +var fetchResolverConfig = controld.FetchResolverConfig + +// apiPreflight is the outcome of the API preflight fetch: the resolver config, the +// error if any, and whether the service was asked to stop while it ran. +type apiPreflight struct { + rc *controld.ResolverConfig + err error + stopRequested bool +} + +// runAPIPreflight fetches the ControlD resolver config bounded by the service +// lifetime, and reports whether a stop was requested while it ran. +// +// The distinction matters because the caller does very different things with it: a stop +// exits quietly, while a failure self-uninstalls a deleted device, surfaces the error to +// a mobile app, and reports a failed start to the service manager. +// +// stopRequested must not be derived from the context once it has been cancelled. +// context.CancelFunc sets ctx.Err() unconditionally, so reading it after the cancel +// classifies *every* failure - a deleted device, an exhausted retry, a mobile caller +// with no stop channel - as an operator stop. Reading the stop channel directly is also +// independent of whether the context's watcher goroutine has been scheduled yet. +func runAPIPreflight(stopCh <-chan struct{}, cfg *ctrld.Config) apiPreflight { + rc, err := fetchCDConfigBoundedBy(stopCh, cfg, processCDFlagsFn) + return apiPreflight{rc: rc, err: err, stopRequested: stopRequested(stopCh)} +} + +// permanentAPIRejection reports whether err is the API refusing this request in a way +// that a restart cannot change, and returns the rejection when it is. +// +// The type alone does not answer this. controld builds an *ErrorResponse for *any* +// non-200 whose body decodes, so a 502 from a load balancer and a 404 for a deleted +// device arrive as the same Go type. Treating both as permanent would let a few minutes +// of API trouble stop ctrld on every host with no service-manager retry behind it, which +// is strictly worse than the abnormal exit it replaced. +// +// So the HTTP status decides, and only a client-error status counts: +// +// - 4xx: the API examined this request and refused it - a deleted device, a revoked +// token, a malformed UID. The same request will be refused again. +// - 408 and 429 are the exceptions: they are the API asking for another attempt later. +// - 5xx, or no recorded status, says nothing about this configuration. Retry. +func permanentAPIRejection(err error) (*controld.ErrorResponse, bool) { + var uer *controld.ErrorResponse + if !errors.As(err, &uer) { + return nil, false + } + // Maintenance is the API being away, not the API refusing this request, so + // it stays retryable even when it arrives with a client-error status. + if controld.IsMaintenance(err) { + return nil, false + } + switch uer.StatusCode { + case http.StatusRequestTimeout, http.StatusTooManyRequests: + return nil, false + } + if uer.StatusCode < 400 || uer.StatusCode >= 500 { + return nil, false + } + return uer, true +} + +// apiFailureCode maps a bootstrap preflight error to its provisioning code. +// A deleted device gets its own code because it triggers self-uninstall; a +// permanent rejection with a known token reason gets its own code; other +// permanent rejections are generic; anything else counts as reachability +// trouble worth retrying. +func apiFailureCode(err error) (provisionFailureCode, bool) { + if err == nil { + return "", false + } + // Before the device check: maintenance must never reach the self-uninstall + // path, because it says nothing about whether this device still exists. + if controld.IsMaintenance(err) { + return provisionCodeAPIMaintenance, true + } + var uer *controld.ErrorResponse + if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { + return provisionCodeAPIDeviceInvalid, true + } + if rejection, ok := permanentAPIRejection(err); ok { + if code, ok := tokenFailureCodeForReason(rejection.ErrorField.Metadata.Reason); ok { + return code, true + } + return provisionCodeAPIRejected, true + } + return provisionCodeAPIUnreachable, true +} + +// tokenFailureCodeForReason maps error.metadata.reason on a provisioning-token +// rejection to its failure code. An empty or unrecognized reason is not +// classified here; the caller falls back to the generic rejection code. +func tokenFailureCodeForReason(reason string) (provisionFailureCode, bool) { + switch reason { + case controld.ReasonTokenInvalid: + return provisionCodeTokenInvalid, true + case controld.ReasonTokenExpired: + return provisionCodeTokenExpired, true + case controld.ReasonTokenLimitReached: + return provisionCodeTokenLimitReached, true + case controld.ReasonTokenDisabled: + return provisionCodeTokenDisabled, true + default: + return "", false + } +} + +// tokenRejectionMessage returns the human message for a TOKEN_* code: what is +// wrong with the provisioning code and one concrete next action. Empty for +// any other code, so callers know to fall back to a generic message. Never +// includes the token itself. +func tokenRejectionMessage(code provisionFailureCode) string { + switch code { + case provisionCodeTokenInvalid: + return "the provisioning code is not valid; check the code and re-enter it exactly as given" + case provisionCodeTokenExpired: + return "the provisioning code has expired; get a new provisioning code from your administrator" + case provisionCodeTokenLimitReached: + return "the provisioning code reached its device limit; free up a device slot or use a different code" + case provisionCodeTokenDisabled: + return "the provisioning code was invalidated; download a profile from an active provisioning code" + default: + return "" + } +} + +// provisionTokenFailureMessage returns the result-file message for a failed +// provision-token exchange: a token-specific message when the API gave a +// known reason, the generic exchange-failure summary otherwise. +func provisionTokenFailureMessage(code provisionFailureCode, err error) string { + if msg := tokenRejectionMessage(code); msg != "" { + return msg + } + return fmt.Sprintf("provision token exchange failed: %v", err) +} + +// apiRejectionSummary reports the HTTP status only. The API's raw error body +// can echo back the value the caller sent, so it stays out of the artifact. +func apiRejectionSummary(statusCode int) string { + return fmt.Sprintf("ControlD API rejected this configuration (HTTP status %d)", statusCode) +} + +// provisionSecrets lists every secret-bearing value to strip from provisioning +// artifacts, including both parts of a composite "/" --cd +// value, which the API may echo back separately. redactSecrets drops the +// values that are too short to redact, so a client ID such as "os" leaves the +// words of the message alone. +func provisionSecrets() []string { + uid, clientID := controld.ParseRawUID(cdUID) + return []string{cdUID, cdOrg, uid, clientID} +} + +// redactedArgs formats command-line arguments for a log line with the +// provision secrets removed. "ctrld start --cd-org " carries the code +// in its arguments, and a code enrolls endpoints into the organization until +// it expires or runs out. +func redactedArgs(args []string) string { + return redactSecrets(fmt.Sprint(args), provisionSecrets()...) +} + +// interceptUpgradeCheckLine is the debug line "ctrld start" logs before its +// intercept upgrade check. It is built here, not inline, so a test can hold the +// line itself, not only redactedArgs, to never carrying a provision secret. +func interceptUpgradeCheckLine(args []string, interceptOnly, svcConfigExists bool, mode string) string { + return fmt.Sprintf("intercept upgrade check: args=%s interceptOnly=%v svcConfigExists=%v interceptMode=%q", + redactedArgs(args), interceptOnly, svcConfigExists, mode) +} + +// uninstallInvalidCdUIDFn is a var so tests can observe the self-uninstall +// without driving the OS service manager. +var uninstallInvalidCdUIDFn = uninstallInvalidCdUID + +// handleAPIPreflightFailure reports a failed resolver-config fetch. A deleted +// device self-uninstalls; it and any other permanent rejection return cleanly +// so a config problem cannot burn the service manager's restart budget (on +// Windows those restarts are what bring enforcement back after a real crash). +// Anything else exits nonzero through failProvision so the manager retries. +func handleAPIPreflightFailure(p *prog, err error, notify func()) { + logger := p.logger.Load() + if logger == nil { + logger = mainLog.Load() + } + cdLogger := logger.With().Str("mode", "cd") + code, _ := apiFailureCode(err) + if code == provisionCodeAPIMaintenance { + // The caller already established there is nothing to fall back on. + cdLogger.Error().Err(err).Msg("Failed to fetch resolver config, the Control D API is in maintenance") + failProvision(newProvisionResult(code, maintenanceNoConfigDetail, nil, provisionSecrets()...), notify) + return + } + var uer *controld.ErrorResponse + if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { + r := newProvisionResult(code, apiRejectionSummary(uer.StatusCode), nil, provisionSecrets()...) + if werr := writeProvisionResult(r); werr != nil { + cdLogger.Warn().Err(werr).Msg("could not persist provision result") + } + _ = uninstallInvalidCdUIDFn(p, cdLogger, false) + cdLogger.Error().Err(err).Int("status", uer.StatusCode).Msg("Failed to fetch resolver config; the device no longer exists") + cdLogger.Error().Msg(r.failureLine()) + notify() + return + } + if rejection, ok := permanentAPIRejection(err); ok { + r := newProvisionResult(code, apiRejectionSummary(rejection.StatusCode), nil, provisionSecrets()...) + if werr := writeProvisionResult(r); werr != nil { + cdLogger.Warn().Err(werr).Msg("could not persist provision result") + } + cdLogger.Error().Err(err).Int("status", rejection.StatusCode).Msg("Failed to fetch resolver config; the API rejected this configuration") + cdLogger.Error().Msg(r.failureLine()) + notify() + return + } + cdLogger.Error().Err(err).Msg("Failed to fetch resolver config") + failProvision(newProvisionResult(code, fmt.Sprintf("failed to fetch resolver config: %v", err), nil, provisionSecrets()...), notify) +} + +// The startup (full metadata) and reload (runtime metadata) API fetches are indirected so +// their lifetime binding can be tested without reaching the network. +var ( + processCDFlagsFn = processCDFlags + processCDFlagsRuntimeFn = processCDFlagsRuntime +) + +// fetchCDConfigBoundedBy runs fetch bounded by stopCh, so a fetch that cannot reach the +// API stops when the service is asked to stop instead of working on behalf of a service +// the OS already considers stopped. The derived context is always cancelled, which +// releases the goroutine watching stopCh. +func fetchCDConfigBoundedBy( + stopCh <-chan struct{}, + cfg *ctrld.Config, + fetch func(context.Context, *ctrld.Config) (*controld.ResolverConfig, error), +) (*controld.ResolverConfig, error) { + ctx, cancel := contextFromStopCh(stopCh) + defer cancel() + return fetch(ctx, cfg) +} + +// fetchCDConfigBoundedByLifetime is the reload path's fetch. Reload binds the same stop +// primitives as startup - it used to wire them up itself, where a dropped cancel or the +// wrong channel would have failed nothing. +func (p *prog) fetchCDConfigBoundedByLifetime(cfg *ctrld.Config) (*controld.ResolverConfig, error) { + return fetchCDConfigBoundedBy(p.stopCh, cfg, processCDFlagsRuntimeFn) +} + +// stopRequested reports whether stopCh has been closed. A nil channel - mobile passes +// none - blocks forever, so the default case is taken and it reads as "no stop". +func stopRequested(stopCh <-chan struct{}) bool { + select { + case <-stopCh: + return true + default: + return false + } +} + +// contextFromStopCh returns a context that is cancelled when stopCh closes, so +// long-running startup work stops as soon as the service is asked to stop. The +// returned cancel func must be called to release the watcher goroutine. +func contextFromStopCh(stopCh <-chan struct{}) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + if stopCh == nil { + return ctx, cancel + } + go func() { + select { + case <-stopCh: + cancel() + case <-ctx.Done(): + } + }() + return ctx, cancel +} + +// processCDFlags fetches the ControlD configuration for cdUID and applies it to cfg. +// +// ctx bounds the whole operation, including the bootstrap-DNS retry loop below. That +// loop retries indefinitely by design (a device with no network yet must eventually +// come up), so it must be cancellable: otherwise a stop request during preflight is +// ignored and the process keeps retrying after the service reports itself stopped. +func processCDFlags(ctx context.Context, cfg *ctrld.Config) (*controld.ResolverConfig, error) { + return processCDFlagsWithMetadata(ctx, cfg, startupMetadata, true) +} + +// startupMetadata is the daemon-start metadata source, indirected so tests can +// prove that startup, and only startup, uses it. +var startupMetadata = ctrld.SystemMetadataStartup + +func processCDFlagsRuntime(ctx context.Context, cfg *ctrld.Config) (*controld.ResolverConfig, error) { + return processCDFlagsWithMetadata(ctx, cfg, ctrld.SystemMetadataRuntime, false) +} + +func processCDFlagsWithMetadata( + ctx context.Context, + cfg *ctrld.Config, + metadata func(context.Context) map[string]string, + includeHostnameHints bool, +) (*controld.ResolverConfig, error) { + logger := mainLog.Load().With().Str("mode", "cd") + logger.Info().Msgf("Fetching Controld D configuration from API: %s", cdUID) bo := backoff.NewBackoff("processCDFlags", logf, 30*time.Second) bo.LogLongerThan = 30 * time.Second - ctx := context.Background() - resolverConfig, err := controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev) + if ctx == nil { + ctx = context.Background() + } + ctx = ctrld.LoggerCtx(ctx, logger) + req := &controld.ResolverConfigRequest{ + RawUID: cdUID, + Version: appVersion, + Metadata: metadata(ctx), + IncludeHostnameHints: includeHostnameHints, + } + resolverConfig, err := fetchResolverConfig(ctx, req, cdDev) + + // Retry logic for network errors using bootstrap DNS + // This is needed because the initial DNS resolution might fail due to network issues + // or DNS server unavailability, but bootstrap DNS can provide alternative resolution for { + if ctxErr := ctx.Err(); ctxErr != nil { + logger.Debug().Msg("Resolver config fetch cancelled") + return nil, ctxErr + } if errUrlNetworkError(err) { bo.BackOff(ctx, err) - logger.Warn().Msg("could not fetch resolver using bootstrap DNS, retrying...") - resolverConfig, err = controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev) + if ctxErr := ctx.Err(); ctxErr != nil { + logger.Debug().Msg("Resolver config fetch cancelled during backoff") + return nil, ctxErr + } + logger.Warn().Msg("Could not fetch resolver using bootstrap DNS, retrying...") + resolverConfig, err = fetchResolverConfig(ctx, req, cdDev) continue } break @@ -657,21 +1180,34 @@ func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { if isMobile() { return nil, err } - logger.Warn().Err(err).Msg("could not fetch resolver config") + if controld.IsMaintenance(err) { + // The caller reports this: only it knows whether a fallback is + // available. Saying "Could not fetch resolver config" here puts a + // connectivity-shaped failure at the top of the bootstrap log, + // which is how the incident behind this was first mis-triaged. + logger.Debug().Err(err).Msg("Resolver config fetch answered with maintenance") + } else { + logger.Warn().Err(err).Msg("Could not fetch resolver config") + } return nil, err } + // The answer settles the PIN either way: a response without one means the + // API says there is none, which is not the same as never having asked. + deactivationPinKnown.Store(true) if resolverConfig.DeactivationPin != nil { - logger.Debug().Msg("saving deactivation pin") + logger.Debug().Msg("Saving deactivation pin") cdDeactivationPin.Store(*resolverConfig.DeactivationPin) } - logger.Info().Msg("generating ctrld config from Control-D configuration") + logger.Info().Msg("Generating ctrld config from Control-D configuration") + // Reset config to ensure clean state before applying Control-D settings + // This prevents mixing of old configuration with new Control-D settings *cfg = ctrld.Config{} // Fetch config, unmarshal to cfg. if resolverConfig.Ctrld.CustomConfig != "" { - logger.Info().Msg("using defined custom config of Control-D resolver") + logger.Info().Msg("Using defined custom config of Control-D resolver") var cfgErr error if cfgErr = validateCdRemoteConfig(resolverConfig, cfg); cfgErr == nil { setListenerDefaultValue(cfg) @@ -680,13 +1216,16 @@ func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { return resolverConfig, nil } } - mainLog.Load().Warn().Err(err).Msg("disregarding invalid custom config") + // cfgErr, not err: err is the resolver-config fetch error from above, which is + // nil on every path that reaches here, so logging it said nothing about why the + // custom config was rejected. + mainLog.Load().Warn().Err(cfgErr).Msg("Disregarding invalid custom config") } bootstrapIP := func(endpoint string) string { u, err := url.Parse(endpoint) if err != nil { - logger.Warn().Err(err).Msgf("no bootstrap IP for invalid endpoint: %s", endpoint) + logger.Warn().Err(err).Msgf("No bootstrap ip for invalid endpoint: %s", endpoint) return "" } switch { @@ -698,6 +1237,8 @@ func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { return "" } + // Initialize upstream configuration with Control-D resolver settings + // This creates the primary DNS resolver configuration for the proxy cfg.Upstream = make(map[string]*ctrld.UpstreamConfig) cfg.Upstream["0"] = &ctrld.UpstreamConfig{ BootstrapIP: bootstrapIP(resolverConfig.DOH), @@ -705,10 +1246,16 @@ func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { Type: cdUpstreamProto, Timeout: 5000, } + + // Create exclusion rules for domains that should bypass Control-D + // These domains will be resolved using the system's default DNS servers rules := make([]ctrld.Rule, 0, len(resolverConfig.Exclude)) for _, domain := range resolverConfig.Exclude { rules = append(rules, ctrld.Rule{domain: []string{}}) } + + // Initialize listener configuration with policy rules + // This sets up the DNS proxy listener with the exclusion policy cfg.Listener = make(map[string]*ctrld.ListenerConfig) lc := &ctrld.ListenerConfig{ Policy: &ctrld.ListenerPolicyConfig{ @@ -718,6 +1265,14 @@ func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { } cfg.Listener["0"] = lc + // Organization Internal Domains. Applied after the Magic Folder excludes + // above, so an excluded domain keeps its exclusion, and before the + // platform AD auto-detection in addExtraSplitDnsRule, which skips any + // domain that already has a rule. + if summary := applyInternalDomains(cfg, resolverConfig.SplitDNS); !summary.empty() { + logInternalDomainsSummary(logger, summary) + } + // Set default value. setListenerDefaultValue(cfg) setNetworkDefaultValue(cfg) @@ -725,6 +1280,24 @@ func processCDFlags(cfg *ctrld.Config) (*controld.ResolverConfig, error) { return resolverConfig, nil } +// logInternalDomainsSummary reports what the Internal Domains list produced, +// as counts only. Domain names and resolver addresses are private to the +// organization and stay at debug level. +func logInternalDomainsSummary(logger *ctrld.Logger, summary internalDomainsSummary) { + event := logger.Info() + if summary.skipped > 0 || summary.preempted > 0 { + event = logger.Warn() + } + event.Int("domains", summary.domains). + Int("os_resolver", summary.osMode). + Int("explicit_resolver", summary.explicit). + Int("explicit_resolver_only", summary.only). + Int("resolvers", summary.resolvers). + Int("skipped", summary.skipped). + Int("preempted", summary.preempted). + Msg("applied organization Internal Domains") +} + // setListenerDefaultValue sets the default value for cfg.Listener if none existed. func setListenerDefaultValue(cfg *ctrld.Config) { if len(cfg.Listener) == 0 { @@ -759,17 +1332,20 @@ func validateCdRemoteConfig(rc *controld.ResolverConfig, cfg *ctrld.Config) erro return v.Unmarshal(&cfg) } +// processListenFlag processes the listen flag func processListenFlag() { if listenAddress == "" { return } + mainLog.Load().Debug().Str("listen_address", listenAddress).Msg("Processing listen flag") + host, portStr, err := net.SplitHostPort(listenAddress) if err != nil { - mainLog.Load().Fatal().Msgf("invalid listener address: %v", err) + mainLog.Load().Fatal().Msgf("Invalid listener address: %v", err) } port, err := strconv.Atoi(portStr) if err != nil { - mainLog.Load().Fatal().Msgf("invalid port number: %v", err) + mainLog.Load().Fatal().Msgf("Invalid port number: %v", err) } lc := &ctrld.ListenerConfig{ IP: host, @@ -778,46 +1354,70 @@ func processListenFlag() { v.Set("listener", map[string]*ctrld.ListenerConfig{ "0": lc, }) + + mainLog.Load().Debug().Str("host", host).Int("port", port).Msg("Listen flag processed successfully") } -func processLogAndCacheFlags() { +// processLogAndCacheFlags processes log and cache related flags +func processLogAndCacheFlags(v *viper.Viper, cfg *ctrld.Config) { + mainLog.Load().Debug().Msg("Processing log and cache flags") + if logPath != "" { cfg.Service.LogPath = logPath + mainLog.Load().Debug().Str("log_path", logPath).Msg("Log path flag processed") } if logPath != "" && cfg.Service.LogLevel == "" { cfg.Service.LogLevel = "debug" + mainLog.Load().Debug().Msg("Log level set to debug") } if cacheSize != 0 { cfg.Service.CacheEnable = true cfg.Service.CacheSize = cacheSize + mainLog.Load().Debug().Int("cache_size", cacheSize).Msg("Cache flag processed") } v.Set("service", cfg.Service) + + mainLog.Load().Debug().Msg("Log and cache flags processed successfully") } +// errInterfaceNotFound reports that the named interface does not exist on the +// host. Callers use errors.Is to tell this apart from a lookup that failed for +// another reason, because an interface that is simply gone (an unplugged +// adapter, a torn down tether) is an expected outcome rather than a failure. +var errInterfaceNotFound = errors.New("interface not found") + +// foreachInterface enumerates the host network interfaces. It is a variable so +// tests can make enumeration fail without depending on the host. +var foreachInterface = netmon.ForeachInterface + +// netInterface returns the network interface by name func netInterface(ifaceName string) (*net.Interface, error) { - if ifaceName == "auto" { + if ifaceName == autoIface { ifaceName = defaultIfaceName() } var iface *net.Interface - err := netmon.ForeachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { + err := foreachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { if i.Name == ifaceName { iface = i.Interface } }) + if err != nil { + // Enumeration itself failed, so the interface cannot be reported as + // missing: nothing was positively established about it. + return nil, err + } if iface == nil { - return nil, errors.New("interface not found") + return nil, errInterfaceNotFound } if _, err := patchNetIfaceName(iface); err != nil { return nil, err } - return iface, err + return iface, nil } +// defaultIfaceName returns the default interface name func defaultIfaceName() string { - if ifaceName := router.DefaultInterfaceName(); ifaceName != "" { - return ifaceName - } dri, err := netmon.DefaultRouteInterface() if err != nil { // On WSL 1, the route table does not have any default route. But the fact that @@ -830,7 +1430,7 @@ func defaultIfaceName() string { if runtime.GOOS == "linux" { return "lo" } - mainLog.Load().Debug().Err(err).Msg("no default route interface found") + mainLog.Load().Debug().Err(err).Msg("No default route interface found") return "" } return dri @@ -849,7 +1449,7 @@ func defaultIfaceName() string { func selfCheckStatus(ctx context.Context, s service.Service, sockDir string) (bool, service.Status, error) { status, err := s.Status() if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not get service status") + mainLog.Load().Warn().Err(err).Msg("Could not get service status") return false, service.StatusUnknown, err } // If ctrld is not running, do nothing, just return the status as-is. @@ -861,7 +1461,7 @@ func selfCheckStatus(ctx context.Context, s service.Service, sockDir string) (bo return true, status, nil } - mainLog.Load().Debug().Msg("waiting for ctrld listener to be ready") + mainLog.Load().Debug().Msg("Waiting for ctrld listener to be ready") cc := newSocketControlClient(ctx, s, sockDir) if cc == nil { return false, status, errors.New("could not connect to control server") @@ -874,13 +1474,13 @@ func selfCheckStatus(ctx context.Context, s service.Service, sockDir string) (bo v.SetConfigFile(defaultConfigFile) } if err := v.ReadInConfig(); err != nil { - mainLog.Load().Error().Err(err).Msgf("failed to re-read configuration file: %s", v.ConfigFileUsed()) + mainLog.Load().Error().Err(err).Msgf("Failed to re-read configuration file: %s", v.ConfigFileUsed()) return false, status, err } cfg = ctrld.Config{} if err := v.Unmarshal(&cfg); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to update new config") + mainLog.Load().Error().Err(err).Msg("Failed to update new config") return false, status, err } @@ -890,12 +1490,12 @@ func selfCheckStatus(ctx context.Context, s service.Service, sockDir string) (bo return true, status, nil } - mainLog.Load().Debug().Msg("ctrld listener is ready") + mainLog.Load().Debug().Msg("Ctrld listener is ready") lc := cfg.FirstListener() addr := net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port)) - mainLog.Load().Debug().Msgf("performing listener test, sending queries to %s", addr) + mainLog.Load().Debug().Msgf("Performing listener test, sending queries to %s", addr) if err := selfCheckResolveDomain(context.TODO(), addr, "internal", selfCheckInternalTestDomain); err != nil { return false, status, err @@ -945,19 +1545,17 @@ func selfCheckResolveDomain(ctx context.Context, addr, scope string, domain stri lastErr = exErr bo.BackOff(ctx, fmt.Errorf("ExchangeContext: %w", exErr)) } - mainLog.Load().Debug().Msgf("self-check against %q failed", domain) + mainLog.Load().Debug().Msgf("Self-check against %q failed", domain) + loggerCtx := ctrld.LoggerCtx(ctx, mainLog.Load()) // Ping all upstreams to provide better error message to users. - for name, uc := range cfg.Upstream { - if err := uc.ErrorPing(); err != nil { - mainLog.Load().Err(err).Msgf("failed to connect to upstream.%s, endpoint: %s", name, uc.Endpoint) - } - } + logUpstreamPingFailures(loggerCtx, cfg.Upstream) marker := strings.Repeat("=", 32) mainLog.Load().Debug().Msg(marker) - mainLog.Load().Debug().Msgf("listener address : %s", addr) - mainLog.Load().Debug().Msgf("last error : %v", lastErr) + + mainLog.Load().Debug().Msgf("Listener address : %s", addr) + mainLog.Load().Debug().Msgf("Last error : %v", lastErr) if lastAnswer != nil { - mainLog.Load().Debug().Msgf("last answer from ctrld :") + mainLog.Load().Debug().Msgf("Last answer from ctrld :") mainLog.Load().Debug().Msg(marker) for _, s := range strings.Split(lastAnswer.String(), "\n") { mainLog.Load().Debug().Msgf("%s", s) @@ -966,39 +1564,63 @@ func selfCheckResolveDomain(ctx context.Context, addr, scope string, domain stri return errSelfCheckNoAnswer } -func userHomeDir() (string, error) { - dir, err := router.HomeDir() - if err != nil { - return "", err - } - if dir != "" { - return dir, nil - } - // viper will expand for us. - if runtime.GOOS == "windows" { - // If we're on windows, use the install path for this. - exePath, err := os.Executable() - if err != nil { - return "", err +// upstreamPingFn is a var so tests can drive the self-check report without a +// network probe. +var upstreamPingFn = (*ctrld.UpstreamConfig).ErrorPing + +// logUpstreamPingFailures names the upstreams that do not answer after a +// failed self-check, so the user learns which one is unreachable. The config +// key and the endpoint are operator text that can hold a token, so both stay +// at debug and the retained line names the upstream by its bounded name. The +// self-check runs before the program exists, so a bare prog carries the main +// logger into the probe report. +func logUpstreamPingFailures(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig) { + reporter := &prog{} + reporter.logger.Store(mainLog.Load()) + for name, uc := range upstreams { + err := upstreamPingFn(uc, ctx) + if err == nil { + continue } - - return filepath.Dir(exePath), nil + reporter.logUpstreamProbeFailure(upstreamPrefix+name, uc, err, reporter.Error, "Failed to connect to the upstream") } +} + +// userHomeDir returns the user's home directory +func userHomeDir() (string, error) { // Mobile platform should provide a rw dir path for this. if isMobile() { return homedir, nil } - dir = "/etc/controld" - if err := os.MkdirAll(dir, 0750); err != nil { - return os.UserHomeDir() // fallback to user home directory + return ctrld.UserHomeDir() +} + +// serviceHomeDir returns the directory where a ctrld service with root or +// administrator rights keeps its files. Unlike userHomeDir, it has no +// fallback to the home directory of the current user, so a caller without +// root that only reads looks where the service wrote. +func serviceHomeDir() (string, error) { + if isMobile() { + return homedir, nil } - if ok, _ := dirWritable(dir); !ok { - return os.UserHomeDir() + return ctrld.ServiceHomeDir() +} + +// absHomeDir returns the absolute path of filename in the ctrld home +// directory. A custom homedir wins, so a file lands next to the log and +// the config that obey the same override. +func absHomeDir(filename string) string { + if homedir != "" { + return filepath.Join(homedir, filename) } - return dir, nil + dir, err := userHomeDir() + if err != nil { + return filename + } + return filepath.Join(dir, filename) } -// socketDir returns directory that ctrld will create socket file for running controlServer. +// socketDir returns directory to use for the controlServer socket. func socketDir() (string, error) { switch { case runtime.GOOS == "windows", isMobile(): @@ -1051,7 +1673,7 @@ func readConfigWithNotice(writeDefaultConfig, notice bool) { dir, err := userHomeDir() if err != nil { - mainLog.Load().Fatal().Msgf("failed to get user home dir: %v", err) + mainLog.Load().Fatal().Msgf("Failed to get user home dir: %v", err) } for _, config := range configs { ctrld.SetConfigNameWithPath(v, config.name, dir) @@ -1073,54 +1695,58 @@ func uninstall(p *prog, s service.Service) { } initInteractiveLogging() if doTasks(tasks) { - if err := p.router.ConfigureService(svcConfig); err != nil { - mainLog.Load().Fatal().Err(err).Msg("could not configure service") - } - if err := p.router.Uninstall(svcConfig); err != nil { - mainLog.Load().Warn().Err(err).Msg("post uninstallation failed, please check system/service log for details error") - return - } // restore static DNS settings or DHCP p.resetDNS(false, true) + restoreSavedStaticDNS(p.runningIface, true) - // Iterate over all physical interfaces and restore DNS if a saved static config exists. - withEachPhysicalInterfaces(p.runningIface, "restore static DNS", func(i *net.Interface) error { - file := savedStaticDnsSettingsFilePath(i) - if _, err := os.Stat(file); err == nil { - if err := restoreDNS(i); err != nil { - mainLog.Load().Error().Err(err).Msgf("Could not restore static DNS on interface %s", i.Name) - } else { - mainLog.Load().Debug().Msgf("Restored static DNS on interface %s successfully", i.Name) - err = os.Remove(file) - if err != nil { + mainLog.Load().Notice().Msg("Service uninstalled") + return + } +} + +var restoreSavedStaticDNSInterfacesFn = withEachPhysicalInterfaces +var restoreSavedStaticDNSRestoreFn = restoreDNS +var savedStaticDNSPathFn = ctrld.SavedStaticDnsSettingsFilePath + +// restoreSavedStaticDNS restores DNS from saved static config files on physical interfaces. +func restoreSavedStaticDNS(excludeIfaceName string, removeSaved bool) { + restoreSavedStaticDNSInterfacesFn(excludeIfaceName, "restore static DNS", func(i *net.Interface) error { + if !interceptTargetAllowsStaticRestore(i.Name) { + mainLog.Load().Debug().Msgf("Saved static DNS restore skipped on interface %s: intercept target cleanup is pending or ownership is unreadable", i.Name) + return nil + } + file := savedStaticDNSPathFn(i) + if _, err := os.Stat(file); err == nil { + if err := restoreSavedStaticDNSRestoreFn(i); err != nil { + mainLog.Load().Error().Err(err).Msgf("Could not restore static DNS on interface %s", i.Name) + } else { + mainLog.Load().Debug().Msgf("Restored static DNS on interface %s successfully", i.Name) + if removeSaved { + if err := os.Remove(file); err != nil { mainLog.Load().Debug().Err(err).Msgf("Could not remove saved static DNS file for interface %s", i.Name) } } } - return nil - }) - - if router.Name() != "" { - mainLog.Load().Debug().Msg("Router cleanup") } - // Stop already did router.Cleanup and report any error if happens, - // ignoring error here to prevent false positive. - _ = p.router.Cleanup() - mainLog.Load().Notice().Msg("Service uninstalled") - return - } + return nil + }) } func validateConfig(cfg *ctrld.Config) error { + mainLog.Load().Debug().Msg("Validating configuration") + if err := ctrld.ValidateConfig(validator.New(), cfg); err != nil { var ve validator.ValidationErrors if errors.As(err, &ve) { for _, fe := range ve { - mainLog.Load().Error().Msgf("invalid config: %s: %s", fe.Namespace(), fieldErrorMsg(fe)) + mainLog.Load().Error().Msgf("Invalid config: %s: %s", fe.Namespace(), fieldErrorMsg(fe)) } } + mainLog.Load().Error().Err(err).Msg("Configuration validation failed") return err } + + mainLog.Load().Debug().Msg("Configuration validation completed successfully") return nil } @@ -1146,6 +1772,8 @@ func fieldErrorMsg(fe validator.FieldError) string { return fmt.Sprintf("minimum len: %q", fe.Param()) case "gte": return fmt.Sprintf("must be greater than or equal to: %s", fe.Param()) + case "lte": + return fmt.Sprintf("must be less than or equal to: %s", fe.Param()) case "cidr": return fmt.Sprintf("invalid value: %s", fe.Value()) case "required_unless", "required": @@ -1206,36 +1834,150 @@ func mobileListenerIp() string { // or defined but invalid to be used, e.g: using loopback address other // than 127.0.0.1 with systemd-resolved. func updateListenerConfig(cfg *ctrld.Config, notifyToLogServerFunc func()) bool { - updated, _ := tryUpdateListenerConfig(cfg, nil, notifyToLogServerFunc, true) + updated, _ := tryUpdateListenerConfig(cfg, notifyToLogServerFunc, true) if addExtraSplitDnsRule(cfg) { updated = true } return updated } +// tryUpdateListenerConfigIntercept handles listener binding for dns-intercept mode on macOS. +// In intercept mode, pf redirects all outbound port-53 traffic to ctrld's listener, +// so ctrld can safely listen on a non-standard port if port 53 is unavailable +// (e.g., mDNSResponder holds *:53). +// +// Flow: +// 1. If config has explicit (non-default) IP:port → use exactly that, no fallback +// 2. Otherwise → try 127.0.0.1:53, then 127.0.0.1:5354, then fatal +func tryUpdateListenerConfigIntercept(cfg *ctrld.Config, notifyFunc func(), fatal bool) (updated, ok bool) { + ok = true + lc := cfg.FirstListener() + if lc == nil { + return false, true + } + + hasExplicitConfig := isExplicitInterceptListener(lc.IP, lc.Port) + if !hasExplicitConfig { + // Set defaults for intercept mode + if lc.IP == "" || lc.IP == "0.0.0.0" { + lc.IP = "127.0.0.1" + updated = true + } + if lc.Port == 0 { + lc.Port = 53 + updated = true + } + } + + // bindAttempts feeds the provisioning result detail. newProvisionResult + // caps it, so it grows freely here. + var bindAttempts []provisionBindAttempt + recordBindAttempt := func(addr, proto string, err error) { + if err != nil { + bindAttempts = append(bindAttempts, provisionBindAttempt{Addr: addr, Proto: proto, OSError: err.Error()}) + } + } + + tryListen := func(ip string, port int) bool { + addr := net.JoinHostPort(ip, strconv.Itoa(port)) + udpLn, udpErr := net.ListenPacket("udp", addr) + if udpLn != nil { + udpLn.Close() + } + recordBindAttempt(addr, "udp", udpErr) + tcpLn, tcpErr := net.Listen("tcp", addr) + if tcpLn != nil { + tcpLn.Close() + } + recordBindAttempt(addr, "tcp", tcpErr) + return udpErr == nil && tcpErr == nil + } + + addr := net.JoinHostPort(lc.IP, strconv.Itoa(lc.Port)) + if tryListen(lc.IP, lc.Port) { + mainLog.Load().Debug().Msgf("DNS intercept: listener available at %s", addr) + return updated, true + } + + mainLog.Load().Info().Msgf("DNS intercept: cannot bind %s", addr) + + if hasExplicitConfig { + // User specified explicit address — don't guess, just fail + if fatal { + msg := fmt.Sprintf("DNS intercept: cannot listen on configured address %s", addr) + mainLog.Load().Error().Msg(msg) + failProvision(newProvisionResult(provisionCodeListenerAddrUnavail, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false + } + return updated, false + } + + // Fallback: try port 5354 (mDNSResponder likely holds *:53) + if tryListen("127.0.0.1", 5354) { + mainLog.Load().Info().Msg("DNS intercept: port 53 unavailable (likely mDNSResponder), using 127.0.0.1:5354") + lc.IP = "127.0.0.1" + lc.Port = 5354 + return true, true + } + + if fatal { + const msg = "DNS intercept: cannot bind 127.0.0.1:53 or 127.0.0.1:5354" + mainLog.Load().Error().Msg(msg) + failProvision(newProvisionResult(provisionCodeListenerBindFailed, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false + } + return updated, false +} + +func isExplicitInterceptListener(ip string, port int) bool { + if ip == "" || ip == "0.0.0.0" || port == 0 { + return false + } + // 127.0.0.1:53 is the default macOS DNS-intercept listener. It can appear + // in generated/custom Control D configs, but it should still be allowed to + // fall back to 127.0.0.1:5354 when mDNSResponder already owns port 53. + return !(ip == "127.0.0.1" && port == 53) +} + +// listenerInterceptMode resolves the mode that selects the listener binding +// strategy, with the same precedence as setDNS. An explicit "off" is final +// here: a fallback to the config value would select the intercept strategy from +// a stale persisted mode on the first start after a revert to standard mode. +// With neither set, the platform default applies, so a macOS start that +// intercepts by default also binds the way intercept mode needs. +func listenerInterceptMode(cfg *ctrld.Config) string { + mode, _ := resolveInterceptMode(interceptMode, cfg) + return mode +} + // tryUpdateListenerConfig tries updating listener config with a working one. // If fatal is true, and there's listen address conflicted, the function do // fatal error. -func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, notifyFunc func(), fatal bool) (updated, ok bool) { +func tryUpdateListenerConfig(cfg *ctrld.Config, notifyFunc func(), fatal bool) (updated, ok bool) { + // In intercept mode (macOS), pf redirects all port-53 traffic to ctrld's listener, + // so ctrld can safely listen on a non-standard port. Use a simple two-attempt flow: + // 1. If config has explicit non-default IP:port, use exactly that + // 2. Otherwise: try 127.0.0.1:53, then 127.0.0.1:5354, then fatal + // This bypasses the full cd-mode listener probing loop entirely. + // dnsIntercept bool is derived later in prog.run(), but we need to know + // the intercept mode here to select the right listener probing strategy. + im := listenerInterceptMode(cfg) + if (im == "dns" || im == "hard") && runtime.GOOS == "darwin" { + return tryUpdateListenerConfigIntercept(cfg, notifyFunc, fatal) + } ok = true lcc := make(map[string]*listenerConfigCheck) cdMode := cdUID != "" nextdnsMode := nextdns != "" - // For Windows server with local Dns server running, we can only try on random local IP. - hasLocalDnsServer := hasLocalDnsServerRunning() - notRouter := router.Name() == "" isDesktop := ctrld.IsDesktopPlatform() for n, listener := range cfg.Listener { lcc[n] = &listenerConfigCheck{} if listener.IP == "" { listener.IP = "0.0.0.0" - // Windows Server lies to us that we could listen on 0.0.0.0:53 - // even there's a process already done that, stick to local IP only. - // // For desktop clients, also stick the listener to the local IP only. // Listening on 0.0.0.0 would expose it to the entire local network, potentially // creating security vulnerabilities (such as DNS amplification or abusing). - if hasLocalDnsServer || isDesktop { + if isDesktop { listener.IP = "127.0.0.1" } lcc[n].IP = true @@ -1246,25 +1988,19 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } // In cd mode, we always try to pick an ip:port pair to work. // Same if nextdns resolver is used. - // - // Except on Windows Server with local Dns running, - // we could only listen on random local IP port 53. if cdMode || nextdnsMode { lcc[n].IP = true lcc[n].Port = true - if hasLocalDnsServer { - lcc[n].Port = false - } } updated = updated || lcc[n].IP || lcc[n].Port } il := mainLog.Load() - if infoLogger != nil { - il = infoLogger - } if isMobile() { // On Mobile, only use first listener, ignore others. + // This is needed because mobile platforms have limited resources and + // multiple listeners can cause conflicts with system DNS services and + // likely don't work anyway. firstLn := cfg.FirstListener() for k := range cfg.Listener { if cfg.Listener[k] != firstLn { @@ -1272,6 +2008,8 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } } if cdMode { + // Use mobile-specific listener settings for Control-D mode + // Mobile platforms require specific IP/port combinations to avoid permission issues. firstLn.IP = mobileListenerIp() firstLn.Port = mobileListenerPort() clear(lcc) @@ -1284,6 +2022,15 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti _ = closer.Close() } }() + // bindAttempts feeds the provisioning result detail. newProvisionResult + // caps it, so it grows freely here. + var bindAttempts []provisionBindAttempt + recordBindAttempt := func(addr, proto string, err error) { + if err != nil { + bindAttempts = append(bindAttempts, provisionBindAttempt{Addr: addr, Proto: proto, OSError: err.Error()}) + } + } + // tryListen attempts to listen on given udp and tcp address. // Created listeners will be kept in listeners slice above, and close // before function finished. @@ -1292,16 +2039,21 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti if udpLn != nil { closers = append(closers, udpLn) } + recordBindAttempt(addr, "udp", udpErr) tcpLn, tcpErr := net.Listen("tcp", addr) if tcpLn != nil { closers = append(closers, tcpLn) } + recordBindAttempt(addr, "tcp", tcpErr) return errors.Join(udpErr, tcpErr) } - logMsg := func(e *zerolog.Event, listenerNum int, format string, v ...any) { + listenerMsg := func(listenerNum int, format string, v ...any) string { + return fmt.Sprintf("listener.%d %s", listenerNum, fmt.Sprintf(format, v...)) + } + logMsg := func(e *ctrld.LogEvent, listenerNum int, format string, v ...any) { e.MsgFunc(func() string { - return fmt.Sprintf("listener.%d %s", listenerNum, fmt.Sprintf(format, v...)) + return listenerMsg(listenerNum, format, v...) }) } @@ -1334,27 +2086,20 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti // On firewalla, we don't need to check localhost, because the lo interface is excluded in dnsmasq // config, so we can always listen on localhost port 53, but no traffic could be routed there. - tryLocalhost := !isLoopback(listener.IP) && router.CanListenLocalhost() + tryLocalhost := !isLoopback(listener.IP) tryAllPort53 := true - tryOldIPPort5354 := true - tryPort5354 := true - if hasLocalDnsServer { + if isZeroIP && listener.Port == 53 { tryAllPort53 = false - tryOldIPPort5354 = false - tryPort5354 = false - } - // if not running on a router, we should not try to listen on any port other than 53 - // if we do, this will break the dns resolution for the system. - if notRouter { - tryOldIPPort5354 = false - tryPort5354 = false } + attempts := 0 maxAttempts := 10 for { if attempts == maxAttempts { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "could not find available listen ip and port") + logMsg(mainLog.Load().Error(), n, "could not find available listen ip and port") + msg := listenerMsg(n, "could not find available listen ip and port") + failProvision(newProvisionResult(provisionCodeListenerBindFailed, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } addr := net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)) err := tryListen(addr) @@ -1362,16 +2107,21 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti break } - logMsg(il.Info(), n, "error listening on address: %s, error: %v", addr, err) + logMsg(il.Debug().Err(err), n, "error listening on address: %s", addr) if !check.IP && !check.Port { if fatal { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "failed to listen: %v", err) + logMsg(mainLog.Load().Error(), n, "failed to listen: %v", err) + msg := listenerMsg(n, "failed to listen: %v", err) + failProvision(newProvisionResult(provisionCodeListenerAddrUnavail, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } ok = false break } + + // Try standard port 53 first for better compatibility + // This is the most common DNS port and has the highest chance of working if tryAllPort53 { tryAllPort53 = false if check.IP { @@ -1385,6 +2135,9 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } continue } + + // Try localhost as fallback for security and compatibility + // Localhost is often available even when other addresses are blocked if tryLocalhost { tryLocalhost = false if check.IP { @@ -1398,44 +2151,26 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } continue } - if tryOldIPPort5354 { - tryOldIPPort5354 = false - if check.IP { - listener.IP = oldIP - } - if check.Port { - listener.Port = 5354 - } - logMsg(il.Info(), n, "could not listen on address: %s, trying current ip with port 5354", addr) - continue - } - if tryPort5354 { - tryPort5354 = false - if check.IP { - listener.IP = "0.0.0.0" - } - if check.Port { - listener.Port = 5354 - } - logMsg(il.Info(), n, "could not listen on address: %s, trying 0.0.0.0:5354", addr) - continue - } + + // Try random IP/port combinations as last resort + // This ensures the service can start even in constrained environments if check.IP && !isZeroIP { // for "0.0.0.0" or "::", we only need to try new port. listener.IP = randomLocalIP() } else { listener.IP = oldIP } - // if we are not running on a router, we should not try to listen on any port other than 53 - // if we do, this will break the dns resolution for the system. - if check.Port && !notRouter { + if check.Port { listener.Port = randomPort() } else { listener.Port = oldPort } if listener.IP == oldIP && listener.Port == oldPort { if fatal { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "could not listen on %s: %v", net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)), err) + triedAddr := net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port)) + logMsg(mainLog.Load().Error(), n, "could not listen on %s: %v", triedAddr, err) + msg := listenerMsg(n, "could not listen on %s: %v", triedAddr, err) + failProvision(newProvisionResult(provisionCodeListenerBindFailed, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } ok = false break @@ -1449,6 +2184,7 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } // Specific case for systemd-resolved. + // systemd-resolved has specific requirements for DNS forwarding that we must handle if useSystemdResolved { if listener := cfg.FirstListener(); listener != nil && listener.Port == 53 { n := listeners[0] @@ -1473,8 +2209,10 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti } } if !found { - notifyFunc() - logMsg(mainLog.Load().Fatal(), n, "could not use %q as DNS nameserver with systemd resolved", listener.IP) + logMsg(mainLog.Load().Error(), n, "could not use %q as DNS nameserver with systemd resolved", listener.IP) + msg := listenerMsg(n, "could not use %q as DNS nameserver with systemd resolved", listener.IP) + failProvision(newProvisionResult(provisionCodeListenerAddrUnavail, msg, bindAttempts, provisionSecrets()...), notifyFunc) + return updated, false } } } @@ -1482,6 +2220,7 @@ func tryUpdateListenerConfig(cfg *ctrld.Config, infoLogger *zerolog.Logger, noti return } +// dirWritable checks if a directory is writable func dirWritable(dir string) (bool, error) { f, err := os.CreateTemp(dir, "") if err != nil { @@ -1491,6 +2230,7 @@ func dirWritable(dir string) (bool, error) { return true, f.Close() } +// osVersion returns the operating system version func osVersion() string { oi := osinfo.New() if runtime.GOOS == "freebsd" { @@ -1501,6 +2241,99 @@ func osVersion() string { return oi.String() } +// provisionTokenMinLen, provisionTokenMaxLen, and provisionTokenExpectedPrefix +// describe the shape of a --cd-org value the input stage checks before any +// network call. +const ( + provisionTokenMinLen = 6 + provisionTokenMaxLen = 64 + provisionTokenExpectedPrefix = "org-v1-" +) + +// provisionTokenShapeValid reports whether token looks like something the API +// could parse: a length between provisionTokenMinLen and provisionTokenMaxLen, +// with no whitespace or control characters. It does not require the +// "org-v1-" prefix - legacy provisioning codes predate that convention and +// are handled with a warning, not a failure. +func provisionTokenShapeValid(token string) bool { + if len(token) < provisionTokenMinLen || len(token) > provisionTokenMaxLen { + return false + } + for _, r := range token { + if unicode.IsSpace(r) || unicode.IsControl(r) { + return false + } + } + return true +} + +// validateProvisionTokenShape classifies an obviously malformed --cd-org +// value (PROVISION_TOKEN_MALFORMED) before any network call, so a typo or a +// pasted URL fails fast instead of going through a doomed API round trip. A +// token missing the "org-v1-" prefix only gets a logged warning: legacy +// provisioning codes may lack it and still work. +func validateProvisionTokenShape() bool { + if !provisionTokenShapeValid(cdOrg) { + msg := fmt.Sprintf("--cd-org provisioning code is malformed: must be %d-%d characters with no whitespace or control characters", provisionTokenMinLen, provisionTokenMaxLen) + failProvision(newProvisionResult(provisionCodeProvisionTokenMalformed, msg, nil, provisionSecrets()...), nil) + return false + } + if !strings.HasPrefix(cdOrg, provisionTokenExpectedPrefix) { + mainLog.Load().Warn().Msg("--cd-org provisioning code does not have the expected org-v1- prefix; continuing, since legacy codes may lack it") + } + return true +} + +// validateCustomHostnameFlag classifies an invalid --custom-hostname value +// (CUSTOM_HOSTNAME_INVALID) before any network call. ctrld's accept/reject +// rule (validHostname) is unchanged - this only names the field, the +// offending character(s), and the allowed format instead of a bare fatal +// exit. ControlD's device-name formatting folds or strips some characters +// ctrld still accepts (dot, space, plus), so a hostname using one of those +// gets a one-line notice instead: the registered device name may differ from +// what was requested. +func validateCustomHostnameFlag() bool { + if customHostname == "" { + return true + } + if !validHostname(customHostname) { + failProvision(newProvisionResult(provisionCodeCustomHostnameInvalid, customHostnameFailureMessage(customHostname), nil, provisionSecrets()...), nil) + return false + } + if hostnameMayBeFoldedByServer(customHostname) { + mainLog.Load().Notice().Msgf("device name %q contains characters ControlD may fold or strip when it registers the device; the registered name may differ", customHostname) + } + return true +} + +// validateInterceptModeFlag classifies an invalid --intercept-mode value +// (INTERCEPT_MODE_INVALID) on the provisioning boundary, so a typo fails with +// a stable code instead of a bare fatal exit an installer wrapper cannot tell +// apart from any other crash. An empty mode means the flag was not set. +func validateInterceptModeFlag(mode string) bool { + if mode == "" || validInterceptMode(mode) { + return true + } + msg := fmt.Sprintf("--intercept-mode %q is not valid: must be 'off', 'dns', or 'hard'", mode) + failProvision(newProvisionResult(provisionCodeInterceptModeInvalid, msg, nil, provisionSecrets()...), nil) + return false +} + +// validateFirewallModeFlag classifies an invalid --firewall-mode value on the +// provisioning boundary. Unlike --intercept-mode, no dedicated code exists +// for this flag, so a bad value falls back to UNCLASSIFIED rather than a bare +// fatal exit. changed must be the flag's Cobra Changed() state: the default +// value ("off") is not itself invalid, so an unset flag always proceeds. +// notify unblocks a waiting "ctrld start"; pass nil when there is none. +func validateFirewallModeFlag(changed bool, mode string, notify func()) bool { + if !changed || validFirewallMode(mode) { + return true + } + msg := fmt.Sprintf("--firewall-mode %q is not valid: must be 'off' or 'on'", mode) + failProvisionUnclassified(msg, notify) + return false +} + // cdUIDFromProvToken fetch UID from ControlD API using provision token. func cdUIDFromProvToken() string { // --cd flag supersedes --cd-org, ignore it if both are supplied. @@ -1511,19 +2344,36 @@ func cdUIDFromProvToken() string { if cdOrg == "" { return "" } - // Validate custom hostname if provided. - if customHostname != "" && !validHostname(customHostname) { - mainLog.Load().Fatal().Msgf("invalid custom hostname: %q", customHostname) + if !validateProvisionTokenShape() { + return "" + } + if !validateCustomHostnameFlag() { + return "" + } + loggerCtx := ctrld.LoggerCtx(context.Background(), mainLog.Load()) + req := &controld.UtilityOrgRequest{ + ProvToken: cdOrg, + Hostname: customHostname, + Metadata: ctrld.SystemMetadata(loggerCtx), } - req := &controld.UtilityOrgRequest{ProvToken: cdOrg, Hostname: customHostname} // Process provision token if provided. - resolverConfig, err := controld.FetchResolverUID(req, rootCmd.Version, cdDev) + resolverConfig, err := fetchResolverUIDFn(loggerCtx, req, appVersion, cdDev) if err != nil { - mainLog.Load().Fatal().Err(err).Msgf("failed to fetch resolver uid with provision token: %s", cdOrg) + // The token exchange is the first API call of an org/MDM install, so + // its failure must carry a code like every other bootstrap failure. + code, _ := apiFailureCode(err) + mainLog.Load().Error().Msgf("Failed to fetch resolver uid with provision token: %s: %s", + redactToken(cdOrg), redactSecrets(err.Error(), provisionSecrets()...)) + failProvision(newProvisionResult(code, provisionTokenFailureMessage(code, err), nil, provisionSecrets()...), nil) + return "" } return resolverConfig.UID } +// fetchResolverUIDFn is a var so tests can drive token-exchange failures +// without reaching the network. +var fetchResolverUIDFn = controld.FetchResolverUID + // removeOrgFlagsFromArgs removes organization flags from command line arguments. // The flags are: // @@ -1564,7 +2414,37 @@ func newSocketControlClient(ctx context.Context, s service.Service, dir string) // newSocketControlClientWithTimeout returns new control client after control server was started. // The timeoutDuration controls how long to wait for the server. +// +// Any answer to /started counts, including the 408 of a service whose start +// never finished: callers such as the deactivation pin check need to reach a +// running service whether or not it is ready. A caller that needs it ready uses +// newReadySocketControlClient. func newSocketControlClientWithTimeout(ctx context.Context, s service.Service, dir string, timeoutDuration time.Duration) *controlClient { + return dialSocketControlServer(ctx, s, dir, timeoutDuration, false) +} + +// newReadySocketControlClient is newSocketControlClient for a caller that needs +// the service ready, not just reachable: only a 200 from /started counts. +// +// A service whose start never finishes still runs its control server, and +// answers /started with 408 once its onStarted hooks have not completed in +// time. `ctrld upgrade` must read that as "the new binary did not become +// ready" and roll back, rather than report the upgrade successful. +func newReadySocketControlClient(ctx context.Context, s service.Service, dir string) *controlClient { + return newReadySocketControlClientWithTimeout(ctx, s, dir, dialSocketControlServerTimeout) +} + +// newReadySocketControlClientWithTimeout is newReadySocketControlClient with +// an explicit deadline. +func newReadySocketControlClientWithTimeout(ctx context.Context, s service.Service, dir string, timeoutDuration time.Duration) *controlClient { + return dialSocketControlServer(ctx, s, dir, timeoutDuration, true) +} + +// dialSocketControlServer polls /started until the control server answers, or +// with requireReady until it answers 200, and returns the client. It returns +// nil if the service is not running, or the answer does not come within +// timeoutDuration. +func dialSocketControlServer(ctx context.Context, s service.Service, dir string, timeoutDuration time.Duration, requireReady bool) *controlClient { // Return early if service is not running. if status, err := s.Status(); err != nil || status != service.StatusRunning { return nil @@ -1579,10 +2459,15 @@ func newSocketControlClientWithTimeout(ctx context.Context, s service.Service, d // The socket control server may not start yet, so attempt to ping // it until we got a response. for { - _, err := cc.post(startedPath, nil) + resp, err := cc.post(startedPath, nil) if err == nil { - // Server was started, stop pinging. - break + code := resp.StatusCode + resp.Body.Close() + if !requireReady || code == http.StatusOK { + // Server was started, stop pinging. + break + } + err = fmt.Errorf("control server reported the service not ready: HTTP %d", code) } // The socket control server is not ready yet, backoff for waiting it to be ready. bo.BackOff(ctx, err) @@ -1609,8 +2494,9 @@ func newSocketControlClientMobile(dir string, stopCh chan struct{}) *controlClie case <-stopCh: return nil default: - _, err := cc.post("/", nil) + resp, err := cc.post("/", nil) if err == nil { + resp.Body.Close() return cc } else { bo.BackOff(ctx, err) @@ -1620,32 +2506,55 @@ func newSocketControlClientMobile(dir string, stopCh chan struct{}) *controlClie } // checkStrFlagEmpty validates if a string flag was set to an empty string. -// If yes, emitting a fatal error message. -func checkStrFlagEmpty(cmd *cobra.Command, flagName string) { +// An explicit empty --cd-org is a malformed provisioning token, classified on +// the provisioning boundary; every other flag keeps the bare fatal, since +// this helper is shared with flags outside that boundary. Returns false when +// the caller must stop: provisionExit is stubbed out under test, so nothing +// else stops execution from falling through. +func checkStrFlagEmpty(cmd *cobra.Command, flagName string) bool { fl := cmd.Flags().Lookup(flagName) - if !fl.Changed || fl.Value.Type() != "string" { - return + if !fl.Changed || fl.Value.Type() != "string" || fl.Value.String() != "" { + return true } - if fl.Value.String() == "" { - mainLog.Load().Fatal().Msgf(`flag "--%s" value must be non-empty`, fl.Name) + if flagName == cdOrgFlagName { + msg := fmt.Sprintf("--%s provisioning code is malformed: value must be non-empty", cdOrgFlagName) + failProvision(newProvisionResult(provisionCodeProvisionTokenMalformed, msg, nil, provisionSecrets()...), nil) + return false } + mainLog.Load().Fatal().Msgf(`flag "--%s" value must be non-empty`, fl.Name) + return false } -func validateCdUpstreamProtocol() { +// validateCdUpstreamProtocol validates the Control D upstream protocol, +// classified on the provisioning boundary since it only runs once --cd is +// set. notify unblocks a waiting "ctrld start" on the daemon side; the parent +// process, which has no one waiting, passes nil. Returns false when the +// caller must stop. +func validateCdUpstreamProtocol(notify func()) bool { if cdUID == "" { - return + return true } switch cdUpstreamProto { case ctrld.ResolverTypeDOH, ctrld.ResolverTypeDOH3: + return true default: - mainLog.Load().Fatal().Msg(`flag "--protocol" must be "doh" or "doh3"`) + msg := fmt.Sprintf("--proto %q is not valid: must be 'doh' or 'doh3'", cdUpstreamProto) + failProvision(newProvisionResult(provisionCodeInvalidFlagCombination, msg, nil, provisionSecrets()...), notify) + return false } } -func validateCdAndNextDNSFlags() { +// validateCdAndNextDNSFlags validates that Control D and NextDNS flags are +// not used together, classified on the provisioning boundary since it only +// fires once --cd or --cd-org is set. Returns false when the caller must +// stop. +func validateCdAndNextDNSFlags() bool { if (cdUID != "" || cdOrg != "") && nextdns != "" { - mainLog.Load().Fatal().Msgf("--%s/--%s could not be used with --%s", cdUidFlagName, cdOrgFlagName, nextdnsFlagName) + msg := fmt.Sprintf("--%s/--%s cannot be used together with --%s", cdUidFlagName, cdOrgFlagName, nextdnsFlagName) + failProvision(newProvisionResult(provisionCodeInvalidFlagCombination, msg, nil, provisionSecrets()...), nil) + return false } + return true } // removeNextDNSFromArgs removes the --nextdns from command line arguments. @@ -1682,6 +2591,7 @@ func doGenerateNextDNSConfig(uid string) error { return writeConfigFile(&cfg) } +// noticeWritingControlDConfig logs on notice level that a Control D config is being written func noticeWritingControlDConfig() error { if cdUID != "" { mainLog.Load().Notice().Msgf("Generating controld config: %s", defaultConfigFile) @@ -1698,12 +2608,21 @@ var errInvalidDeactivationPin = errors.New("deactivation pin is invalid") // errRequiredDeactivationPin indicates that the deactivation pin is required but not provided by users. var errRequiredDeactivationPin = errors.New("deactivation pin is required to stop or uninstall the service") +// errTooManyDeactivationPin represents an error indicating excessive deactivation PIN request attempts. +var errTooManyDeactivationPin = errors.New("too many request attempts") + +// errUnverifiableDeactivationPin reports that this run never had an answer from +// the API about the PIN, so it can neither check one nor establish that none is +// required. Refusing is the only safe answer: the alternative hands a +// PIN-protected device to whoever asks for the duration of an API outage. +var errUnverifiableDeactivationPin = errors.New("cannot verify the deactivation pin while the Control D API is unavailable, try again once it returns") + // checkDeactivationPin validates if the deactivation pin matches one in ControlD config. func checkDeactivationPin(s service.Service, stopCh chan struct{}) error { mainLog.Load().Debug().Msg("Checking deactivation pin") dir, err := socketDir() if err != nil { - mainLog.Load().Err(err).Msg("could not check deactivation pin") + mainLog.Load().Err(err).Msg("Could not check deactivation pin") return err } mainLog.Load().Debug().Msg("Creating control client") @@ -1722,14 +2641,21 @@ func checkDeactivationPin(s service.Service, stopCh chan struct{}) error { resp, err := cc.post(deactivationPath, bytes.NewReader(data)) mainLog.Load().Debug().Msg("Posting deactivation request done") if resp != nil { + defer resp.Body.Close() switch resp.StatusCode { case http.StatusBadRequest: mainLog.Load().Error().Msg(errRequiredDeactivationPin.Error()) return errRequiredDeactivationPin // pin is required + case http.StatusTooManyRequests: + mainLog.Load().Error().Msg(errTooManyDeactivationPin.Error()) + return errTooManyDeactivationPin case http.StatusOK: return nil // valid pin case http.StatusNotFound: return nil // the server is running older version of ctrld + case http.StatusServiceUnavailable: + mainLog.Load().Error().Msg(errUnverifiableDeactivationPin.Error()) + return errUnverifiableDeactivationPin } } mainLog.Load().Error().Err(err).Msg(errInvalidDeactivationPin.Error()) @@ -1738,7 +2664,10 @@ func checkDeactivationPin(s service.Service, stopCh chan struct{}) error { // isCheckDeactivationPinErr reports whether there is an error during check deactivation pin process. func isCheckDeactivationPinErr(err error) bool { - return errors.Is(err, errInvalidDeactivationPin) || errors.Is(err, errRequiredDeactivationPin) + return errors.Is(err, errInvalidDeactivationPin) || + errors.Is(err, errRequiredDeactivationPin) || + errors.Is(err, errTooManyDeactivationPin) || + errors.Is(err, errUnverifiableDeactivationPin) } // ensureUninstall ensures that s.Uninstall will remove ctrld service from system completely. @@ -1762,29 +2691,13 @@ func exchangeContextWithTimeout(c *dns.Client, timeout time.Duration, msg *dns.M return c.ExchangeContext(ctx, msg, addr) } -// absHomeDir returns the absolute path to given filename using home directory as root dir. -func absHomeDir(filename string) string { - if homedir != "" { - return filepath.Join(homedir, filename) - } - dir, err := userHomeDir() - if err != nil { - return filename - } - return filepath.Join(dir, filename) -} - -// runInCdMode reports whether ctrld service is running in cd mode. -func runInCdMode() bool { - return curCdUID() != "" -} - // curCdUID returns the current ControlD UID used by running ctrld process. func curCdUID() string { - if s, _ := newService(&prog{}, svcConfig); s != nil { + svcCmd := NewServiceCommand() + if s, _, _ := svcCmd.initializeServiceManager(); s != nil { // Configure Windows service failure actions if err := ConfigureWindowsServiceFailureActions(ctrldServiceName); err != nil { - mainLog.Load().Debug().Err(err).Msgf("failed to configure Windows service %s failure actions", ctrldServiceName) + mainLog.Load().Debug().Err(err).Msgf("Failed to configure windows service %s failure actions", ctrldServiceName) } if dir, _ := socketDir(); dir != "" { cc := newSocketControlClient(context.TODO(), s, dir) @@ -1817,16 +2730,20 @@ func goArm() string { return "5" } -// upgradeUrl returns the url for downloading new ctrld binary. +// upgradeUrl builds the download URL for this platform's binary. +// +// The path carries no version segment: the v1 line publishes as "ctrld" and this +// one as "ctrld-client", so the file name distinguishes them and they can share +// one path. func upgradeUrl(baseUrl string) string { - dlPath := fmt.Sprintf("%s-%s/ctrld", runtime.GOOS, runtime.GOARCH) + dlPath := fmt.Sprintf("%s-%s/%s", runtime.GOOS, runtime.GOARCH, cliName) // Use arm version set during build time, v5 binary can be run on higher arm version system. if armVersion := goArm(); armVersion != "" { - dlPath = fmt.Sprintf("%s-%sv%s/ctrld", runtime.GOOS, runtime.GOARCH, armVersion) + dlPath = fmt.Sprintf("%s-%sv%s/%s", runtime.GOOS, runtime.GOARCH, armVersion, cliName) } // linux/amd64 has nocgo version, to support systems that missing some libc (like openwrt). if !cgoEnabled && runtime.GOOS == "linux" && runtime.GOARCH == "amd64" { - dlPath = fmt.Sprintf("%s-%s-nocgo/ctrld", runtime.GOOS, runtime.GOARCH) + dlPath = fmt.Sprintf("%s-%s-nocgo/%s", runtime.GOOS, runtime.GOARCH, cliName) } dlUrl := fmt.Sprintf("%s/%s", baseUrl, dlPath) if runtime.GOOS == "windows" { @@ -1854,18 +2771,44 @@ func runningIface(s service.Service) *ifaceResponse { return nil } +// apiRejectionMessage returns the result-file message for a permanent API +// rejection: the HTTP status for a device-invalid or generic rejection, or +// the raw error for anything else (network trouble, an unreachable API). +func apiRejectionMessage(err error) string { + var uer *controld.ErrorResponse + if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { + return apiRejectionSummary(uer.StatusCode) + } + if rejection, ok := permanentAPIRejection(err); ok { + return apiRejectionSummary(rejection.StatusCode) + } + return fmt.Sprintf("failed to fetch resolver config: %v", err) +} + // doValidateCdRemoteConfig fetches and validates custom config for cdUID. +// fatal distinguishes the two callers: the direct "--cd " install path +// passes true and classifies a fetch failure on the provisioning boundary; +// the restart path passes false and gets the error back to decide for +// itself, with no process exit. func doValidateCdRemoteConfig(cdUID string, fatal bool) error { - rc, err := controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev) + loggerCtx := ctrld.LoggerCtx(context.Background(), mainLog.Load()) + // Install and restart validation use runtime metadata. The daemon start + // that follows sends the full snapshot. + req := &controld.ResolverConfigRequest{ + RawUID: cdUID, + Version: appVersion, + Metadata: ctrld.SystemMetadataRuntime(loggerCtx), + } + rc, err := fetchResolverConfig(loggerCtx, req, cdDev) if err != nil { - logger := mainLog.Load().Fatal() - if !fatal { - logger = mainLog.Load().Warn() - } - logger.Err(err).Err(err).Msgf("failed to fetch resolver uid: %s", cdUID) if !fatal { + mainLog.Load().Warn().Err(err).Msgf("Failed to fetch resolver config for %s", redactToken(cdUID)) return err } + code, _ := apiFailureCode(err) + mainLog.Load().Error().Err(err).Msgf("Failed to fetch resolver config for %s", redactToken(cdUID)) + failProvision(newProvisionResult(code, apiRejectionMessage(err), nil, provisionSecrets()...), nil) + return err } // return earlier if there's no custom config. @@ -1884,47 +2827,78 @@ func doValidateCdRemoteConfig(cdUID string, fatal bool) error { } else { if errors.As(cfgErr, &viper.ConfigParseError{}) { if configStr, _ := base64.StdEncoding.DecodeString(rc.Ctrld.CustomConfig); len(configStr) > 0 { - tmpDir := os.TempDir() - tmpConfFile := filepath.Join(tmpDir, "ctrld.toml") errorLogged := false - // Write remote config to a temporary file to get details error. - if we := os.WriteFile(tmpConfFile, configStr, 0600); we == nil { + // Write remote config to a uniquely named temporary file to get detailed error. + if tmpFile, tmpErr := os.CreateTemp("", "ctrld-*.toml"); tmpErr == nil { + tmpConfFile := tmpFile.Name() + if _, err := tmpFile.Write(configStr); err != nil { + mainLog.Load().Error().Err(err).Msg("failed to write temporary config file") + } + if err := tmpFile.Close(); err != nil { + mainLog.Load().Error().Err(err).Msg("failed to save temporary config file") + + } if de := decoderErrorFromTomlFile(tmpConfFile); de != nil { row, col := de.Position() - mainLog.Load().Error().Msgf("failed to parse custom config at line: %d, column: %d, error: %s", row, col, de.Error()) + mainLog.Load().Error().Msgf("Failed to parse custom config at line: %d, column: %d, error: %s", row, col, de.Error()) errorLogged = true } - _ = os.Remove(tmpConfFile) + if err := os.Remove(tmpConfFile); err != nil { + mainLog.Load().Error().Err(err).Msg("failed to remove temporary config file") + } } // If we could not log details error, emit what we have already got. if !errorLogged { - mainLog.Load().Error().Msgf("failed to parse custom config: %v", cfgErr) + mainLog.Load().Error().Msgf("Failed to parse custom config: %v", cfgErr) } } } else { - mainLog.Load().Error().Msgf("failed to unmarshal custom config: %v", err) + mainLog.Load().Error().Msgf("Failed to unmarshal custom config: %v", err) } } if cfgErr != nil { - mainLog.Load().Warn().Msg("disregarding invalid custom config") + mainLog.Load().Warn().Msg("Disregarding invalid custom config") } v = oldV return nil } +// ensureRunningIfaceForInvalidUninstall populates p.runningIface before the +// invalid-device self-uninstall resets DNS. This path can run early during +// service startup (e.g. right after a reboot) before the running interface is +// otherwise known. resetDNS, via resetDNSForRunningIface, silently skips DNS +// restoration when p.runningIface is empty, which would leave the OS pointed at +// ctrld's local listener after the service is removed. See issue-556. +func ensureRunningIfaceForInvalidUninstall(p *prog, s service.Service) { + if iface == "" { + iface = autoIface + } + p.preRun() + if ir := runningIface(s); ir != nil { + p.runningIface = ir.Name + p.requiredMultiNICsConfig = ir.All + } +} + // uninstallInvalidCdUID performs self-uninstallation because the ControlD device does not exist. -func uninstallInvalidCdUID(p *prog, logger zerolog.Logger, doStop bool) bool { - s, err := newService(p, svcConfig) +func uninstallInvalidCdUID(p *prog, logger *ctrld.Logger, doStop bool) bool { + svcCmd := NewServiceCommand() + s, _, err := svcCmd.initializeServiceManager() if err != nil { - logger.Warn().Err(err).Msg("failed to create new service") + logger.Warn().Err(err).Msg("Failed to create new service") return false } + ensureRunningIfaceForInvalidUninstall(p, s) // restore static DNS settings or DHCP p.resetDNS(false, true) + // The invalid-device path may run early during service startup before runningIface + // is known. Restore every saved static DNS file so uninstalling does not leave the + // OS pointed at ctrld's local listener after the service is removed. + restoreSavedStaticDNS("", true) tasks := []task{{s.Uninstall, true, "Uninstall"}} if doTasks(tasks) { - logger.Info().Msg("uninstalled service") + logger.Info().Msg("Uninstalled service") if doStop { _ = s.Stop() } @@ -1932,3 +2906,12 @@ func uninstallInvalidCdUID(p *prog, logger zerolog.Logger, doStop bool) bool { } return false } + +// redactToken returns the first 4 characters of a token followed by ***, +// or just *** if the token is 4 characters or shorter. +func redactToken(s string) string { + if len(s) <= 4 { + return "***" + } + return s[:4] + "***" +} diff --git a/cmd/cli/cli_intercept_listener_test.go b/cmd/cli/cli_intercept_listener_test.go new file mode 100644 index 00000000..6787d1e3 --- /dev/null +++ b/cmd/cli/cli_intercept_listener_test.go @@ -0,0 +1,116 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestIsExplicitInterceptListener(t *testing.T) { + tests := []struct { + name string + ip string + port int + want bool + }{ + {name: "empty", ip: "", port: 0, want: false}, + {name: "wildcard", ip: "0.0.0.0", port: 53, want: false}, + {name: "zero port", ip: "127.0.0.1", port: 0, want: false}, + {name: "default intercept listener", ip: "127.0.0.1", port: 53, want: false}, + {name: "fallback port explicit", ip: "127.0.0.1", port: 5354, want: true}, + {name: "custom loopback explicit", ip: "127.0.0.2", port: 53, want: true}, + {name: "custom address explicit", ip: "192.0.2.10", port: 53, want: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isExplicitInterceptListener(tt.ip, tt.port); got != tt.want { + t.Fatalf("isExplicitInterceptListener(%q, %d) = %v, want %v", tt.ip, tt.port, got, tt.want) + } + }) + } +} + +// TestPreserveBoundListeners is a regression test for #551: on reload, the on-disk +// generated config still declares 127.0.0.1:53, but the running listener has fallen back +// to 127.0.0.1:5354. preserveBoundListeners must keep the in-memory config on the actual +// bound port so pf rdr rules and probes do not target the dead default port. +func TestPreserveBoundListeners(t *testing.T) { + // cur = actual running listener (fell back to 5354); newCfg = freshly read from disk (53). + cur := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}} + newListeners := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + + preserveBoundListeners(newListeners, cur) + + if got := newListeners["0"].Port; got != 5354 { + t.Errorf("listener port after reload = %d, want 5354 (actual bound port)", got) + } + if got := newListeners["0"].IP; got != "127.0.0.1" { + t.Errorf("listener IP after reload = %q, want 127.0.0.1", got) + } +} + +// TestPreserveBoundListeners_NoChange verifies that when the on-disk config matches the +// running listener, the config is left untouched (a legitimate reload with the same port). +func TestPreserveBoundListeners_NoChange(t *testing.T) { + cur := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}} + newListeners := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}} + + preserveBoundListeners(newListeners, cur) + + if got := newListeners["0"].Port; got != 5354 { + t.Errorf("listener port = %d, want 5354", got) + } +} + +// TestPreserveBoundListeners_MissingCurrent verifies that a listener present on disk but not +// in the current running set (e.g. newly added) is left as configured. +func TestPreserveBoundListeners_MissingCurrent(t *testing.T) { + cur := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}} + newListeners := map[string]*ctrld.ListenerConfig{ + "0": {IP: "127.0.0.1", Port: 53}, + "1": {IP: "127.0.0.1", Port: 5355}, + } + + preserveBoundListeners(newListeners, cur) + + if got := newListeners["0"].Port; got != 5354 { + t.Errorf("listener 0 port = %d, want 5354 (preserved)", got) + } + if got := newListeners["1"].Port; got != 5355 { + t.Errorf("listener 1 port = %d, want 5355 (unchanged, no current binding)", got) + } +} + +// TestPreserveBoundListeners_ExplicitChangeNotMasked verifies that an explicit, non-default +// listener in the reloaded config is applied rather than reverted to the old bound listener. +// Reverting an explicit change would make the control-server reload comparison return 200 +// instead of 201, silently dropping the new listener. Regression guard for #551 review. +func TestPreserveBoundListeners_ExplicitChangeNotMasked(t *testing.T) { + // Running listener fell back to 5354; user reloads with an explicit new listener. + cur := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}} + newListeners := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.2", Port: 5399}} + + preserveBoundListeners(newListeners, cur) + + if got := newListeners["0"].IP; got != "127.0.0.2" { + t.Errorf("explicit listener IP = %q, want 127.0.0.2 (not reverted)", got) + } + if got := newListeners["0"].Port; got != 5399 { + t.Errorf("explicit listener port = %d, want 5399 (not reverted)", got) + } +} + +// TestPreserveBoundListeners_ExplicitDefaultPreserved verifies that the default +// 127.0.0.1:53 listener remains fallback-eligible: when it diverges from the running +// fallback port it is still preserved (isExplicitInterceptListener treats :53 as non-explicit). +func TestPreserveBoundListeners_ExplicitDefaultPreserved(t *testing.T) { + cur := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}} + newListeners := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + + preserveBoundListeners(newListeners, cur) + + if got := newListeners["0"].Port; got != 5354 { + t.Errorf("default listener port = %d, want 5354 (preserved fallback)", got) + } +} diff --git a/cmd/cli/cli_preflight_test.go b/cmd/cli/cli_preflight_test.go new file mode 100644 index 00000000..328585d7 --- /dev/null +++ b/cmd/cli/cli_preflight_test.go @@ -0,0 +1,442 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func TestContextFromStopCh(t *testing.T) { + t.Run("cancels when stopCh closes", func(t *testing.T) { + stopCh := make(chan struct{}) + ctx, cancel := contextFromStopCh(stopCh) + defer cancel() + + if ctx.Err() != nil { + t.Fatalf("context cancelled before the stop request: %v", ctx.Err()) + } + close(stopCh) + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("context was not cancelled after stopCh closed") + } + if !errors.Is(ctx.Err(), context.Canceled) { + t.Errorf("ctx.Err() = %v, want %v", ctx.Err(), context.Canceled) + } + }) + + t.Run("cancel releases the watcher", func(t *testing.T) { + // stopCh is never closed: cancel() must still end the goroutine watching it. + ctx, cancel := contextFromStopCh(make(chan struct{})) + cancel() + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("context was not cancelled by cancel()") + } + }) + + t.Run("nil stopCh is usable", func(t *testing.T) { + // Mobile callers have no stop channel; preflight must still run. + ctx, cancel := contextFromStopCh(nil) + defer cancel() + if ctx.Err() != nil { + t.Fatalf("context cancelled immediately: %v", ctx.Err()) + } + }) +} + +func TestManagedConfigFetchMetadataScope(t *testing.T) { + oldFetch, oldStartup := fetchResolverConfig, startupMetadata + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig, startupMetadata = oldFetch, oldStartup + cdUID = oldUID + }) + cdUID = "testuid" + startupMetadata = func(context.Context) map[string]string { + return map[string]string{"username": "session-user"} + } + + requests := make(chan *controld.ResolverConfigRequest, 2) + fetchResolverConfig = func(_ context.Context, req *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + requests <- req + return nil, errors.New("stop after capturing metadata") + } + + _, _ = processCDFlags(context.Background(), &ctrld.Config{}) + startup := <-requests + if got := startup.Metadata["username"]; got != "session-user" { + t.Errorf("managed startup username = %q, want the startup metadata source", got) + } + if !startup.IncludeHostnameHints { + t.Error("managed startup did not request hostname hints") + } + + p := &prog{stopCh: make(chan struct{})} + _, _ = p.fetchCDConfigBoundedByLifetime(&ctrld.Config{}) + runtime := <-requests + if _, ok := runtime.Metadata["username"]; ok { + t.Error("runtime reload metadata included username") + } + if runtime.IncludeHostnameHints { + t.Error("runtime reload requested hostname hints") + } +} + +// retryableNetworkErr is the shape processCDFlags treats as "retry with bootstrap +// DNS": a url.Error wrapping a network failure. +func retryableNetworkErr() error { + return &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, + } +} + +func TestProcessCDFlagsStopsWhenCancelled(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + var calls atomic.Int64 + fetchResolverConfig = func(ctx context.Context, req *controld.ResolverConfigRequest, dev bool) (*controld.ResolverConfig, error) { + calls.Add(1) + return nil, retryableNetworkErr() + } + + // A stop request arriving while the API is unreachable. Before this was + // cancellable, the retry loop kept running after the service reported itself + // stopped, which is what kept the incident's process alive and enforcing. + stopCh := make(chan struct{}) + ctx, cancel := contextFromStopCh(stopCh) + defer cancel() + + done := make(chan error, 1) + go func() { + cfg := ctrld.Config{} + _, err := processCDFlags(ctx, &cfg) + done <- err + }() + + // Let it fail at least once and settle into backoff before stopping. + deadline := time.After(10 * time.Second) + for calls.Load() == 0 { + select { + case <-deadline: + t.Fatal("resolver config was never fetched") + case err := <-done: + t.Fatalf("processCDFlags returned before any fetch: %v", err) + default: + time.Sleep(5 * time.Millisecond) + } + } + close(stopCh) + + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Errorf("processCDFlags err = %v, want it to report %v", err, context.Canceled) + } + case <-time.After(30 * time.Second): + t.Fatal("processCDFlags did not return after the stop request") + } +} + +func TestProcessCDFlagsReturnsImmediatelyWhenAlreadyCancelled(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + var calls atomic.Int64 + fetchResolverConfig = func(ctx context.Context, req *controld.ResolverConfigRequest, dev bool) (*controld.ResolverConfig, error) { + calls.Add(1) + return nil, retryableNetworkErr() + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + cfg := ctrld.Config{} + _, err := processCDFlags(ctx, &cfg) + if !errors.Is(err, context.Canceled) { + t.Errorf("processCDFlags err = %v, want %v", err, context.Canceled) + } + // One attempt is made before the loop notices; it must not retry past that. + if got := calls.Load(); got > 1 { + t.Errorf("fetched %d times with a cancelled context, want at most 1", got) + } +} + +// TestRunAPIPreflightClassification is the regression guard for classifying a preflight +// failure as an operator stop. +// +// runAPIPreflight cancels the context it derived from stopCh. Sampling the stop state +// from that context afterwards reports "stopped" unconditionally, because +// context.CancelFunc sets ctx.Err() whether or not anyone asked to stop. run() then +// takes the stop branch for every failure, which skips self-uninstalling a deleted +// device, skips the mobile exit callback, and tells the service manager a failed start +// was a clean exit. +func TestRunAPIPreflightClassification(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + // A deleted ControlD device: non-retryable, so preflight returns promptly. + deletedDevice := func() error { + e := &controld.ErrorResponse{} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = "device does not exist" + return e + } + + openCh := make(chan struct{}) + closedCh := make(chan struct{}) + close(closedCh) + + tests := []struct { + name string + stopCh <-chan struct{} + fetchErr func() error + wantStop bool + }{ + { + // The P1: no stop was requested, so this must reach the failure branch. + name: "api error with no stop request", + stopCh: openCh, + fetchErr: deletedDevice, + }, + { + // Mobile passes no stop channel at all, so it could never have stopped. + name: "api error with a nil stop channel", + stopCh: nil, + fetchErr: deletedDevice, + }, + { + name: "stop requested during preflight", + stopCh: closedCh, + fetchErr: func() error { return retryableNetworkErr() }, + wantStop: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, tc.fetchErr() + } + cfg := ctrld.Config{} + pf := runAPIPreflight(tc.stopCh, &cfg) + + if pf.err == nil { + t.Fatal("expected preflight to fail") + } + if pf.stopRequested != tc.wantStop { + t.Errorf("stopRequested = %v, want %v", pf.stopRequested, tc.wantStop) + } + }) + } +} + +// TestRunAPIPreflightPreservesAPIError verifies the error reaches the caller in a form +// the failure branch can still act on: self-uninstall keys off an *ErrorResponse with +// InvalidConfigCode, and it only runs if that error is both classified as a failure and +// still unwrappable. +func TestRunAPIPreflightPreservesAPIError(t *testing.T) { + oldFetch := fetchResolverConfig + oldUID := cdUID + t.Cleanup(func() { + fetchResolverConfig = oldFetch + cdUID = oldUID + }) + cdUID = "testuid" + + want := &controld.ErrorResponse{} + want.ErrorField.Code = controld.InvalidConfigCode + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, want + } + + cfg := ctrld.Config{} + pf := runAPIPreflight(make(chan struct{}), &cfg) + + if pf.stopRequested { + t.Error("a device-deleted failure must not be reported as an operator stop") + } + var got *controld.ErrorResponse + if !errors.As(pf.err, &got) { + t.Fatalf("error no longer unwraps to *controld.ErrorResponse: %v", pf.err) + } + if got.ErrorField.Code != controld.InvalidConfigCode { + t.Errorf("code = %d, want %d (self-uninstall would not trigger)", got.ErrorField.Code, controld.InvalidConfigCode) + } +} + +// TestPermanentAPIRejectionNarrowsToClientErrors is the regression guard for the clean +// exit added above. +// +// controld builds an *ErrorResponse for any non-200 whose body decodes, so the Go type +// says nothing about whether the API's answer will change on a retry. Keying the clean +// exit off the type alone meant a 502 from a load balancer, or an API having a bad ten +// minutes, stopped ctrld on every affected host with no service-manager retry behind it - +// worse than the abnormal exit it replaced, because a Fatal at least gets restarted. +// +// Only a client-error status may take that path. +func TestPermanentAPIRejectionNarrowsToClientErrors(t *testing.T) { + rejection := func(status, code int) error { + e := &controld.ErrorResponse{StatusCode: status} + e.ErrorField.Code = code + e.ErrorField.Message = "api said no" + return e + } + + tests := []struct { + name string + err error + wantPermanent bool + }{ + { + // The case the clean exit exists for: the device is gone, and every restart + // will be told the same thing. + name: "deleted device", + err: rejection(http.StatusNotFound, controld.InvalidConfigCode), + wantPermanent: true, + }, + {"revoked credentials", rejection(http.StatusUnauthorized, 0), true}, + {"forbidden", rejection(http.StatusForbidden, 0), true}, + {"malformed request", rejection(http.StatusBadRequest, 0), true}, + + // Server-side trouble. These must keep the abnormal exit so the service + // manager's recovery policy retries. + {"bad gateway", rejection(http.StatusBadGateway, 0), false}, + {"internal error", rejection(http.StatusInternalServerError, 0), false}, + {"service unavailable", rejection(http.StatusServiceUnavailable, 0), false}, + + // 4xx, but both are the API asking for a later attempt rather than refusing + // this configuration. + {"request timeout", rejection(http.StatusRequestTimeout, 0), false}, + {"rate limited", rejection(http.StatusTooManyRequests, 0), false}, + + // An *ErrorResponse built without a recorded status carries no verdict. A + // hand-constructed one, or a decode path that forgets to record the status, + // must not silently gain the clean exit. + {"no recorded status", rejection(0, controld.InvalidConfigCode), false}, + + // Not an API answer at all: the incident's denied socket reaches Fatal. + {"network failure", retryableNetworkErr(), false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := permanentAPIRejection(tc.err) + if ok != tc.wantPermanent { + t.Errorf("permanentAPIRejection() = %v, want %v", ok, tc.wantPermanent) + } + if ok && got == nil { + t.Error("a permanent rejection must return the rejection for reporting") + } + }) + } + + // The wrapped form matters too: preflight composes the fetch error, and errors.As has + // to reach through that for either branch to be chosen correctly. + wrapped := fmt.Errorf("processCDFlags: %w", rejection(http.StatusNotFound, controld.InvalidConfigCode)) + if _, ok := permanentAPIRejection(wrapped); !ok { + t.Error("a wrapped API rejection must still be recognised") + } + wrappedTransient := fmt.Errorf("processCDFlags: %w", rejection(http.StatusBadGateway, 0)) + if _, ok := permanentAPIRejection(wrappedTransient); ok { + t.Error("a wrapped 502 must not be treated as a permanent rejection") + } +} + +func TestStopRequested(t *testing.T) { + closedCh := make(chan struct{}) + close(closedCh) + + if stopRequested(nil) { + t.Error("a nil stop channel must read as no stop (mobile passes none)") + } + if stopRequested(make(chan struct{})) { + t.Error("an open stop channel must read as no stop") + } + if !stopRequested(closedCh) { + t.Error("a closed stop channel must read as a stop") + } +} + +// TestReloadFetchIsBoundedByServiceLifetime covers the reload path's stop wiring. +// +// Reload fetches the ControlD config too, and it used to build the bounded context +// itself. Nothing tested that: the wrong channel, or a dropped cancel, would have left a +// reload retrying against an unreachable API after "Service stopped" was logged, and no +// test would have failed. The reload path still binds its runtime-only metadata fetch to +// the service lifetime, so this pins both cancellation and watcher cleanup. +func TestReloadFetchIsBoundedByServiceLifetime(t *testing.T) { + original := processCDFlagsRuntimeFn + t.Cleanup(func() { processCDFlagsRuntimeFn = original }) + + t.Run("a stop request cancels the reload fetch", func(t *testing.T) { + stopCh := make(chan struct{}) + close(stopCh) + + var sawCancelled bool + processCDFlagsRuntimeFn = func(ctx context.Context, _ *ctrld.Config) (*controld.ResolverConfig, error) { + select { + case <-ctx.Done(): + sawCancelled = true + case <-time.After(2 * time.Second): + } + return nil, ctx.Err() + } + + p := &prog{stopCh: stopCh} + if _, err := p.fetchCDConfigBoundedByLifetime(&ctrld.Config{}); !errors.Is(err, context.Canceled) { + t.Errorf("reload fetch err = %v, want %v", err, context.Canceled) + } + if !sawCancelled { + t.Error("the reload fetch did not observe the stop request: it is not bound to the service lifetime") + } + }) + + t.Run("the derived context is always released", func(t *testing.T) { + // stopCh stays open: the fetch's own cancel is what must end the watcher, or + // every reload leaks a goroutine. + var captured context.Context + processCDFlagsRuntimeFn = func(ctx context.Context, _ *ctrld.Config) (*controld.ResolverConfig, error) { + captured = ctx + return nil, nil + } + + p := &prog{stopCh: make(chan struct{})} + if _, err := p.fetchCDConfigBoundedByLifetime(&ctrld.Config{}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + select { + case <-captured.Done(): + case <-time.After(time.Second): + t.Error("the reload fetch left its context uncancelled") + } + }) +} diff --git a/cmd/cli/cli_provision_test.go b/cmd/cli/cli_provision_test.go new file mode 100644 index 00000000..e924d9d2 --- /dev/null +++ b/cmd/cli/cli_provision_test.go @@ -0,0 +1,565 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// TestApiFailureCode covers the preflight-error mapping: a deleted device +// gets its own code (it drives self-uninstall), other permanent rejections +// are generic, anything else is retryable reachability trouble. +func TestApiFailureCode(t *testing.T) { + rejection := func(status, code int) error { + e := &controld.ErrorResponse{StatusCode: status} + e.ErrorField.Code = code + e.ErrorField.Message = "api said no" + return e + } + + tests := []struct { + name string + err error + wantCode provisionFailureCode + wantOk bool + }{ + {name: "nil error", err: nil, wantCode: "", wantOk: false}, + { + name: "deleted device maps to device invalid", + err: rejection(http.StatusNotFound, controld.InvalidConfigCode), + wantCode: provisionCodeAPIDeviceInvalid, + wantOk: true, + }, + { + name: "revoked credentials map to rejected", + err: rejection(http.StatusUnauthorized, 0), + wantCode: provisionCodeAPIRejected, + wantOk: true, + }, + { + name: "server error maps to unreachable", + err: rejection(http.StatusBadGateway, 0), + wantCode: provisionCodeAPIUnreachable, + wantOk: true, + }, + { + name: "network failure maps to unreachable", + err: retryableNetworkErr(), + wantCode: provisionCodeAPIUnreachable, + wantOk: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code, ok := apiFailureCode(tc.err) + if ok != tc.wantOk { + t.Fatalf("apiFailureCode() ok = %v, want %v", ok, tc.wantOk) + } + if code != tc.wantCode { + t.Errorf("apiFailureCode() code = %s, want %s", code, tc.wantCode) + } + }) + } +} + +// TestApiFailureCodeMapsRejectionReason covers the token_* reasons the API sends in +// error.metadata.reason on a provisioning-token rejection. An absent or unknown +// reason must fall back to the generic API_REJECTED exactly as before this reason +// field existed. +func TestApiFailureCodeMapsRejectionReason(t *testing.T) { + rejectionWithReason := func(reason string) error { + e := &controld.ErrorResponse{StatusCode: http.StatusBadRequest} + e.ErrorField.Code = 40003 + e.ErrorField.Message = "invalid token" + e.ErrorField.Metadata.Reason = reason + return e + } + + tests := []struct { + name string + reason string + wantCode provisionFailureCode + }{ + {name: "token_invalid", reason: "token_invalid", wantCode: provisionCodeTokenInvalid}, + {name: "token_expired", reason: "token_expired", wantCode: provisionCodeTokenExpired}, + {name: "token_limit_reached", reason: "token_limit_reached", wantCode: provisionCodeTokenLimitReached}, + {name: "token_disabled", reason: "token_disabled", wantCode: provisionCodeTokenDisabled}, + {name: "reason absent falls back", reason: "", wantCode: provisionCodeAPIRejected}, + {name: "unknown reason falls back", reason: "some_future_reason", wantCode: provisionCodeAPIRejected}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code, ok := apiFailureCode(rejectionWithReason(tc.reason)) + if !ok { + t.Fatal("apiFailureCode() ok = false, want true") + } + if code != tc.wantCode { + t.Errorf("apiFailureCode() code = %s, want %s", code, tc.wantCode) + } + }) + } +} + +// TestApiFailureCodeSurvivesMalformedReasonType covers a rejection body whose +// metadata.reason is the wrong JSON type end to end: decode it exactly as +// internal/controld does (json.Unmarshal into the same exported type), then +// classify it. Before the metadata decode fix, this body failed the whole +// decode and apiFailureCode never saw an *ErrorResponse at all, so it fell +// back to API_UNREACHABLE - the retryable bootstrap code - instead of the +// permanent rejection this HTTP 400 with a known error code actually is. +func TestApiFailureCodeSurvivesMalformedReasonType(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "reason as a number", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":12345}}}`}, + {name: "reason as an object", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":{"inner":"value"}}}}`}, + {name: "reason as null", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":null}}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := &controld.ErrorResponse{StatusCode: http.StatusBadRequest} + if err := json.Unmarshal([]byte(tc.body), e); err != nil { + t.Fatalf("a malformed reason must not fail the whole decode: %v", err) + } + code, ok := apiFailureCode(e) + if !ok { + t.Fatal("apiFailureCode() ok = false, want true") + } + if code != provisionCodeAPIRejected { + t.Errorf("apiFailureCode() code = %s, want %s (not %s)", code, provisionCodeAPIRejected, provisionCodeAPIUnreachable) + } + }) + } +} + +// TestCdUIDFromProvTokenReasonCodes covers the full path from an API rejection +// reason to a persisted result file: each known reason gets its own code, exit +// code, and stage, with a message naming the field and a next action but never +// echoing the token. Absent and unknown reasons keep the generic rejection. +func TestCdUIDFromProvTokenReasonCodes(t *testing.T) { + const secretToken = "org-secret-token-999" + + tests := []struct { + name string + reason string + wantCode provisionFailureCode + wantContains string + }{ + {name: "token_invalid", reason: "token_invalid", wantCode: provisionCodeTokenInvalid, wantContains: "provisioning code"}, + {name: "token_expired", reason: "token_expired", wantCode: provisionCodeTokenExpired, wantContains: "expired"}, + {name: "token_limit_reached", reason: "token_limit_reached", wantCode: provisionCodeTokenLimitReached, wantContains: "limit"}, + {name: "token_disabled", reason: "token_disabled", wantCode: provisionCodeTokenDisabled, wantContains: "invalidated"}, + {name: "reason absent", reason: "", wantCode: provisionCodeAPIRejected, wantContains: ""}, + {name: "unknown reason", reason: "brand_new_reason", wantCode: provisionCodeAPIRejected, wantContains: ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + cdUID = "" + cdOrg = secretToken + customHostname = "" + + rejected := &controld.ErrorResponse{StatusCode: http.StatusBadRequest} + rejected.ErrorField.Code = 40003 + rejected.ErrorField.Message = "invalid token " + secretToken + rejected.ErrorField.Metadata.Reason = tc.reason + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + return nil, rejected + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if *exitCode != provisionExitCodeForCode[tc.wantCode] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[tc.wantCode]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(tc.wantCode) { + t.Errorf("code = %q, want %q", r.Code, tc.wantCode) + } + if r.Stage != string(provisionStageBootstrap) { + t.Errorf("stage = %q, want bootstrap", r.Stage) + } + if tc.wantContains != "" && !strings.Contains(r.Message, tc.wantContains) { + t.Errorf("message = %q, want it to contain %q", r.Message, tc.wantContains) + } + if strings.Contains(r.Message, secretToken) { + t.Errorf("token leaked into result message: %q", r.Message) + } + }) + } +} + +func stubProvisionGlobals(t *testing.T) (exitCode *int, notified *bool) { + t.Helper() + oldCdUID, oldCdOrg := cdUID, cdOrg + oldExit, oldUninstall := provisionExit, uninstallInvalidCdUIDFn + t.Cleanup(func() { + cdUID, cdOrg = oldCdUID, oldCdOrg + provisionExit, uninstallInvalidCdUIDFn = oldExit, oldUninstall + }) + overrideProvisionResultPath(t) + code := -1 + provisionExit = func(c int) { code = c } + n := false + return &code, &n +} + +func TestHandleAPIPreflightFailure(t *testing.T) { + deviceInvalid := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusNotFound} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = "device does not exist" + return e + } + rejected := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusUnauthorized} + e.ErrorField.Message = "bad token" + return e + } + + t.Run("permanent rejection returns cleanly", func(t *testing.T) { + exitCode, notified := stubProvisionGlobals(t) + handleAPIPreflightFailure(&prog{}, rejected(), func() { *notified = true }) + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want a clean return", *exitCode) + } + if !*notified { + t.Error("notify not called") + } + r, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if r.Code != string(provisionCodeAPIRejected) { + t.Errorf("code = %q, want API_REJECTED", r.Code) + } + }) + + t.Run("deleted device self-uninstalls and returns cleanly", func(t *testing.T) { + exitCode, notified := stubProvisionGlobals(t) + uninstalled := false + uninstallInvalidCdUIDFn = func(_ *prog, _ *ctrld.Logger, _ bool) bool { + uninstalled = true + return true + } + handleAPIPreflightFailure(&prog{}, deviceInvalid(), func() { *notified = true }) + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want a clean return", *exitCode) + } + if !uninstalled { + t.Error("self-uninstall not attempted") + } + if !*notified { + t.Error("notify not called") + } + r, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if r.Code != string(provisionCodeAPIDeviceInvalid) { + t.Errorf("code = %q, want API_DEVICE_INVALID", r.Code) + } + }) + + t.Run("unreachable exits nonzero", func(t *testing.T) { + exitCode, notified := stubProvisionGlobals(t) + handleAPIPreflightFailure(&prog{}, retryableNetworkErr(), func() { *notified = true }) + if *exitCode != provisionExitCodeForCode[provisionCodeAPIUnreachable] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeAPIUnreachable]) + } + if !*notified { + t.Error("notify not called") + } + r, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if r.Code != string(provisionCodeAPIUnreachable) { + t.Errorf("code = %q, want API_UNREACHABLE", r.Code) + } + }) + + t.Run("bare uid from a composite --cd value is redacted", func(t *testing.T) { + _, _ = stubProvisionGlobals(t) + cdUID = "deviceabc/clientxyz" + cdOrg = "" + err := fmt.Errorf("failed: api says deviceabc is unknown") + handleAPIPreflightFailure(&prog{}, err, func() {}) + r, rerr := readProvisionResult() + if rerr != nil { + t.Fatal(rerr) + } + if strings.Contains(r.Message, "deviceabc") { + t.Errorf("bare uid leaked into message: %q", r.Message) + } + }) +} + +// TestDoValidateCdRemoteConfigClassifiesAPIFailure covers the direct +// "--cd " install path (fatal=true): a fetch failure must classify on +// the provisioning boundary with the same per-class codes as the daemon-side +// preflight, instead of a bare fatal. +func TestDoValidateCdRemoteConfigClassifiesAPIFailure(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch := fetchResolverConfig + t.Cleanup(func() { fetchResolverConfig = oldFetch }) + + deviceInvalid := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusNotFound} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = "device does not exist" + return e + } + rejected := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusUnauthorized} + e.ErrorField.Message = "bad token" + return e + } + + tests := []struct { + name string + err error + wantCode provisionFailureCode + }{ + {name: "device invalid", err: deviceInvalid(), wantCode: provisionCodeAPIDeviceInvalid}, + {name: "permanent rejection", err: rejected(), wantCode: provisionCodeAPIRejected}, + {name: "unreachable", err: retryableNetworkErr(), wantCode: provisionCodeAPIUnreachable}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, tc.err + } + if err := doValidateCdRemoteConfig("device-uid-123", true); err == nil { + t.Error("doValidateCdRemoteConfig() error = nil, want the fetch error back") + } + if *exitCode != provisionExitCodeForCode[tc.wantCode] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[tc.wantCode]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(tc.wantCode) { + t.Errorf("code = %q, want %q", r.Code, tc.wantCode) + } + }) + } +} + +// TestDoValidateCdRemoteConfigNonFatalReturnsError proves the restart path +// (fatal=false) is unaffected: it still just warns and hands the error back, +// with no process exit and no result file. +func TestDoValidateCdRemoteConfigNonFatalReturnsError(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch := fetchResolverConfig + t.Cleanup(func() { fetchResolverConfig = oldFetch }) + + wantErr := errors.New("network unreachable") + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, wantErr + } + + if err := doValidateCdRemoteConfig("device-uid-123", false); !errors.Is(err, wantErr) { + t.Errorf("doValidateCdRemoteConfig() error = %v, want %v", err, wantErr) + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + if _, err := readProvisionResult(); err == nil { + t.Error("expected no provision result written for the non-fatal path") + } +} + +// TestDoValidateCdRemoteConfigDoesNotSelfUninstall proves the direct-cd +// install path never triggers self-uninstall on a device-invalid failure: +// this runs before the service is installed, so there is nothing to remove. +func TestDoValidateCdRemoteConfigDoesNotSelfUninstall(t *testing.T) { + _, _ = stubProvisionGlobals(t) + oldFetch, oldUninstall := fetchResolverConfig, uninstallInvalidCdUIDFn + t.Cleanup(func() { fetchResolverConfig, uninstallInvalidCdUIDFn = oldFetch, oldUninstall }) + + uninstallCalled := false + uninstallInvalidCdUIDFn = func(*prog, *ctrld.Logger, bool) bool { + uninstallCalled = true + return true + } + e := &controld.ErrorResponse{StatusCode: http.StatusNotFound} + e.ErrorField.Code = controld.InvalidConfigCode + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, e + } + + _ = doValidateCdRemoteConfig("device-uid-123", true) + + if uninstallCalled { + t.Error("doValidateCdRemoteConfig triggered self-uninstall; nothing is installed yet on this path") + } +} + +func TestCdUIDFromProvTokenFailureEmitsCode(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + cdUID = "" + cdOrg = "org-secret-token-123" + customHostname = "" + + rejected := &controld.ErrorResponse{StatusCode: http.StatusUnauthorized} + rejected.ErrorField.Message = "bad provision token org-secret-token-123" + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + return nil, rejected + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if *exitCode != provisionExitCodeForCode[provisionCodeAPIRejected] { + t.Errorf("exit = %d, want API_REJECTED exit %d", *exitCode, provisionExitCodeForCode[provisionCodeAPIRejected]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeAPIRejected) { + t.Errorf("code = %q, want API_REJECTED", r.Code) + } + if strings.Contains(r.Message, cdOrg) { + t.Errorf("token leaked into result message: %q", r.Message) + } +} + +// Regression test: an explicit ip:port that fails to bind used to die with a +// bare fatal log automation could not tell apart from any other crash. It +// must report a stable code through the provisioning result instead. +func TestTryUpdateListenerConfigConfiguredAddrUnavailable(t *testing.T) { + // Occupy one localhost port on both udp and tcp, and hold both for the + // whole test so ctrld's own bind attempt is guaranteed to fail. + udpConn, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("could not reserve a udp port: %v", err) + } + defer udpConn.Close() + + host, portStr, err := net.SplitHostPort(udpConn.LocalAddr().String()) + if err != nil { + t.Fatalf("could not parse reserved address: %v", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("could not parse reserved port: %v", err) + } + + tcpLn, err := net.Listen("tcp", net.JoinHostPort(host, portStr)) + if err != nil { + t.Fatalf("could not reserve the same port on tcp: %v", err) + } + defer tcpLn.Close() + + oldCdUID, oldCdOrg, oldNextdns, oldIntercept := cdUID, cdOrg, nextdns, interceptMode + oldPath, oldExit := provisionResultPath, provisionExit + t.Cleanup(func() { + cdUID, cdOrg, nextdns, interceptMode = oldCdUID, oldCdOrg, oldNextdns, oldIntercept + provisionResultPath, provisionExit = oldPath, oldExit + }) + // Non-cd, non-nextdns mode with an explicit ip:port: no fallback checks, + // the path that used to reach the fatal exit directly. + cdUID = "" + cdOrg = "" + nextdns = "" + interceptMode = "" + + tmpDir := t.TempDir() + provisionResultPath = func() string { return filepath.Join(tmpDir, "provision_result.json") } + + var exitCode int + var exited bool + provisionExit = func(code int) { exitCode = code; exited = true } + + cfg := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: host, Port: port}, + }, + } + + notified := false + _, ok := tryUpdateListenerConfig(cfg, func() { notified = true }, true) + + if ok { + t.Error("tryUpdateListenerConfig ok = true, want false") + } + if !notified { + t.Error("expected notifyFunc to run before the recorded exit") + } + if !exited { + t.Fatal("expected provisionExit to be called") + } + if exitCode != 42 { + t.Errorf("exit code = %d, want 42 (LISTENER_CONFIGURED_ADDR_UNAVAILABLE)", exitCode) + } + + result, err := readProvisionResult() + if err != nil { + t.Fatalf("could not read provision result: %v", err) + } + if result.Code != string(provisionCodeListenerAddrUnavail) { + t.Errorf("result code = %s, want %s", result.Code, provisionCodeListenerAddrUnavail) + } + if result.Stage != string(provisionStageListener) { + t.Errorf("result stage = %s, want %s", result.Stage, provisionStageListener) + } + if result.ExitCode != 42 { + t.Errorf("result exit code = %d, want 42", result.ExitCode) + } + if result.Detail == nil || len(result.Detail.Attempts) == 0 { + t.Fatal("expected the occupied address to appear as a recorded bind attempt") + } + + occupiedAddr := net.JoinHostPort(host, portStr) + // Windows words WSAEADDRINUSE differently, so only require the canonical + // message on platforms that produce it. + requireInUseText := runtime.GOOS != "windows" + var sawUDP, sawTCP bool + for _, a := range result.Detail.Attempts { + if a.Addr != occupiedAddr || a.OSError == "" { + continue + } + if requireInUseText && !strings.Contains(strings.ToLower(a.OSError), "address already in use") { + continue + } + switch a.Proto { + case "udp": + sawUDP = true + case "tcp": + sawTCP = true + } + } + if !sawUDP { + t.Error("expected a udp attempt on the occupied address with a bind error") + } + if !sawTCP { + t.Error("expected a tcp attempt on the occupied address with a bind error") + } +} + +// The exhaustion path (exit 41) is not covered: forcing every fallback, +// including a freshly randomized ip/port, to fail has no deterministic seam, +// so a test would race whatever ports are free on the host. diff --git a/cmd/cli/cli_test.go b/cmd/cli/cli_test.go index eae2673d..232a6b72 100644 --- a/cmd/cli/cli_test.go +++ b/cmd/cli/cli_test.go @@ -5,8 +5,11 @@ import ( "path/filepath" "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" ) func Test_writeConfigFile(t *testing.T) { @@ -22,6 +25,31 @@ func Test_writeConfigFile(t *testing.T) { require.NoError(t, err) } +// A managed start rewrites the config file only when the API answer differs +// from it, e.g. a custom config added after install. +func Test_configFileDiffers(t *testing.T) { + oldV, oldFile, oldPath, oldUID := v, defaultConfigFile, configPath, cdUID + t.Cleanup(func() { v, defaultConfigFile, configPath, cdUID = oldV, oldFile, oldPath, oldUID }) + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + configPath = filepath.Join(t.TempDir(), "ctrld.toml") + cdUID = "test-uid" + + generated := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: "doh", Endpoint: "https://dns.controld.com/test-uid"}}} + assert.True(t, configFileDiffers(generated), "missing file must differ") + require.NoError(t, writeConfigFile(generated)) + assert.False(t, configFileDiffers(generated), "file written from the same config must not differ") + + custom := *generated + custom.Service.AllowUnprivilegedLogSend = true + assert.True(t, configFileDiffers(&custom), "custom config added after install must differ") + require.NoError(t, writeConfigFile(&custom)) + assert.False(t, configFileDiffers(&custom)) + data, err := os.ReadFile(configPath) + require.NoError(t, err) + assert.Contains(t, string(data), "allow_unprivileged_log_send = true") + assert.True(t, configFileDiffers(generated), "custom config removed must differ") +} + func Test_isStableVersion(t *testing.T) { tests := []struct { name string @@ -44,3 +72,55 @@ func Test_isStableVersion(t *testing.T) { }) } } + +// Test_displayVersion pins the tag-to-display transform. master carries v2.x.x +// tags so its releases can be tracked next to the v1.x.x line still cut from the +// v1.0 branch, and the client reports the tag with its major decremented. +// +// The v1.0-branch cases are the ones that make the split safe without a +// build-time branch signal: a major of 1 has to pass through untouched, or the +// v1.0 line would start reporting v0.x.x. +func Test_displayVersion(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"master tag", "v2.0.0", "v1.0.0"}, + {"master tag with minor and patch", "v2.3.1", "v1.3.1"}, + {"master prerelease keeps its suffix", "v2.1.0-rc1", "v1.1.0-rc1"}, + {"master tag with build metadata", "v2.1.0+build.5", "v1.1.0+build.5"}, + {"a later major still decrements by one", "v3.2.1", "v2.2.1"}, + // v1.0 branch: untouched, which is what scopes the transform to master. + {"v1.0 branch tag", "v1.3.5", "v1.3.5"}, + {"v1.0 branch prerelease", "v1.3.5-next", "v1.3.5-next"}, + // Not semantic versions: dev and commit-suffixed builds pass through. + {"dev", "dev", "dev"}, + {"dev with commit", "dev-abc1234", "dev-abc1234"}, + {"empty", "", ""}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := displayVersion(tc.in); got != tc.want { + t.Errorf("displayVersion(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// Test_displayVersionKeepsStabilityClassification guards the coupling between the +// transform and isStableVersion, which selects the self-upgrade channel: a tag +// must not change from prerelease to stable (or back) by being renumbered. +func Test_displayVersionKeepsStabilityClassification(t *testing.T) { + for _, ver := range []string{"v2.0.0", "v2.1.0-rc1", "v1.3.5", "v1.3.5-next", "dev"} { + ver := ver + t.Run(ver, func(t *testing.T) { + t.Parallel() + if got, want := isStableVersion(displayVersion(ver)), isStableVersion(ver); got != want { + t.Errorf("isStableVersion(displayVersion(%q)) = %v, want %v", ver, got, want) + } + }) + } +} diff --git a/cmd/cli/commands.go b/cmd/cli/commands.go deleted file mode 100644 index a1074f29..00000000 --- a/cmd/cli/commands.go +++ /dev/null @@ -1,1397 +0,0 @@ -package cli - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net" - "net/http" - "os" - "os/exec" - "path/filepath" - "runtime" - "slices" - "sort" - "strconv" - "strings" - "time" - - "github.com/docker/go-units" - "github.com/kardianos/service" - "github.com/minio/selfupdate" - "github.com/olekukonko/tablewriter" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/clientinfo" - "github.com/Control-D-Inc/ctrld/internal/router" -) - -// dialSocketControlServerTimeout is the default timeout to wait when ping control server. -const dialSocketControlServerTimeout = 30 * time.Second - -func initLogCmd() *cobra.Command { - warnRuntimeLoggingNotEnabled := func() { - mainLog.Load().Warn().Msg("runtime debug logging is not enabled") - mainLog.Load().Warn().Msg(`ctrld may be running without "--cd" flag or logging is already enabled`) - } - logSendCmd := &cobra.Command{ - Use: "send", - Short: "Send runtime debug logs to ControlD", - Args: cobra.NoArgs, - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Run: func(cmd *cobra.Command, args []string) { - - p := &prog{router: router.New(&cfg, false)} - s, _ := newService(p, svcConfig) - - status, err := s.Status() - if errors.Is(err, service.ErrNotInstalled) { - mainLog.Load().Warn().Msg("service not installed") - return - } - if status == service.StatusStopped { - mainLog.Load().Warn().Msg("service is not running") - return - } - - dir, err := socketDir() - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to find ctrld home dir") - } - cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) - resp, err := cc.post(sendLogsPath, nil) - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to send logs") - } - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusServiceUnavailable: - mainLog.Load().Warn().Msg("runtime logs could only be sent once per minute") - return - case http.StatusMovedPermanently: - warnRuntimeLoggingNotEnabled() - return - } - var logs logSentResponse - if err := json.NewDecoder(resp.Body).Decode(&logs); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to decode sent logs result") - } - size := units.BytesSize(float64(logs.Size)) - if logs.Error == "" { - mainLog.Load().Notice().Msgf("runtime logs sent successfully (%s)", size) - } else { - mainLog.Load().Error().Msgf("failed to send logs (%s)", size) - mainLog.Load().Error().Msg(logs.Error) - } - }, - } - logViewCmd := &cobra.Command{ - Use: "view", - Short: "View current runtime debug logs", - Args: cobra.NoArgs, - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Run: func(cmd *cobra.Command, args []string) { - - p := &prog{router: router.New(&cfg, false)} - s, _ := newService(p, svcConfig) - - status, err := s.Status() - if errors.Is(err, service.ErrNotInstalled) { - mainLog.Load().Warn().Msg("service not installed") - return - } - if status == service.StatusStopped { - mainLog.Load().Warn().Msg("service is not running") - return - } - - dir, err := socketDir() - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to find ctrld home dir") - } - cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) - resp, err := cc.post(viewLogsPath, nil) - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to get logs") - } - defer resp.Body.Close() - - switch resp.StatusCode { - case http.StatusMovedPermanently: - warnRuntimeLoggingNotEnabled() - return - case http.StatusBadRequest: - mainLog.Load().Warn().Msg("runtime debugs log is not available") - buf, err := io.ReadAll(resp.Body) - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to read response body") - } - mainLog.Load().Warn().Msgf("ctrld process response:\n\n%s\n", string(buf)) - return - case http.StatusOK: - } - var logs logViewResponse - if err := json.NewDecoder(resp.Body).Decode(&logs); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to decode view logs result") - } - fmt.Println(logs.Data) - }, - } - logCmd := &cobra.Command{ - Use: "log", - Short: "Manage runtime debug logs", - Args: cobra.OnlyValidArgs, - ValidArgs: []string{ - logSendCmd.Use, - }, - } - logCmd.AddCommand(logSendCmd) - logCmd.AddCommand(logViewCmd) - rootCmd.AddCommand(logCmd) - - return logCmd -} - -func initRunCmd() *cobra.Command { - runCmd := &cobra.Command{ - Use: "run", - Short: "Run the DNS proxy server", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - RunCobraCommand(cmd) - }, - } - runCmd.Flags().BoolVarP(&daemon, "daemon", "d", false, "Run as daemon") - runCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file") - runCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config") - runCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port") - runCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint") - runCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint") - runCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy") - runCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file") - runCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items") - runCmd.Flags().StringVarP(&cdUID, cdUidFlagName, "", "", "Control D resolver uid") - runCmd.Flags().StringVarP(&cdOrg, cdOrgFlagName, "", "", "Control D provision token") - runCmd.Flags().StringVarP(&customHostname, customHostnameFlagName, "", "", "Custom hostname passed to ControlD API") - runCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain") - _ = runCmd.Flags().MarkHidden("dev") - runCmd.Flags().StringVarP(&homedir, "homedir", "", "", "") - _ = runCmd.Flags().MarkHidden("homedir") - runCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`) - _ = runCmd.Flags().MarkHidden("iface") - runCmd.Flags().StringVarP(&cdUpstreamProto, "proto", "", ctrld.ResolverTypeDOH, `Control D upstream type, either "doh" or "doh3"`) - runCmd.Flags().BoolVarP(&rfc1918, "rfc1918", "", false, "Listen on RFC1918 addresses when 127.0.0.1 is the only listener") - - runCmd.FParseErrWhitelist = cobra.FParseErrWhitelist{UnknownFlags: true} - rootCmd.AddCommand(runCmd) - - return runCmd -} - -func initStartCmd() *cobra.Command { - startCmd := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "start", - Short: "Install and start the ctrld service", - Long: `Install and start the ctrld service - -NOTE: running "ctrld start" without any arguments will start already installed ctrld service.`, - Args: func(cmd *cobra.Command, args []string) error { - args = filterEmptyStrings(args) - if len(args) > 0 { - return fmt.Errorf("'ctrld start' doesn't accept positional arguments\n" + - "Use flags instead (e.g. --cd, --iface) or see 'ctrld start --help' for all options") - } - return nil - }, - Run: func(cmd *cobra.Command, args []string) { - checkStrFlagEmpty(cmd, cdUidFlagName) - checkStrFlagEmpty(cmd, cdOrgFlagName) - validateCdAndNextDNSFlags() - sc := &service.Config{} - *sc = *svcConfig - osArgs := os.Args[2:] - osArgs = filterEmptyStrings(osArgs) - if os.Args[1] == "service" { - osArgs = os.Args[3:] - } - setDependencies(sc) - sc.Arguments = append([]string{"run"}, osArgs...) - - p := &prog{ - router: router.New(&cfg, cdUID != ""), - cfg: &cfg, - } - s, err := newService(p, sc) - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - return - } - p.preRun() - - status, err := s.Status() - isCtrldRunning := status == service.StatusRunning - isCtrldInstalled := !errors.Is(err, service.ErrNotInstalled) - - // Get current running iface, if any. - var currentIface *ifaceResponse - - // If pin code was set, do not allow running start command. - if isCtrldRunning { - if err := checkDeactivationPin(s, nil); isCheckDeactivationPinErr(err) { - os.Exit(deactivationPinInvalidExitCode) - } - currentIface = runningIface(s) - mainLog.Load().Debug().Msgf("current interface on start: %v", currentIface) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - reportSetDnsOk := func(sockDir string) { - if cc := newSocketControlClient(ctx, s, sockDir); cc != nil { - if resp, _ := cc.post(ifacePath, nil); resp != nil && resp.StatusCode == http.StatusOK { - if iface == "auto" { - iface = defaultIfaceName() - } - res := &ifaceResponse{} - if err := json.NewDecoder(resp.Body).Decode(res); err != nil { - mainLog.Load().Warn().Err(err).Msg("failed to get iface info") - return - } - if res.OK { - name := res.Name - if iff, err := net.InterfaceByName(name); err == nil { - _, _ = patchNetIfaceName(iff) - name = iff.Name - } - logger := mainLog.Load().With().Str("iface", name).Logger() - logger.Debug().Msg("setting DNS successfully") - if res.All { - // Log that DNS is set for other interfaces. - withEachPhysicalInterfaces( - name, - "set DNS", - func(i *net.Interface) error { return nil }, - ) - } - } - } - } - } - - // No config path, generating config in HOME directory. - noConfigStart := isNoConfigStart(cmd) - writeDefaultConfig := !noConfigStart && configBase64 == "" - - logServerStarted := make(chan struct{}) - // A buffer channel to gather log output from runCmd and report - // to user in case self-check process failed. - runCmdLogCh := make(chan string, 256) - ud, err := userHomeDir() - sockDir := ud - if err != nil { - mainLog.Load().Warn().Msg("log server did not start") - close(logServerStarted) - } else { - setWorkingDirectory(sc, ud) - if configPath == "" && writeDefaultConfig { - defaultConfigFile = filepath.Join(ud, defaultConfigFile) - } - sc.Arguments = append(sc.Arguments, "--homedir="+ud) - if d, err := socketDir(); err == nil { - sockDir = d - } - sockPath := filepath.Join(sockDir, ctrldLogUnixSock) - _ = os.Remove(sockPath) - go func() { - defer func() { - close(runCmdLogCh) - _ = os.Remove(sockPath) - }() - close(logServerStarted) - if conn := runLogServer(sockPath); conn != nil { - // Enough buffer for log message, we don't produce - // such long log message, but just in case. - buf := make([]byte, 1024) - for { - n, err := conn.Read(buf) - if err != nil { - return - } - msg := string(buf[:n]) - if _, _, found := strings.Cut(msg, msgExit); found { - cancel() - } - runCmdLogCh <- msg - } - } - }() - } - <-logServerStarted - - if !startOnly { - startOnly = len(osArgs) == 0 - } - // If user run "ctrld start" and ctrld is already installed, starting existing service. - if startOnly && isCtrldInstalled { - tryReadingConfigWithNotice(false, true) - if err := v.Unmarshal(&cfg); err != nil { - mainLog.Load().Fatal().Msgf("failed to unmarshal config: %v", err) - } - - // if already running, dont restart - if isCtrldRunning { - mainLog.Load().Notice().Msg("service is already running") - return - } - - initInteractiveLogging() - tasks := []task{ - {func() error { - // Save current DNS so we can restore later. - withEachPhysicalInterfaces("", "saveCurrentStaticDNS", func(i *net.Interface) error { - if err := saveCurrentStaticDNS(i); !errors.Is(err, errSaveCurrentStaticDNSNotSupported) && err != nil { - return err - } - return nil - }) - return nil - }, false, "Save current DNS"}, - {func() error { - return ConfigureWindowsServiceFailureActions(ctrldServiceName) - }, false, "Configure service failure actions"}, - {s.Start, true, "Start"}, - {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, - } - mainLog.Load().Notice().Msg("Starting existing ctrld service") - if doTasks(tasks) { - mainLog.Load().Notice().Msg("Service started") - sockDir, err := socketDir() - if err != nil { - mainLog.Load().Warn().Err(err).Msg("Failed to get socket directory") - os.Exit(1) - } - reportSetDnsOk(sockDir) - } else { - mainLog.Load().Error().Err(err).Msg("Failed to start existing ctrld service") - os.Exit(1) - } - return - } - - if cdUID != "" { - _ = doValidateCdRemoteConfig(cdUID, true) - } else if uid := cdUIDFromProvToken(); uid != "" { - cdUID = uid - mainLog.Load().Debug().Msg("using uid from provision token") - removeOrgFlagsFromArgs(sc) - // Pass --cd flag to "ctrld run" command, so the provision token takes no effect. - sc.Arguments = append(sc.Arguments, "--cd="+cdUID) - } - if cdUID != "" { - validateCdUpstreamProtocol() - } - - if err := p.router.ConfigureService(sc); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to configure service on router") - } - - if configPath != "" { - v.SetConfigFile(configPath) - } - - tryReadingConfigWithNotice(writeDefaultConfig, true) - - if err := v.Unmarshal(&cfg); err != nil { - mainLog.Load().Fatal().Msgf("failed to unmarshal config: %v", err) - } - - initInteractiveLogging() - - if nextdns != "" { - removeNextDNSFromArgs(sc) - } - - // Explicitly passing config, so on system where home directory could not be obtained, - // or sub-process env is different with the parent, we still behave correctly and use - // the expected config file. - if configPath == "" { - sc.Arguments = append(sc.Arguments, "--config="+defaultConfigFile) - } - - if router.Name() != "" && iface != "" { - mainLog.Load().Debug().Msg("cleaning up router before installing") - _ = p.router.Cleanup() - } - - tasks := []task{ - {s.Stop, false, "Stop"}, - {func() error { return doGenerateNextDNSConfig(nextdns) }, true, "Checking config"}, - {func() error { return ensureUninstall(s) }, false, "Ensure uninstall"}, - //resetDnsTask(p, s, isCtrldInstalled, currentIface), - {func() error { - // Save current DNS so we can restore later. - withEachPhysicalInterfaces("", "saveCurrentStaticDNS", func(i *net.Interface) error { - if err := saveCurrentStaticDNS(i); !errors.Is(err, errSaveCurrentStaticDNSNotSupported) && err != nil { - return err - } - return nil - }) - return nil - }, false, "Save current DNS"}, - {s.Install, false, "Install"}, - {func() error { - return ConfigureWindowsServiceFailureActions(ctrldServiceName) - }, false, "Configure Windows service failure actions"}, - {s.Start, true, "Start"}, - // Note that startCmd do not actually write ControlD config, but the config file was - // generated after s.Start, so we notice users here for consistent with nextdns mode. - {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, - } - mainLog.Load().Notice().Msg("Starting service") - if doTasks(tasks) { - if err := p.router.Install(sc); err != nil { - mainLog.Load().Warn().Err(err).Msg("post installation failed, please check system/service log for details error") - return - } - - // add a small delay to ensure the service is started and did not crash - time.Sleep(1 * time.Second) - - ok, status, err := selfCheckStatus(ctx, s, sockDir) - switch { - case ok && status == service.StatusRunning: - mainLog.Load().Notice().Msg("Service started") - default: - marker := bytes.Repeat([]byte("="), 32) - // If ctrld service is not running, emitting log obtained from ctrld process. - if status != service.StatusRunning || ctx.Err() != nil { - mainLog.Load().Error().Msg("ctrld service may not have started due to an error or misconfiguration, service log:") - _, _ = mainLog.Load().Write(marker) - haveLog := false - for msg := range runCmdLogCh { - _, _ = mainLog.Load().Write([]byte(strings.ReplaceAll(msg, msgExit, ""))) - haveLog = true - } - // If we're unable to get log from "ctrld run", notice users about it. - if !haveLog { - mainLog.Load().Write([]byte(`"`)) - } - } - // Report any error if occurred. - if err != nil { - _, _ = mainLog.Load().Write(marker) - msg := fmt.Sprintf("An error occurred while performing test query: %s", err) - mainLog.Load().Write([]byte(msg)) - } - // If ctrld service is running but selfCheckStatus failed, it could be related - // to user's system firewall configuration, notice users about it. - if status == service.StatusRunning && err == nil { - _, _ = mainLog.Load().Write(marker) - mainLog.Load().Write([]byte(`ctrld service was running, but a DNS query could not be sent to its listener`)) - mainLog.Load().Write([]byte(`Please check your system firewall if it is configured to block/intercept/redirect DNS queries`)) - } - - _, _ = mainLog.Load().Write(marker) - uninstall(p, s) - os.Exit(1) - } - reportSetDnsOk(sockDir) - } - }, - } - // Keep these flags in sync with runCmd above, except for "-d"/"--nextdns". - startCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file") - startCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config") - startCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port") - startCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint") - startCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint") - startCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy") - startCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file") - startCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items") - startCmd.Flags().StringVarP(&cdUID, cdUidFlagName, "", "", "Control D resolver uid") - startCmd.Flags().StringVarP(&cdOrg, cdOrgFlagName, "", "", "Control D provision token") - startCmd.Flags().StringVarP(&customHostname, customHostnameFlagName, "", "", "Custom hostname passed to ControlD API") - startCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain") - _ = startCmd.Flags().MarkHidden("dev") - startCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`) - startCmd.Flags().StringVarP(&nextdns, nextdnsFlagName, "", "", "NextDNS resolver id") - startCmd.Flags().StringVarP(&cdUpstreamProto, "proto", "", ctrld.ResolverTypeDOH, `Control D upstream type, either "doh" or "doh3"`) - startCmd.Flags().BoolVarP(&skipSelfChecks, "skip_self_checks", "", false, `Skip self checks after installing ctrld service`) - startCmd.Flags().BoolVarP(&startOnly, "start_only", "", false, "Do not install new service") - _ = startCmd.Flags().MarkHidden("start_only") - startCmd.Flags().BoolVarP(&rfc1918, "rfc1918", "", false, "Listen on RFC1918 addresses when 127.0.0.1 is the only listener") - - routerCmd := &cobra.Command{ - Use: "setup", - Run: func(cmd *cobra.Command, _ []string) { - exe, err := os.Executable() - if err != nil { - mainLog.Load().Fatal().Msgf("could not find executable path: %v", err) - os.Exit(1) - } - flags := make([]string, 0) - cmd.Flags().Visit(func(flag *pflag.Flag) { - flags = append(flags, fmt.Sprintf("--%s=%s", flag.Name, flag.Value)) - }) - cmdArgs := []string{"start"} - cmdArgs = append(cmdArgs, flags...) - command := exec.Command(exe, cmdArgs...) - command.Stdout = os.Stdout - command.Stderr = os.Stderr - command.Stdin = os.Stdin - if err := command.Run(); err != nil { - mainLog.Load().Fatal().Msg(err.Error()) - } - }, - } - routerCmd.Flags().AddFlagSet(startCmd.Flags()) - routerCmd.Hidden = true - rootCmd.AddCommand(routerCmd) - - startCmdAlias := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "start", - Short: "Quick start service and configure DNS on interface", - Long: `Quick start service and configure DNS on interface - -NOTE: running "ctrld start" without any arguments will start already installed ctrld service.`, - Args: func(cmd *cobra.Command, args []string) error { - args = filterEmptyStrings(args) - if len(args) > 0 { - return fmt.Errorf("'ctrld start' doesn't accept positional arguments\n" + - "Use flags instead (e.g. --cd, --iface) or see 'ctrld start --help' for all options") - } - return nil - }, - Run: func(cmd *cobra.Command, args []string) { - if len(os.Args) == 2 { - startOnly = true - } - if !cmd.Flags().Changed("iface") { - os.Args = append(os.Args, "--iface="+ifaceStartStop) - } - iface = ifaceStartStop - startCmd.Run(cmd, args) - }, - } - startCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", "auto", `Update DNS setting for iface, "auto" means the default interface gateway`) - startCmdAlias.Flags().AddFlagSet(startCmd.Flags()) - rootCmd.AddCommand(startCmdAlias) - - return startCmd -} - -func initStopCmd() *cobra.Command { - stopCmd := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "stop", - Short: "Stop the ctrld service", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - readConfig(false) - v.Unmarshal(&cfg) - p := &prog{router: router.New(&cfg, runInCdMode())} - s, err := newService(p, svcConfig) - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - return - } - p.preRun() - if ir := runningIface(s); ir != nil { - p.runningIface = ir.Name - p.requiredMultiNICsConfig = ir.All - } - - initInteractiveLogging() - - status, err := s.Status() - if errors.Is(err, service.ErrNotInstalled) { - mainLog.Load().Warn().Msg("service not installed") - return - } - if status == service.StatusStopped { - mainLog.Load().Warn().Msg("service is already stopped") - return - } - - if err := checkDeactivationPin(s, nil); isCheckDeactivationPinErr(err) { - os.Exit(deactivationPinInvalidExitCode) - } - if doTasks([]task{{s.Stop, true, "Stop"}}) { - if router.WaitProcessExited() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - for { - select { - case <-ctx.Done(): - mainLog.Load().Error().Msg("timeout while waiting for service to stop") - return - default: - } - time.Sleep(time.Second) - if status, _ := s.Status(); status == service.StatusStopped { - break - } - } - } - mainLog.Load().Notice().Msg("Service stopped") - } - }, - } - stopCmd.Flags().StringVarP(&iface, "iface", "", "", `Reset DNS setting for iface, "auto" means the default interface gateway`) - stopCmd.Flags().Int64VarP(&deactivationPin, "pin", "", defaultDeactivationPin, `Pin code for stopping ctrld`) - _ = stopCmd.Flags().MarkHidden("pin") - - stopCmdAlias := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "stop", - Short: "Quick stop service and remove DNS from interface", - Run: func(cmd *cobra.Command, args []string) { - if !cmd.Flags().Changed("iface") { - os.Args = append(os.Args, "--iface="+ifaceStartStop) - } - iface = ifaceStartStop - stopCmd.Run(cmd, args) - }, - } - stopCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", "auto", `Reset DNS setting for iface, "auto" means the default interface gateway`) - stopCmdAlias.Flags().AddFlagSet(stopCmd.Flags()) - rootCmd.AddCommand(stopCmdAlias) - - return stopCmd -} - -func initRestartCmd() *cobra.Command { - restartCmd := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "restart", - Short: "Restart the ctrld service", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - readConfig(false) - v.Unmarshal(&cfg) - cdUID = curCdUID() - cdMode := cdUID != "" - - p := &prog{router: router.New(&cfg, cdMode)} - s, err := newService(p, svcConfig) - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - return - } - if _, err := s.Status(); errors.Is(err, service.ErrNotInstalled) { - mainLog.Load().Warn().Msg("service not installed") - return - } - if iface == "" { - iface = "auto" - } - p.preRun() - if ir := runningIface(s); ir != nil { - p.runningIface = ir.Name - p.requiredMultiNICsConfig = ir.All - } - - initInteractiveLogging() - - var validateConfigErr error - if cdMode { - validateConfigErr = doValidateCdRemoteConfig(cdUID, false) - } - - if ir := runningIface(s); ir != nil { - iface = ir.Name - } - - doRestart := func() bool { - tasks := []task{ - {s.Stop, true, "Stop"}, - {func() error { - p.router.Cleanup() - // restore static DNS settings or DHCP - p.resetDNS(false, true) - return nil - }, false, "Cleanup"}, - {func() error { - time.Sleep(time.Second * 1) - return nil - }, false, "Waiting for service to stop"}, - } - if doTasks(tasks) { - - if router.WaitProcessExited() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - loop: - for { - select { - case <-ctx.Done(): - mainLog.Load().Error().Msg("timeout while waiting for service to stop") - break loop - default: - } - time.Sleep(time.Second) - if status, _ := s.Status(); status == service.StatusStopped { - break - } - } - } - } else { - return false - } - - tasks = []task{ - {s.Start, true, "Start"}, - } - - return doTasks(tasks) - - } - - if doRestart() { - if dir, err := socketDir(); err == nil { - timeout := dialSocketControlServerTimeout - // If we failed to validate remote config above, it's likely that - // we are having problem with network connection. So using a shorter - // timeout than default one for better UX. - if validateConfigErr != nil { - timeout = 5 * time.Second - } - if cc := newSocketControlClientWithTimeout(context.TODO(), s, dir, timeout); cc != nil { - _, _ = cc.post(ifacePath, nil) - } else { - mainLog.Load().Warn().Err(err).Msg("Service was restarted, but ctrld process may not be ready yet") - } - } else { - mainLog.Load().Warn().Err(err).Msg("Service was restarted, but could not ping the control server") - } - mainLog.Load().Notice().Msg("Service restarted") - } else { - mainLog.Load().Error().Msg("Service restart failed") - } - }, - } - - restartCmdAlias := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "restart", - Short: "Restart the ctrld service", - Run: func(cmd *cobra.Command, args []string) { - restartCmd.Run(cmd, args) - }, - } - rootCmd.AddCommand(restartCmdAlias) - - return restartCmd -} - -func initReloadCmd(restartCmd *cobra.Command) *cobra.Command { - reloadCmd := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "reload", - Short: "Reload the ctrld service", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - - p := &prog{router: router.New(&cfg, false)} - s, _ := newService(p, svcConfig) - - status, err := s.Status() - if errors.Is(err, service.ErrNotInstalled) { - mainLog.Load().Warn().Msg("service not installed") - return - } - if status == service.StatusStopped { - mainLog.Load().Warn().Msg("service is not running") - return - } - - dir, err := socketDir() - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to find ctrld home dir") - } - cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) - resp, err := cc.post(reloadPath, nil) - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to send reload signal to ctrld") - } - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusOK: - mainLog.Load().Notice().Msg("Service reloaded") - case http.StatusCreated: - s, err := newService(&prog{}, svcConfig) - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - return - } - mainLog.Load().Warn().Msg("Service was reloaded, but new config requires service restart.") - mainLog.Load().Warn().Msg("Restarting service") - if _, err := s.Status(); errors.Is(err, service.ErrNotInstalled) { - mainLog.Load().Warn().Msg("Service not installed") - return - } - restartCmd.Run(cmd, args) - default: - buf, err := io.ReadAll(resp.Body) - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("could not read response from control server") - } - mainLog.Load().Error().Err(err).Msgf("failed to reload ctrld: %s", string(buf)) - } - }, - } - - reloadCmdAlias := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "reload", - Short: "Reload the ctrld service", - Run: func(cmd *cobra.Command, args []string) { - reloadCmd.Run(cmd, args) - }, - } - rootCmd.AddCommand(reloadCmdAlias) - - return reloadCmd -} - -func initStatusCmd() *cobra.Command { - statusCmd := &cobra.Command{ - Use: "status", - Short: "Show status of the ctrld service", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - s, err := newService(&prog{}, svcConfig) - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - return - } - status, err := s.Status() - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - os.Exit(1) - } - switch status { - case service.StatusUnknown: - mainLog.Load().Notice().Msg("Unknown status") - os.Exit(2) - case service.StatusRunning: - mainLog.Load().Notice().Msg("Service is running") - os.Exit(0) - case service.StatusStopped: - mainLog.Load().Notice().Msg("Service is stopped") - os.Exit(1) - } - }, - } - if runtime.GOOS == "darwin" { - // On darwin, running status command without privileges may return wrong information. - statusCmd.PreRun = func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - } - } - - statusCmdAlias := &cobra.Command{ - Use: "status", - Short: "Show status of the ctrld service", - Args: cobra.NoArgs, - Run: statusCmd.Run, - } - rootCmd.AddCommand(statusCmdAlias) - - return statusCmd -} - -func initUninstallCmd() *cobra.Command { - uninstallCmd := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "uninstall", - Short: "Stop and uninstall the ctrld service", - Long: `Stop and uninstall the ctrld service. - -NOTE: Uninstalling will set DNS to values provided by DHCP.`, - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - readConfig(false) - v.Unmarshal(&cfg) - p := &prog{router: router.New(&cfg, runInCdMode())} - s, err := newService(p, svcConfig) - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - return - } - if iface == "" { - iface = "auto" - } - p.preRun() - if ir := runningIface(s); ir != nil { - p.runningIface = ir.Name - p.requiredMultiNICsConfig = ir.All - } - if err := checkDeactivationPin(s, nil); isCheckDeactivationPinErr(err) { - os.Exit(deactivationPinInvalidExitCode) - } - uninstall(p, s) - if cleanup { - var files []string - // Config file. - files = append(files, v.ConfigFileUsed()) - // Log file and backup log file. - // For safety, only process if log file path is absolute. - if logFile := normalizeLogFilePath(cfg.Service.LogPath); filepath.IsAbs(logFile) { - files = append(files, logFile) - oldLogFile := logFile + oldLogSuffix - if _, err := os.Stat(oldLogFile); err == nil { - files = append(files, oldLogFile) - } - } - // Socket files. - if dir, _ := socketDir(); dir != "" { - files = append(files, filepath.Join(dir, ctrldControlUnixSock)) - files = append(files, filepath.Join(dir, ctrldLogUnixSock)) - } - // Static DNS settings files. - withEachPhysicalInterfaces("", "", func(i *net.Interface) error { - file := savedStaticDnsSettingsFilePath(i) - if _, err := os.Stat(file); err == nil { - files = append(files, file) - } - return nil - }) - // Windows forwarders file. - if hasLocalDnsServerRunning() { - files = append(files, absHomeDir(windowsForwardersFilename)) - } - // Binary itself. - bin, _ := os.Executable() - if bin != "" && supportedSelfDelete { - files = append(files, bin) - } - // Backup file after upgrading. - oldBin := bin + oldBinSuffix - if _, err := os.Stat(oldBin); err == nil { - files = append(files, oldBin) - } - for _, file := range files { - if file == "" { - continue - } - if err := os.Remove(file); err != nil { - if os.IsNotExist(err) { - continue - } - mainLog.Load().Warn().Err(err).Msgf("failed to remove file: %s", file) - } else { - mainLog.Load().Debug().Msgf("file removed: %s", file) - } - } - if err := selfDeleteExe(); err != nil { - mainLog.Load().Warn().Err(err).Msg("failed to delete ctrld binary") - } else { - if !supportedSelfDelete { - mainLog.Load().Debug().Msgf("file removed: %s", bin) - } - } - } - }, - } - uninstallCmd.Flags().StringVarP(&iface, "iface", "", "", `Reset DNS setting for iface, use "auto" for the default gateway interface`) - uninstallCmd.Flags().Int64VarP(&deactivationPin, "pin", "", defaultDeactivationPin, `Pin code for uninstalling ctrld`) - _ = uninstallCmd.Flags().MarkHidden("pin") - uninstallCmd.Flags().BoolVarP(&cleanup, "cleanup", "", false, `Removing ctrld binary and config files`) - - uninstallCmdAlias := &cobra.Command{ - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Use: "uninstall", - Short: "Stop and uninstall the ctrld service", - Long: `Stop and uninstall the ctrld service. - -NOTE: Uninstalling will set DNS to values provided by DHCP.`, - Run: func(cmd *cobra.Command, args []string) { - if !cmd.Flags().Changed("iface") { - os.Args = append(os.Args, "--iface="+ifaceStartStop) - } - iface = ifaceStartStop - uninstallCmd.Run(cmd, args) - }, - } - uninstallCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", "auto", `Reset DNS setting for iface, "auto" means the default interface gateway`) - uninstallCmdAlias.Flags().AddFlagSet(uninstallCmd.Flags()) - rootCmd.AddCommand(uninstallCmdAlias) - - return uninstallCmd -} - -func initInterfacesCmd() *cobra.Command { - listIfacesCmd := &cobra.Command{ - Use: "list", - Short: "List network interfaces of the host", - Args: cobra.NoArgs, - Run: func(cmd *cobra.Command, args []string) { - withEachPhysicalInterfaces("", "Interface list", func(i *net.Interface) error { - fmt.Printf("Index : %d\n", i.Index) - fmt.Printf("Name : %s\n", i.Name) - var status string - if i.Flags&net.FlagUp != 0 { - status = "Up" - } else { - status = "Down" - } - fmt.Printf("Status: %s\n", status) - addrs, _ := i.Addrs() - for i, ipaddr := range addrs { - if i == 0 { - fmt.Printf("Addrs : %v\n", ipaddr) - continue - } - fmt.Printf(" %v\n", ipaddr) - } - nss, err := currentStaticDNS(i) - if err != nil { - mainLog.Load().Warn().Err(err).Msg("failed to get DNS") - } - if len(nss) == 0 { - nss = currentDNS(i) - } - for i, dns := range nss { - if i == 0 { - fmt.Printf("DNS : %s\n", dns) - continue - } - fmt.Printf(" : %s\n", dns) - } - println() - return nil - }) - }, - } - interfacesCmd := &cobra.Command{ - Use: "interfaces", - Short: "Manage network interfaces", - Args: cobra.OnlyValidArgs, - ValidArgs: []string{ - listIfacesCmd.Use, - }, - } - interfacesCmd.AddCommand(listIfacesCmd) - - return interfacesCmd -} - -func initClientsCmd() *cobra.Command { - listClientsCmd := &cobra.Command{ - Use: "list", - Short: "List clients that ctrld discovered", - Args: cobra.NoArgs, - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Run: func(cmd *cobra.Command, args []string) { - - p := &prog{router: router.New(&cfg, false)} - s, _ := newService(p, svcConfig) - - status, err := s.Status() - if errors.Is(err, service.ErrNotInstalled) { - mainLog.Load().Warn().Msg("service not installed") - return - } - if status == service.StatusStopped { - mainLog.Load().Warn().Msg("service is not running") - return - } - - dir, err := socketDir() - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to find ctrld home dir") - } - cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) - resp, err := cc.post(listClientsPath, nil) - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to get clients list") - } - defer resp.Body.Close() - - var clients []*clientinfo.Client - if err := json.NewDecoder(resp.Body).Decode(&clients); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to decode clients list result") - } - map2Slice := func(m map[string]struct{}) []string { - s := make([]string, 0, len(m)) - for k := range m { - if k == "" { // skip empty source from output. - continue - } - s = append(s, k) - } - sort.Strings(s) - return s - } - // If metrics is enabled, server set this for all clients, so we can check only the first one. - // Ideally, we may have a field in response to indicate that query count should be shown, but - // it would break earlier version of ctrld, which only look list of clients in response. - withQueryCount := len(clients) > 0 && clients[0].IncludeQueryCount - data := make([][]string, len(clients)) - for i, c := range clients { - row := []string{ - c.IP.String(), - c.Hostname, - c.Mac, - strings.Join(map2Slice(c.Source), ","), - } - if withQueryCount { - row = append(row, strconv.FormatInt(c.QueryCount, 10)) - } - data[i] = row - } - table := tablewriter.NewWriter(os.Stdout) - headers := []string{"IP", "Hostname", "Mac", "Discovered"} - if withQueryCount { - headers = append(headers, "Queries") - } - table.SetHeader(headers) - table.SetAutoFormatHeaders(false) - table.AppendBulk(data) - table.Render() - }, - } - clientsCmd := &cobra.Command{ - Use: "clients", - Short: "Manage clients", - Args: cobra.OnlyValidArgs, - ValidArgs: []string{ - listClientsCmd.Use, - }, - } - clientsCmd.AddCommand(listClientsCmd) - rootCmd.AddCommand(clientsCmd) - - return clientsCmd -} - -func initUpgradeCmd() *cobra.Command { - const ( - upgradeChannelDev = "dev" - upgradeChannelProd = "prod" - upgradeChannelDefault = "default" - ) - upgradeChannel := map[string]string{ - upgradeChannelDefault: "https://dl.controld.dev", - upgradeChannelDev: "https://dl.controld.dev", - upgradeChannelProd: "https://dl.controld.com", - } - if isStableVersion(curVersion()) { - upgradeChannel[upgradeChannelDefault] = upgradeChannel[upgradeChannelProd] - } - upgradeCmd := &cobra.Command{ - Use: "upgrade", - Short: "Upgrading ctrld to latest version", - ValidArgs: []string{upgradeChannelDev, upgradeChannelProd}, - Args: cobra.MaximumNArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { - checkHasElevatedPrivilege() - }, - Run: func(cmd *cobra.Command, args []string) { - bin, err := os.Executable() - if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to get current ctrld binary path") - } - sc := &service.Config{} - *sc = *svcConfig - sc.Executable = bin - readConfig(false) - v.Unmarshal(&cfg) - p := &prog{router: router.New(&cfg, runInCdMode())} - s, err := newService(p, sc) - if err != nil { - mainLog.Load().Error().Msg(err.Error()) - return - } - if iface == "" { - iface = "auto" - } - p.preRun() - if ir := runningIface(s); ir != nil { - p.runningIface = ir.Name - p.requiredMultiNICsConfig = ir.All - } - - svcInstalled := true - if _, err := s.Status(); errors.Is(err, service.ErrNotInstalled) { - svcInstalled = false - } - oldBin := bin + oldBinSuffix - baseUrl := upgradeChannel[upgradeChannelDefault] - if len(args) > 0 { - channel := args[0] - switch channel { - case upgradeChannelProd, upgradeChannelDev: // ok - default: - mainLog.Load().Fatal().Msgf("uprade argument must be either %q or %q", upgradeChannelProd, upgradeChannelDev) - } - baseUrl = upgradeChannel[channel] - } - dlUrl := upgradeUrl(baseUrl) - mainLog.Load().Debug().Msgf("Downloading binary: %s", dlUrl) - - resp, err := getWithRetry(dlUrl, downloadServerIp) - if err != nil { - - mainLog.Load().Fatal().Err(err).Msg("failed to download binary") - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - mainLog.Load().Fatal().Msgf("could not download binary: %s", http.StatusText(resp.StatusCode)) - } - mainLog.Load().Debug().Msg("Updating current binary") - if err := selfupdate.Apply(resp.Body, selfupdate.Options{OldSavePath: oldBin}); err != nil { - if rerr := selfupdate.RollbackError(err); rerr != nil { - mainLog.Load().Error().Err(rerr).Msg("could not rollback old binary") - } - mainLog.Load().Fatal().Err(err).Msg("failed to update current binary") - } - - doRestart := func() bool { - if !svcInstalled { - return true - } - tasks := []task{ - {s.Stop, true, "Stop"}, - {func() error { - p.router.Cleanup() - // restore static DNS settings or DHCP - p.resetDNS(false, true) - return nil - }, false, "Cleanup"}, - {func() error { - time.Sleep(time.Second * 1) - return nil - }, false, "Waiting for service to stop"}, - } - if doTasks(tasks) { - - if router.WaitProcessExited() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - loop: - for { - select { - case <-ctx.Done(): - mainLog.Load().Error().Msg("timeout while waiting for service to stop") - break loop - default: - } - time.Sleep(time.Second) - if status, _ := s.Status(); status == service.StatusStopped { - break - } - } - } - } - - tasks = []task{ - {s.Start, true, "Start"}, - } - if doTasks(tasks) { - if dir, err := socketDir(); err == nil { - if cc := newSocketControlClient(context.TODO(), s, dir); cc != nil { - _, _ = cc.post(ifacePath, nil) - return true - } - } - } - return false - } - if svcInstalled { - mainLog.Load().Debug().Msg("Restarting ctrld service using new binary") - } - if doRestart() { - _ = os.Remove(oldBin) - _ = os.Chmod(bin, 0755) - ver := "unknown version" - out, err := exec.Command(bin, "--version").CombinedOutput() - if err != nil { - mainLog.Load().Warn().Err(err).Msg("Failed to get new binary version") - } - if after, found := strings.CutPrefix(string(out), "ctrld version "); found { - ver = after - } - mainLog.Load().Notice().Msgf("Upgrade successful - %s", ver) - return - } - - mainLog.Load().Warn().Msgf("Upgrade failed, restoring previous binary: %s", oldBin) - if err := os.Remove(bin); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to remove new binary") - } - if err := os.Rename(oldBin, bin); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to restore old binary") - } - if doRestart() { - mainLog.Load().Notice().Msg("Restored previous binary successfully") - return - } - }, - } - rootCmd.AddCommand(upgradeCmd) - - return upgradeCmd -} - -func initServicesCmd(commands ...*cobra.Command) *cobra.Command { - serviceCmd := &cobra.Command{ - Use: "service", - Short: "Manage ctrld service", - Args: cobra.OnlyValidArgs, - } - serviceCmd.ValidArgs = make([]string, len(commands)) - for i, cmd := range commands { - serviceCmd.ValidArgs[i] = cmd.Use - serviceCmd.AddCommand(cmd) - } - rootCmd.AddCommand(serviceCmd) - - return serviceCmd -} - -// filterEmptyStrings removes empty strings from a slice of strings. -// It returns a new slice containing only non-empty strings. -func filterEmptyStrings(slice []string) []string { - return slices.DeleteFunc(slice, func(s string) bool { - return s == "" - }) -} diff --git a/cmd/cli/commands_cleanup_test.go b/cmd/cli/commands_cleanup_test.go new file mode 100644 index 00000000..36b50abb --- /dev/null +++ b/cmd/cli/commands_cleanup_test.go @@ -0,0 +1,71 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" +) + +// writeCleanupTestFile creates a file that a cleanup test can watch. +func writeCleanupTestFile(t *testing.T, path string) string { + t.Helper() + if err := os.WriteFile(path, []byte("line\n"), 0600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + return path +} + +func assertRemoved(t *testing.T, paths []string) { + t.Helper() + for _, path := range paths { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("%s is still there: %v", path, err) + } + } +} + +func Test_removeLogFiles(t *testing.T) { + dir := t.TempDir() + logPath := writeCleanupTestFile(t, filepath.Join(dir, "syslog")) + chain := []string{ + writeCleanupTestFile(t, filepath.Join(dir, "syslog.1")), + writeCleanupTestFile(t, filepath.Join(dir, "syslog.2")), + } + // A log_path can name a file in a directory of another program, so a + // number above the configured count belongs to that program. + foreign := writeCleanupTestFile(t, filepath.Join(dir, "syslog.5")) + home := t.TempDir() + internal := []string{ + writeCleanupTestFile(t, filepath.Join(home, logFileName)), + writeCleanupTestFile(t, filepath.Join(home, journalLogFileName)), + } + internalBackups := []string{ + writeCleanupTestFile(t, filepath.Join(home, logFileName+".3")), + writeCleanupTestFile(t, filepath.Join(home, journalLogFileName+".1")), + } + + if errs := removeLogFiles(logPath, len(chain), internal); len(errs) > 0 { + t.Fatalf("removeLogFiles: %v", errs) + } + + assertRemoved(t, append([]string{logPath}, chain...)) + assertRemoved(t, append(internal, internalBackups...)) + if _, err := os.Stat(foreign); err != nil { + t.Fatalf("the cleanup removed %s, which sits above the backup count: %v", foreign, err) + } +} + +func Test_removeLogFilesWithoutALogPath(t *testing.T) { + home := t.TempDir() + internal := []string{writeCleanupTestFile(t, filepath.Join(home, logFileName))} + kept := writeCleanupTestFile(t, filepath.Join(home, "other.log")) + + if errs := removeLogFiles("", 4, internal); len(errs) > 0 { + t.Fatalf("removeLogFiles: %v", errs) + } + + assertRemoved(t, internal) + if _, err := os.Stat(kept); err != nil { + t.Fatalf("the cleanup removed %s, which is not a ctrld log: %v", kept, err) + } +} diff --git a/cmd/cli/commands_clients.go b/cmd/cli/commands_clients.go new file mode 100644 index 00000000..9f577758 --- /dev/null +++ b/cmd/cli/commands_clients.go @@ -0,0 +1,141 @@ +package cli + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/kardianos/service" + "github.com/olekukonko/tablewriter" + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld/internal/clientinfo" +) + +// ClientsCommand handles clients-related operations +type ClientsCommand struct { + controlClient *controlClient +} + +// NewClientsCommand creates a new clients command handler +func NewClientsCommand() (*ClientsCommand, error) { + dir, err := socketDir() + if err != nil { + return nil, fmt.Errorf("failed to find ctrld home dir: %w", err) + } + + cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) + return &ClientsCommand{ + controlClient: cc, + }, nil +} + +// ListClients lists all connected clients +func (cc *ClientsCommand) ListClients(cmd *cobra.Command, args []string) error { + // Check service status first + sc := NewServiceCommand() + s, _, err := sc.initializeServiceManager() + if err != nil { + return err + } + + status, err := s.Status() + if errors.Is(err, service.ErrNotInstalled) { + mainLog.Load().Warn().Msg("Service not installed") + return nil + } + if status == service.StatusStopped { + mainLog.Load().Warn().Msg("Service is not running") + return nil + } + + resp, err := cc.controlClient.post(listClientsPath, nil) + if err != nil { + return fmt.Errorf("failed to get clients: %w", err) + } + defer resp.Body.Close() + + var clients []*clientinfo.Client + if err := json.NewDecoder(resp.Body).Decode(&clients); err != nil { + return fmt.Errorf("failed to decode clients result: %w", err) + } + + map2Slice := func(m map[string]struct{}) []string { + s := make([]string, 0, len(m)) + for k := range m { + if k == "" { // skip empty source from output. + continue + } + s = append(s, k) + } + sort.Strings(s) + return s + } + + // If metrics is enabled, server set this for all clients, so we can check only the first one. + // Ideally, we may have a field in response to indicate that query count should be shown, but + // it would break earlier version of ctrld, which only look list of clients in response. + withQueryCount := len(clients) > 0 && clients[0].IncludeQueryCount + data := make([][]string, len(clients)) + for i, c := range clients { + row := []string{ + c.IP.String(), + c.Hostname, + c.Mac, + strings.Join(map2Slice(c.Source), ","), + } + if withQueryCount { + row = append(row, strconv.FormatInt(c.QueryCount, 10)) + } + data[i] = row + } + + table := tablewriter.NewWriter(os.Stdout) + headers := []string{"IP", "Hostname", "Mac", "Discovered"} + if withQueryCount { + headers = append(headers, "Queries") + } + table.SetHeader(headers) + table.SetAutoFormatHeaders(false) + table.AppendBulk(data) + table.Render() + + return nil +} + +// InitClientsCmd creates the clients command with proper logic +func InitClientsCmd(rootCmd *cobra.Command) *cobra.Command { + listClientsCmd := &cobra.Command{ + Use: "list", + Short: "List clients that ctrld discovered", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: func(cmd *cobra.Command, args []string) error { + cc, err := NewClientsCommand() + if err != nil { + return err + } + return cc.ListClients(cmd, args) + }, + } + + clientsCmd := &cobra.Command{ + Use: "clients", + Short: "Manage clients", + Args: cobra.OnlyValidArgs, + ValidArgs: []string{ + listClientsCmd.Use, + }, + } + clientsCmd.AddCommand(listClientsCmd) + rootCmd.AddCommand(clientsCmd) + + return clientsCmd +} diff --git a/cmd/cli/commands_diag.go b/cmd/cli/commands_diag.go new file mode 100644 index 00000000..b4cd17f5 --- /dev/null +++ b/cmd/cli/commands_diag.go @@ -0,0 +1,466 @@ +package cli + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + "unicode/utf8" + + "github.com/kardianos/service" + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// diag reports the facts that support asks for on every provisioning +// ticket. A customer or admin can then paste the output of one command +// instead of a hunt through logs and preference panes. It runs without +// root. A section that needs data the current user cannot read reports +// that, instead of a failure of the whole command. On an installed device +// those are the provisioning-result and the service-state sections, because +// the service wrote the file as root and the service manager answers only +// root. +const diagCmdLong = `Collect diagnostics for a provisioning failure. + +Reports the client version, MDM-managed preferences (macOS only), the last +provisioning result, service state, and whether the Control D API is +reachable. Safe to paste into a support ticket: it never prints the +provisioning token itself, only whether one is present. + +Run it with root or administrator rights on an installed device to include +the last provisioning result and the service state. Without those rights, +the two sections report permission denied.` + +// diagAPIProbeTimeout bounds the API reachability check. diag must return +// promptly even when the network is unreachable. +const diagAPIProbeTimeout = 5 * time.Second + +// diagOverallTimeout bounds the whole report. No single probe, however +// wedged, may keep "ctrld diag" from returning. +const diagOverallTimeout = 15 * time.Second + +// diagServiceStateTimeout bounds one service-state probe. systemctl or +// launchctl can hang against a wedged service manager; the kardianos +// service package gives us no way to cancel that call, so we race it +// against this timer in a goroutine instead. +const diagServiceStateTimeout = 5 * time.Second + +// diagFieldMaxLen bounds any single field pulled from outside ctrld's own +// control (a managed-preferences value), so a misconfigured profile cannot +// blow up the report's size. +const diagFieldMaxLen = 256 + +// managedPrefsDomain is the MDM-managed preferences domain ctrld reads its +// provisioning settings from. +const managedPrefsDomain = "/Library/Managed Preferences/com.controld.ctrld" + +// managedPrefsBin is invoked with an absolute path so diag never depends on +// PATH. +const managedPrefsBin = "/usr/bin/defaults" + +// managedPrefsSupported reports whether this platform has managed +// preferences to read. A var so tests can exercise the macOS-shaped report +// on any OS. +var managedPrefsSupported = func() bool { return runtime.GOOS == "darwin" } + +// managedPrefsRead runs `defaults read [key]` and returns the +// trimmed value, or ok=false if the domain or key does not exist. A var so +// tests seam it instead of shelling out for real. It honors the caller's +// ctx so a near-expired overall deadline cuts this short too. +var managedPrefsRead = func(ctx context.Context, domain, key string) (string, bool) { + args := []string{"read", domain} + if key != "" { + args = append(args, key) + } + probeCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + out, err := exec.CommandContext(probeCtx, managedPrefsBin, args...).Output() + if err != nil { + return "", false + } + return strings.TrimSpace(string(out)), true +} + +// diagProbeReachability makes the one HTTPS probe api_reachability reports +// on. A var so tests replace it instead of hitting the network. +var diagProbeReachability = controld.ProbeReachability + +type diagReport struct { + ClientVersion string `json:"client_version"` + Commit string `json:"commit"` + ManagedPreferences diagManagedPreferences `json:"managed_preferences"` + ProvisionResult diagProvisionResult `json:"provision_result"` + ServiceState diagServiceState `json:"service_state"` + APIReachability diagAPIReachability `json:"api_reachability"` +} + +type diagManagedPreferences struct { + Applicable bool `json:"applicable"` + ProfilePresent bool `json:"profile_present"` + ProvisionToken string `json:"provision_token"` // "present" or "absent"; never the value + CustomHostname string `json:"custom_hostname"` + InterceptMode string `json:"intercept_mode"` + Note string `json:"note"` +} + +// diagProvisionResult mirrors the on-disk provision_result.json for report +// purposes. AgeSeconds is -1 when no age applies (nothing recorded, or the +// timestamp did not parse). Message and Attempts are re-bounded on read: the +// file was written bounded, but diag must not trust that a file on disk +// still is (a mismatched version, or hand-edited). +type diagProvisionResult struct { + Status string `json:"status"` // none, recorded, untrusted, unreadable, corrupt + Stage string `json:"stage"` + Code string `json:"code"` + ExitCode int `json:"exit_code"` + Message string `json:"message"` + Attempts []provisionBindAttempt `json:"attempts,omitempty"` + AgeSeconds int64 `json:"age_seconds"` +} + +type diagServiceState struct { + Status string `json:"status"` // running, stopped, not_installed, unknown + Note string `json:"note"` +} + +type diagAPIReachability struct { + Reachable bool `json:"reachable"` + ErrorClass string `json:"error_class"` // empty when reachable +} + +// diagProvisionResultPath locates the result file where the service wrote +// it, with the same homedir override the writer obeys. The user home-dir +// resolver has a fallback to the home directory of the current user when +// /etc/controld is not writable, and every run without root hits that +// fallback. A diag that used it would report "none recorded" for a file it +// never opened. A var so tests can point it at a temp dir. +var diagProvisionResultPath = func() string { + if homedir != "" { + return filepath.Join(homedir, provisionResultFileName) + } + dir, err := serviceHomeDir() + if err != nil { + return provisionResultPath() + } + return filepath.Join(dir, provisionResultFileName) +} + +// diagServiceStateFn collects service_state. A var so tests can supply a +// fixed state instead of depending on whatever service happens to be +// installed on the machine running the tests. +var diagServiceStateFn = collectServiceStateReal + +// buildDiagReport bounds the whole report at diagOverallTimeout: whatever +// deadline the caller passed in, a single probe wedging past this must not +// keep "ctrld diag" from returning. +func buildDiagReport(ctx context.Context) diagReport { + ctx, cancel := context.WithTimeout(ctx, diagOverallTimeout) + defer cancel() + return diagReport{ + ClientVersion: appVersion, + Commit: commit, + ManagedPreferences: collectManagedPreferences(ctx), + ProvisionResult: collectProvisionResultDiag(), + ServiceState: collectServiceStateBounded(ctx), + APIReachability: collectAPIReachability(ctx), + } +} + +func collectManagedPreferences(ctx context.Context) diagManagedPreferences { + if !managedPrefsSupported() { + return diagManagedPreferences{Note: "not applicable on this platform"} + } + m := diagManagedPreferences{Applicable: true} + if _, ok := managedPrefsRead(ctx, managedPrefsDomain, ""); !ok { + m.Note = "configuration profile not found" + return m + } + m.ProfilePresent = true + // An empty value reads as absent: the postinstall refuses to provision + // on an empty ProvisionToken, so "present" would send support the wrong + // way. + if v, ok := managedPrefsRead(ctx, managedPrefsDomain, "ProvisionToken"); ok && v != "" { + m.ProvisionToken = "present" + } else { + m.ProvisionToken = "absent" + } + if v, ok := managedPrefsRead(ctx, managedPrefsDomain, "CustomHostname"); ok { + m.CustomHostname = boundedDiagField(v) + } + if v, ok := managedPrefsRead(ctx, managedPrefsDomain, "InterceptMode"); ok { + m.InterceptMode = boundedDiagField(v) + } + return m +} + +// boundedDiagField caps a value from outside ctrld's control to a size that +// keeps the report bounded, cutting on a rune boundary so it never splits a +// multi-byte character. +func boundedDiagField(s string) string { + if utf8.RuneCountInString(s) <= diagFieldMaxLen { + return s + } + runes := []rune(s) + return string(runes[:diagFieldMaxLen]) +} + +// collectProvisionResultDiag reads provision_result.json from the service +// home through the same trusted reader "ctrld start" uses, so diag never +// reports a code that contract validation would reject. A read the current +// user is not permitted to make is its own status: the file is not +// corrupt, the reader lacks root. +func collectProvisionResultDiag() diagProvisionResult { + r, err := readProvisionResultAt(diagProvisionResultPath()) + if err != nil { + if os.IsNotExist(err) { + return diagProvisionResult{Status: "none", AgeSeconds: -1} + } + if os.IsPermission(err) { + return diagProvisionResult{Status: "unreadable", AgeSeconds: -1} + } + return diagProvisionResult{Status: "corrupt", AgeSeconds: -1} + } + if !provisionResultTrusted(r) { + return diagProvisionResult{Status: "untrusted", AgeSeconds: -1} + } + age := int64(-1) + if ts, parseErr := time.Parse(time.RFC3339, r.Timestamp); parseErr == nil { + if d := time.Since(ts); d >= 0 { + age = int64(d.Round(time.Second).Seconds()) + } else { + age = 0 + } + } + var attempts []provisionBindAttempt + if r.Detail != nil { + attempts = boundedDiagAttempts(r.Detail.Attempts) + } + return diagProvisionResult{ + Status: "recorded", + Stage: r.Stage, + Code: r.Code, + ExitCode: r.ExitCode, + Message: boundedDiagField(r.Message), + Attempts: attempts, + AgeSeconds: age, + } +} + +// boundedDiagAttempts re-applies the same attempt-count cap the file was +// written with, and bounds each attempt's fields, so a result file from a +// mismatched or tampered version cannot make the report unbounded. +func boundedDiagAttempts(attempts []provisionBindAttempt) []provisionBindAttempt { + if len(attempts) > maxProvisionBindAttempts { + attempts = attempts[:maxProvisionBindAttempts] + } + bounded := make([]provisionBindAttempt, len(attempts)) + for i, a := range attempts { + bounded[i] = provisionBindAttempt{ + Addr: boundedDiagField(a.Addr), + Proto: boundedDiagField(a.Proto), + OSError: boundedDiagField(a.OSError), + } + } + return bounded +} + +// collectServiceStateReal is diagServiceStateFn's production implementation. +// It reuses the same service-manager wrapper "ctrld status" does, so a +// permission-limited run reports "requires elevated privileges" rather than +// a wrong status (see the launchd wrapper in service.go). +func collectServiceStateReal() diagServiceState { + sc := NewServiceCommand() + s, _, err := sc.initializeServiceManager() + if err != nil { + return diagServiceState{Status: "unknown", Note: "could not set up the service manager"} + } + status, statusErr := s.Status() + switch { + case errors.Is(statusErr, service.ErrNotInstalled): + return diagServiceState{Status: "not_installed"} + case statusErr != nil: + return diagServiceState{Status: "unknown", Note: boundedDiagField(statusErr.Error())} + case status == service.StatusRunning: + return diagServiceState{Status: "running"} + case status == service.StatusStopped: + return diagServiceState{Status: "stopped"} + default: + return diagServiceState{Status: "unknown"} + } +} + +// collectServiceStateBounded runs diagServiceStateFn in the background and +// races it against ctx and diagServiceStateTimeout, so a wedged service +// manager reports "timed out" instead of hanging the whole diag report. The +// goroutine is left running if the probe never returns; that is harmless +// since the process exits shortly after diag prints its report. +func collectServiceStateBounded(ctx context.Context) diagServiceState { + done := make(chan diagServiceState, 1) + go func() { done <- diagServiceStateFn() }() + + timer := time.NewTimer(diagServiceStateTimeout) + defer timer.Stop() + + select { + case s := <-done: + return s + case <-ctx.Done(): + return diagServiceState{Status: "unknown", Note: "timed out"} + case <-timer.C: + return diagServiceState{Status: "unknown", Note: "timed out"} + } +} + +func collectAPIReachability(ctx context.Context) diagAPIReachability { + probeCtx, cancel := context.WithTimeout(ctx, diagAPIProbeTimeout) + defer cancel() + if err := diagProbeReachability(probeCtx, cdDev); err != nil { + return diagAPIReachability{Reachable: false, ErrorClass: classifyReachabilityError(err)} + } + return diagAPIReachability{Reachable: true} +} + +// classifyReachabilityError turns a probe failure into a coarse class safe +// to print: no host, no address, no request details, just what kind of +// failure it was. +func classifyReachabilityError(err error) string { + if err == nil { + return "" + } + if errors.Is(err, context.DeadlineExceeded) { + return "timeout" + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return "timeout" + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return "dns" + } + var certErr *tls.CertificateVerificationError + if errors.As(err, &certErr) { + return "tls" + } + var opErr *net.OpError + if errors.As(err, &opErr) { + return "connection" + } + return "other" +} + +// diagElevateHint names the step that gets root or administrator rights on +// this OS, for the text report. +func diagElevateHint() string { + if runtime.GOOS == "windows" { + return "run again from an administrator prompt" + } + return "run again with sudo" +} + +func displayOrDefault(s, def string) string { + if s == "" { + return def + } + return s +} + +// renderDiagText writes the text-mode report in a fixed section order, so +// output stays stable across runs and safe to diff or paste into a ticket. +func renderDiagText(w io.Writer, r diagReport) { + fmt.Fprintf(w, "client version: %s (commit %s)\n\n", r.ClientVersion, r.Commit) + + fmt.Fprintln(w, "managed preferences:") + switch { + case !r.ManagedPreferences.Applicable: + fmt.Fprintf(w, " %s\n", displayOrDefault(r.ManagedPreferences.Note, "not applicable")) + case !r.ManagedPreferences.ProfilePresent: + fmt.Fprintf(w, " %s\n", r.ManagedPreferences.Note) + default: + fmt.Fprintf(w, " provision token: %s\n", r.ManagedPreferences.ProvisionToken) + fmt.Fprintf(w, " custom hostname: %s\n", displayOrDefault(r.ManagedPreferences.CustomHostname, "(not set)")) + fmt.Fprintf(w, " intercept mode: %s\n", displayOrDefault(r.ManagedPreferences.InterceptMode, "(not set)")) + } + fmt.Fprintln(w) + + fmt.Fprintln(w, "last provisioning result:") + switch r.ProvisionResult.Status { + case "none": + fmt.Fprintln(w, " none recorded") + case "untrusted": + fmt.Fprintln(w, " result file present but not trusted (contents ignored)") + case "unreadable": + fmt.Fprintf(w, " permission denied (%s)\n", diagElevateHint()) + case "corrupt": + fmt.Fprintln(w, " result file present but could not be read (contents ignored)") + default: + fmt.Fprintf(w, " stage: %s\n", r.ProvisionResult.Stage) + fmt.Fprintf(w, " code: %s\n", r.ProvisionResult.Code) + fmt.Fprintf(w, " exit code: %d\n", r.ProvisionResult.ExitCode) + fmt.Fprintf(w, " message: %s\n", r.ProvisionResult.Message) + for _, a := range r.ProvisionResult.Attempts { + fmt.Fprintf(w, " attempt: %s/%s: %s\n", a.Addr, a.Proto, a.OSError) + } + fmt.Fprintf(w, " age: %s\n", (time.Duration(r.ProvisionResult.AgeSeconds) * time.Second).String()) + } + fmt.Fprintln(w) + + fmt.Fprintln(w, "service state:") + fmt.Fprintf(w, " status: %s\n", r.ServiceState.Status) + if r.ServiceState.Note != "" { + fmt.Fprintf(w, " note: %s\n", r.ServiceState.Note) + } + fmt.Fprintln(w) + + fmt.Fprintln(w, "api reachability:") + fmt.Fprintf(w, " reachable: %t\n", r.APIReachability.Reachable) + if r.APIReachability.ErrorClass != "" { + fmt.Fprintf(w, " error class: %s\n", r.APIReachability.ErrorClass) + } +} + +func writeDiagJSON(w io.Writer, r diagReport) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(r) +} + +// InitDiagCmd registers "ctrld diag" alongside the other top-level commands. +func InitDiagCmd(rootCmd *cobra.Command) *cobra.Command { + var asJSON bool + diagCmd := &cobra.Command{ + Use: "diag", + Short: "Collect diagnostics for a provisioning failure", + Long: diagCmdLong, + Args: cobra.NoArgs, + // diag always exits 0 once it ran: a failure it finds is reported, + // not turned into a nonzero exit. RunE returning an error would exit + // 1 (see Main), so every branch below reports instead of erroring. + RunE: func(cmd *cobra.Command, args []string) error { + report := buildDiagReport(context.Background()) + if asJSON { + if err := writeDiagJSON(cmd.OutOrStdout(), report); err != nil { + // A closed pipe (e.g. `ctrld diag --json | head -1`) must + // not turn into a nonzero exit; the report already ran. + mainLog.Load().Debug().Err(err).Msg("could not write diag JSON report") + } + return nil + } + renderDiagText(cmd.OutOrStdout(), report) + return nil + }, + } + diagCmd.Flags().BoolVar(&asJSON, "json", false, "print the report as JSON") + rootCmd.AddCommand(diagCmd) + return diagCmd +} diff --git a/cmd/cli/commands_diag_test.go b/cmd/cli/commands_diag_test.go new file mode 100644 index 00000000..baf6a449 --- /dev/null +++ b/cmd/cli/commands_diag_test.go @@ -0,0 +1,429 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/spf13/cobra" +) + +// fakeProvisionToken stands in for a real provisioning code. Tests assert it +// never reaches either output mode. +const fakeProvisionToken = "org-v1-FAKE00000000000000000000TOKEN" + +func withManagedPrefsSeam(t *testing.T, supported bool, values map[string]string) { + t.Helper() + oldSupported := managedPrefsSupported + oldRead := managedPrefsRead + managedPrefsSupported = func() bool { return supported } + managedPrefsRead = func(_ context.Context, _, key string) (string, bool) { + v, ok := values[key] + return v, ok + } + t.Cleanup(func() { + managedPrefsSupported = oldSupported + managedPrefsRead = oldRead + }) +} + +// overrideDiagProvisionResultPath points the writer and diag at one temp +// file, so a test can write a result and read it back through diag. +func overrideDiagProvisionResultPath(t *testing.T) string { + t.Helper() + path := overrideProvisionResultPath(t) + old := diagProvisionResultPath + diagProvisionResultPath = func() string { return path } + t.Cleanup(func() { diagProvisionResultPath = old }) + return path +} + +func withServiceStateSeam(t *testing.T, state diagServiceState) { + t.Helper() + old := diagServiceStateFn + diagServiceStateFn = func() diagServiceState { return state } + t.Cleanup(func() { diagServiceStateFn = old }) +} + +func withAPIProbeSeam(t *testing.T, err error) { + t.Helper() + old := diagProbeReachability + diagProbeReachability = func(context.Context, bool) error { return err } + t.Cleanup(func() { diagProbeReachability = old }) +} + +func seedTrustedProvisionResult(t *testing.T, age time.Duration) { + t.Helper() + overrideDiagProvisionResultPath(t) + r := newProvisionResult(provisionCodeTokenExpired, "the provisioning code has expired", nil) + r.Timestamp = time.Now().Add(-age).UTC().Format(time.RFC3339) + if err := writeProvisionResult(r); err != nil { + t.Fatal(err) + } +} + +// seedOversizedProvisionResult writes a trusted result file straight to disk +// (bypassing newProvisionResult's own bounding), standing in for a file left +// by a mismatched or tampered version of ctrld. +func seedOversizedProvisionResult(t *testing.T) { + t.Helper() + overrideDiagProvisionResultPath(t) + attempts := make([]provisionBindAttempt, maxProvisionBindAttempts*3) + for i := range attempts { + attempts[i] = provisionBindAttempt{ + Addr: "0.0.0.0:53", + Proto: "udp", + OSError: strings.Repeat("e", diagFieldMaxLen*3), + } + } + r := &provisionResult{ + Version: 1, + Timestamp: time.Now().UTC().Format(time.RFC3339), + Stage: string(provisionStageListener), + Code: string(provisionCodeListenerBindFailed), + ExitCode: provisionExitCodeForCode[provisionCodeListenerBindFailed], + Message: strings.Repeat("m", diagFieldMaxLen*3), + Detail: &provisionDetail{Attempts: attempts}, + } + if err := writeProvisionResult(r); err != nil { + t.Fatal(err) + } +} + +func TestDiagProvisionResultBoundsOversizedFields(t *testing.T) { + seedOversizedProvisionResult(t) + + got := collectProvisionResultDiag() + + if n := utf8.RuneCountInString(got.Message); n > diagFieldMaxLen { + t.Errorf("message length = %d, want <= %d", n, diagFieldMaxLen) + } + if len(got.Attempts) > maxProvisionBindAttempts { + t.Errorf("attempts length = %d, want <= %d", len(got.Attempts), maxProvisionBindAttempts) + } + for _, a := range got.Attempts { + if n := utf8.RuneCountInString(a.OSError); n > diagFieldMaxLen { + t.Errorf("attempt os_error length = %d, want <= %d", n, diagFieldMaxLen) + } + } +} + +func TestDiagTextReportNeverLeaksToken(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{ + "": "", // domain probe: profile present + "ProvisionToken": fakeProvisionToken, + "CustomHostname": "corp-laptop.example.com", + "InterceptMode": "intercept-dns", + }) + seedTrustedProvisionResult(t, 3*time.Minute+12*time.Second) + withServiceStateSeam(t, diagServiceState{Status: "stopped"}) + withAPIProbeSeam(t, nil) + + report := buildDiagReport(context.Background()) + var buf bytes.Buffer + renderDiagText(&buf, report) + out := buf.String() + + if strings.Contains(out, fakeProvisionToken) { + t.Fatalf("text output leaked the provision token: %s", out) + } + wantLines := []string{ + "provision token: present", + "custom hostname: corp-laptop.example.com", + "intercept mode: intercept-dns", + "stage: bootstrap", + "code: TOKEN_EXPIRED", + "exit code: 34", + "status: stopped", + "reachable: true", + } + for _, want := range wantLines { + if !strings.Contains(out, want) { + t.Errorf("text output missing %q, got:\n%s", want, out) + } + } +} + +func TestDiagJSONReportNeverLeaksToken(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{ + "": "", + "ProvisionToken": fakeProvisionToken, + "CustomHostname": "corp-laptop.example.com", + "InterceptMode": "standard", + }) + seedTrustedProvisionResult(t, time.Minute) + withServiceStateSeam(t, diagServiceState{Status: "running"}) + withAPIProbeSeam(t, errors.New("dial tcp: connect: connection refused")) + + report := buildDiagReport(context.Background()) + var buf bytes.Buffer + if err := writeDiagJSON(&buf, report); err != nil { + t.Fatal(err) + } + out := buf.String() + if strings.Contains(out, fakeProvisionToken) { + t.Fatalf("JSON output leaked the provision token: %s", out) + } + + var decoded diagReport + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("could not decode JSON report: %v", err) + } + if decoded.ManagedPreferences.ProvisionToken != "present" { + t.Errorf("provision_token = %q, want present", decoded.ManagedPreferences.ProvisionToken) + } + if decoded.ManagedPreferences.CustomHostname != "corp-laptop.example.com" { + t.Errorf("custom_hostname = %q", decoded.ManagedPreferences.CustomHostname) + } + if decoded.ProvisionResult.Status != "recorded" || decoded.ProvisionResult.Code != "TOKEN_EXPIRED" { + t.Errorf("provision_result = %+v", decoded.ProvisionResult) + } + if decoded.ServiceState.Status != "running" { + t.Errorf("service_state = %+v", decoded.ServiceState) + } + if decoded.APIReachability.Reachable { + t.Error("api_reachability.reachable = true, want false") + } + if decoded.APIReachability.ErrorClass == "" { + t.Error("api_reachability.error_class empty for an unreachable API") + } +} + +func TestDiagEmptyMachine(t *testing.T) { + overrideDiagProvisionResultPath(t) // temp dir, no result file written + withManagedPrefsSeam(t, false, nil) + withServiceStateSeam(t, diagServiceState{Status: "not_installed"}) + withAPIProbeSeam(t, context.DeadlineExceeded) + + report := buildDiagReport(context.Background()) + + if report.ManagedPreferences.Applicable { + t.Error("managed preferences reported applicable with no profile on this platform") + } + if report.ProvisionResult.Status != "none" { + t.Errorf("provision result status = %q, want none", report.ProvisionResult.Status) + } + if report.ServiceState.Status != "not_installed" { + t.Errorf("service state = %q, want not_installed", report.ServiceState.Status) + } + if report.APIReachability.Reachable { + t.Error("api reachability reported reachable with a forced timeout") + } + if report.APIReachability.ErrorClass != "timeout" { + t.Errorf("error class = %q, want timeout", report.APIReachability.ErrorClass) + } + + var buf bytes.Buffer + renderDiagText(&buf, report) + if !strings.Contains(buf.String(), "none recorded") { + t.Errorf("text output missing 'none recorded': %s", buf.String()) + } +} + +// Without root, diag must look where the root-run service wrote the result +// file, not in the home directory of the current user. +func TestDiagProvisionResultPathIgnoresUserHome(t *testing.T) { + want := "/etc/controld/" + provisionResultFileName + if runtime.GOOS == "windows" { + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + want = filepath.Join(filepath.Dir(exe), provisionResultFileName) + } + if got := diagProvisionResultPath(); got != want { + t.Errorf("diag provision result path = %q, want %q", got, want) + } +} + +// Diag must read where a daemon started with --homedir wrote, so it obeys +// the same override as the writer. +func TestDiagProvisionResultPathHonorsHomedir(t *testing.T) { + old := homedir + homedir = t.TempDir() + t.Cleanup(func() { homedir = old }) + want := filepath.Join(homedir, provisionResultFileName) + if got := diagProvisionResultPath(); got != want { + t.Errorf("diag provision result path = %q, want %q", got, want) + } +} + +// A result file the current user cannot read must report that, not +// "corrupt": the file is fine, the reader lacks root. +func TestDiagProvisionResultUnreadable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("file modes do not deny reads on Windows") + } + if os.Geteuid() == 0 { + t.Skip("root can read a 0000 file") + } + path := overrideDiagProvisionResultPath(t) + if err := os.WriteFile(path, []byte("{}"), 0o000); err != nil { + t.Fatal(err) + } + + r := collectProvisionResultDiag() + if r.Status != "unreadable" { + t.Errorf("status = %q, want unreadable", r.Status) + } + if r.AgeSeconds != -1 { + t.Errorf("age_seconds = %d, want -1", r.AgeSeconds) + } + + var buf bytes.Buffer + renderDiagText(&buf, diagReport{ProvisionResult: r}) + if !strings.Contains(buf.String(), "permission denied ("+diagElevateHint()+")") { + t.Errorf("text output does not name permission denied and the elevation step: %s", buf.String()) + } +} + +func TestDiagServiceStateHangYieldsTimeout(t *testing.T) { + old := diagServiceStateFn + // started closes the instant the background probe goroutine reads and + // invokes our stub. Cleanup waits for that before restoring the global: + // otherwise a slow-to-schedule goroutine can still be reading + // diagServiceStateFn when Cleanup writes to it, a data race on the shared + // package var (this test's stub is left running past the test's own + // return, same as production - see collectServiceStateBounded's doc). + started := make(chan struct{}) + diagServiceStateFn = func() diagServiceState { + close(started) + time.Sleep(2 * time.Second) // stand in for a wedged systemctl/launchctl + return diagServiceState{Status: "running"} + } + t.Cleanup(func() { + <-started + diagServiceStateFn = old + }) + + withManagedPrefsSeam(t, false, nil) + overrideDiagProvisionResultPath(t) + withAPIProbeSeam(t, nil) + + // A short deadline stands in for the overall 15s budget already having + // run low; the probe must still yield within it instead of hanging. + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + + start := time.Now() + report := buildDiagReport(ctx) + elapsed := time.Since(start) + + if elapsed > time.Second { + t.Fatalf("buildDiagReport took %s, want bounded well under the 2s hang", elapsed) + } + if report.ServiceState.Status != "unknown" { + t.Errorf("service state status = %q, want unknown", report.ServiceState.Status) + } + if !strings.Contains(report.ServiceState.Note, "timed out") { + t.Errorf("service state note = %q, want it to mention timing out", report.ServiceState.Note) + } +} + +func TestDiagManagedPrefsProfileAbsent(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{}) // domain read fails: profile absent + m := collectManagedPreferences(context.Background()) + if m.ProfilePresent { + t.Error("profile reported present when the domain read failed") + } + if m.Note == "" { + t.Error("expected a note explaining the absent profile") + } +} + +func TestDiagManagedPrefsTokenAbsent(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{"": ""}) // profile present, no keys set + m := collectManagedPreferences(context.Background()) + if !m.ProfilePresent { + t.Fatal("profile should be present") + } + if m.ProvisionToken != "absent" { + t.Errorf("provision token = %q, want absent", m.ProvisionToken) + } +} + +// An empty ProvisionToken value must read as absent: the postinstall refuses +// to provision on an empty token, so diag must not call it present. +func TestDiagManagedPrefsTokenEmpty(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{"": "", "ProvisionToken": ""}) + m := collectManagedPreferences(context.Background()) + if !m.ProfilePresent { + t.Fatal("profile should be present") + } + if m.ProvisionToken != "absent" { + t.Errorf("provision token = %q, want absent", m.ProvisionToken) + } +} + +func TestClassifyReachabilityError(t *testing.T) { + if got := classifyReachabilityError(nil); got != "" { + t.Errorf("nil error class = %q, want empty", got) + } + if got := classifyReachabilityError(context.DeadlineExceeded); got != "timeout" { + t.Errorf("deadline exceeded class = %q, want timeout", got) + } +} + +func TestDiagCommandJSONFlag(t *testing.T) { + withManagedPrefsSeam(t, false, nil) + overrideDiagProvisionResultPath(t) + withServiceStateSeam(t, diagServiceState{Status: "not_installed"}) + withAPIProbeSeam(t, nil) + + rootCmd := &cobra.Command{Use: "ctrld"} + InitDiagCmd(rootCmd) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetArgs([]string{"diag", "--json"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("diag --json returned error: %v", err) + } + var decoded diagReport + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("diag --json did not print valid JSON: %v\n%s", err, buf.String()) + } +} + +// writerFailingAfter accepts the first n bytes written to it, then fails +// every write after that - standing in for a pipe closed by a downstream +// reader (`ctrld diag --json | head -1`). +type writerFailingAfter struct { + n int + written int +} + +func (w *writerFailingAfter) Write(p []byte) (int, error) { + if w.written >= w.n { + return 0, errors.New("write: broken pipe") + } + remaining := w.n - w.written + if len(p) > remaining { + w.written += remaining + return remaining, errors.New("write: broken pipe") + } + w.written += len(p) + return len(p), nil +} + +func TestDiagJSONWriteErrorStillExitsZero(t *testing.T) { + withManagedPrefsSeam(t, false, nil) + overrideDiagProvisionResultPath(t) + withServiceStateSeam(t, diagServiceState{Status: "not_installed"}) + withAPIProbeSeam(t, nil) + + rootCmd := &cobra.Command{Use: "ctrld"} + InitDiagCmd(rootCmd) + rootCmd.SetOut(&writerFailingAfter{n: 10}) + rootCmd.SetArgs([]string{"diag", "--json"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("diag --json with a failing writer returned error %v, want nil per the always-exit-0 contract", err) + } +} diff --git a/cmd/cli/commands_interfaces.go b/cmd/cli/commands_interfaces.go new file mode 100644 index 00000000..e4565725 --- /dev/null +++ b/cmd/cli/commands_interfaces.go @@ -0,0 +1,87 @@ +package cli + +import ( + "fmt" + "net" + + "github.com/spf13/cobra" +) + +// InterfacesCommand handles interfaces-related operations +type InterfacesCommand struct{} + +// NewInterfacesCommand creates a new interfaces command handler +func NewInterfacesCommand() (*InterfacesCommand, error) { + return &InterfacesCommand{}, nil +} + +// ListInterfaces lists all network interfaces +func (ic *InterfacesCommand) ListInterfaces(cmd *cobra.Command, args []string) error { + withEachPhysicalInterfaces("", "Interface list", func(i *net.Interface) error { + fmt.Printf("Index : %d\n", i.Index) + fmt.Printf("Name : %s\n", i.Name) + var status string + if i.Flags&net.FlagUp != 0 { + status = "Up" + } else { + status = "Down" + } + fmt.Printf("Status: %s\n", status) + addrs, _ := i.Addrs() + for i, ipaddr := range addrs { + if i == 0 { + fmt.Printf("Addrs : %v\n", ipaddr) + continue + } + fmt.Printf(" %v\n", ipaddr) + } + nss, err := currentStaticDNS(i) + if err != nil { + mainLog.Load().Warn().Err(err).Msg("Failed to get DNS") + } + if len(nss) == 0 { + nss = currentDNS(i) + } + for i, dns := range nss { + if i == 0 { + fmt.Printf("DNS : %s\n", dns) + continue + } + fmt.Printf(" : %s\n", dns) + } + println() + return nil + }) + return nil +} + +// InitInterfacesCmd creates the interfaces command with proper logic +func InitInterfacesCmd(_ *cobra.Command) *cobra.Command { + listInterfacesCmd := &cobra.Command{ + Use: "list", + Short: "List network interfaces", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: func(cmd *cobra.Command, args []string) error { + ic, err := NewInterfacesCommand() + if err != nil { + return err + } + return ic.ListInterfaces(cmd, args) + }, + } + + interfacesCmd := &cobra.Command{ + Use: "interfaces", + Short: "Manage network interfaces", + Args: cobra.OnlyValidArgs, + ValidArgs: []string{ + listInterfacesCmd.Use, + }, + } + interfacesCmd.AddCommand(listInterfacesCmd) + + return interfacesCmd +} diff --git a/cmd/cli/commands_log.go b/cmd/cli/commands_log.go new file mode 100644 index 00000000..56c38b79 --- /dev/null +++ b/cmd/cli/commands_log.go @@ -0,0 +1,295 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + + "github.com/docker/go-units" + "github.com/kardianos/service" + "github.com/spf13/cobra" +) + +// LogCommand handles log-related operations +type LogCommand struct { + controlClient *controlClient +} + +// NewLogCommand creates a new log command handler +func NewLogCommand() (*LogCommand, error) { + dir, err := socketDir() + if err != nil { + return nil, fmt.Errorf("failed to find ctrld home dir: %w", err) + } + + cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) + return &LogCommand{ + controlClient: cc, + }, nil +} + +// logRequestPath adds the query that makes the control server read every log +// file, not only the newest debug bytes. +func logRequestPath(path string, full bool) string { + if !full { + return path + } + return path + "?full=1" +} + +// warnRuntimeLoggingNotEnabled logs a warning about runtime logging not being enabled +func (lc *LogCommand) warnRuntimeLoggingNotEnabled() { + mainLog.Load().Warn().Msg("Runtime debug logging is not enabled") + mainLog.Load().Warn().Msg(`ctrld may be running without "--cd" flag or logging is already enabled`) +} + +// SendLogs sends runtime debug logs to ControlD +func (lc *LogCommand) SendLogs(cmd *cobra.Command, args []string) error { + sc := NewServiceCommand() + s, _, err := sc.initializeServiceManager() + if err != nil { + return err + } + + status, err := s.Status() + if errors.Is(err, service.ErrNotInstalled) { + mainLog.Load().Warn().Msg("Service not installed") + return nil + } + if status == service.StatusStopped { + mainLog.Load().Warn().Msg("Service is not running") + return nil + } + + full, _ := cmd.Flags().GetBool("full") + resp, err := lc.controlClient.post(logRequestPath(sendLogsPath, full), nil) + if err != nil { + return fmt.Errorf("failed to send logs: %w", err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusServiceUnavailable: + mainLog.Load().Warn().Msg("Runtime logs could only be sent once per minute") + return nil + case http.StatusMovedPermanently: + lc.warnRuntimeLoggingNotEnabled() + return nil + } + + var logs logSentResponse + if err := json.NewDecoder(resp.Body).Decode(&logs); err != nil { + return fmt.Errorf("failed to decode sent logs result: %w", err) + } + + if logs.Error != "" { + return fmt.Errorf("failed to send logs: %s", logs.Error) + } + + mainLog.Load().Notice().Msgf("Sent %s of runtime logs", units.BytesSize(float64(logs.Size))) + return nil +} + +// ViewLogs views current runtime debug logs +func (lc *LogCommand) ViewLogs(cmd *cobra.Command, args []string) error { + sc := NewServiceCommand() + s, _, err := sc.initializeServiceManager() + if err != nil { + return err + } + + status, err := s.Status() + if errors.Is(err, service.ErrNotInstalled) { + mainLog.Load().Warn().Msg("Service not installed") + return nil + } + if status == service.StatusStopped { + mainLog.Load().Warn().Msg("Service is not running") + return nil + } + + full, _ := cmd.Flags().GetBool("full") + resp, err := lc.controlClient.post(logRequestPath(viewLogsPath, full), nil) + if err != nil { + return fmt.Errorf("failed to get logs: %w", err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusMovedPermanently: + lc.warnRuntimeLoggingNotEnabled() + return nil + case http.StatusBadRequest: + mainLog.Load().Warn().Msg("Runtime debug logs are not available") + buf, err := io.ReadAll(resp.Body) + if err != nil { + mainLog.Load().Fatal().Err(err).Msg("Failed to read response body") + } + mainLog.Load().Warn().Msgf("ctrld process response:\n\n%s\n", string(buf)) + return nil + case http.StatusOK: + } + + var logs logViewResponse + if err := json.NewDecoder(resp.Body).Decode(&logs); err != nil { + return fmt.Errorf("failed to decode view logs result: %w", err) + } + + fmt.Print(logs.Data) + return nil +} + +// TailLogs streams live runtime debug logs to the terminal +func (lc *LogCommand) TailLogs(cmd *cobra.Command, args []string) error { + sc := NewServiceCommand() + s, _, err := sc.initializeServiceManager() + if err != nil { + return err + } + + status, err := s.Status() + if errors.Is(err, service.ErrNotInstalled) { + mainLog.Load().Warn().Msg("Service not installed") + return nil + } + if status == service.StatusStopped { + mainLog.Load().Warn().Msg("Service is not running") + return nil + } + + tailLines, _ := cmd.Flags().GetInt("lines") + tailPath := fmt.Sprintf("%s?lines=%d", tailLogsPath, tailLines) + resp, err := lc.controlClient.postStream(tailPath, nil) + if err != nil { + return fmt.Errorf("failed to connect for log tailing: %w", err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusMovedPermanently: + lc.warnRuntimeLoggingNotEnabled() + return nil + case http.StatusOK: + default: + return fmt.Errorf("unexpected response status: %d", resp.StatusCode) + } + + // Set up signal handling for clean shutdown. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + done := make(chan struct{}) + go func() { + defer close(done) + // Stream output to stdout. + buf := make([]byte, 4096) + for { + n, readErr := resp.Body.Read(buf) + if n > 0 { + os.Stdout.Write(buf[:n]) + } + if readErr != nil { + if readErr != io.EOF { + mainLog.Load().Error().Err(readErr).Msg("Error reading log stream") + } + return + } + } + }() + + select { + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + msg := fmt.Sprintf("\nexiting: %s\n", context.Cause(ctx).Error()) + os.Stdout.WriteString(msg) + } + case <-done: + } + + return nil +} + +// InitLogCmd creates the log command with proper logic +func InitLogCmd(rootCmd *cobra.Command) *cobra.Command { + lc, err := NewLogCommand() + if err != nil { + panic(fmt.Sprintf("failed to create log command: %v", err)) + } + + logSendCmd := &cobra.Command{ + Use: "send", + Short: "Send runtime debug logs to ControlD", + Long: "Send runtime debug logs to ControlD. On macOS, an administrator can enable service.allow_unprivileged_log_send and restart ctrld to allow standard users to send bounded diagnostics. --full still requires elevation.", + Args: cobra.NoArgs, + PreRunE: func(cmd *cobra.Command, args []string) error { + if delegatedLogSendCLI() { + if full, _ := cmd.Flags().GetBool("full"); full { + return errors.New("log send --full requires administrator privileges") + } + return nil + } + checkHasElevatedPrivilege() + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + // The standard-user macOS route must precede service.Status and the + // administrative socket dial: both depend on the calling user's + // launchd context. socketDir already ran when InitLogCmd built the + // command; for a standard user it falls back to the home directory + // without error, so it is harmless here. + if delegatedLogSendCLI() { + full, _ := cmd.Flags().GetBool("full") + return runDelegatedLogSend(cmd.Context(), full) + } + return lc.SendLogs(cmd, args) + }, + } + logSendCmd.Flags().Bool("full", false, "Send every log file, not only the newest 10 MB of debug") + + logViewCmd := &cobra.Command{ + Use: "view", + Short: "View current runtime debug logs", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: lc.ViewLogs, + } + logViewCmd.Flags().Bool("full", false, "Show every log file, not only the newest 10 MB of debug") + + logTailCmd := &cobra.Command{ + Use: "tail", + Short: "Tail live runtime debug logs", + Long: "Stream live runtime debug logs to the terminal, similar to tail -f. Press Ctrl+C to stop.", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: lc.TailLogs, + } + logTailCmd.Flags().IntP("lines", "n", 10, "Number of historical lines to show on connect") + + logCmd := &cobra.Command{ + Use: "log", + Short: "Manage runtime debug logs", + Args: cobra.OnlyValidArgs, + ValidArgs: []string{ + logSendCmd.Use, + logViewCmd.Use, + logTailCmd.Use, + }, + } + logCmd.AddCommand(logSendCmd) + logCmd.AddCommand(logViewCmd) + logCmd.AddCommand(logTailCmd) + rootCmd.AddCommand(logCmd) + + return logCmd +} diff --git a/cmd/cli/commands_run.go b/cmd/cli/commands_run.go new file mode 100644 index 00000000..5b1ec2ed --- /dev/null +++ b/cmd/cli/commands_run.go @@ -0,0 +1,68 @@ +package cli + +import ( + "strings" + + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld" +) + +// RunCommand handles run-related operations +type RunCommand struct { + // Add any dependencies here if needed in the future +} + +// NewRunCommand creates a new run command handler +func NewRunCommand() *RunCommand { + return &RunCommand{} +} + +// Run implements the logic for the run command +func (rc *RunCommand) Run(cmd *cobra.Command, args []string) { + RunCobraCommand(cmd) +} + +// InitRunCmd creates the run command with proper logic +func InitRunCmd(rootCmd *cobra.Command) *cobra.Command { + rc := NewRunCommand() + + runCmd := &cobra.Command{ + Use: "run", + Short: "Run the DNS proxy server", + Args: cobra.NoArgs, + Run: rc.Run, + } + runCmd.Flags().BoolVarP(&daemon, "daemon", "d", false, "Run as daemon") + runCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file") + runCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config") + runCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port") + runCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint") + runCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint") + runCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy") + runCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file") + runCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items") + runCmd.Flags().StringVarP(&cdUID, cdUidFlagName, "", "", "Control D resolver uid") + runCmd.Flags().StringVarP(&cdOrg, cdOrgFlagName, "", "", "Control D provision token") + runCmd.Flags().StringVarP(&customHostname, customHostnameFlagName, "", "", "Custom hostname passed to ControlD API") + runCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain") + _ = runCmd.Flags().MarkHidden("dev") + runCmd.Flags().StringVarP(&homedir, "homedir", "", "", "") + _ = runCmd.Flags().MarkHidden("homedir") + runCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`) + _ = runCmd.Flags().MarkHidden("iface") + runCmd.Flags().StringVarP(&cdUpstreamProto, "proto", "", ctrld.ResolverTypeDOH, `Control D upstream type, either "doh" or "doh3"`) + runCmd.Flags().BoolVarP(&rfc1918, "rfc1918", "", false, "Listen on RFC1918 addresses when 127.0.0.1 is the only listener") + runCmd.Flags().StringVarP(&interceptMode, "intercept-mode", "", "", "OS-level DNS interception mode: 'dns' (with VPN split routing), 'hard' (all DNS through ctrld, no VPN split routing), or 'off' (interface DNS mode, persisted as intercept_mode = \"off\"). Unset uses service.intercept_mode, then the platform default: 'dns' on macOS and Windows desktop editions, interface DNS mode elsewhere (Windows Server included)") + + // Set by "ctrld-client start" on a service it installs with no intercept + // mode selected; not for users. See interceptModeDefaultFlag. + runCmd.Flags().BoolVar(&interceptModeDefault, strings.TrimPrefix(interceptModeDefaultFlag, "--"), false, "Apply the platform default intercept mode (set by ctrld-client start)") + _ = runCmd.Flags().MarkHidden(strings.TrimPrefix(interceptModeDefaultFlag, "--")) + runCmd.Flags().StringVarP(&firewallMode, "firewall-mode", "", "off", "DNS-resolved IP allowlist: 'on' blocks connections to IPs not resolved by ctrld, 'off' allows all") + + runCmd.FParseErrWhitelist = cobra.FParseErrWhitelist{UnknownFlags: true} + rootCmd.AddCommand(runCmd) + + return runCmd +} diff --git a/cmd/cli/commands_service.go b/cmd/cli/commands_service.go new file mode 100644 index 00000000..deabc66f --- /dev/null +++ b/cmd/cli/commands_service.go @@ -0,0 +1,318 @@ +package cli + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/kardianos/service" + "github.com/spf13/cobra" +) + +// filterEmptyStrings removes empty strings from a slice +// This is used to clean up command line arguments and configuration values +func filterEmptyStrings(slice []string) []string { + var result []string + for _, s := range slice { + if s != "" { + result = append(result, s) + } + } + return result +} + +// ServiceCommand handles service-related operations +// This encapsulates all service management functionality for the CLI +type ServiceCommand struct { + serviceManager *ServiceManager +} + +// initializeServiceManager creates a service manager with default configuration +// This sets up the basic service infrastructure needed for all service operations +func (sc *ServiceCommand) initializeServiceManager() (service.Service, *prog, error) { + svcConfig := sc.createServiceConfig() + return sc.initializeServiceManagerWithServiceConfig(svcConfig) +} + +// initializeServiceManagerWithServiceConfig creates a service manager with the given configuration +// This allows for custom service configuration while maintaining the same initialization pattern +func (sc *ServiceCommand) initializeServiceManagerWithServiceConfig(svcConfig *service.Config) (service.Service, *prog, error) { + p := &prog{} + + s, err := sc.newService(p, svcConfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to create service: %w", err) + } + + sc.serviceManager = &ServiceManager{prog: p, svc: s} + return s, p, nil +} + +// newService creates a new service instance using the provided program and configuration. +// This abstracts the service creation process for different operating systems +func (sc *ServiceCommand) newService(p *prog, svcConfig *service.Config) (service.Service, error) { + s, err := newService(p, svcConfig) + if err != nil { + return nil, fmt.Errorf("failed to create service: %w", err) + } + return s, nil +} + +// NewServiceCommand creates a new service command handler +// This provides a clean factory method for creating service command instances +func NewServiceCommand() *ServiceCommand { + return &ServiceCommand{} +} + +// createServiceConfig creates a properly initialized service configuration +// This ensures consistent service naming and description across all platforms +func (sc *ServiceCommand) createServiceConfig() *service.Config { + return &service.Config{ + Name: ctrldServiceName, + DisplayName: ctrldServiceDisplayName, + Description: "A highly configurable, multi-protocol DNS forwarding proxy", + Option: service.KeyValue{}, + } +} + +// InitServiceCmd creates the service command with proper logic and aliases +// This sets up all service-related subcommands with appropriate permissions and flags +func InitServiceCmd(rootCmd *cobra.Command) *cobra.Command { + // Create service command handlers + sc := NewServiceCommand() + + startCmd, startCmdAlias := createStartCommands(sc) + rootCmd.AddCommand(startCmdAlias) + + // Stop command + stopCmd := &cobra.Command{ + Use: "stop", + Short: "Stop the ctrld service", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: sc.Stop, + } + stopCmd.Flags().StringVarP(&iface, "iface", "", "", `Reset DNS setting for iface, "auto" means the default interface gateway`) + stopCmd.Flags().Int64VarP(&deactivationPin, "pin", "", defaultDeactivationPin, `Pin code for stopping ctrld`) + _ = stopCmd.Flags().MarkHidden("pin") + + // Restart command + restartCmd := &cobra.Command{ + Use: "restart", + Short: "Restart the ctrld service", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: sc.Restart, + } + + // Status command + statusCmd := &cobra.Command{ + Use: "status", + Short: "Show status of the ctrld service", + Long: statusCmdLong, + Args: cobra.NoArgs, + RunE: sc.Status, + } + if runtime.GOOS == "darwin" { + // On darwin, running status command without privileges may return wrong information. + statusCmd.PreRun = func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + } + } + + // Reload command + reloadCmd := &cobra.Command{ + Use: "reload", + Short: "Reload the ctrld service", + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: sc.Reload, + } + + // Uninstall command + uninstallCmd := &cobra.Command{ + Use: "uninstall", + Short: "Stop and uninstall the ctrld service", + Long: `Stop and uninstall the ctrld service. + +NOTE: Uninstalling will set DNS to values provided by DHCP.`, + Args: cobra.NoArgs, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: sc.Uninstall, + } + uninstallCmd.Flags().StringVarP(&iface, "iface", "", "", `Reset DNS setting for iface, "auto" means the default interface gateway`) + uninstallCmd.Flags().Int64VarP(&deactivationPin, "pin", "", defaultDeactivationPin, `Pin code for stopping ctrld`) + _ = uninstallCmd.Flags().MarkHidden("pin") + uninstallCmd.Flags().BoolVarP(&cleanup, "cleanup", "", false, `Removing ctrld binary and config files`) + + // Interfaces command - use the existing InitInterfacesCmd function + interfacesCmd := InitInterfacesCmd(rootCmd) + + stopCmdAlias := &cobra.Command{ + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + Use: "stop", + Short: "Quick stop service and remove DNS from interface", + RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("iface") { + os.Args = append(os.Args, "--iface="+ifaceStartStop) + } + iface = ifaceStartStop + return stopCmd.RunE(cmd, args) + }, + } + stopCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", "auto", `Reset DNS setting for iface, "auto" means the default interface gateway`) + stopCmdAlias.Flags().AddFlagSet(stopCmd.Flags()) + rootCmd.AddCommand(stopCmdAlias) + + // Create aliases for other service commands + restartCmdAlias := &cobra.Command{ + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + Use: "restart", + Short: "Restart the ctrld service", + RunE: func(cmd *cobra.Command, args []string) error { + return restartCmd.RunE(cmd, args) + }, + } + rootCmd.AddCommand(restartCmdAlias) + + reloadCmdAlias := &cobra.Command{ + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + Use: "reload", + Short: "Reload the ctrld service", + RunE: func(cmd *cobra.Command, args []string) error { + return reloadCmd.RunE(cmd, args) + }, + } + rootCmd.AddCommand(reloadCmdAlias) + + statusCmdAlias := &cobra.Command{ + Use: "status", + Short: "Show status of the ctrld service", + Long: statusCmdLong, + Args: cobra.NoArgs, + RunE: statusCmd.RunE, + } + rootCmd.AddCommand(statusCmdAlias) + + uninstallCmdAlias := &cobra.Command{ + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + Use: "uninstall", + Short: "Stop and uninstall the ctrld service", + Long: `Stop and uninstall the ctrld service. + +NOTE: Uninstalling will set DNS to values provided by DHCP.`, + RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("iface") { + os.Args = append(os.Args, "--iface="+ifaceStartStop) + } + iface = ifaceStartStop + return uninstallCmd.RunE(cmd, args) + }, + } + uninstallCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", "auto", `Reset DNS setting for iface, "auto" means the default interface gateway`) + uninstallCmdAlias.Flags().AddFlagSet(uninstallCmd.Flags()) + rootCmd.AddCommand(uninstallCmdAlias) + + // Create service command + serviceCmd := &cobra.Command{ + Use: "service", + Short: "Manage ctrld service", + Args: cobra.OnlyValidArgs, + } + serviceCmd.ValidArgs = make([]string, 7) + serviceCmd.ValidArgs[0] = startCmd.Use + serviceCmd.ValidArgs[1] = stopCmd.Use + serviceCmd.ValidArgs[2] = restartCmd.Use + serviceCmd.ValidArgs[3] = reloadCmd.Use + serviceCmd.ValidArgs[4] = statusCmd.Use + serviceCmd.ValidArgs[5] = uninstallCmd.Use + serviceCmd.ValidArgs[6] = interfacesCmd.Use + + serviceCmd.AddCommand(startCmd) + serviceCmd.AddCommand(stopCmd) + serviceCmd.AddCommand(restartCmd) + serviceCmd.AddCommand(reloadCmd) + serviceCmd.AddCommand(statusCmd) + serviceCmd.AddCommand(uninstallCmd) + serviceCmd.AddCommand(interfacesCmd) + + rootCmd.AddCommand(serviceCmd) + + return serviceCmd +} + +// validInterceptMode reports whether the given value is a recognized --intercept-mode. +// This is the single source of truth for mode validation — used by the early start +// command check, the runtime validation in prog.go, and onlyInterceptFlags below. +// Add new modes here to have them recognized everywhere. +func validInterceptMode(mode string) bool { + switch mode { + case "off", "dns", "hard": + return true + } + return false +} + +// validFirewallMode reports whether the given value is a recognized --firewall-mode. +func validFirewallMode(mode string) bool { + switch mode { + case "off", "on": + return true + } + return false +} + +// onlyInterceptFlags reports whether args contain only intercept mode +// flags (--intercept-mode ) and flags that are auto-added by the +// start command alias (--iface). This is used to detect "ctrld start --intercept-mode dns" +// (or "off" to disable) on an existing installation, where the intent is to modify the +// intercept flag on the existing service without replacing other arguments. +// +// Note: the startCmdAlias appends "--iface=auto" to os.Args when --iface isn't +// explicitly provided, so we must allow it here. +func onlyInterceptFlags(args []string) bool { + hasIntercept := false + for i := 0; i < len(args); i++ { + arg := args[i] + switch { + case arg == "--intercept-mode": + // Next arg must be a valid mode value. + if i+1 < len(args) && validInterceptMode(args[i+1]) { + hasIntercept = true + i++ // skip the value + } else { + return false + } + case strings.HasPrefix(arg, "--intercept-mode="): + val := strings.TrimPrefix(arg, "--intercept-mode=") + if validInterceptMode(val) { + hasIntercept = true + } else { + return false + } + case arg == "--iface="+autoIface || arg == "--iface" || arg == autoIface: + // Auto-added by startCmdAlias or its value; safe to ignore. + continue + default: + return false + } + } + return hasIntercept +} diff --git a/cmd/cli/commands_service_manager.go b/cmd/cli/commands_service_manager.go new file mode 100644 index 00000000..ffdbca3a --- /dev/null +++ b/cmd/cli/commands_service_manager.go @@ -0,0 +1,41 @@ +package cli + +import ( + "fmt" + "time" + + "github.com/kardianos/service" +) + +// dialSocketControlServerTimeout is the default timeout to wait when ping control server. +const dialSocketControlServerTimeout = 30 * time.Second + +// ServiceManager handles service operations +type ServiceManager struct { + prog *prog + svc service.Service +} + +// NewServiceManager creates a new service manager +func NewServiceManager() (*ServiceManager, error) { + p := &prog{} + + // Create a proper service configuration + svcConfig := &service.Config{ + Name: ctrldServiceName, + DisplayName: ctrldServiceDisplayName, + Description: "A highly configurable, multi-protocol DNS forwarding proxy", + Option: service.KeyValue{}, + } + + s, err := newService(p, svcConfig) + if err != nil { + return nil, fmt.Errorf("failed to create service: %w", err) + } + return &ServiceManager{prog: p, svc: s}, nil +} + +// Status returns the current service status +func (sm *ServiceManager) Status() (service.Status, error) { + return sm.svc.Status() +} diff --git a/cmd/cli/commands_service_reload.go b/cmd/cli/commands_service_reload.go new file mode 100644 index 00000000..5ddf4ff6 --- /dev/null +++ b/cmd/cli/commands_service_reload.go @@ -0,0 +1,67 @@ +package cli + +import ( + "errors" + "io" + "net/http" + "path/filepath" + + "github.com/kardianos/service" + "github.com/spf13/cobra" +) + +// Reload implements the logic from cmdReload.Run +func (sc *ServiceCommand) Reload(cmd *cobra.Command, args []string) error { + logger := mainLog.Load() + logger.Debug().Msg("Service reload command started") + + s, _, err := sc.initializeServiceManager() + if err != nil { + logger.Error().Err(err).Msg("Failed to initialize service manager") + return err + } + + status, err := s.Status() + if errors.Is(err, service.ErrNotInstalled) { + logger.Warn().Msg("Service not installed") + return nil + } + if status == service.StatusStopped { + logger.Warn().Msg("Service is not running") + return nil + } + + dir, err := socketDir() + if err != nil { + logger.Fatal().Err(err).Msg("Failed to find ctrld home dir") + } + + cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) + resp, err := cc.post(reloadPath, nil) + if err != nil { + logger.Fatal().Err(err).Msg("Failed to send reload signal to ctrld") + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + logger.Notice().Msg("Service reloaded") + case http.StatusCreated: + logger.Warn().Msg("Service was reloaded, but new config requires service restart.") + logger.Warn().Msg("Restarting service") + if _, err := s.Status(); errors.Is(err, service.ErrNotInstalled) { + logger.Warn().Msg("Service not installed") + return nil + } + return sc.Restart(cmd, args) + default: + buf, err := io.ReadAll(resp.Body) + if err != nil { + logger.Fatal().Err(err).Msg("Could not read response from control server") + } + logger.Error().Err(err).Msgf("Failed to reload ctrld: %s", string(buf)) + } + + logger.Debug().Msg("Service reload command completed") + return nil +} diff --git a/cmd/cli/commands_service_restart.go b/cmd/cli/commands_service_restart.go new file mode 100644 index 00000000..d2876574 --- /dev/null +++ b/cmd/cli/commands_service_restart.go @@ -0,0 +1,117 @@ +package cli + +import ( + "context" + "errors" + "time" + + "github.com/kardianos/service" + "github.com/spf13/cobra" +) + +// Restart implements the logic from cmdRestart.Run +func (sc *ServiceCommand) Restart(cmd *cobra.Command, args []string) error { + logger := mainLog.Load() + logger.Debug().Msg("Service restart command started") + + readConfig(false) + v.Unmarshal(&cfg) + cdUID = curCdUID() + cdMode := cdUID != "" + + s, p, err := sc.initializeServiceManager() + if err != nil { + logger.Error().Err(err).Msg("Failed to initialize service manager") + return err + } + + if _, err := s.Status(); errors.Is(err, service.ErrNotInstalled) { + logger.Warn().Msg("Service not installed") + return nil + } + + p.cfg = &cfg + if iface == "" { + iface = autoIface + } + p.preRun() + if ir := runningIface(s); ir != nil { + p.runningIface = ir.Name + p.requiredMultiNICsConfig = ir.All + } + + initInteractiveLogging() + + var validateConfigErr error + if cdMode { + logger.Debug().Msg("Validating ControlD remote config") + validateConfigErr = doValidateCdRemoteConfig(cdUID, false) + if validateConfigErr != nil { + logger.Warn().Err(validateConfigErr).Msg("ControlD remote config validation failed") + } + } + + if ir := runningIface(s); ir != nil { + iface = ir.Name + } + + doRestart := func() bool { + logger.Debug().Msg("Starting service restart sequence") + + tasks := []task{ + {s.Stop, true, "Stop"}, + {func() error { + // restore static DNS settings or DHCP + p.resetDNS(false, true) + return nil + }, false, "Cleanup"}, + {func() error { + time.Sleep(time.Second * 1) + return nil + }, false, "Waiting for service to stop"}, + } + if !doTasks(tasks) { + logger.Error().Msg("Service stop tasks failed") + return false + } + tasks = []task{ + {s.Start, true, "Start"}, + } + success := doTasks(tasks) + if success { + logger.Debug().Msg("Service restart sequence completed successfully") + } else { + logger.Error().Msg("Service restart sequence failed") + } + return success + } + + if doRestart() { + if dir, err := socketDir(); err == nil { + timeout := dialSocketControlServerTimeout + if validateConfigErr != nil { + timeout = 5 * time.Second + } + // Only a ready service counts. Restart replaces nothing, so + // there is nothing to roll back, but a service that came + // back without finishing its start must not be reported as + // restarted cleanly. + if cc := newReadySocketControlClientWithTimeout(context.TODO(), s, dir, timeout); cc != nil { + _, _ = cc.post(ifacePath, nil) + logger.Debug().Msg("Control server ping successful") + } else { + logger.Warn().Msgf("Service was restarted, but ctrld did not report ready within %s; check `ctrld status`", timeout) + return nil + } + } else { + logger.Warn().Err(err).Msg("Service was restarted, but could not ping the control server") + return nil + } + logger.Notice().Msg("Service restarted") + } else { + logger.Error().Msg("Service restart failed") + } + + logger.Debug().Msg("Service restart command completed") + return nil +} diff --git a/cmd/cli/commands_service_start.go b/cmd/cli/commands_service_start.go new file mode 100644 index 00000000..a6579eef --- /dev/null +++ b/cmd/cli/commands_service_start.go @@ -0,0 +1,596 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/kardianos/service" + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld" +) + +// serviceStageFailureCode maps an aborted service-manager task to its +// provisioning code. Other abortOnError tasks (like config validation) keep +// their own error paths. +func serviceStageFailureCode(taskName string) (provisionFailureCode, bool) { + switch taskName { + case "Install": + return provisionCodeServiceInstall, true + case "Start": + return provisionCodeServiceStartFailed, true + default: + return "", false + } +} + +// serviceTaskErrorSummary describes which service-manager task failed and why, +// for use as a provisioning result message. +func serviceTaskErrorSummary(taskName string, err error) string { + return fmt.Sprintf("%s failed: %v", taskName, err) +} + +// resultStalenessTolerance absorbs clock granularity between "ctrld start" +// recording its start time and the daemon writing its result file. +const resultStalenessTolerance = 2 * time.Second + +// reportStartFailure reports why "ctrld start" failed after install/start +// looked fine. A result file the daemon wrote during this attempt names the +// failure better than a generic self-check code, so it wins. +func reportStartFailure(startedAt time.Time, fallbackMsg string) { + if r, err := readProvisionResult(); err == nil && provisionResultTrusted(r) { + if ts, err := time.Parse(time.RFC3339, r.Timestamp); err == nil { + if !ts.Before(startedAt.Add(-resultStalenessTolerance)) { + mainLog.Load().Error().Msg(r.failureLine()) + provisionExit(r.ExitCode) + return + } + } + } + failProvision(newProvisionResult(provisionCodeServiceSelfCheck, fallbackMsg, nil, provisionSecrets()...), nil) +} + +// Start implements the logic from cmdStart.Run +func (sc *ServiceCommand) Start(cmd *cobra.Command, args []string) error { + logger := mainLog.Load() + logger.Debug().Msg("Service start command started") + + // Clear before any check runs, not just before doTasksE: a result from a + // previous attempt must never survive to mislead diag/postinstall on this + // one, even if this attempt fails before reaching doTasksE. + clearProvisionResult() + + firewallModeFlagChanged = cmd.Flags().Changed("firewall-mode") + if !checkStrFlagEmpty(cmd, cdUidFlagName) { + return nil + } + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + return nil + } + if !validateCdAndNextDNSFlags() { + return nil + } + + svcConfig := sc.createServiceConfig() + osArgs := os.Args[2:] + osArgs = filterEmptyStrings(osArgs) + if os.Args[1] == "service" { + osArgs = os.Args[3:] + } + setDependencies(svcConfig) + svcConfig.Arguments = append([]string{"run"}, osArgs...) + + // Validate --intercept-mode early, before installing the service. + // Without this, a typo like "--intercept-mode fds" would install the service, + // the child process would Fatal() on the invalid value, and the parent would + // then uninstall — confusing and destructive. + if !validateInterceptModeFlag(interceptMode) { + return nil + } + if !validateFirewallModeFlag(firewallModeFlagChanged, firewallMode, nil) { + return nil + } + + // Initialize service manager with proper configuration + s, p, err := sc.initializeServiceManagerWithServiceConfig(svcConfig) + if err != nil { + logger.Error().Err(err).Msg("Failed to initialize service manager") + // A bare error return would exit 1 with no result file, so support + // could not tell this failure from a start that never ran. + failProvisionUnclassified("initialize service manager: "+err.Error(), nil) + return nil + } + + p.cfg = &cfg + p.preRun() + + status, err := s.Status() + isCtrldRunning := status == service.StatusRunning + isCtrldInstalled := !errors.Is(err, service.ErrNotInstalled) + + // Get current running iface, if any. + var currentIface *ifaceResponse + + // Handle "ctrld start --intercept-mode dns|hard" on an existing + // service BEFORE the pin check. Adding intercept mode is an enhancement, not + // deactivation, so it doesn't require the deactivation pin. We modify the + // plist/registry directly and restart the service via the OS service manager. + osArgsEarly := os.Args[2:] + if os.Args[1] == "service" { + osArgsEarly = os.Args[3:] + } + osArgsEarly = filterEmptyStrings(osArgsEarly) + interceptOnly := onlyInterceptFlags(osArgsEarly) + svcExists := serviceConfigFileExists() + logger.Debug().Msg(interceptUpgradeCheckLine(osArgsEarly, interceptOnly, svcExists, interceptMode)) + if interceptOnly && svcExists { + // An explicit "off" argument must override a previously persisted config + // value while the service clears that value on startup. + if err := removeServiceFlag("--intercept-mode"); err != nil { + failRunUnclassified(logger.Error().Err(err), fmt.Sprintf("failed to remove existing intercept mode from service arguments: %v", err), nil) + return nil + } + + if interceptMode == "off" { + logger.Notice().Msg("Existing service detected — disabling intercept mode") + } else { + logger.Notice().Msgf("Existing service detected — appending --intercept-mode %s to service arguments", interceptMode) + } + if err := appendServiceFlag("--intercept-mode"); err != nil { + failRunUnclassified(logger.Error().Err(err), fmt.Sprintf("failed to append intercept flag to service arguments: %v", err), nil) + return nil + } + if err := appendServiceFlag(interceptMode); err != nil { + failRunUnclassified(logger.Error().Err(err), fmt.Sprintf("failed to append intercept mode value to service arguments: %v", err), nil) + return nil + } + + // Stop the service if running (bypasses ctrld pin — this is an + // enhancement, not deactivation). Then fall through to the normal + // startOnly path which handles start, self-check, and reporting. + if isCtrldRunning { + logger.Notice().Msg("Stopping service for intercept mode upgrade") + _ = s.Stop() + isCtrldRunning = false + } + startOnly = true + isCtrldInstalled = true + // Fall through to startOnly path below. + } + + // If pin code was set, do not allow running start command. + if isCtrldRunning { + if err := checkDeactivationPin(s, nil); isCheckDeactivationPinErr(err) { + logger.Error().Msg("Deactivation pin check failed") + os.Exit(deactivationPinInvalidExitCode) + } + currentIface = runningIface(s) + logger.Debug().Msgf("Current interface on start: %v", currentIface) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + reportSetDnsOk := func(sockDir string) { + if cc := newSocketControlClient(ctx, s, sockDir); cc != nil { + if resp, _ := cc.post(ifacePath, nil); resp != nil && resp.StatusCode == http.StatusOK { + if iface == autoIface { + iface = defaultIfaceName() + } + res := &ifaceResponse{} + if err := json.NewDecoder(resp.Body).Decode(res); err != nil { + logger.Warn().Err(err).Msg("Failed to get iface info") + return + } + if res.OK { + // In intercept mode, show intercept-specific status instead of + // per-interface DNS messages (which are irrelevant). + if res.InterceptMode != "" { + switch res.InterceptMode { + case "hard": + logger.Notice().Msg("DNS hard intercept mode active — all DNS traffic intercepted, no VPN split routing") + default: + logger.Notice().Msg("DNS intercept mode active — all DNS traffic intercepted via OS packet filter") + } + if res.InterceptModeSource == interceptModeSourceDefault { + logger.Notice().Msg("Intercept mode is the default on this platform; run \"ctrld-client start --intercept-mode off\" to use interface DNS mode instead") + } + } else { + name := res.Name + if iff, err := net.InterfaceByName(name); err == nil { + _, _ = patchNetIfaceName(iff) + name = iff.Name + } + ifaceLogger := logger.With().Str("iface", name) + ifaceLogger.Debug().Msg("Setting DNS successfully") + if res.All { + // Log that DNS is set for other interfaces. + withEachPhysicalInterfaces( + name, + "set DNS", + func(i *net.Interface) error { return nil }, + ) + } + } + } + } + } + } + + // No config path, generating config in HOME directory. + noConfigStart := isNoConfigStart(cmd) + writeDefaultConfig := !noConfigStart && configBase64 == "" + + logServerStarted := make(chan struct{}) + stopLogCh := make(chan struct{}) + ud, err := userHomeDir() + sockDir := ud + var logServerSocketPath string + if err != nil { + logger.Warn().Err(err).Msg("Failed to get user home directory") + logger.Warn().Msg("Log server did not start") + close(logServerStarted) + } else { + setWorkingDirectory(svcConfig, ud) + if configPath == "" && writeDefaultConfig { + defaultConfigFile = filepath.Join(ud, defaultConfigFile) + } + svcConfig.Arguments = append(svcConfig.Arguments, "--homedir="+ud) + if d, err := socketDir(); err == nil { + sockDir = d + } + logServerSocketPath = filepath.Join(sockDir, ctrldLogUnixSock) + _ = os.Remove(logServerSocketPath) + go func() { + defer os.Remove(logServerSocketPath) + + close(logServerStarted) + + // Start HTTP log server + if err := httpLogServer(logServerSocketPath, stopLogCh); err != nil && err != http.ErrServerClosed { + logger.Warn().Err(err).Msg("Failed to serve HTTP log server") + return + } + }() + } + <-logServerStarted + + if !startOnly { + startOnly = len(osArgs) == 0 + } + // If user run "ctrld start" and ctrld is already installed, starting existing service. + if startOnly && isCtrldInstalled { + tryReadingConfigWithNotice(false, true) + if err := v.Unmarshal(&cfg); err != nil { + failRunUnclassified(logger.Error(), fmt.Sprintf("failed to unmarshal config: %v", err), nil) + return nil + } + + // if already running, dont restart + if isCtrldRunning { + logger.Notice().Msg("Service is already running") + return nil + } + + initInteractiveLogging() + tasks := []task{ + {func() error { + // Save current DNS so we can restore later. + withEachPhysicalInterfaces("", "saveCurrentStaticDNS", func(i *net.Interface) error { + if err := saveCurrentStaticDNS(i); !errors.Is(err, errSaveCurrentStaticDNSNotSupported) && err != nil { + return err + } + return nil + }) + return nil + }, false, "Save current DNS"}, + {func() error { + return ConfigureWindowsServiceFailureActions(ctrldServiceName) + }, false, "Configure service failure actions"}, + {s.Start, true, "Start"}, + {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, + } + startAttemptAt := time.Now() + logger.Notice().Msg("Starting existing ctrld service") + failedTask, taskErr := doTasksE(tasks) + if taskErr != nil { + if code, ok := serviceStageFailureCode(failedTask); ok { + failProvision(newProvisionResult(code, serviceTaskErrorSummary(failedTask, taskErr), nil, provisionSecrets()...), nil) + return nil + } + failProvisionUnclassified(serviceTaskErrorSummary(failedTask, taskErr), nil) + return nil + } + sockDir, err := socketDir() + if err != nil { + failRunUnclassified(logger.Error(), fmt.Sprintf("failed to get socket directory: %v", err), nil) + return nil + } + + // The daemon can start and still fail provisioning (for example a + // listener bind conflict). Self-check like a fresh install so this + // path reports the daemon's failure code instead of a false + // "Service started" — but never uninstall an existing service. + time.Sleep(1 * time.Second) + ok, status, err := selfCheckStatus(ctx, s, sockDir) + if !ok || status != service.StatusRunning { + fallbackMsg := "ctrld service did not pass its post-start self-check" + if err != nil { + fallbackMsg = fmt.Sprintf("An error occurred while performing test query: %s", err) + logger.Error().Msg(fallbackMsg) + } + if status == service.StatusRunning && err == nil { + fallbackMsg = "ctrld service was running, but a DNS query could not be sent to its listener; check firewall rules blocking/intercepting/redirecting DNS queries" + logger.Error().Msg(fallbackMsg) + } + reportStartFailure(startAttemptAt, fallbackMsg) + return nil + } + logger.Notice().Msg("Service started") + clearProvisionResult() + reportSetDnsOk(sockDir) + // Verify service registration after successful start. + if err := verifyServiceRegistration(); err != nil { + logger.Warn().Err(err).Msg("Service registry verification failed") + } + return nil + } + + if cdUID != "" { + if err := doValidateCdRemoteConfig(cdUID, true); err != nil { + return nil + } + } else if uid := cdUIDFromProvToken(); uid != "" { + cdUID = uid + logger.Debug().Msg("Using uid from provision token") + removeOrgFlagsFromArgs(svcConfig) + // Pass --cd flag to "ctrld run" command, so the provision token takes no effect. + svcConfig.Arguments = append(svcConfig.Arguments, "--cd="+cdUID) + } + if cdUID != "" { + if !validateCdUpstreamProtocol(nil) { + return nil + } + } + + if configPath != "" { + v.SetConfigFile(configPath) + } + + tryReadingConfigWithNotice(writeDefaultConfig, true) + + if err := v.Unmarshal(&cfg); err != nil { + failRunUnclassified(logger.Error(), fmt.Sprintf("failed to unmarshal config: %v", err), nil) + return nil + } + + initInteractiveLogging() + + // Record the intercept mode in the service's arguments, so the installed + // service can tell a fresh install from one that predates the dns default + // (#589). See installInterceptMode. + var existingArgs []string + if isCtrldInstalled && interceptMode == "" { + args, err := installedServiceArgs() + if err != nil { + logger.Debug().Err(err).Msg("Could not read the arguments of the existing service") + } + existingArgs = args + } + mode, markDefault := installInterceptMode(interceptMode, &cfg, isCtrldInstalled, existingArgs) + switch { + case mode != "": + if _, hadMode := interceptModeFromArgs(existingArgs); !hadMode { + logger.Notice().Msg("Existing service selects no intercept mode; keeping interface DNS mode (--intercept-mode off)") + } + svcConfig.Arguments = append(svcConfig.Arguments, "--intercept-mode", mode) + interceptMode = mode + case markDefault: + // One argument with an inline value: see interceptModeDefaultFlag. + svcConfig.Arguments = append(svcConfig.Arguments, interceptModeDefaultFlag+"=true") + } + + if nextdns != "" { + removeNextDNSFromArgs(svcConfig) + } + + // Explicitly passing config, so on system where home directory could not be obtained, + // or sub-process env is different with the parent, we still behave correctly and use + // the expected config file. + if configPath == "" { + svcConfig.Arguments = append(svcConfig.Arguments, "--config="+defaultConfigFile) + } + + tasks := []task{ + {s.Stop, false, "Stop"}, + {func() error { return doGenerateNextDNSConfig(nextdns) }, true, "Checking config"}, + {func() error { return ensureUninstall(s) }, false, "Ensure uninstall"}, + //resetDnsTask(p, s, isCtrldInstalled, currentIface), + {func() error { + // Save current DNS so we can restore later. + withEachPhysicalInterfaces("", "saveCurrentStaticDNS", func(i *net.Interface) error { + if err := saveCurrentStaticDNS(i); !errors.Is(err, errSaveCurrentStaticDNSNotSupported) && err != nil { + return err + } + return nil + }) + return nil + }, false, "Save current DNS"}, + {s.Install, true, "Install"}, + {func() error { + return ConfigureWindowsServiceFailureActions(ctrldServiceName) + }, false, "Configure Windows service failure actions"}, + {s.Start, true, "Start"}, + // Note that startCmd do not actually write ControlD config, but the config file was + // generated after s.Start, so we notice users here for consistent with nextdns mode. + {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, + } + startAttemptAt := time.Now() + logger.Notice().Msg("Starting service") + failedTask, taskErr := doTasksE(tasks) + if taskErr != nil { + if code, ok := serviceStageFailureCode(failedTask); ok { + failProvision(newProvisionResult(code, serviceTaskErrorSummary(failedTask, taskErr), nil, provisionSecrets()...), nil) + return nil + } + // Not a service-stage task. doTasksE already logged the cause; classify + // UNCLASSIFIED instead of the old silent fall-through that exited 0. + failProvisionUnclassified(serviceTaskErrorSummary(failedTask, taskErr), nil) + return nil + } + + // add a small delay to ensure the service is started and did not crash + time.Sleep(1 * time.Second) + + ok, status, err := selfCheckStatus(ctx, s, sockDir) + switch { + case ok && status == service.StatusRunning: + logger.Notice().Msg("Service started") + clearProvisionResult() + default: + marker := append(bytes.Repeat([]byte("="), 32), '\n') + fallbackMsg := "ctrld service did not pass its post-start self-check" + // If ctrld service is not running, emitting log obtained from ctrld process. + if status != service.StatusRunning || ctx.Err() != nil { + logger.Error().Msg("Ctrld service may not have started due to an error or misconfiguration, service log:") + _, _ = logger.Write(marker) + + // Wait for log collection to complete + <-stopLogCh + + // Retrieve logs from HTTP server if available + if logServerSocketPath != "" { + hlc := newHTTPLogClient(logServerSocketPath) + logs, err := hlc.GetLogs() + if err != nil { + logger.Warn().Err(err).Msg("Failed to get logs from HTTP log server") + } + if len(logs) == 0 { + logger.Write([]byte("\n")) + } else { + logger.Write(logs) + logger.Write([]byte("\n")) + } + } else { + logger.Write([]byte("\n")) + } + } + // Report any error if occurred. + if err != nil { + _, _ = logger.Write(marker) + msg := fmt.Sprintf("An error occurred while performing test query: %s\n", err) + logger.Write([]byte(msg)) + fallbackMsg = msg + } + // If ctrld service is running but selfCheckStatus failed, it could be related + // to user's system firewall configuration, notice users about it. + if status == service.StatusRunning && err == nil { + _, _ = logger.Write(marker) + logger.Write([]byte("ctrld service was running, but a DNS query could not be sent to its listener\n")) + logger.Write([]byte("Please check your system firewall if it is configured to block/intercept/redirect DNS queries\n")) + fallbackMsg = "ctrld service was running, but a DNS query could not be sent to its listener; check firewall rules blocking/intercepting/redirecting DNS queries" + } + + _, _ = logger.Write(marker) + uninstall(p, s) + reportStartFailure(startAttemptAt, fallbackMsg) + return nil + } + reportSetDnsOk(sockDir) + // Verify service registration after successful start. + if err := verifyServiceRegistration(); err != nil { + logger.Warn().Err(err).Msg("Service registry verification failed") + } + + logger.Debug().Msg("Service start command completed") + return nil +} + +// createStartCommands creates the start command and its alias +func createStartCommands(sc *ServiceCommand) (*cobra.Command, *cobra.Command) { + // Start command + startCmd := &cobra.Command{ + Use: "start", + Short: "Install and start the ctrld service", + Long: `Install and start the ctrld service + +NOTE: running "ctrld start" without any arguments will start already installed ctrld service.`, + Args: func(cmd *cobra.Command, args []string) error { + args = filterEmptyStrings(args) + if len(args) > 0 { + return fmt.Errorf("'ctrld start' doesn't accept positional arguments\n" + + "Use flags instead (e.g. --cd, --iface) or see 'ctrld start --help' for all options") + } + return nil + }, + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: sc.Start, + } + // Keep these flags in sync with runCmd above, except for "-d"/"--nextdns". + startCmd.Flags().StringVarP(&configPath, "config", "c", "", "Path to config file") + startCmd.Flags().StringVarP(&configBase64, "base64_config", "", "", "Base64 encoded config") + startCmd.Flags().StringVarP(&listenAddress, "listen", "", "", "Listener address and port, in format: address:port") + startCmd.Flags().StringVarP(&primaryUpstream, "primary_upstream", "", "", "Primary upstream endpoint") + startCmd.Flags().StringVarP(&secondaryUpstream, "secondary_upstream", "", "", "Secondary upstream endpoint") + startCmd.Flags().StringSliceVarP(&domains, "domains", "", nil, "List of domain to apply in a split DNS policy") + startCmd.Flags().StringVarP(&logPath, "log", "", "", "Path to log file") + startCmd.Flags().IntVarP(&cacheSize, "cache_size", "", 0, "Enable cache with size items") + startCmd.Flags().StringVarP(&cdUID, cdUidFlagName, "", "", "Control D resolver uid") + startCmd.Flags().StringVarP(&cdOrg, cdOrgFlagName, "", "", "Control D provision token") + startCmd.Flags().StringVarP(&customHostname, customHostnameFlagName, "", "", "Custom hostname passed to ControlD API") + startCmd.Flags().BoolVarP(&cdDev, "dev", "", false, "Use Control D dev resolver/domain") + _ = startCmd.Flags().MarkHidden("dev") + startCmd.Flags().StringVarP(&iface, "iface", "", "", `Update DNS setting for iface, "auto" means the default interface gateway`) + startCmd.Flags().StringVarP(&nextdns, nextdnsFlagName, "", "", "NextDNS resolver id") + startCmd.Flags().StringVarP(&cdUpstreamProto, "proto", "", ctrld.ResolverTypeDOH, `Control D upstream type, either "doh" or "doh3"`) + startCmd.Flags().BoolVarP(&skipSelfChecks, "skip_self_checks", "", false, `Skip self checks after installing ctrld service`) + startCmd.Flags().BoolVarP(&startOnly, "start_only", "", false, "Do not install new service") + _ = startCmd.Flags().MarkHidden("start_only") + startCmd.Flags().BoolVarP(&rfc1918, "rfc1918", "", false, "Listen on RFC1918 addresses when 127.0.0.1 is the only listener") + startCmd.Flags().StringVarP(&interceptMode, "intercept-mode", "", "", "OS-level DNS interception mode: 'dns' (with VPN split routing), 'hard' (all DNS through ctrld, no VPN split routing), or 'off' (interface DNS mode, persisted as intercept_mode = \"off\"). Unset uses service.intercept_mode, then the platform default: 'dns' on macOS and Windows desktop editions, interface DNS mode elsewhere (Windows Server included)") + startCmd.Flags().StringVarP(&firewallMode, "firewall-mode", "", "off", "DNS-resolved IP allowlist: 'on' blocks connections to IPs not resolved by ctrld, 'off' allows all") + + // Start command alias + startCmdAlias := &cobra.Command{ + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + Use: "start", + Short: "Quick start service and configure DNS on interface", + Long: `Quick start service and configure DNS on interface + +NOTE: running "ctrld start" without any arguments will start already installed ctrld service.`, + Args: func(cmd *cobra.Command, args []string) error { + args = filterEmptyStrings(args) + if len(args) > 0 { + return fmt.Errorf("'ctrld start' doesn't accept positional arguments\n" + + "Use flags instead (e.g. --cd, --iface) or see 'ctrld start --help' for all options") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + if len(os.Args) == 2 { + startOnly = true + } + if !cmd.Flags().Changed("iface") { + os.Args = append(os.Args, "--iface="+ifaceStartStop) + } + iface = ifaceStartStop + return startCmd.RunE(cmd, args) + }, + } + startCmdAlias.Flags().StringVarP(&ifaceStartStop, "iface", "", autoIface, `Update DNS setting for iface, "auto" means the default interface gateway`) + startCmdAlias.Flags().AddFlagSet(startCmd.Flags()) + + return startCmd, startCmdAlias +} diff --git a/cmd/cli/commands_service_start_test.go b/cmd/cli/commands_service_start_test.go new file mode 100644 index 00000000..c2401760 --- /dev/null +++ b/cmd/cli/commands_service_start_test.go @@ -0,0 +1,153 @@ +package cli + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// startFunctionSource extracts the source text of ServiceCommand.Start's body +// from commands_service_start.go. Driving Start() itself end-to-end for every +// early-return branch is not practical in a unit test: within a few lines of +// any check failing, Start() reaches into the real OS service manager. Some +// invariants about its shape are cheaper and more reliable to pin by reading +// the source than by executing it. +func startFunctionSource(t *testing.T) string { + t.Helper() + file := packageSourcePath(t, "commands_service_start.go") + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, file, nil, 0) + if err != nil { + t.Fatalf("could not parse %s: %v", file, err) + } + for _, decl := range node.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != "Start" || fn.Recv == nil { + continue + } + src, err := os.ReadFile(file) + if err != nil { + t.Fatalf("could not read %s: %v", file, err) + } + start := fset.Position(fn.Body.Lbrace).Offset + end := fset.Position(fn.Body.Rbrace).Offset + return string(src[start:end]) + } + t.Fatalf("ServiceCommand.Start not found in %s", file) + return "" +} + +// TestServiceCommandStartClearsProvisionResultBeforeAnyCheck pins the +// ordering fix: clearProvisionResult() must run before every check in +// Start() that can fail or return early, not just before doTasksE. Without +// this, a check between the top of Start() and the old call sites could +// return early (whether by writing its own classified failure or, like the +// "service already running" and service-manager-init-error paths, by writing +// nothing at all) while a previous attempt's result file was still sitting +// there to mislead diag/postinstall on retry. +func TestServiceCommandStartClearsProvisionResultBeforeAnyCheck(t *testing.T) { + body := startFunctionSource(t) + + clearIdx := strings.Index(body, "clearProvisionResult()") + if clearIdx == -1 { + t.Fatal("Start() no longer calls clearProvisionResult()") + } + + // Every check or step that can return out of Start() before reaching + // doTasksE. Each must appear after the entry clear. + earlyChecks := []string{ + "checkStrFlagEmpty(", + "validateCdAndNextDNSFlags(", + "validateInterceptModeFlag(", + "validateFirewallModeFlag(", + "initializeServiceManagerWithServiceConfig(", + "doTasksE(", + } + for _, check := range earlyChecks { + idx := strings.Index(body, check) + if idx == -1 { + t.Fatalf("expected Start() to still call %s", check) + } + if idx < clearIdx { + t.Errorf("%s appears before clearProvisionResult(): a failure there could leave a stale result file behind", check) + } + } +} + +// TestServiceCommandStartClassifiesServiceManagerInitFailure pins the fix for +// a bare error return: a service-manager init failure in Start() must fail +// through failProvisionUnclassified, so a result file and the identifier line +// exist, instead of returning the error for a plain exit 1. +func TestServiceCommandStartClassifiesServiceManagerInitFailure(t *testing.T) { + body := startFunctionSource(t) + initIdx := strings.Index(body, "initializeServiceManagerWithServiceConfig(") + if initIdx == -1 { + t.Fatal("Start() no longer calls initializeServiceManagerWithServiceConfig") + } + branchEnd := strings.Index(body[initIdx:], "p.cfg = &cfg") + if branchEnd == -1 { + t.Fatal("could not find the end of the service-manager init branch") + } + branch := body[initIdx : initIdx+branchEnd] + if !strings.Contains(branch, "failProvisionUnclassified(") { + t.Error("service-manager init failure does not fail through failProvisionUnclassified") + } + if strings.Contains(branch, "return err") { + t.Error("service-manager init failure still returns the bare error, which exits 1 with no result file") + } +} + +// startTestCommand builds the minimal cobra.Command ServiceCommand.Start needs +// before it can reach its early --intercept-mode check: the --cd/--cd-org +// flags must exist (checkStrFlagEmpty looks them up unconditionally) but stay +// unchanged, so neither Fatals. +func startTestCommand() *cobra.Command { + cmd := &cobra.Command{} + cmd.Flags().String(cdUidFlagName, "", "") + cmd.Flags().String(cdOrgFlagName, "", "") + return cmd +} + +// TestServiceCommandStartReplacesStaleResultOnEarlyClassifiedFailure is a +// behavioral companion to the structural test above: it drives the real +// Start() through its earliest classified failure (an invalid +// --intercept-mode) and checks the file left behind names the new attempt, +// not a stale one seeded beforehand. +func TestServiceCommandStartReplacesStaleResultOnEarlyClassifiedFailure(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldIntercept, oldNextdns, oldFirewallChanged := interceptMode, nextdns, firewallModeFlagChanged + t.Cleanup(func() { + interceptMode, nextdns, firewallModeFlagChanged = oldIntercept, oldNextdns, oldFirewallChanged + }) + cdUID, cdOrg, nextdns = "", "", "" + interceptMode = "bogus" // fails validateInterceptModeFlag before any OS work + + if err := writeProvisionResult(newProvisionResult(provisionCodeServiceStartFailed, "a previous failed attempt", nil)); err != nil { + t.Fatal(err) + } + + sc := NewServiceCommand() + if err := sc.Start(startTestCommand(), nil); err != nil { + t.Fatalf("Start() error = %v", err) + } + + wantExit := provisionExitCodeForCode[provisionCodeInterceptModeInvalid] + if *exitCode != wantExit { + t.Fatalf("exit = %d, want %d (validateInterceptModeFlag should have run)", *exitCode, wantExit) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code == string(provisionCodeServiceStartFailed) { + t.Fatal("stale result from a previous attempt survived the new attempt") + } + if r.Code != string(provisionCodeInterceptModeInvalid) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInterceptModeInvalid) + } +} diff --git a/cmd/cli/commands_service_status.go b/cmd/cli/commands_service_status.go new file mode 100644 index 00000000..6a5bf147 --- /dev/null +++ b/cmd/cli/commands_service_status.go @@ -0,0 +1,226 @@ +package cli + +import ( + "errors" + "fmt" + "io/fs" + "net/http" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/kardianos/service" + "github.com/spf13/cobra" +) + +// Exit codes reported by "ctrld status". +const ( + statusExitRunning = 0 + statusExitStopped = 1 + statusExitUnknown = 2 + // statusExitNotReady means the service manager considers the service running, + // but the process has not finished starting up, so it is not serving DNS or + // applying policy. This is a distinct code because it needs a distinct response: + // the process exists, so restarting the service is what recovers it, while a + // stopped service needs starting and an unknown state needs investigation. + statusExitNotReady = 3 +) + +// serviceReadinessTimeout bounds the control-socket probe. Status must answer +// quickly, and a service that cannot respond within this window is not usefully +// "running" from a caller's point of view either way. +const serviceReadinessTimeout = 3 * time.Second + +// statusCmdLong documents what the reported states mean, including that a service the +// OS calls running is not necessarily serving. +const statusCmdLong = `Show status of the ctrld service. + +Reports both what the OS service manager thinks and whether ctrld has finished +starting up, since a service can be registered as running while its process is +still in startup and serving nothing. + +Exit codes: + 0 running and serving, or running with startup not verified + 1 stopped + 2 status unknown + 3 registered as running, but startup has not completed + +Verifying startup requires reaching ctrld's control socket. On Linux, BSD and macOS +that socket lives in a directory only the privileged user resolves, so an +unprivileged "ctrld status" reports the service manager's view and says startup was +not verified rather than claiming the service is unhealthy. Exit 3 is only reported +when the check could actually be made.` + +// Status implements the logic from cmdStatus.Run +func (sc *ServiceCommand) Status(cmd *cobra.Command, args []string) error { + logger := mainLog.Load() + logger.Debug().Msg("Service status command started") + + s, _, err := sc.initializeServiceManager() + if err != nil { + logger.Error().Err(err).Msg("Failed to initialize service manager") + return err + } + + status, err := s.Status() + if err != nil { + logger.Error().Msg(err.Error()) + os.Exit(1) + } + + switch status { + case service.StatusUnknown: + logger.Notice().Msg("Unknown status") + os.Exit(statusExitUnknown) + case service.StatusRunning: + // The service manager only knows a process was created. During the Windows + // Firewall Mode incident it reported "running" for a process stuck in API + // preflight that had never created its control socket, started a DNS + // listener, or applied any policy - and "Service is running" sent the + // investigation looking elsewhere. Probe readiness before claiming it. + ready, probeErr := serviceReady() + if probeErr != nil { + logger.Debug().Err(probeErr).Msg("Readiness probe did not confirm startup") + } + r := classifyReadiness(ready, probeErr, readinessVerifiable()) + for _, msg := range r.messages { + logger.Notice().Msg(msg) + } + os.Exit(r.exitCode) + case service.StatusStopped: + logger.Notice().Msg("Service is stopped") + os.Exit(statusExitStopped) + } + + logger.Debug().Msg("Service status command completed") + return nil +} + +// readiness is what "ctrld status" reports for a service the service manager +// considers running. +type readiness struct { + messages []string + exitCode int +} + +// readinessVerifiable reports whether a failed control-socket probe can be trusted to +// mean "the service has not finished starting up". +// +// It can only mean that if this process resolves the same socket path the daemon +// created, and socketDir() is caller-relative on unix: it returns the system directory +// only when that is writable, and the caller's home directory otherwise. So a +// root-owned daemon listens on /var/run/ctrld_control.sock while an unprivileged +// "ctrld status" looks under $HOME, finds nothing, and gets ENOENT - which means "wrong +// path", not "not ready". Reporting exit 3 there would tell a monitoring check to +// restart a perfectly healthy daemon. +// +// On Windows and mobile socketDir() is the install/home directory for every caller, so +// the probe is comparable - which matters because Windows is where the hung-start this +// exit code exists for was seen. On Windows that only holds while this binary is the +// installed one: a copy run from elsewhere resolves a different socket directory, so its +// failed probe would say nothing about the service. installedServiceDirMatches() checks +// that, and answers true when it cannot tell, preserving the previous behaviour. +func readinessVerifiable() bool { + if isMobile() { + return true + } + if runtime.GOOS == "windows" { + return installedServiceDirMatches() + } + elevated, err := hasElevatedPrivilege() + return err == nil && elevated +} + +// classifyReadiness turns a control-socket probe result into the report for a service +// the service manager calls running. +// +// verifiable comes from readinessVerifiable: when it is false a failed probe says +// nothing about the service, so the report falls back to the service manager's view. +// A *successful* probe is still conclusive either way - reaching the socket at all is +// positive evidence, whoever the caller is. +func classifyReadiness(ready bool, err error, verifiable bool) readiness { + switch { + case ready: + return readiness{ + messages: []string{"Service is running"}, + exitCode: statusExitRunning, + } + case !verifiable: + return readiness{ + messages: []string{"Service is running (startup not verified: re-run with elevated privileges to check readiness)"}, + exitCode: statusExitRunning, + } + case errors.Is(err, errReadinessNotReported): + // The service answered, just not with a verdict - an older daemon without the + // /started route. It is alive and reachable, so the service manager's view is + // the best available answer. + return readiness{ + messages: []string{"Service is running (startup not verified: this ctrld build does not report readiness)"}, + exitCode: statusExitRunning, + } + case errors.Is(err, fs.ErrPermission): + // Without access to the control socket there is nothing to report beyond the + // service manager's view. Do not call a service unhealthy because the caller + // lacks privilege. + return readiness{ + messages: []string{"Service is running (startup not verified: control socket requires elevated privileges)"}, + exitCode: statusExitRunning, + } + default: + return readiness{ + messages: []string{ + "Service is registered as running, but has not completed startup: it is not serving DNS", + "Check the ctrld log for why startup did not finish, then restart the service", + }, + exitCode: statusExitNotReady, + } + } +} + +// serviceReady reports whether a running ctrld has finished starting up, by asking +// its control server. The control server answers /started only once the onStarted +// hooks have completed, which is after the DNS listeners are up, so a successful +// probe means the process is actually serving rather than merely alive. +// +// An error means "could not confirm readiness" and is returned for the caller to +// classify: a refused connection or missing socket is a process that never got that +// far, while a permission error says nothing about the service's health. +func serviceReady() (bool, error) { + dir, err := socketDir() + if err != nil { + return false, err + } + return serviceReadyAt(filepath.Join(dir, ControlSocketName()), serviceReadinessTimeout) +} + +// errReadinessNotReported marks a control server that answered without a readiness +// verdict. +// +// http.Client.Post returns (resp, nil) for any status, so a daemon with no /started +// route answers 404 and an internal failure answers 5xx - neither says the service has +// not started. Reporting "not ready" there tells a monitoring check to restart a healthy +// service, and it happens in normal operation: after an upgrade replaces the binary on +// disk but before the service restarts, and throughout a mixed-version rollout. +var errReadinessNotReported = errors.New("control server did not report readiness") + +// serviceReadyAt is serviceReady against an explicit socket path and timeout. +func serviceReadyAt(sockPath string, timeout time.Duration) (bool, error) { + cc := newControlClient(sockPath) + cc.c.Timeout = timeout + resp, err := cc.post(startedPath, nil) + if err != nil { + return false, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusRequestTimeout: + // The daemon's own verdict: its onStarted hooks have not completed. This is the + // hung start statusExitNotReady exists for. + return false, nil + default: + return false, fmt.Errorf("%w: HTTP %d", errReadinessNotReported, resp.StatusCode) + } +} diff --git a/cmd/cli/commands_service_status_test.go b/cmd/cli/commands_service_status_test.go new file mode 100644 index 00000000..2d4d3b3e --- /dev/null +++ b/cmd/cli/commands_service_status_test.go @@ -0,0 +1,281 @@ +package cli + +import ( + "errors" + "io/fs" + "net" + "net/http" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +// startControlSocket serves handler on a unix socket and returns its path. +func startControlSocket(t *testing.T, handler http.HandlerFunc) string { + t.Helper() + // Keep the path short: unix socket paths have a low length limit. + dir, err := os.MkdirTemp("", "ctrldsock") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + sockPath := filepath.Join(dir, "s.sock") + ln, err := net.Listen("unix", sockPath) + if err != nil { + t.Skipf("cannot listen on a unix socket: %v", err) + } + mux := http.NewServeMux() + mux.Handle(startedPath, handler) + srv := &http.Server{Handler: mux} + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return sockPath +} + +func TestServiceReadyAt(t *testing.T) { + t.Run("ready when the control server reports started", func(t *testing.T) { + sock := startControlSocket(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + ready, err := serviceReadyAt(sock, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ready { + t.Error("ready = false, want true") + } + }) + + t.Run("not ready when startup has not finished", func(t *testing.T) { + // What /started returns when the onStarted hooks have not completed. + sock := startControlSocket(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusRequestTimeout) + }) + ready, err := serviceReadyAt(sock, time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ready { + t.Error("ready = true for a control server that has not finished startup") + } + }) + + t.Run("not ready when there is no control socket", func(t *testing.T) { + // The incident: the process was alive but had never created the socket, so + // every control request was refused. + ready, err := serviceReadyAt(filepath.Join(t.TempDir(), "absent.sock"), time.Second) + if ready { + t.Error("ready = true with no control socket") + } + if err == nil { + t.Error("expected an error when the control socket does not exist") + } + }) + + t.Run("not ready when the probe times out", func(t *testing.T) { + sock := startControlSocket(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + }) + ready, err := serviceReadyAt(sock, 50*time.Millisecond) + if ready { + t.Error("ready = true for a probe that timed out") + } + if err == nil { + t.Error("expected an error when the probe times out") + } + }) +} + +func TestClassifyReadiness(t *testing.T) { + tests := []struct { + name string + ready bool + err error + verifiable bool + wantCode int + }{ + { + name: "ready", + ready: true, + verifiable: true, + wantCode: statusExitRunning, + }, + { + // The service manager says running, the process is not serving. This + // must not report success. + name: "running but never finished startup", + err: errors.New("connect: connection refused"), + verifiable: true, + wantCode: statusExitNotReady, + }, + { + // A caller without privilege cannot probe; that is not evidence of a + // broken service, so it must not be reported as one. + name: "probe not permitted", + err: fs.ErrPermission, + verifiable: true, + wantCode: statusExitRunning, + }, + { + name: "wrapped permission error", + err: &net.OpError{Op: "dial", Err: fs.ErrPermission}, + verifiable: true, + wantCode: statusExitRunning, + }, + { + // The P2: an unprivileged caller on unix resolves a socket path the + // daemon never used, so the probe fails with ENOENT rather than a + // permission error. That says nothing about the service and must not be + // reported as unhealthy - a monitoring check acting on exit 3 would + // restart a healthy daemon. + name: "missing socket at an unverifiable path", + err: &net.OpError{Op: "dial", Err: os.ErrNotExist}, + verifiable: false, + wantCode: statusExitRunning, + }, + { + name: "connection refused at an unverifiable path", + err: errors.New("connect: connection refused"), + verifiable: false, + wantCode: statusExitRunning, + }, + { + // A probe that actually reached the socket is conclusive whoever ran it. + name: "successful probe is trusted even when unverifiable", + ready: true, + verifiable: false, + wantCode: statusExitRunning, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := classifyReadiness(tc.ready, tc.err, tc.verifiable) + if got.exitCode != tc.wantCode { + t.Errorf("exitCode = %d, want %d", got.exitCode, tc.wantCode) + } + if len(got.messages) == 0 { + t.Error("no message to report") + } + }) + } +} + +// TestReadinessVerifiableMatchesSocketVisibility is the closure test for the P2: the +// not-ready verdict must only be reachable when this process resolves the same socket +// directory the daemon uses. +// +// On unix that is the privileged user's path, so an unprivileged run - which is how +// "ctrld status" is normally invoked, since only darwin has an elevation PreRun and the +// root-level alias has none - must not be able to reach exit 3. +func TestReadinessVerifiableMatchesSocketVisibility(t *testing.T) { + verifiable := readinessVerifiable() + + if runtime.GOOS == "windows" { + if !verifiable { + t.Error("on Windows every caller resolves the install directory, so the probe is always verifiable") + } + return + } + + elevated, err := hasElevatedPrivilege() + if err != nil { + t.Skipf("cannot determine privilege: %v", err) + } + if verifiable != elevated { + t.Errorf("readinessVerifiable() = %v, want %v (elevated)", verifiable, elevated) + } + + if !elevated { + // The shape the review asked to assert: unprivileged, healthy daemon, and a + // probe that cannot see its socket must still report running. + dir, err := socketDir() + if err != nil { + t.Fatalf("socketDir(): %v", err) + } + if dir == "/var/run" { + t.Skip("unprivileged but /var/run is writable, so the probe path does match") + } + r := classifyReadiness(false, &net.OpError{Op: "dial", Err: os.ErrNotExist}, verifiable) + if r.exitCode == statusExitNotReady { + t.Errorf("unprivileged status probing %q reported not-ready (exit %d) for a healthy service", dir, r.exitCode) + } + } +} + +// Every status must map to its own exit code: a caller that cannot tell a hung +// service from a healthy or a stopped one is back to the incident's diagnostics. +// +// The literal values are the contract. statusCmdLong documents them and monitoring +// scripts key off them, so asserting the constants against each other would let a +// renumbering keep the suite green while silently breaking every caller. +func TestStatusExitCodesAreDistinct(t *testing.T) { + for _, tc := range []struct { + name string + got int + want int + }{ + {"running", statusExitRunning, 0}, + {"stopped", statusExitStopped, 1}, + {"unknown", statusExitUnknown, 2}, + {"not ready", statusExitNotReady, 3}, + } { + if tc.got != tc.want { + t.Errorf("%s exit code = %d, want %d: statusCmdLong and monitoring scripts document this value", tc.name, tc.got, tc.want) + } + } + + codes := map[int]string{ + statusExitRunning: "running", + statusExitStopped: "stopped", + statusExitUnknown: "unknown", + statusExitNotReady: "not ready", + } + if len(codes) != 4 { + t.Errorf("status exit codes collide, only %d distinct: %v", len(codes), codes) + } +} + +// TestReadinessProbeStatusHandling covers what each control-server answer means. +// +// http.Client.Post returns (resp, nil) for any status code, so a daemon without the +// /started route answers 404 and the probe must report "cannot confirm" rather than "not +// started". That state is reached in normal operation - after an upgrade replaces the +// binary but before the service restarts, and throughout a mixed-version rollout - and +// reporting exit 3 there tells monitoring to restart a healthy service. +func TestReadinessProbeStatusHandling(t *testing.T) { + tests := []struct { + name string + status int + wantReady bool + wantReported bool // whether the answer carries a readiness verdict + wantExitCode int + }{ + {"started", http.StatusOK, true, true, statusExitRunning}, + {"still starting", http.StatusRequestTimeout, false, true, statusExitNotReady}, + {"no readiness route", http.StatusNotFound, false, false, statusExitRunning}, + {"control server error", http.StatusInternalServerError, false, false, statusExitRunning}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + status := tc.status + sock := startControlSocket(t, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + }) + + ready, err := serviceReadyAt(sock, time.Second) + if ready != tc.wantReady { + t.Errorf("ready = %v, want %v", ready, tc.wantReady) + } + if reported := !errors.Is(err, errReadinessNotReported); reported != tc.wantReported { + t.Errorf("readiness reported = %v, want %v (err: %v)", reported, tc.wantReported, err) + } + if got := classifyReadiness(ready, err, true).exitCode; got != tc.wantExitCode { + t.Errorf("exit code = %d, want %d", got, tc.wantExitCode) + } + }) + } +} diff --git a/cmd/cli/commands_service_stop.go b/cmd/cli/commands_service_stop.go new file mode 100644 index 00000000..c388fcd9 --- /dev/null +++ b/cmd/cli/commands_service_stop.go @@ -0,0 +1,61 @@ +package cli + +import ( + "errors" + "os" + + "github.com/kardianos/service" + "github.com/spf13/cobra" +) + +// Stop implements the logic from cmdStop.Run +func (sc *ServiceCommand) Stop(cmd *cobra.Command, args []string) error { + logger := mainLog.Load() + logger.Debug().Msg("Service stop command started") + + readConfig(false) + v.Unmarshal(&cfg) + + s, p, err := sc.initializeServiceManager() + if err != nil { + logger.Error().Err(err).Msg("Failed to initialize service manager") + return err + } + + p.cfg = &cfg + if iface == "" { + iface = autoIface + } + p.preRun() + if ir := runningIface(s); ir != nil { + p.runningIface = ir.Name + p.requiredMultiNICsConfig = ir.All + } + + initInteractiveLogging() + + status, err := s.Status() + if errors.Is(err, service.ErrNotInstalled) { + logger.Warn().Msg("Service not installed") + return nil + } + if status == service.StatusStopped { + logger.Warn().Msg("Service is already stopped") + return nil + } + + if err := checkDeactivationPin(s, nil); isCheckDeactivationPinErr(err) { + logger.Error().Msg("Deactivation pin check failed") + os.Exit(deactivationPinInvalidExitCode) + } + + logger.Debug().Msg("Stopping service") + if doTasks([]task{{s.Stop, true, "Stop"}}) { + logger.Notice().Msg("Service stopped") + } else { + logger.Error().Msg("Service stop failed") + } + + logger.Debug().Msg("Service stop command completed") + return nil +} diff --git a/cmd/cli/commands_service_uninstall.go b/cmd/cli/commands_service_uninstall.go new file mode 100644 index 00000000..f1801fc2 --- /dev/null +++ b/cmd/cli/commands_service_uninstall.go @@ -0,0 +1,139 @@ +package cli + +import ( + "fmt" + "net" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld" +) + +// Uninstall implements the logic from cmdUninstall.Run +func (sc *ServiceCommand) Uninstall(cmd *cobra.Command, args []string) error { + logger := mainLog.Load() + logger.Debug().Msg("Service uninstall command started") + + readConfig(false) + v.Unmarshal(&cfg) + + s, p, err := sc.initializeServiceManager() + if err != nil { + logger.Error().Err(err).Msg("Failed to initialize service manager") + return err + } + + p.cfg = &cfg + if iface == "" { + iface = autoIface + } + p.preRun() + if ir := runningIface(s); ir != nil { + p.runningIface = ir.Name + p.requiredMultiNICsConfig = ir.All + } + + if err := checkDeactivationPin(s, nil); isCheckDeactivationPinErr(err) { + logger.Error().Msg("Deactivation pin check failed") + os.Exit(deactivationPinInvalidExitCode) + } + + logger.Debug().Msg("Starting service uninstall") + uninstall(p, s) + + if cleanup { + logger.Debug().Msg("Performing cleanup operations") + var files []string + // Config file. + files = append(files, v.ConfigFileUsed()) + // Log files. For safety, only remove the log_path chain if that path + // is absolute. + logFile := normalizeLogFilePath(cfg.Service.LogPath) + if !filepath.IsAbs(logFile) { + logFile = "" + } + internalLogs := []string{absHomeDir(logFileName), absHomeDir(journalLogFileName)} + for _, err := range removeLogFiles(logFile, debugLogBudget(&cfg.Service).backups, internalLogs) { + logger.Warn().Err(err).Msg("Failed to remove log file") + } + // Socket files. + if dir, _ := socketDir(); dir != "" { + files = append(files, filepath.Join(dir, ctrldControlUnixSock)) + files = append(files, filepath.Join(dir, ctrldLogUnixSock)) + } + // Static DNS settings files. + withEachPhysicalInterfaces("", "", func(i *net.Interface) error { + file := ctrld.SavedStaticDnsSettingsFilePath(i) + files = append(files, file) + return nil + }) + bin, err := os.Executable() + if err != nil { + logger.Warn().Err(err).Msg("Failed to get executable path") + } + if bin != "" && supportedSelfDelete { + files = append(files, bin) + } + // Backup file after upgrading. + oldBin := bin + oldBinSuffix + if _, err := os.Stat(oldBin); err == nil { + files = append(files, oldBin) + } + for _, file := range files { + if file == "" { + continue + } + if err := os.Remove(file); err == nil { + logger.Notice().Str("file", file).Msg("File removed during cleanup") + } else { + logger.Debug().Err(err).Str("file", file).Msg("Failed to remove file during cleanup") + } + } + // Self-delete the ctrld binary if supported + if err := selfDeleteExe(); err != nil { + logger.Warn().Err(err).Msg("Failed to delete ctrld binary") + } else { + if !supportedSelfDelete { + logger.Debug().Msgf("File removed: %s", bin) + } + } + + logger.Debug().Msg("Cleanup operations completed") + } + + logger.Debug().Msg("Service uninstall command completed") + return nil +} + +// removeLogFiles deletes the log files of this installation and returns what +// it could not remove. A missing file is not an error. +func removeLogFiles(logPath string, backups int, internalPaths []string) []error { + for _, path := range internalPaths { + pruneNumberedBackups(path, 0) + } + var errs []error + for _, path := range logFilesToRemove(logPath, backups, internalPaths) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + errs = append(errs, fmt.Errorf("remove %s: %w", path, err)) + } + } + return errs +} + +// logFilesToRemove names the log files of this installation. The log_path +// backups come from the configured count, because log_path can name a file in +// a directory that holds the files of other programs, and a scan of that +// directory would take their files too. An empty logPath leaves the internal +// files alone. +func logFilesToRemove(logPath string, backups int, internalPaths []string) []string { + var paths []string + if logPath != "" { + paths = append(paths, logPath) + for index := 1; index <= backups; index++ { + paths = append(paths, fmt.Sprintf("%s.%d", logPath, index)) + } + } + return append(paths, internalPaths...) +} diff --git a/cmd/cli/commands_start_test.go b/cmd/cli/commands_start_test.go new file mode 100644 index 00000000..882fac2f --- /dev/null +++ b/cmd/cli/commands_start_test.go @@ -0,0 +1,122 @@ +package cli + +import ( + "testing" + "time" +) + +func TestServiceStageFailureCode(t *testing.T) { + tests := []struct { + taskName string + wantCode provisionFailureCode + wantOK bool + }{ + {"Install", provisionCodeServiceInstall, true}, + {"Start", provisionCodeServiceStartFailed, true}, + {"Checking config", "", false}, + {"", "", false}, + } + for _, tc := range tests { + code, ok := serviceStageFailureCode(tc.taskName) + if code != tc.wantCode || ok != tc.wantOK { + t.Errorf("serviceStageFailureCode(%q) = (%q, %v), want (%q, %v)", tc.taskName, code, ok, tc.wantCode, tc.wantOK) + } + } +} + +func stubProvisionExit(t *testing.T) *int { + t.Helper() + exitCode := -1 + old := provisionExit + provisionExit = func(code int) { exitCode = code } + t.Cleanup(func() { provisionExit = old }) + return &exitCode +} + +func TestReportStartFailureUsesFreshDaemonResult(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + startedAt := time.Now() + daemonResult := newProvisionResult(provisionCodeAPIUnreachable, "daemon could not reach the API", nil) + if err := writeProvisionResult(daemonResult); err != nil { + t.Fatal(err) + } + + reportStartFailure(startedAt, "generic self-check failure") + + if *exitCode != provisionExitCodeForCode[provisionCodeAPIUnreachable] { + t.Errorf("exit code = %d, want the daemon's own exit code %d", *exitCode, provisionExitCodeForCode[provisionCodeAPIUnreachable]) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != string(provisionCodeAPIUnreachable) { + t.Errorf("persisted code = %q, want the daemon's own code untouched", out.Code) + } +} + +func TestReportStartFailureFallsBackOnStaleDaemonResult(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + stale := newProvisionResult(provisionCodeAPIUnreachable, "an old failure", nil) + stale.Timestamp = time.Now().Add(-1 * time.Hour).UTC().Format(time.RFC3339) + if err := writeProvisionResult(stale); err != nil { + t.Fatal(err) + } + + startedAt := time.Now() + reportStartFailure(startedAt, "test query failed: timeout") + + if *exitCode != provisionExitCodeForCode[provisionCodeServiceSelfCheck] { + t.Errorf("exit code = %d, want SERVICE_SELFCHECK_FAILED exit %d", *exitCode, provisionExitCodeForCode[provisionCodeServiceSelfCheck]) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != string(provisionCodeServiceSelfCheck) { + t.Errorf("persisted code = %q, want %q", out.Code, provisionCodeServiceSelfCheck) + } + if out.Message != "test query failed: timeout" { + t.Errorf("persisted message = %q, want the fallback message", out.Message) + } +} + +func TestReportStartFailureRejectsUntrustedFile(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + planted := newProvisionResult(provisionCodeAPIUnreachable, "planted", nil) + planted.Code = "FAKE_CODE" + planted.ExitCode = 99 + if err := writeProvisionResult(planted); err != nil { + t.Fatal(err) + } + + reportStartFailure(time.Now().Add(-time.Minute), "self-check failed") + + if *exitCode != provisionExitCodeForCode[provisionCodeServiceSelfCheck] { + t.Errorf("exit = %d, want the fallback %d, never the planted 99", *exitCode, provisionExitCodeForCode[provisionCodeServiceSelfCheck]) + } +} + +func TestReportStartFailureFallsBackWhenResultFileMissing(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := stubProvisionExit(t) + + reportStartFailure(time.Now(), "firewall hint") + + if *exitCode != provisionExitCodeForCode[provisionCodeServiceSelfCheck] { + t.Errorf("exit code = %d, want SERVICE_SELFCHECK_FAILED exit %d", *exitCode, provisionExitCodeForCode[provisionCodeServiceSelfCheck]) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Message != "firewall hint" { + t.Errorf("persisted message = %q, want the fallback message", out.Message) + } +} diff --git a/cmd/cli/commands_test.go b/cmd/cli/commands_test.go new file mode 100644 index 00000000..91a6c1ca --- /dev/null +++ b/cmd/cli/commands_test.go @@ -0,0 +1,202 @@ +package cli + +import ( + "bytes" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestBasicCommandStructure tests the actual root command structure +func TestBasicCommandStructure(t *testing.T) { + // Test the actual root command that's returned from initCLI() + rootCmd := initCLI() + + // Test that root command has basic properties + assert.Equal(t, "ctrld-client", rootCmd.Use) + assert.NotEmpty(t, rootCmd.Short, "Root command should have a short description") + + // Test that root command has subcommands + commands := rootCmd.Commands() + assert.NotNil(t, commands, "Root command should have subcommands") + assert.Greater(t, len(commands), 0, "Root command should have at least one subcommand") + + // Test that expected commands exist + expectedCommands := []string{"run", "service", "clients", "upgrade", "log"} + for _, cmdName := range expectedCommands { + found := false + for _, cmd := range commands { + if cmd.Name() == cmdName { + found = true + break + } + } + assert.True(t, found, "Expected command %s not found in root command", cmdName) + } +} + +// TestServiceCommandCreation tests service command creation +func TestServiceCommandCreation(t *testing.T) { + sc := NewServiceCommand() + require.NotNil(t, sc, "ServiceCommand should be created") + + // Test service config creation + config := sc.createServiceConfig() + require.NotNil(t, config, "Service config should be created") + assert.Equal(t, ctrldServiceName, config.Name) + assert.Equal(t, ctrldServiceDisplayName, config.DisplayName) + // Windows requires service display names to be unique and rejects a second + // registration with ERROR_DUPLICATE_SERVICE_NAME. Reusing the v1 service's + // display name would make "ctrld-client start" unable to install on hosts + // that still have v1 installed, so pin that they stay distinct (#565). + assert.NotEqual(t, "Control-D Helper Service", config.DisplayName) + assert.Equal(t, "A highly configurable, multi-protocol DNS forwarding proxy", config.Description) +} + +// TestServiceCommandSubCommands tests service command sub commands +func TestServiceCommandSubCommands(t *testing.T) { + rootCmd := &cobra.Command{ + Use: "ctrld-client", + Short: "DNS forwarding proxy", + } + + serviceCmd := InitServiceCmd(rootCmd) + require.NotNil(t, serviceCmd, "Service command should be created") + + // Test that service command has subcommands + subcommands := serviceCmd.Commands() + assert.Greater(t, len(subcommands), 0, "Service command should have subcommands") + + // Test specific subcommands exist + expectedCommands := []string{"start", "stop", "restart", "reload", "status", "uninstall", "interfaces"} + + for _, cmdName := range expectedCommands { + found := false + for _, cmd := range subcommands { + if cmd.Name() == cmdName { + found = true + break + } + } + assert.True(t, found, "Expected service subcommand %s not found", cmdName) + } +} + +// TestCommandHelp tests basic help functionality +func TestCommandHelp(t *testing.T) { + // Initialize the CLI to set up the root command + rootCmd := initCLI() + + // Test help command execution + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + + rootCmd.SetArgs([]string{"--help"}) + err := rootCmd.Execute() + assert.NoError(t, err, "Help command should execute without error") + assert.Contains(t, buf.String(), "dns forwarding proxy", "Help output should contain description") +} + +// TestCommandVersion tests version command +func TestCommandVersion(t *testing.T) { + // Initialize the CLI to set up the root command + rootCmd := initCLI() + + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + + // Test version command + rootCmd.SetArgs([]string{"--version"}) + err := rootCmd.Execute() + assert.NoError(t, err, "Version command should execute without error") + assert.Contains(t, buf.String(), "version", "Version output should contain version information") +} + +// TestCommandErrorHandling tests error handling +func TestCommandErrorHandling(t *testing.T) { + // Initialize the CLI to set up the root command + rootCmd := initCLI() + + // Test invalid flag instead of invalid command + rootCmd.SetArgs([]string{"--invalid-flag"}) + err := rootCmd.Execute() + assert.Error(t, err, "Invalid flag should return error") +} + +// TestCommandFlags tests flag functionality +func TestCommandFlags(t *testing.T) { + // Initialize the CLI to set up the root command + rootCmd := initCLI() + + // Test that root command has expected flags + verboseFlag := rootCmd.PersistentFlags().Lookup("verbose") + assert.NotNil(t, verboseFlag, "Verbose flag should exist") + assert.Equal(t, "v", verboseFlag.Shorthand) + + silentFlag := rootCmd.PersistentFlags().Lookup("silent") + assert.NotNil(t, silentFlag, "Silent flag should exist") + assert.Equal(t, "s", silentFlag.Shorthand) +} + +// TestCommandExecution tests basic command execution +func TestCommandExecution(t *testing.T) { + // Initialize the CLI to set up the root command + rootCmd := initCLI() + + // Test that root command can be executed (help command) + var buf bytes.Buffer + rootCmd.SetOut(&buf) + rootCmd.SetErr(&buf) + + rootCmd.SetArgs([]string{"--help"}) + err := rootCmd.Execute() + assert.NoError(t, err, "Root command should execute without error") + assert.Contains(t, buf.String(), "dns forwarding proxy", "Help output should contain description") +} + +// TestCommandArgs tests argument handling +func TestCommandArgs(t *testing.T) { + // Initialize the CLI to set up the root command + rootCmd := initCLI() + + // Test that root command can handle arguments properly + // Test with no args (should succeed) + err := rootCmd.Execute() + assert.NoError(t, err, "Root command with no args should execute") + + // Test with help flag (should succeed) + rootCmd.SetArgs([]string{"--help"}) + err = rootCmd.Execute() + assert.NoError(t, err, "Root command with help flag should execute") +} + +// TestCommandSubcommands tests subcommand functionality +func TestCommandSubcommands(t *testing.T) { + // Initialize the CLI to set up the root command + rootCmd := initCLI() + + // Test that root command has subcommands + commands := rootCmd.Commands() + assert.Greater(t, len(commands), 0, "Root command should have subcommands") + + // Test that specific subcommands exist and can be executed + expectedSubcommands := []string{"run", "service", "clients", "upgrade", "log"} + for _, subCmdName := range expectedSubcommands { + // Find the subcommand + var subCmd *cobra.Command + for _, cmd := range commands { + if cmd.Name() == subCmdName { + subCmd = cmd + break + } + } + assert.NotNil(t, subCmd, "Subcommand %s should exist", subCmdName) + + // Test that subcommand has help + assert.NotEmpty(t, subCmd.Short, "Subcommand %s should have a short description", subCmdName) + } +} diff --git a/cmd/cli/commands_upgrade.go b/cmd/cli/commands_upgrade.go new file mode 100644 index 00000000..067d9daf --- /dev/null +++ b/cmd/cli/commands_upgrade.go @@ -0,0 +1,396 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "strings" + "time" + + "github.com/kardianos/service" + "github.com/minio/selfupdate" + "github.com/spf13/cobra" +) + +const ( + upgradeChannelDev = "dev" + upgradeChannelProd = "prod" + upgradeChannelDefault = "default" +) + +// UpgradeCommand handles upgrade-related operations +type UpgradeCommand struct { +} + +// NewUpgradeCommand creates a new upgrade command handler +func NewUpgradeCommand() (*UpgradeCommand, error) { + return &UpgradeCommand{}, nil +} + +// Upgrade performs the upgrade operation +func (uc *UpgradeCommand) Upgrade(cmd *cobra.Command, args []string) error { + upgradeChannel := map[string]string{ + upgradeChannelDefault: "https://dl.controld.dev", + upgradeChannelDev: "https://dl.controld.dev", + upgradeChannelProd: "https://dl.controld.com", + } + if isStableVersion(curVersion()) { + upgradeChannel[upgradeChannelDefault] = upgradeChannel[upgradeChannelProd] + } + + bin, err := os.Executable() + if err != nil { + mainLog.Load().Fatal().Err(err).Msg("Failed to get current ctrld binary path") + } + + readConfig(false) + v.Unmarshal(&cfg) + svcCmd := NewServiceCommand() + s, p, err := svcCmd.initializeServiceManager() + if err != nil { + mainLog.Load().Error().Msg(err.Error()) + return nil + } + + if iface == "" { + iface = autoIface + } + p.preRun() + if ir := runningIface(s); ir != nil { + p.runningIface = ir.Name + p.requiredMultiNICsConfig = ir.All + } + + svcInstalled := true + if _, err := s.Status(); errors.Is(err, service.ErrNotInstalled) { + svcInstalled = false + } + + oldBin := bin + oldBinSuffix + baseUrl := upgradeChannel[upgradeChannelDefault] + if len(args) > 0 { + channel := args[0] + switch channel { + case upgradeChannelProd, upgradeChannelDev: // ok + default: + mainLog.Load().Fatal().Msgf("Upgrade argument must be either %q or %q", upgradeChannelProd, upgradeChannelDev) + } + baseUrl = upgradeChannel[channel] + } + + dlUrl := upgradeUrl(baseUrl) + mainLog.Load().Debug().Msgf("Downloading binary: %s", dlUrl) + + resp, err := getWithRetry(dlUrl, downloadServerIp) + if err != nil { + mainLog.Load().Fatal().Err(err).Msg("Failed to download binary") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + mainLog.Load().Fatal().Msgf("Could not download binary: %s", http.StatusText(resp.StatusCode)) + } + + mainLog.Load().Debug().Msg("Updating current binary") + if err := selfupdate.Apply(resp.Body, selfupdate.Options{OldSavePath: oldBin}); err != nil { + if rerr := selfupdate.RollbackError(err); rerr != nil { + mainLog.Load().Error().Err(rerr).Msg("Could not rollback old binary") + } + mainLog.Load().Fatal().Err(err).Msg("Failed to update current binary") + } + + doRestart := func() bool { + if !svcInstalled { + return true + } + tasks := []task{ + {s.Stop, true, "Stop"}, + {func() error { + // restore static DNS settings or DHCP + p.resetDNS(false, true) + return nil + }, false, "Cleanup"}, + {func() error { + time.Sleep(time.Second * 1) + return nil + }, false, "Waiting for service to stop"}, + } + doTasks(tasks) + + tasks = []task{ + {s.Start, true, "Start"}, + } + if doTasks(tasks) { + if dir, err := socketDir(); err == nil { + // Only a ready service counts: a build that starts but + // never becomes ready must be rolled back. + if cc := newReadySocketControlClient(context.TODO(), s, dir); cc != nil { + _, _ = cc.post(ifacePath, nil) + return true + } + } + } + return false + } + + if svcInstalled { + mainLog.Load().Debug().Msg("Restarting ctrld service using new binary") + } + + if doRestart() { + _ = os.Remove(oldBin) + _ = os.Chmod(bin, 0755) + ver, err := binaryVersion(bin) + if err != nil { + mainLog.Load().Warn().Err(err).Msg("Failed to get new binary version") + ver = "unknown version" + } + mainLog.Load().Notice().Msgf("Upgrade successful - %s", ver) + return nil + } + + stop := func() error { + if !svcInstalled { + return nil + } + if err := stopServiceAndWait(s, upgradeStopTimeout); err != nil { + return err + } + // Mirror the Cleanup task in doRestart: leave DNS settings as the OS had them, + // not as a half-started ctrld left them. + p.resetDNS(false, true) + return nil + } + return rollbackToPreviousBinary(bin, oldBin, stop, doRestart) +} + +// rollbackToPreviousBinary restores oldBin over bin after the replacement failed to +// become ready, and restarts the service on the restored binary. +// +// stop must leave the replacement's process gone, because every step here modifies +// the executable that process is running from. It is called first for that reason: +// readiness failing does not mean the process exited - the service manager can report +// a started service whose process never became operational, which is what the Windows +// Firewall Mode incident produced. Windows holds an exclusive lock on a running +// executable's image, so the previous code's os.Remove(bin) failed with "Access is +// denied", and because that was fatal the restore never ran: the broken binary stayed +// installed with the previous one stranded at its _previous name. +// +// Stopping first also puts the host back in a known state. A stopped ctrld holds no +// WFP or NRPT enforcement, so a replacement that was blocking traffic stops blocking +// it here instead of at the next reboot. +func rollbackToPreviousBinary(bin, oldBin string, stop func() error, restart func() bool) error { + if err := stop(); err != nil { + mainLog.Load().Error().Err(err).Msg("Could not confirm the service stopped; not modifying its binary") + return err + } + + // Only restore a previous binary that actually runs. During the incident the + // _previous file existed but produced no version output; renaming that over the + // current binary would have replaced a service that starts but hangs with one that + // cannot start at all. + // + // The probe is retried for the same reason removeBinaryWithRetry is: on Windows a + // single exec can fail transiently while antivirus scans the file or the disk is + // busy, and treating that as "no usable previous binary" leaves the host stopped + // with the broken binary installed and no enforcement - an end state worse than + // restoring a binary that turns out to be bad, which the restart check below + // catches. + // + // Running "--version" proves the file executes. It is not an authenticity check: + // nothing here compares a signature or checksum before a file becomes the installed + // service binary. That is acceptable only because the install directory is writable + // by administrators alone, which is this command's standing assumption. + prevVer, err := binaryVersionWithRetry(oldBin, upgradeStopTimeout) + if err != nil { + mainLog.Load().Error().Err(err).Msgf("Previous binary at %s is not usable, keeping it for inspection", oldBin) + mainLog.Load().Notice().Msgf("Service is stopped and %s is still the installed binary - no ctrld enforcement is active", bin) + return fmt.Errorf("upgrade failed and no usable previous binary to restore: %w", err) + } + + mainLog.Load().Warn().Msgf("Restoring previous binary: %s (%s)", oldBin, prevVer) + if err := removeBinaryWithRetry(bin, upgradeStopTimeout); err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to remove new binary") + mainLog.Load().Notice().Msg("Service is stopped - no ctrld enforcement is active") + return err + } + if err := os.Rename(oldBin, bin); err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to restore old binary") + mainLog.Load().Notice().Msgf("Service is stopped and %s is missing; reinstall ctrld to recover", bin) + return err + } + if restart() { + mainLog.Load().Notice().Msgf("Restored previous binary successfully - %s", prevVer) + return nil + } + + mainLog.Load().Error().Msg("Restored the previous binary but it did not become ready either") + return errors.New("upgrade failed and the restored binary did not become ready") +} + +const ( + // upgradeStopTimeout bounds how long rollback waits for the replacement process to + // exit, and for Windows to release the lock on its image afterwards. + upgradeStopTimeout = 30 * time.Second + // upgradeStopPollInterval is how often the service status is re-checked while + // waiting for the process to exit. + upgradeStopPollInterval = 500 * time.Millisecond + // binaryVersionTimeout bounds the "--version" probe, so a binary that hangs on + // startup cannot hang the upgrade. + binaryVersionTimeout = 10 * time.Second +) + +// stopServiceAndWait stops the service and waits until the service manager reports +// it stopped. Rollback needs the process gone, not merely asked to stop: a stop +// request returns before the process exits, and on Windows the executable stays +// locked until it does. +func stopServiceAndWait(s service.Service, timeout time.Duration) error { + if err := s.Stop(); err != nil { + // Not fatal: the service may already be stopped, or stopping may fail while + // the process is exiting anyway. The status poll below decides. + mainLog.Load().Debug().Err(err).Msg("Stop request failed, waiting for the process to exit anyway") + } + deadline := time.Now().Add(timeout) + statusReadable := false + var lastErr error + for { + status, err := s.Status() + switch { + case errors.Is(err, service.ErrNotInstalled): + return nil + case err == nil: + statusReadable = true + if status == service.StatusStopped { + return nil + } + default: + lastErr = err + } + if !time.Now().Before(deadline) { + if !statusReadable { + // The status was never readable, so "did not stop" was never observed - + // only "could not be observed". Refusing to continue here would leave the + // broken binary installed with the service stopped and no enforcement, + // which is the outcome rollback exists to avoid. Let the caller proceed: + // the remove is retried while the image is locked, and the restart check + // still has to pass before this reports success. + mainLog.Load().Warn().Err(lastErr).Msgf("Could not read service status within %s; continuing with rollback", timeout) + return nil + } + return fmt.Errorf("service did not stop within %s", timeout) + } + time.Sleep(upgradeStopPollInterval) + } +} + +// binaryVersionWithRetry probes a binary's version, retrying transient exec failures +// until timeout. Only the last error is reported: the earlier attempts are noise once a +// retry has been made. +func binaryVersionWithRetry(path string, timeout time.Duration) (string, error) { + deadline := time.Now().Add(timeout) + for { + version, err := binaryVersionFn(path) + if err == nil { + return version, nil + } + if !time.Now().Before(deadline) { + return "", err + } + mainLog.Load().Debug().Err(err).Msgf("Version probe of %s failed, retrying", path) + time.Sleep(upgradeStopPollInterval) + } +} + +// removeBinaryWithRetry removes path, retrying while it is still locked. Windows +// releases the lock on an executable's image asynchronously after its process exits, +// so a remove issued immediately after the service reports stopped can still fail +// with "Access is denied". +func removeBinaryWithRetry(path string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := os.Remove(path) + if err == nil || errors.Is(err, os.ErrNotExist) { + return nil + } + if !time.Now().Before(deadline) { + return fmt.Errorf("could not remove %s within %s: %w", path, timeout, err) + } + time.Sleep(upgradeStopPollInterval) + } +} + +// binaryVersionFn is indirected so rollback can be tested without staging a runnable +// executable per platform. The probe itself is covered directly against the test +// binary; see TestBinaryVersion. +var binaryVersionFn = binaryVersion + +// binaryVersion runs path with "--version" and returns the version it reports. It +// answers "can this binary actually run on this host", which is what rollback needs +// to know before making a file the installed ctrld. +// +// On Windows path is ctrld-client.exe_previous, whose extension is not in PATHEXT. That +// resolves because os/exec only falls back to appending PATHEXT entries when the path +// has no extension at all (lp_windows.go findExecutable): with one present and the +// file on disk, it is used as-is. A suffix that left no extension - renaming +// oldBinSuffix such that the result is "ctrld-client_previous" - would break this probe with +// "executable file not found in %PATH%", and rollback would then refuse to restore a +// perfectly good binary. +func binaryVersion(path string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), binaryVersionTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, path, "--version").CombinedOutput() + if err != nil { + return "", fmt.Errorf("running %s --version: %w", path, err) + } + ver, ok := parseVersionOutput(string(out)) + if !ok { + return "", fmt.Errorf("unexpected --version output from %s: %q", path, strings.TrimSpace(string(out))) + } + return ver, nil +} + +// parseVersionOutput extracts the version from a binary's "--version" output. +// +// The expected prefix is built from cliName, which is also what the root command +// is named, because Cobra renders "--version" as " version " from +// that same name. Spelling the prefix out here instead would make a rename of the +// client silently break this parser - and with it rollback, which refuses to +// restore a previous binary whose version it cannot read. That failure mode is +// the worst one this code has: the service is already stopped, so a wrongly +// rejected previous binary leaves the host with no ctrld enforcement at all. +func parseVersionOutput(out string) (string, bool) { + // Not CutPrefix's own return: on a miss it hands back the whole input, which a + // caller that forgot to check the bool would store as if it were a version. + ver, ok := strings.CutPrefix(strings.TrimSpace(out), cliName+" version ") + if !ok { + return "", false + } + return ver, true +} + +// InitUpgradeCmd creates the upgrade command with proper logic +func InitUpgradeCmd(rootCmd *cobra.Command) *cobra.Command { + upgradeCmd := &cobra.Command{ + Use: "upgrade", + Short: "Upgrading ctrld to latest version", + ValidArgs: []string{upgradeChannelDev, upgradeChannelProd}, + Args: cobra.MaximumNArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + checkHasElevatedPrivilege() + }, + RunE: func(cmd *cobra.Command, args []string) error { + uc, err := NewUpgradeCommand() + if err != nil { + return err + } + return uc.Upgrade(cmd, args) + }, + } + + rootCmd.AddCommand(upgradeCmd) + + return upgradeCmd +} diff --git a/cmd/cli/commands_upgrade_test.go b/cmd/cli/commands_upgrade_test.go new file mode 100644 index 00000000..3b46db68 --- /dev/null +++ b/cmd/cli/commands_upgrade_test.go @@ -0,0 +1,365 @@ +package cli + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kardianos/service" +) + +// fakeService implements the parts of service.Service that rollback uses. Any other +// method panics, which keeps accidental dependencies visible. +type fakeService struct { + service.Service + + stopErr error + stopCalls int + statuses []service.Status // consumed one per Status() call; the last repeats + statusErr error + onStopCall func() +} + +func (f *fakeService) Stop() error { + f.stopCalls++ + if f.onStopCall != nil { + f.onStopCall() + } + return f.stopErr +} + +func (f *fakeService) Status() (service.Status, error) { + if f.statusErr != nil { + return service.StatusUnknown, f.statusErr + } + if len(f.statuses) == 0 { + return service.StatusStopped, nil + } + st := f.statuses[0] + if len(f.statuses) > 1 { + f.statuses = f.statuses[1:] + } + return st, nil +} + +func TestStopServiceAndWait(t *testing.T) { + tests := []struct { + name string + svc *fakeService + timeout time.Duration + wantErr bool + }{ + { + name: "stops after a few polls", + svc: &fakeService{statuses: []service.Status{service.StatusRunning, service.StatusRunning, service.StatusStopped}}, + timeout: 5 * time.Second, + }, + { + name: "already stopped", + svc: &fakeService{statuses: []service.Status{service.StatusStopped}}, + timeout: 5 * time.Second, + }, + { + // A stop request that errors is not fatal on its own: the process may be + // exiting anyway, so the status poll decides. + name: "stop errors but service is stopped", + svc: &fakeService{stopErr: errors.New("already stopped"), statuses: []service.Status{service.StatusStopped}}, + timeout: 5 * time.Second, + }, + { + name: "not installed", + svc: &fakeService{statusErr: service.ErrNotInstalled}, + timeout: 5 * time.Second, + }, + { + // The process never exits. Rollback must be told so, because modifying a + // running executable is what produced "Access is denied". + name: "never stops", + svc: &fakeService{statuses: []service.Status{service.StatusRunning}}, + timeout: time.Millisecond, + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := stopServiceAndWait(tc.svc, tc.timeout) + if tc.wantErr && err == nil { + t.Fatal("expected an error, got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tc.svc.stopCalls != 1 { + t.Errorf("Stop() called %d times, want 1", tc.svc.stopCalls) + } + }) + } +} + +func TestRemoveBinaryWithRetry(t *testing.T) { + t.Run("removes an existing file", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "ctrld") + if err := os.WriteFile(path, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + if err := removeBinaryWithRetry(path, time.Second); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("file still exists after removal: %v", err) + } + }) + + t.Run("missing file is not an error", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "absent") + if err := removeBinaryWithRetry(path, time.Second); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("gives up and reports when the path cannot be removed", func(t *testing.T) { + // A non-empty directory stands in for a locked executable: os.Remove keeps + // failing, so the retry loop must surface the error rather than hang. + dir := filepath.Join(t.TempDir(), "locked") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "child"), nil, 0o644); err != nil { + t.Fatal(err) + } + if err := removeBinaryWithRetry(dir, time.Millisecond); err == nil { + t.Fatal("expected an error for a path that cannot be removed") + } + }) +} + +func TestBinaryVersion(t *testing.T) { + t.Run("reports the version", func(t *testing.T) { + t.Setenv(envFakeVersionOutput, cliName+" version dev-94fbd3f") + got, err := binaryVersion(os.Args[0]) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "dev-94fbd3f" { + t.Errorf("binaryVersion() = %q, want %q", got, "dev-94fbd3f") + } + }) + + t.Run("rejects a binary that prints no version", func(t *testing.T) { + // The incident's ctrld.exe_previous: the file exists and runs, but produces no + // version output. Restoring it would have replaced a hung service with one + // that cannot start at all. + t.Setenv(envFakeVersionOutput, envFakeVersionSilent) + if _, err := binaryVersion(os.Args[0]); err == nil { + t.Fatal("expected an error for a binary with no version output") + } + }) + + t.Run("rejects a missing binary", func(t *testing.T) { + if _, err := binaryVersion(filepath.Join(t.TempDir(), "absent")); err == nil { + t.Fatal("expected an error for a missing binary") + } + }) +} + +// stubBinaryVersion makes the version probe report ver for any path, so a rollback +// test does not have to stage a runnable executable. +// +// Staging one is not portable: oldBin is bin+"_previous", so a fixture named "ctrld" +// yields the extension-less "ctrld_previous", which Windows refuses to execute +// ("executable file not found in %PATH%"), and a symlink to the test binary needs a +// privilege Windows does not grant by default. The probe itself is covered against the +// real test binary in TestBinaryVersion; these tests are about rollback's ordering. +func stubBinaryVersion(t *testing.T, ver string, err error) { + t.Helper() + prev := binaryVersionFn + binaryVersionFn = func(string) (string, error) { return ver, err } + t.Cleanup(func() { binaryVersionFn = prev }) +} + +func TestRollbackToPreviousBinaryStopsBeforeTouchingTheBinary(t *testing.T) { + dir := t.TempDir() + bin := filepath.Join(dir, "ctrld") + oldBin := bin + oldBinSuffix + if err := os.WriteFile(bin, []byte("replacement"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(oldBin, []byte("previous"), 0o755); err != nil { + t.Fatal(err) + } + stubBinaryVersion(t, "dev-a75d669", nil) + + // The invariant: when stop runs, the replacement's executable is still untouched. + // Reversing these two is exactly the "Access is denied" defect. + var stopped bool + var binExistedAtStop bool + stop := func() error { + stopped = true + _, err := os.Stat(bin) + binExistedAtStop = err == nil + return nil + } + restarted := false + restart := func() bool { restarted = true; return true } + + if err := rollbackToPreviousBinary(bin, oldBin, stop, restart); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !stopped { + t.Error("rollback did not stop the service") + } + if !binExistedAtStop { + t.Error("the binary was modified before the service was stopped") + } + if !restarted { + t.Error("rollback did not restart the service") + } + if _, err := os.Stat(oldBin); !errors.Is(err, os.ErrNotExist) { + t.Errorf("previous binary was not moved into place: %v", err) + } + if _, err := os.Stat(bin); err != nil { + t.Errorf("restored binary is missing: %v", err) + } +} + +func TestRollbackToPreviousBinaryKeepsUnusablePrevious(t *testing.T) { + dir := t.TempDir() + bin := filepath.Join(dir, "ctrld") + oldBin := bin + oldBinSuffix + if err := os.WriteFile(bin, []byte("replacement"), 0o755); err != nil { + t.Fatal(err) + } + // A previous binary that exists but does not report a version, as in the incident. + if err := os.WriteFile(oldBin, []byte("not a working binary"), 0o755); err != nil { + t.Fatal(err) + } + // Stubbed rather than left to the real probe: that would fail here for the right + // reason on unix (not an executable) but the wrong one on Windows (the fixture's + // name has no extension), so the assertion would not be about usability at all. + stubBinaryVersion(t, "", errors.New("unexpected --version output")) + + stopped := false + restarted := false + err := rollbackToPreviousBinary(bin, oldBin, + func() error { stopped = true; return nil }, + func() bool { restarted = true; return true }, + ) + if err == nil { + t.Fatal("expected an error when the previous binary is unusable") + } + if !stopped { + t.Error("the service must still be stopped: a broken replacement holds enforcement") + } + if restarted { + t.Error("must not restart the service with an unusable binary") + } + // Nothing was swapped, and the previous file is kept for inspection. + if _, err := os.Stat(oldBin); err != nil { + t.Errorf("unusable previous binary was not preserved: %v", err) + } + if _, err := os.Stat(bin); err != nil { + t.Errorf("installed binary was removed despite having nothing to restore: %v", err) + } +} + +func TestRollbackToPreviousBinaryAbortsWhenStopFails(t *testing.T) { + dir := t.TempDir() + bin := filepath.Join(dir, "ctrld") + oldBin := bin + oldBinSuffix + for _, p := range []string{bin, oldBin} { + if err := os.WriteFile(p, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + } + + stopErr := errors.New("service did not stop within 30s") + err := rollbackToPreviousBinary(bin, oldBin, + func() error { return stopErr }, + func() bool { t.Error("must not restart after a failed stop"); return false }, + ) + if !errors.Is(err, stopErr) { + t.Fatalf("error = %v, want %v", err, stopErr) + } + // The executable of a process that may still be running must be left alone. + if _, err := os.Stat(bin); err != nil { + t.Errorf("binary was modified even though the stop failed: %v", err) + } +} + +// TestVersionOutputParsesThroughRollbackProbe ties the "--version" output the root +// command actually produces to the parser rollback reads it with. +// +// These are two halves of one contract that live in different files: Cobra renders +// " version ", and binaryVersion cuts a prefix off it. Renaming the +// client moved the first half; if the second half had kept its literal, every +// upgrade would have logged "unknown version" and - the part that matters - +// rollbackToPreviousBinary would have judged a perfectly good previous binary +// "not usable" and left the host stopped with the broken one installed. +// +// The version output is taken from the real root command rather than assembled +// here, so a future change to the name, the template, or the parser has to keep +// them agreeing. +func TestVersionOutputParsesThroughRollbackProbe(t *testing.T) { + rootCmd := initCLI() + rootCmd.SetVersionTemplate(rootCmd.VersionTemplate()) + + var out bytes.Buffer + rootCmd.SetOut(&out) + rootCmd.SetErr(&out) + rootCmd.SetArgs([]string{"--version"}) + if err := rootCmd.Execute(); err != nil { + t.Fatalf("running --version: %v", err) + } + + got := out.String() + if strings.TrimSpace(got) == "" { + t.Fatal("--version produced no output") + } + ver, ok := parseVersionOutput(got) + if !ok { + t.Fatalf("the version probe cannot parse the root command's own --version output %q; "+ + "rollback would reject a working previous binary as unusable", strings.TrimSpace(got)) + } + if ver != appVersion { + t.Errorf("parsed version = %q, want %q", ver, appVersion) + } +} + +// TestParseVersionOutput covers the shapes the probe must accept and reject. The +// rejected ones are what a genuinely broken previous binary produces - the +// incident's ctrld.exe_previous printed nothing at all - and rollback depends on +// telling those apart from a healthy binary under a new name. +func TestParseVersionOutput(t *testing.T) { + tests := []struct { + name string + out string + want string + ok bool + }{ + {"current identity", cliName + " version v1.0.0", "v1.0.0", true}, + {"trailing newline", cliName + " version v1.0.0\n", "v1.0.0", true}, + {"dev build", cliName + " version dev-94fbd3f", "dev-94fbd3f", true}, + // The pre-rename identity: a v1-line binary is not a valid rollback target + // for this client, and must not be read as one. + {"previous identity", "ctrld version v1.3.5", "", false}, + {"no output", "", "", false}, + {"unrelated output", "some other program", "", false}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, ok := parseVersionOutput(tc.out) + if ok != tc.ok { + t.Fatalf("parseVersionOutput(%q) ok = %v, want %v", tc.out, ok, tc.ok) + } + if got != tc.want { + t.Errorf("parseVersionOutput(%q) = %q, want %q", tc.out, got, tc.want) + } + }) + } +} diff --git a/cmd/cli/conn.go b/cmd/cli/conn.go deleted file mode 100644 index 82e64688..00000000 --- a/cmd/cli/conn.go +++ /dev/null @@ -1,51 +0,0 @@ -package cli - -import ( - "net" - "time" -) - -// logConn wraps a net.Conn, override the Write behavior. -// runCmd uses this wrapper, so as long as startCmd finished, -// ctrld log won't be flushed with un-necessary write errors. -type logConn struct { - conn net.Conn -} - -func (lc *logConn) Read(b []byte) (n int, err error) { - return lc.conn.Read(b) -} - -func (lc *logConn) Close() error { - return lc.conn.Close() -} - -func (lc *logConn) LocalAddr() net.Addr { - return lc.conn.LocalAddr() -} - -func (lc *logConn) RemoteAddr() net.Addr { - return lc.conn.RemoteAddr() -} - -func (lc *logConn) SetDeadline(t time.Time) error { - return lc.conn.SetDeadline(t) -} - -func (lc *logConn) SetReadDeadline(t time.Time) error { - return lc.conn.SetReadDeadline(t) -} - -func (lc *logConn) SetWriteDeadline(t time.Time) error { - return lc.conn.SetWriteDeadline(t) -} - -func (lc *logConn) Write(b []byte) (int, error) { - // Write performs writes with underlying net.Conn, ignore any errors happen. - // "ctrld run" command use this wrapper to report errors to "ctrld start". - // If no error occurred, "ctrld start" may finish before "ctrld run" attempt - // to close the connection, so ignore errors conservatively here, prevent - // un-necessary error "write to closed connection" flushed to ctrld log. - _, _ = lc.conn.Write(b) - return len(b), nil -} diff --git a/cmd/cli/control_client.go b/cmd/cli/control_client.go index 7382d4e8..7a5693bd 100644 --- a/cmd/cli/control_client.go +++ b/cmd/cli/control_client.go @@ -5,13 +5,17 @@ import ( "io" "net" "net/http" + "net/url" + "strings" "time" ) +// controlClient represents an HTTP client for communicating with the control server type controlClient struct { c *http.Client } +// newControlClient creates a new control client with Unix socket transport func newControlClient(addr string) *controlClient { return &controlClient{c: &http.Client{ Transport: &http.Transport{ @@ -25,13 +29,31 @@ func newControlClient(addr string) *controlClient { } func (c *controlClient) post(path string, data io.Reader) (*http.Response, error) { - // for log/send, set the timeout to 5 minutes - if path == sendLogsPath { + // A log send uploads many megabytes, so it needs more time than the other + // calls. + if isSendLogsPath(path) { c.c.Timeout = time.Minute * 5 } return c.c.Post("http://unix"+path, contentTypeJson, data) } +// isSendLogsPath reports whether path addresses the log send handler. The path +// can carry a query, so the comparison takes the path alone. A path that does +// not parse keeps the prefix rule, because it still reaches that handler. +func isSendLogsPath(path string) bool { + parsed, err := url.Parse(path) + if err != nil { + return strings.HasPrefix(path, sendLogsPath) + } + return parsed.Path == sendLogsPath +} + +// postStream sends a POST request with no timeout, suitable for long-lived streaming connections. +func (c *controlClient) postStream(path string, data io.Reader) (*http.Response, error) { + c.c.Timeout = 0 + return c.c.Post("http://unix"+path, contentTypeJson, data) +} + // deactivationRequest represents request for validating deactivation pin. type deactivationRequest struct { Pin int64 `json:"pin"` diff --git a/cmd/cli/control_server.go b/cmd/cli/control_server.go index 9281b904..8ea015cc 100644 --- a/cmd/cli/control_server.go +++ b/cmd/cli/control_server.go @@ -10,6 +10,8 @@ import ( "os" "reflect" "sort" + "strconv" + "sync/atomic" "time" "github.com/kardianos/service" @@ -29,20 +31,27 @@ const ( ifacePath = "/iface" viewLogsPath = "/log/view" sendLogsPath = "/log/send" + tailLogsPath = "/log/tail" ) type ifaceResponse struct { - Name string `json:"name"` - All bool `json:"all"` - OK bool `json:"ok"` + Name string `json:"name"` + All bool `json:"all"` + OK bool `json:"ok"` + InterceptMode string `json:"intercept_mode,omitempty"` // "dns", "hard", or "" (not intercepting) + // InterceptModeSource is where InterceptMode came from: "flag", "config", + // or "default" (the platform default). Empty from a service that predates it. + InterceptModeSource string `json:"intercept_mode_source,omitempty"` } +// controlServer represents an HTTP server for handling control requests type controlServer struct { server *http.Server mux *http.ServeMux addr string } +// newControlServer creates a new control server instance func newControlServer(addr string) (*controlServer, error) { mux := http.NewServeMux() s := &controlServer{ @@ -56,18 +65,23 @@ func newControlServer(addr string) (*controlServer, error) { func (s *controlServer) start() error { _ = os.Remove(s.addr) unixListener, err := net.Listen("unix", s.addr) - if l, ok := unixListener.(*net.UnixListener); ok { - l.SetUnlinkOnClose(true) - } if err != nil { return err } + // Restrict socket permissions to owner-only (0600) so that only the + // process owner (typically root) can connect. Defense-in-depth since + // the control server endpoints carry no authentication of their own. + if err := os.Chmod(s.addr, 0600); err != nil { + return err + } + if l, ok := unixListener.(*net.UnixListener); ok { + l.SetUnlinkOnClose(true) + } go s.server.Serve(unixListener) return nil } func (s *controlServer) stop() error { - _ = os.Remove(s.addr) ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() return s.server.Shutdown(ctx) @@ -79,34 +93,34 @@ func (s *controlServer) register(pattern string, handler http.Handler) { func (p *prog) registerControlServerHandler() { p.cs.register(listClientsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - mainLog.Load().Debug().Msg("handling list clients request") + p.Debug().Msg("Handling list clients request") clients := p.ciTable.ListClients() - mainLog.Load().Debug().Int("client_count", len(clients)).Msg("retrieved clients list") + p.Debug().Int("client_count", len(clients)).Msg("Retrieved clients list") sort.Slice(clients, func(i, j int) bool { return clients[i].IP.Less(clients[j].IP) }) - mainLog.Load().Debug().Msg("sorted clients by IP address") + p.Debug().Msg("Sorted clients by IP address") if p.metricsQueryStats.Load() { - mainLog.Load().Debug().Msg("metrics query stats enabled, collecting query counts") + p.Debug().Msg("Metrics query stats enabled, collecting query counts") for idx, client := range clients { - mainLog.Load().Debug(). + p.Debug(). Int("index", idx). Str("ip", client.IP.String()). Str("mac", client.Mac). Str("hostname", client.Hostname). - Msg("processing client metrics") + Msg("Processing client metrics") client.IncludeQueryCount = true dm := &dto.Metric{} if statsClientQueriesCount.MetricVec == nil { - mainLog.Load().Debug(). + p.Debug(). Str("client_ip", client.IP.String()). - Msg("skipping metrics collection: MetricVec is nil") + Msg("Skipping metrics collection: MetricVec is nil") continue } @@ -116,44 +130,44 @@ func (p *prog) registerControlServerHandler() { client.Hostname, ) if err != nil { - mainLog.Load().Debug(). + p.Debug(). Err(err). Str("client_ip", client.IP.String()). Str("mac", client.Mac). Str("hostname", client.Hostname). - Msg("failed to get metrics for client") + Msg("Failed to get metrics for client") continue } if err := m.Write(dm); err == nil && dm.Counter != nil { client.QueryCount = int64(dm.Counter.GetValue()) - mainLog.Load().Debug(). + p.Debug(). Str("client_ip", client.IP.String()). Int64("query_count", client.QueryCount). - Msg("successfully collected query count") + Msg("Successfully collected query count") } else if err != nil { - mainLog.Load().Debug(). + p.Debug(). Err(err). Str("client_ip", client.IP.String()). - Msg("failed to write metric") + Msg("Failed to write metric") } } } else { - mainLog.Load().Debug().Msg("metrics query stats disabled, skipping query counts") + p.Debug().Msg("Metrics query stats disabled, skipping query counts") } if err := json.NewEncoder(w).Encode(&clients); err != nil { - mainLog.Load().Error(). + p.Error(). Err(err). Int("client_count", len(clients)). - Msg("failed to encode clients response") + Msg("Failed to encode clients response") http.Error(w, err.Error(), http.StatusInternalServerError) return } - mainLog.Load().Debug(). + p.Debug(). Int("client_count", len(clients)). - Msg("successfully sent clients list response") + Msg("Successfully sent clients list response") })) p.cs.register(startedPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { select { @@ -175,14 +189,14 @@ func (p *prog) registerControlServerHandler() { oldSvc := p.cfg.Service p.mu.Unlock() if err := p.sendReloadSignal(); err != nil { - mainLog.Load().Err(err).Msg("could not send reload signal") + p.Error().Err(err).Msg("Could not send reload signal") http.Error(w, err.Error(), http.StatusInternalServerError) return } select { case <-p.reloadDoneCh: case <-time.After(5 * time.Second): - http.Error(w, "timeout waiting for ctrld reload", http.StatusInternalServerError) + http.Error(w, "Timeout waiting for ctrld reload", http.StatusInternalServerError) return } @@ -216,19 +230,52 @@ func (p *prog) registerControlServerHandler() { return } + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + + // Reject further attempts while locked out due to repeated wrong PINs. + if now := time.Now().Unix(); now < deactivationLockedUntil.Load() { + w.WriteHeader(http.StatusTooManyRequests) + return + } + // Re-fetch pin code from API. - if rc, err := controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev); rc != nil { + rcReq := &controld.ResolverConfigRequest{ + RawUID: cdUID, + Version: appVersion, + Metadata: ctrld.SystemMetadataRuntime(context.Background()), + } + if rc, err := fetchResolverConfig(loggerCtx, rcReq, cdDev); rc != nil { + deactivationPinKnown.Store(true) if rc.DeactivationPin != nil { cdDeactivationPin.Store(*rc.DeactivationPin) } else { cdDeactivationPin.Store(defaultDeactivationPin) } + // Every resolver-config response carries the organization's allowed + // destinations, including this one, so apply them rather than discarding + // a fresher list until the next scheduled refresh converges. + // + // Only the destinations: p.rc is deliberately left alone. The scheduled + // refresh decides whether to reload ctrld by comparing the response + // against p.rc, so storing this one here would let an exclude-list change + // be compared away and never reloaded. The destination set needs no + // reload - it is enforced directly and applying it is idempotent. + p.applyAllowedDestinations(p.firewallAllowList(), rc.DestinationIPs) } else { - mainLog.Load().Warn().Err(err).Msg("could not re-fetch deactivation pin code") + p.Warn().Err(err).Msg("Could not re-fetch deactivation pin code") } - // If pin code not set, allowing deactivation. + // If pin code not set, allowing deactivation - but only when the API + // actually said so. A run that started on the configuration on disk + // during maintenance never asked, and the re-fetch above leaves the + // default untouched while the API stays away, so an unknown PIN would + // otherwise read as no PIN and authorize the deactivation it protects. if !deactivationPinSet() { + if !deactivationPinKnown.Load() { + p.Warn().Msg("Cannot verify deactivation pin: no answer from the Control D API this run") + w.WriteHeader(http.StatusServiceUnavailable) + return + } w.WriteHeader(http.StatusOK) return } @@ -236,7 +283,7 @@ func (p *prog) registerControlServerHandler() { var req deactivationRequest if err := json.NewDecoder(request.Body).Decode(&req); err != nil { w.WriteHeader(http.StatusPreconditionFailed) - mainLog.Load().Err(err).Msg("invalid deactivation request") + p.Error().Err(err).Msg("Invalid deactivation request") return } @@ -244,6 +291,7 @@ func (p *prog) registerControlServerHandler() { switch req.Pin { case cdDeactivationPin.Load(): code = http.StatusOK + deactivationFailedAttempts.Store(0) select { case p.pinCodeValidCh <- struct{}{}: default: @@ -251,6 +299,11 @@ func (p *prog) registerControlServerHandler() { case defaultDeactivationPin: // If the pin code was set, but users do not provide --pin, return proper code to client. code = http.StatusBadRequest + default: + if deactivationFailedAttempts.Add(1) >= deactivationMaxFailedAttempts { + deactivationLockedUntil.Store(time.Now().Unix() + deactivationLockoutSeconds) + deactivationFailedAttempts.Store(0) + } } w.WriteHeader(code) })) @@ -271,6 +324,11 @@ func (p *prog) registerControlServerHandler() { res.Name = p.runningIface res.All = p.requiredMultiNICsConfig res.OK = true + // Report intercept mode to the start command for proper log output. + if interceptMode == "dns" || interceptMode == "hard" { + res.InterceptMode = interceptMode + res.InterceptModeSource = interceptModeSource + } } } if err := json.NewEncoder(w).Encode(res); err != nil { @@ -279,63 +337,301 @@ func (p *prog) registerControlServerHandler() { return } })) - p.cs.register(viewLogsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - lr, err := p.logReader() - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - defer lr.r.Close() - if lr.size == 0 { - w.WriteHeader(http.StatusMovedPermanently) - return - } - data, err := io.ReadAll(lr.r) - if err != nil { - http.Error(w, fmt.Sprintf("could not read log: %v", err), http.StatusInternalServerError) - return - } - if err := json.NewEncoder(w).Encode(&logViewResponse{Data: string(data)}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - http.Error(w, fmt.Sprintf("could not marshal log data: %v", err), http.StatusInternalServerError) - return - } - })) + p.cs.register(viewLogsPath, http.HandlerFunc(p.handleLogView)) p.cs.register(sendLogsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - if time.Since(p.internalLogSent) < logWriterSentInterval { + if !p.logUpload.reserve() { w.WriteHeader(http.StatusServiceUnavailable) return } - r, err := p.logReader() + defer p.logUpload.release() + r, err := p.logReader(wantFullLogs(request), true) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } if r.size == 0 { + r.r.Close() w.WriteHeader(http.StatusMovedPermanently) return } + // The upload loses its color codes on the way, so the answer reports + // the bytes that went out, not the bytes that the files hold. + upload := &countingReadCloser{rc: r.r} req := &controld.LogsRequest{ UID: cdUID, - Data: r.r, + Data: upload, } - mainLog.Load().Debug().Msg("sending log file to ControlD server") - resp := logSentResponse{Size: r.size} - if err := controld.SendLogs(req, cdDev); err != nil { - mainLog.Load().Error().Msgf("could not send log file to ControlD server: %v", err) + p.Debug().Msg("Sending log file to ControlD server") + resp := logSentResponse{} + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + if err := controld.SendLogs(loggerCtx, req, cdDev); err != nil { + p.Error().Msgf("Could not send log file to ControlD server: %v", err) resp.Error = err.Error() w.WriteHeader(http.StatusInternalServerError) } else { - mainLog.Load().Debug().Msg("sending log file successfully") + p.Debug().Msg("Sending log file successfully") w.WriteHeader(http.StatusOK) } + resp.Size = upload.count() if err := json.NewEncoder(w).Encode(&resp); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } - p.internalLogSent = time.Now() })) + p.cs.register(tailLogsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + + // Determine logging mode and validate before starting the stream. + var lw *logWriter + useInternalLog := p.needInternalLogging() + if useInternalLog { + p.mu.Lock() + lw = p.internalLogWriter + p.mu.Unlock() + if lw == nil { + w.WriteHeader(http.StatusMovedPermanently) + return + } + } else if p.cfg.Service.LogPath == "" { + // No logging configured at all. + w.WriteHeader(http.StatusMovedPermanently) + return + } + + // Parse optional "lines" query param for initial context. + numLines := 10 + if v := request.URL.Query().Get("lines"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + numLines = n + } + } + + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Transfer-Encoding", "chunked") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + + if useInternalLog { + // Internal logging mode: subscribe to the logWriter. + + // Send last N lines as initial context. + if numLines > 0 { + if tail := lw.tailLastLines(numLines); len(tail) > 0 { + w.Write(tail) + flusher.Flush() + } + } + + ch, unsub := lw.Subscribe() + defer unsub() + for { + select { + case data, ok := <-ch: + if !ok { + return + } + if _, err := w.Write(data); err != nil { + return + } + flusher.Flush() + case <-request.Context().Done(): + return + } + } + } else { + // File-based logging mode: tail the log file. + followLogFile(request.Context(), w, flusher, normalizeLogFilePath(p.cfg.Service.LogPath), numLines) + } + })) +} + +// jsonStringChunkSize bounds the memory that one log view answer needs. A log +// of tens of megabytes must not get a second copy in the answer. +const jsonStringChunkSize = 32 * 1024 + +// wantFullLogs reports whether the request asks for every log file instead of +// the newest debug bytes. +func wantFullLogs(r *http.Request) bool { + return r.URL.Query().Get("full") == "1" +} + +// handleLogView answers with the log bytes in the data field. It writes the +// body while it reads the log, so the answer needs no copy of the whole log. +func (p *prog) handleLogView(w http.ResponseWriter, r *http.Request) { + lr, err := p.logReader(wantFullLogs(r), false) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer lr.r.Close() + if lr.size == 0 { + w.WriteHeader(http.StatusMovedPermanently) + return + } + w.Header().Set("Content-Type", contentTypeJson) + if _, err := io.WriteString(w, `{"data":`); err != nil { + return + } + if err := writeJSONString(w, lr.r); err != nil { + // The answer is on its way, so it can only end short. This line tells + // the operator why. + p.Error().Err(err).Msg("Could not send log view answer") + return + } + _, _ = io.WriteString(w, `}`) +} + +// writeJSONString writes what r yields as one JSON string, in chunks. +func writeJSONString(w io.Writer, r io.Reader) error { + if _, err := io.WriteString(w, `"`); err != nil { + return err + } + chunk := make([]byte, jsonStringChunkSize) + for { + n, readErr := r.Read(chunk) + if n > 0 { + if err := writeJSONStringChunk(w, chunk[:n]); err != nil { + return err + } + } + if readErr == io.EOF { + break + } + if readErr != nil { + return readErr + } + } + _, err := io.WriteString(w, `"`) + return err +} + +// writeJSONStringChunk writes one chunk and escapes the bytes that a JSON +// string cannot carry. It writes the bytes between two escapes in one call. +func writeJSONStringChunk(w io.Writer, chunk []byte) error { + start := 0 + for i, b := range chunk { + escaped := jsonStringEscape(b) + if escaped == "" { + continue + } + if _, err := w.Write(chunk[start:i]); err != nil { + return err + } + if _, err := io.WriteString(w, escaped); err != nil { + return err + } + start = i + 1 + } + _, err := w.Write(chunk[start:]) + return err +} + +// jsonStringEscape returns the escape of one byte, or an empty string for a +// byte that a JSON string carries as it is. Every byte of a multi-byte +// character is above 0x1f, so a chunk that ends inside a character still +// writes the character whole. +func jsonStringEscape(b byte) string { + switch b { + case '"': + return `\"` + case '\\': + return `\\` + case '\n': + return `\n` + case '\r': + return `\r` + case '\t': + return `\t` + } + if b < 0x20 { + return fmt.Sprintf(`\u%04x`, b) + } + return "" +} + +// countingReadCloser counts the bytes that the HTTP client reads from an +// upload. It closes the log files behind the upload, because the client only +// closes the body it was given. +type countingReadCloser struct { + rc io.ReadCloser + n atomic.Int64 +} + +func (c *countingReadCloser) Read(p []byte) (int, error) { + n, err := c.rc.Read(p) + c.n.Add(int64(n)) + return n, err +} + +func (c *countingReadCloser) Close() error { + return c.rc.Close() +} + +// count reports the bytes read so far. +func (c *countingReadCloser) count() int64 { + return c.n.Load() +} + +// tailFileLastLines reads the last n lines from a file and returns them. +// The file position is left at the end of the file after this call. +func tailFileLastLines(f *os.File, n int) []byte { + stat, err := f.Stat() + if err != nil || stat.Size() == 0 { + return nil + } + + // Read from the end in chunks to find the last n lines. + const chunkSize = 4096 + fileSize := stat.Size() + var lines []byte + offset := fileSize + count := 0 + + for offset > 0 && count <= n { + readSize := int64(chunkSize) + if readSize > offset { + readSize = offset + } + offset -= readSize + buf := make([]byte, readSize) + nRead, err := f.ReadAt(buf, offset) + if err != nil && err != io.EOF { + break + } + buf = buf[:nRead] + lines = append(buf, lines...) + + // Count newlines in this chunk. + for _, b := range buf { + if b == '\n' { + count++ + } + } + } + + // Trim to last n lines. + idx := 0 + nlCount := 0 + for i := len(lines) - 1; i >= 0; i-- { + if lines[i] == '\n' { + nlCount++ + if nlCount == n+1 { + idx = i + 1 + break + } + } + } + lines = lines[idx:] + + // Seek to end of file for subsequent reads. + f.Seek(0, io.SeekEnd) + return lines } +// jsonResponse wraps an HTTP handler to set JSON content type func jsonResponse(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/cmd/cli/control_server_tail_test.go b/cmd/cli/control_server_tail_test.go new file mode 100644 index 00000000..ec678e50 --- /dev/null +++ b/cmd/cli/control_server_tail_test.go @@ -0,0 +1,104 @@ +package cli + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// streamRecorder collects a streamed response under a lock, because the +// handler writes from its own goroutine while the test reads. +type streamRecorder struct { + mu sync.Mutex + header http.Header + body strings.Builder +} + +func (r *streamRecorder) Header() http.Header { return r.header } + +func (r *streamRecorder) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.body.Write(p) +} + +func (r *streamRecorder) WriteHeader(int) {} + +func (r *streamRecorder) Flush() {} + +func (r *streamRecorder) String() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.body.String() +} + +// waitForText polls the recorder until want arrives or the deadline passes. +func waitForText(rec *streamRecorder, want string, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if strings.Contains(rec.String(), want) { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} + +// Test_logTailFollowsTheRotatedLogPath keeps one tail request open while the +// log_path file rotates. The lines after the rotation must reach the client. +func Test_logTailFollowsTheRotatedLogPath(t *testing.T) { + origUID := cdUID + cdUID = "" + t.Cleanup(func() { cdUID = origUID }) + + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 64, backups: 1}, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + if _, err := rf.Write([]byte("before rotation\n")); err != nil { + t.Fatal(err) + } + + p := &prog{ + cfg: &ctrld.Config{Service: ctrld.ServiceConfig{LogPath: path}}, + cs: &controlServer{mux: http.NewServeMux()}, + } + p.registerControlServerHandler() + + ctx, cancel := context.WithCancel(context.Background()) + req := httptest.NewRequest(http.MethodGet, tailLogsPath+"?lines=1", nil).WithContext(ctx) + rec := &streamRecorder{header: http.Header{}} + done := make(chan struct{}) + go func() { + defer close(done) + p.cs.mux.ServeHTTP(rec, req) + }() + t.Cleanup(func() { + cancel() + <-done + }) + + if !waitForText(rec, "before rotation", 2*time.Second) { + t.Fatalf("the first line did not arrive: %q", rec.String()) + } + for i := 0; i < 4; i++ { + if _, err := rf.Write([]byte("filler line that crosses the budget\n")); err != nil { + t.Fatal(err) + } + } + if _, err := rf.Write([]byte("after rotation sentinel\n")); err != nil { + t.Fatal(err) + } + if !waitForText(rec, "after rotation sentinel", 3*time.Second) { + t.Fatalf("the sentinel did not arrive after the rotation: %q", rec.String()) + } +} diff --git a/cmd/cli/control_server_test.go b/cmd/cli/control_server_test.go index 297b37dd..bdafe7b3 100644 --- a/cmd/cli/control_server_test.go +++ b/cmd/cli/control_server_test.go @@ -2,10 +2,14 @@ package cli import ( "bytes" + "encoding/json" "io" "net/http" "os" + "strings" "testing" + + "github.com/Control-D-Inc/ctrld" ) func TestControlServer(t *testing.T) { @@ -52,3 +56,107 @@ func TestControlServer(t *testing.T) { t.Fatal(err) } } + +// jsonStringTestPayload holds every byte class that a JSON string must escape +// and splits a multi-byte character across the chunk boundary, so a chunk that +// ends inside a character is covered. +func jsonStringTestPayload() []byte { + var payload bytes.Buffer + payload.WriteString("quotes \" backslash \\ tab\t return\r newline\n") + payload.Write([]byte{0x00, 0x01, 0x1f}) + for payload.Len() < jsonStringChunkSize-1 { + payload.WriteByte('x') + } + payload.WriteString("é日本語 and an emoji 🙂\n") + payload.WriteString("the tail after the boundary\n") + return payload.Bytes() +} + +func Test_writeJSONString(t *testing.T) { + payload := jsonStringTestPayload() + var written bytes.Buffer + + if err := writeJSONString(&written, bytes.NewReader(payload)); err != nil { + t.Fatalf("writeJSONString: %v", err) + } + + var got string + if err := json.Unmarshal(written.Bytes(), &got); err != nil { + t.Fatalf("unmarshal %d bytes: %v", written.Len(), err) + } + if got != string(payload) { + t.Fatalf("the round trip returned %d bytes, want the %d bytes of the payload", len(got), len(payload)) + } +} + +// newLogViewTestServer serves the log view handler of p on a temporary socket +// and returns a client of it. +func newLogViewTestServer(t *testing.T, p *prog) *controlClient { + t.Helper() + f, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("temporary socket: %v", err) + } + f.Close() + t.Cleanup(func() { os.Remove(f.Name()) }) + s, err := newControlServer(f.Name()) + if err != nil { + t.Fatalf("newControlServer: %v", err) + } + s.register(viewLogsPath, http.HandlerFunc(p.handleLogView)) + if err := s.start(); err != nil { + t.Fatalf("start: %v", err) + } + t.Cleanup(func() { _ = s.stop() }) + return newControlClient(f.Name()) +} + +func Test_logViewHandler(t *testing.T) { + t.Run("answers the log as one JSON string", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + line := "a line with \"quotes\", a tab\there and 日本語\n" + if _, err := p.internalLogWriter.Write([]byte(line)); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + + resp, err := newLogViewTestServer(t, p).post(viewLogsPath, nil) + + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if ct := resp.Header.Get("content-type"); ct != contentTypeJson { + t.Fatalf("content type = %s, want %s", ct, contentTypeJson) + } + var view logViewResponse + if err := json.NewDecoder(resp.Body).Decode(&view); err != nil { + t.Fatalf("decode the answer: %v", err) + } + if !strings.Contains(view.Data, line) { + t.Fatalf("the answer holds %.64q, want the line %q in it", view.Data, line) + } + if !strings.Contains(view.Data, logWriterLogEndMarker) { + t.Fatalf("the answer holds no log end marker: %.64q", view.Data) + } + }) + + t.Run("answers 301 without a log", func(t *testing.T) { + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + silent, cdUID = false, "" + p := &prog{cfg: &ctrld.Config{}} + + resp, err := newLogViewTestServer(t, p).post(viewLogsPath, nil) + + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusMovedPermanently { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMovedPermanently) + } + }) +} diff --git a/cmd/cli/dns.go b/cmd/cli/dns.go deleted file mode 100644 index cf9d779e..00000000 --- a/cmd/cli/dns.go +++ /dev/null @@ -1,4 +0,0 @@ -package cli - -//lint:ignore U1000 use in os_linux.go -type getDNS func(iface string) []string diff --git a/cmd/cli/dns64.go b/cmd/cli/dns64.go new file mode 100644 index 00000000..f0334daa --- /dev/null +++ b/cmd/cli/dns64.go @@ -0,0 +1,501 @@ +package cli + +import ( + "context" + "net" + "net/netip" + "sync" + "time" + + "github.com/miekg/dns" + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" +) + +// DNS64 synthesis for IPv6-only networks WITHOUT client-side 464XLAT (no +// CLAT). On such networks the carrier's DNS64 resolver is load-bearing: it +// synthesizes AAAA records mapping IPv4-only destinations into the NAT64 +// prefix, and there is no CLAT interface to carry real IPv4 traffic. When +// ctrld answers with genuine A records there, IPv4-only destinations become +// unreachable — DNS resolves but connectivity fails (issue companion to +// #533; tethering is unaffected because Apple/Android always provide CLAT). +// +// ctrld therefore performs its own RFC 6147-style synthesis after filtering: +// when the network is IPv6-only with no CLAT and a NAT64 prefix is known, +// an AAAA query whose (policy-approved) answer contains no AAAA records is +// re-resolved as an A query through the same upstream, and the A records are +// mapped into the NAT64 prefix. Blocked answers are never synthesized — +// synthesis runs on the answer the policy engine already approved. +// +// NAT64 prefix discovery uses RFC 7050: resolve AAAA for ipv4only.arpa +// through the network's own resolvers and derive the prefix from the +// embedded well-known IPv4 addresses (192.0.0.170/171). PREF64 router +// advertisements (RFC 8781) are not parsed; RFC 7050 covers the same +// networks without OS-specific RA plumbing. + +const ( + // dns64RecheckInterval bounds how often network state (CLAT presence, + // IPv4 availability, NAT64 prefix) is re-evaluated. + dns64RecheckInterval = 5 * time.Minute + // dns64WellKnownName is the RFC 7050 discovery name. + dns64WellKnownName = "ipv4only.arpa." + // dns64DiscoverTimeout bounds one background discovery attempt. + dns64DiscoverTimeout = 5 * time.Second +) + +// rfc7050WellKnown are the IPv4 addresses embedded in ipv4only.arpa AAAA +// answers on DNS64 networks (RFC 7050). +var rfc7050WellKnown = []netip.Addr{ + netip.AddrFrom4([4]byte{192, 0, 0, 170}), + netip.AddrFrom4([4]byte{192, 0, 0, 171}), +} + +var dns64WellKnownPrefix = netip.MustParsePrefix("64:ff9b::/96") + +// clatPrefix is the RFC 7335 IPv4 service-continuity prefix used by +// client-side translators (CLAT). +var clatPrefix = netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 0, 0, 0}), 29) + +type dns64State struct { + mu sync.Mutex + checkedAt time.Time + active bool // network is v6-only, no CLAT, prefix known + prefix netip.Prefix // discovered NAT64 prefix (/96) + discovering bool + generation uint64 +} + +// dns64NetworkClassFn is a seam for tests. +var dns64NetworkClassFn = currentDNS64NetworkClass + +func addrFromNetAddr(a net.Addr) (netip.Addr, bool) { + var ip net.IP + switch v := a.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + default: + return netip.Addr{}, false + } + nip, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Addr{}, false + } + return nip.Unmap(), true +} + +// dns64NetworkClass classifies the host addressing state from interface +// addresses. Only IPv4 on the default-route interface counts as usable, so +// RFC1918 addresses owned by Docker, Parallels, VMware, and similar virtual +// interfaces do not disable DNS64. CLAT is detected across all interfaces. +func dns64NetworkClass(defaultRouteAddrs, allAddrs []net.Addr) (hasUsableIPv4, hasCLAT bool) { + for _, a := range allAddrs { + nip, ok := addrFromNetAddr(a) + if !ok || !nip.Is4() { + continue + } + if clatPrefix.Contains(nip) { + hasCLAT = true + } + } + for _, a := range defaultRouteAddrs { + nip, ok := addrFromNetAddr(a) + if !ok || !nip.Is4() || clatPrefix.Contains(nip) { + continue + } + if nip.IsLoopback() || nip.IsLinkLocalUnicast() || nip.IsUnspecified() { + continue + } + hasUsableIPv4 = true + } + return hasUsableIPv4, hasCLAT +} + +func currentDNS64NetworkClass() (hasUsableIPv4, hasCLAT bool, err error) { + defaultRouteInterface, err := netmon.DefaultRouteInterface() + if err != nil { + return false, false, err + } + iface, err := net.InterfaceByName(defaultRouteInterface) + if err != nil { + return false, false, err + } + defaultRouteAddrs, err := iface.Addrs() + if err != nil { + return false, false, err + } + allAddrs, err := net.InterfaceAddrs() + if err != nil { + return false, false, err + } + hasUsableIPv4, hasCLAT = dns64NetworkClass(defaultRouteAddrs, allAddrs) + return hasUsableIPv4, hasCLAT, nil +} + +// nat64PrefixFromAnswer derives the NAT64 prefix from an ipv4only.arpa AAAA +// answer per RFC 7050: find an AAAA embedding a well-known IPv4 address in +// its last 4 bytes and take the leading /96. +func nat64PrefixFromAnswer(answer *dns.Msg) (netip.Prefix, bool) { + if answer == nil { + return netip.Prefix{}, false + } + for _, rr := range answer.Answer { + aaaa, ok := rr.(*dns.AAAA) + if !ok { + continue + } + v6, ok := netip.AddrFromSlice(aaaa.AAAA.To16()) + if !ok || v6.Is4() { + continue + } + b := v6.As16() + embedded := netip.AddrFrom4([4]byte{b[12], b[13], b[14], b[15]}) + for _, wk := range rfc7050WellKnown { + if embedded == wk { + var p [16]byte + copy(p[:12], b[:12]) + return netip.PrefixFrom(netip.AddrFrom16(p), 96), true + } + } + } + return netip.Prefix{}, false +} + +// synthesizeAAAAFromA returns a copy of aAnswer converted into an AAAA +// answer for the original AAAA request: every A record is mapped into the +// NAT64 prefix; other records (CNAMEs etc.) are preserved. +func synthesizeAAAAFromA(req *dns.Msg, aAnswer *dns.Msg, prefix netip.Prefix) *dns.Msg { + if aAnswer == nil { + return nil + } + out := aAnswer.Copy() + out.SetReply(req) + out.Rcode = aAnswer.Rcode + out.Compress = true + // An authenticated A RRset alone cannot authenticate synthesized AAAA. + // maybeDNS64 also checks the original AAAA denial before setting AD. + out.AuthenticatedData = false + answers := make([]dns.RR, 0, len(aAnswer.Answer)) + pb := prefix.Addr().As16() + for _, rr := range aAnswer.Answer { + a, ok := rr.(*dns.A) + if !ok { + // Preserve CNAME chain records unchanged. + answers = append(answers, dns.Copy(rr)) + continue + } + v4 := a.A.To4() + if v4 == nil { + continue + } + v4Addr, ok := netip.AddrFromSlice(v4) + if !ok || !v4Addr.IsGlobalUnicast() || (prefix == dns64WellKnownPrefix && v4Addr.IsPrivate()) { + continue + } + var b [16]byte + copy(b[:12], pb[:12]) + copy(b[12:], v4) + aaaa := &dns.AAAA{ + Hdr: dns.RR_Header{ + Name: a.Hdr.Name, + Rrtype: dns.TypeAAAA, + Class: a.Hdr.Class, + Ttl: a.Hdr.Ttl, + }, + AAAA: net.IP(b[:]), + } + answers = append(answers, aaaa) + } + out.Answer = answers + return out +} + +// answerHasAAAA reports whether the answer section contains any AAAA record. +func answerHasAAAA(answer *dns.Msg) bool { + if answer == nil { + return false + } + for _, rr := range answer.Answer { + if _, ok := rr.(*dns.AAAA); ok { + return true + } + } + return false +} + +// dns64Eligible reports whether an answer qualifies for DNS64 synthesis: +// an AAAA query answered NOERROR with no AAAA records. NXDOMAIN and error +// rcodes are never synthesized (RFC 6147 §5.1.2: the name genuinely does +// not exist or the query failed). +func dns64Eligible(req, answer *dns.Msg) bool { + if !dns64RequestAllowed(req) || answer == nil { + return false + } + if answer.Rcode != dns.RcodeSuccess { + return false + } + return !answerHasAAAA(answer) +} + +// Apply the same client gate to cold synthesis and fresh/stale variants. +// In particular, RFC 6147 section 5.5 forbids synthesis for CD+DO clients. +// Retain the existing conservative bypass for all CD requests. +func dns64RequestAllowed(req *dns.Msg) bool { + return req != nil && len(req.Question) > 0 && req.Question[0].Qtype == dns.TypeAAAA && !req.CheckingDisabled +} + +// dns64Active reports whether synthesis should currently run, re-evaluating +// network class and (if needed) kicking off background prefix discovery at +// most every dns64RecheckInterval. +func (p *prog) dns64Active() bool { + s := &p.dns64 + s.mu.Lock() + if time.Since(s.checkedAt) < dns64RecheckInterval { + active := s.active + s.mu.Unlock() + return active + } + s.checkedAt = time.Now() + generation := s.generation + s.mu.Unlock() + + hasV4, hasCLAT, err := dns64NetworkClassFn() + s.mu.Lock() + defer s.mu.Unlock() + if generation != s.generation { + return s.active + } + if err != nil { + s.active = false + return false + } + if hasV4 || hasCLAT { + // Dual-stack or 464XLAT: the OS/CLAT handles IPv4 reachability; + // synthesis would be unnecessary (and on CLAT networks, harmful — + // real A records are preferable so traffic uses the CLAT). + s.generation++ + s.active = false + s.prefix = netip.Prefix{} + s.discovering = false + return false + } + s.active = s.prefix.IsValid() + if !s.discovering { + s.discovering = true + generation := s.generation + go p.discoverNAT64Prefix(generation) + } + return s.active +} + +func (p *prog) activeDNS64Prefix() (netip.Prefix, bool) { + if !p.dns64Active() { + return netip.Prefix{}, false + } + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + return p.dns64.prefix, p.dns64.prefix.IsValid() +} + +// dns64PrefixSnapshot reports the stored NAT64 prefix and whether synthesis +// runs. It reads the state only, because a report of the network must not +// reclassify it or start a discovery query. +func (p *prog) dns64PrefixSnapshot() (netip.Prefix, bool) { + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + return p.dns64.prefix, p.dns64.active +} + +func dns64CacheVariant(prefix netip.Prefix) string { + return "dns64:" + prefix.String() +} + +func dns64CacheKey(msg *dns.Msg, upstream string, prefix netip.Prefix) dnscache.Key { + variant := dns64CacheVariant(prefix) + // DO changes the authentication and DNSSEC data in a synthesized reply. + // Do not reuse a non-DO result for a DO client (or the reverse). + if opt := msg.IsEdns0(); opt != nil && opt.Do() { + variant += ":do" + } + return dnscache.NewVariantKey(msg, upstream, variant) +} + +func (p *prog) resetDNS64State() { + p.dns64.mu.Lock() + p.dns64.generation++ + p.dns64.checkedAt = time.Time{} + p.dns64.active = false + p.dns64.prefix = netip.Prefix{} + p.dns64.discovering = false + p.dns64.mu.Unlock() +} + +func dns64RouteStateChanged(delta *netmon.ChangeDelta) bool { + if delta == nil || delta.Old == nil || delta.New == nil { + return true + } + if delta.Old.DefaultRouteInterface != delta.New.DefaultRouteInterface || + delta.Old.HaveV4 != delta.New.HaveV4 || delta.Old.HaveV6 != delta.New.HaveV6 { + return true + } + if dns64StateHasCLAT(delta.Old) != dns64StateHasCLAT(delta.New) { + return true + } + iface := delta.New.DefaultRouteInterface + oldPrefixes := delta.Old.InterfaceIPs[iface] + newPrefixes := delta.New.InterfaceIPs[iface] + if len(oldPrefixes) != len(newPrefixes) { + return true + } + newPrefixSet := make(map[netip.Prefix]struct{}, len(newPrefixes)) + for _, prefix := range newPrefixes { + newPrefixSet[prefix] = struct{}{} + } + for _, prefix := range oldPrefixes { + if _, ok := newPrefixSet[prefix]; !ok { + return true + } + } + return false +} + +func dns64StateHasCLAT(state *netmon.State) bool { + for _, prefixes := range state.InterfaceIPs { + for _, prefix := range prefixes { + if clatPrefix.Contains(prefix.Addr().Unmap()) { + return true + } + } + } + return false +} + +func (p *prog) handleDNS64NetworkChange(delta *netmon.ChangeDelta, major bool) { + if major || dns64RouteStateChanged(delta) { + p.resetDNS64State() + } +} + +func (p *prog) storeDiscoveredNAT64Prefix(generation uint64, prefix netip.Prefix) bool { + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + if p.dns64.generation != generation { + return false + } + p.dns64.prefix = prefix + p.dns64.active = true + p.dns64.checkedAt = time.Now() + return true +} + +// discoverNAT64Prefix resolves ipv4only.arpa AAAA through the OS-discovered +// resolvers (the network's own DNS64 resolver) and stores the derived +// prefix. Runs in the background; failures leave synthesis inactive until +// the next recheck window. +func (p *prog) discoverNAT64Prefix(generation uint64) { + defer func() { + p.dns64.mu.Lock() + if p.dns64.generation == generation { + p.dns64.discovering = false + } + p.dns64.mu.Unlock() + }() + ctx, cancel := context.WithTimeout(context.Background(), dns64DiscoverTimeout) + defer cancel() + ctx = ctrld.LoggerCtx(ctx, mainLog.Load()) + + msg := new(dns.Msg) + msg.SetQuestion(dns64WellKnownName, dns.TypeAAAA) + resolver, err := ctrld.NewResolver(ctx, osUpstreamConfig) + if err != nil { + mainLog.Load().Debug().Err(err).Msg("dns64: could not create OS resolver for NAT64 discovery") + return + } + answer, err := resolver.Resolve(ctx, msg) + if err != nil { + mainLog.Load().Debug().Err(err).Msg("dns64: NAT64 prefix discovery query failed") + return + } + prefix, ok := nat64PrefixFromAnswer(answer) + if !ok { + p.logNAT64Result("") + return + } + if !p.storeDiscoveredNAT64Prefix(generation, prefix) { + return + } + p.logNAT64Result(prefix.String()) +} + +// logNAT64Result reports one discovery result. An empty prefix means that the +// network has none. Both results report a change of the network, so both go to +// the journal at info. +func (p *prog) logNAT64Result(prefix string) { + if !p.noteNAT64Result(prefix) { + return + } + if prefix == "" { + journal(mainLog.Load().Info()).Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + return + } + journal(mainLog.Load().Info()).Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) +} + +// noteNAT64Result reports whether this NAT64 result differs from the one that +// the journal already holds. An empty prefix means no prefix. Discovery runs +// every recheck window, and the journal keeps the changes only. +func (p *prog) noteNAT64Result(prefix string) bool { + p.lastNAT64Mu.Lock() + defer p.lastNAT64Mu.Unlock() + if p.lastNAT64Logged == prefix { + return false + } + p.lastNAT64Logged = prefix + return true +} + +// maybeDNS64 applies DNS64 synthesis to an already-filtered answer when the +// network requires it. resolveA re-resolves the question as an A query +// through the same upstream that produced the answer. +func (p *prog) maybeDNS64(ctx context.Context, req *dns.Msg, answer *dns.Msg, resolveA func(*dns.Msg) *dns.Msg) (*dns.Msg, netip.Prefix) { + if !dns64Eligible(req, answer) || !p.dns64Active() { + return answer, netip.Prefix{} + } + p.dns64.mu.Lock() + prefix := p.dns64.prefix + generation := p.dns64.generation + p.dns64.mu.Unlock() + if !prefix.IsValid() { + return answer, netip.Prefix{} + } + aReq := req.Copy() + aReq.Question[0].Qtype = dns.TypeA + aAnswer := resolveA(aReq) + if aAnswer == nil || aAnswer.Rcode != dns.RcodeSuccess || !sameQuestion(aReq, aAnswer) { + return answer, netip.Prefix{} + } + synth := synthesizeAAAAFromA(req, aAnswer, prefix) + p.dns64.mu.Lock() + current := p.dns64.active && p.dns64.generation == generation && p.dns64.prefix == prefix + p.dns64.mu.Unlock() + if !current { + return answer, netip.Prefix{} + } + if synth == nil || !answerHasAAAA(synth) { + // The companion A lookup completed successfully, so this passthrough + // answer is definitive for the current prefix and may be cached in the + // DNS64 variant to avoid repeating both upstream lookups. + return answer, prefix + } + // RFC 6147 section 5.5: AD may describe synthesis only when both the + // AAAA denial and the A response were authenticated by our upstream. + // Without DO, synthesized responses must not assert AD. + if opt := req.IsEdns0(); opt != nil && opt.Do() { + synth.AuthenticatedData = answer.AuthenticatedData && aAnswer.AuthenticatedData + } + ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: synthesized AAAA from A records via NAT64 prefix %s", prefix) + return synth, prefix +} diff --git a/cmd/cli/dns64_dnssec_test.go b/cmd/cli/dns64_dnssec_test.go new file mode 100644 index 00000000..a6032420 --- /dev/null +++ b/cmd/cli/dns64_dnssec_test.go @@ -0,0 +1,143 @@ +package cli + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/miekg/dns" + ctrld "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" +) + +func TestDNS64DNSSECCacheAndSynthesis(t *testing.T) { + captureDebugMainLog(t) + for _, do := range []bool{false, true} { + for _, denialAD := range []bool{false, true} { + for _, aAD := range []bool{false, true} { + t.Run(fmt.Sprintf("DO=%v/denialAD=%v/A_AD=%v", do, denialAD, aAD), func(t *testing.T) { + req := mkAAAAReq("dnssec.example.") + req.SetEdns0(1232, do) + denial := new(dns.Msg) + denial.SetReply(req) + denial.AuthenticatedData = denialAD + cache, err := dnscache.NewLRUCache(10) + if err != nil { + t.Fatal(err) + } + p := &prog{cache: cache} + p.logger.Store(mainLog.Load()) + p.dns64.active = true + p.dns64.prefix = dns64WellKnownPrefix + p.dns64.checkedAt = time.Now() + calls := 0 + resolveA := func(q *dns.Msg) *dns.Msg { + calls++ + if q.Question[0].Qtype != dns.TypeA { + t.Fatal("not an A lookup") + } + a := new(dns.Msg) + a.SetReply(q) + a.AuthenticatedData = aAD + a.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: q.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP("198.51.100.7")}} + return a + } + out, prefix := p.maybeDNS64(context.Background(), req, denial, resolveA) + wantAD := do && denialAD && aAD + if calls != 1 || !answerHasAAAA(out) || !prefix.IsValid() || out.AuthenticatedData != wantAD { + t.Fatalf("cold: calls=%d AAAA=%v prefix=%v AD=%v wantAD=%v", calls, answerHasAAAA(out), prefix, out.AuthenticatedData, wantAD) + } + now := time.Now() + for _, stale := range []bool{false, true} { + expiry := now.Add(time.Minute) + if stale { + expiry = now.Add(-time.Minute) + } + cache.Add(dns64CacheKey(req, "upstream.0", prefix), dnscache.NewValue(out, expiry)) + answer, old, hit, variant, _ := p.cachedResponse(req, "upstream.0", prefix, true, now) + got := answer + if stale { + got = old + } + if got == nil || !answerHasAAAA(got) || got.AuthenticatedData != wantAD || hit == stale || variant == stale { + t.Fatalf("normal cache stale=%v: answer=%v old=%v hit=%v variant=%v", stale, answer, old, hit, variant) + } + cd := req.Copy() + cd.CheckingDisabled = true + answer, old, hit, variant, _ = p.cachedResponse(cd, "upstream.0", prefix, true, now) + if answerHasAAAA(answer) || answerHasAAAA(old) || hit || variant { + t.Errorf("CD cache stale=%v leaked synthesis: answer=%v old=%v hit=%v variant=%v", stale, answer, old, hit, variant) + } + cold, coldPrefix := p.maybeDNS64(context.Background(), cd, denial, resolveA) + if cold != denial || coldPrefix.IsValid() || calls != 1 { + t.Fatal("CD cold path performed synthesis") + } + } + if denial.AuthenticatedData != denialAD { + t.Fatal("mutated denial") + } + }) + } + } + } +} + +func TestDNS64DNSSECOrdinaryCacheFallback(t *testing.T) { + req := mkAAAAReq("dnssec.example.") + req.CheckingDisabled = true + req.SetEdns0(1232, true) + ordinary := new(dns.Msg) + ordinary.SetReply(req) + cache, err := dnscache.NewLRUCache(10) + if err != nil { + t.Fatal(err) + } + p := &prog{cache: cache} + p.logger.Store(mainLog.Load()) + now := time.Now() + cache.Add(dnscache.NewKey(req, "upstream.0"), dnscache.NewValue(ordinary, now.Add(time.Minute))) + answer, _, hit, variant, bypass := p.cachedResponse(req, "upstream.0", dns64WellKnownPrefix, true, now) + if !hit || variant || bypass || answerHasAAAA(answer) { + t.Fatalf("ordinary cache: hit=%v variant=%v bypass=%v answer=%v", hit, variant, bypass, answer) + } +} + +func TestRecoveryOSOnlyCandidatePool(t *testing.T) { + for _, reason := range []RecoveryReason{RecoveryReasonNetworkChange, RecoveryReasonRegularFailure} { + p := &prog{cfg: &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: ctrld.ResolverTypeOS}, "nil": nil}}} + p.logger.Store(mainLog.Load()) + pool := p.buildRecoveryUpstreams(reason) + if len(pool) != 1 || pool[upstreamOS] != osUpstreamConfig { + t.Errorf("reason=%v: OS-only recovery candidates=%v", reason, pool) + } + } +} + +func TestDNS64DNSSECCacheDOIsolation(t *testing.T) { + for _, do := range []bool{false, true} { + req := mkAAAAReq("dnssec.example.") + req.SetEdns0(1232, do) + response := new(dns.Msg) + response.SetReply(req) + response.AuthenticatedData = do + response.Answer = []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{Name: req.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}, AAAA: net.ParseIP("64:ff9b::c633:6407")}} + cache, err := dnscache.NewLRUCache(10) + if err != nil { + t.Fatal(err) + } + p := &prog{cache: cache} + p.logger.Store(mainLog.Load()) + now := time.Now() + other := req.Copy() + other.IsEdns0().SetDo(!do) + for _, expiry := range []time.Time{now.Add(time.Minute), now.Add(-time.Minute)} { + cache.Add(dns64CacheKey(req, "upstream.0", dns64WellKnownPrefix), dnscache.NewValue(response, expiry)) + answer, stale, hit, variant, _ := p.cachedResponse(other, "upstream.0", dns64WellKnownPrefix, true, now) + if answer != nil || stale != nil || hit || variant { + t.Fatalf("DO=%v result reused for DO=%v: answer=%v stale=%v", do, !do, answer, stale) + } + } + } +} diff --git a/cmd/cli/dns64_test.go b/cmd/cli/dns64_test.go new file mode 100644 index 00000000..3985aeaa --- /dev/null +++ b/cmd/cli/dns64_test.go @@ -0,0 +1,424 @@ +package cli + +import ( + "net" + "net/netip" + "testing" + "time" + + "github.com/miekg/dns" + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld/internal/dnscache" +) + +func mkAAAAReq(name string) *dns.Msg { + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(name), dns.TypeAAAA) + return m +} + +func mkNetAddr(cidr string) net.Addr { + ip, n, err := net.ParseCIDR(cidr) + if err != nil { + panic(err) + } + n.IP = ip + return n +} + +func TestDNS64NetworkClass(t *testing.T) { + tests := []struct { + name string + defaultRouteAddrs []net.Addr + allAddrs []net.Addr + wantV4 bool + wantCLAT bool + }{ + {"dual stack", []net.Addr{mkNetAddr("10.0.11.61/23"), mkNetAddr("2605:8d80::1/64")}, []net.Addr{mkNetAddr("10.0.11.61/23"), mkNetAddr("2605:8d80::1/64")}, true, false}, + {"virtual rfc1918 does not imply ipv4 connectivity", []net.Addr{mkNetAddr("2605:8d80::1/64")}, []net.Addr{mkNetAddr("2605:8d80::1/64"), mkNetAddr("192.168.65.1/24")}, false, false}, + {"464xlat tether (customer case)", []net.Addr{mkNetAddr("2605:8d80:6b41:122::1/64")}, []net.Addr{mkNetAddr("192.0.0.2/32"), mkNetAddr("2605:8d80:6b41:122::1/64")}, false, true}, + {"v6 only no clat (dns64 network)", []net.Addr{mkNetAddr("2001:db8::1/64")}, []net.Addr{mkNetAddr("2001:db8::1/64")}, false, false}, + {"loopback only", []net.Addr{mkNetAddr("127.0.0.1/8"), mkNetAddr("::1/128")}, []net.Addr{mkNetAddr("127.0.0.1/8"), mkNetAddr("::1/128")}, false, false}, + {"link local v4 ignored", []net.Addr{mkNetAddr("169.254.10.1/16"), mkNetAddr("2001:db8::1/64")}, []net.Addr{mkNetAddr("169.254.10.1/16"), mkNetAddr("2001:db8::1/64")}, false, false}, + {"clat plus real v4", []net.Addr{mkNetAddr("10.0.0.5/24")}, []net.Addr{mkNetAddr("192.0.0.2/32"), mkNetAddr("10.0.0.5/24")}, true, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + v4, clat := dns64NetworkClass(tc.defaultRouteAddrs, tc.allAddrs) + if v4 != tc.wantV4 || clat != tc.wantCLAT { + t.Errorf("dns64NetworkClass() = (v4=%v, clat=%v), want (v4=%v, clat=%v)", v4, clat, tc.wantV4, tc.wantCLAT) + } + }) + } +} + +func TestNAT64PrefixFromAnswer(t *testing.T) { + mkAnswer := func(v6 string) *dns.Msg { + m := new(dns.Msg) + m.SetQuestion(dns64WellKnownName, dns.TypeAAAA) + r := new(dns.Msg) + r.SetReply(m) + if v6 != "" { + r.Answer = append(r.Answer, &dns.AAAA{ + Hdr: dns.RR_Header{Name: dns64WellKnownName, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 300}, + AAAA: net.ParseIP(v6), + }) + } + return r + } + tests := []struct { + name string + answer *dns.Msg + wantPrefix string + wantOK bool + }{ + {"well-known prefix + 192.0.0.170", mkAnswer("64:ff9b::c000:aa"), "64:ff9b::/96", true}, + {"well-known prefix + 192.0.0.171", mkAnswer("64:ff9b::c000:ab"), "64:ff9b::/96", true}, + {"carrier-specific prefix", mkAnswer("2001:db8:64::c000:aa"), "2001:db8:64::/96", true}, + {"non-dns64 answer (real aaaa)", mkAnswer("2001:db8::1"), "", false}, + {"empty answer", mkAnswer(""), "", false}, + {"nil answer", nil, "", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p, ok := nat64PrefixFromAnswer(tc.answer) + if ok != tc.wantOK { + t.Fatalf("nat64PrefixFromAnswer() ok = %v, want %v", ok, tc.wantOK) + } + if ok && p != netip.MustParsePrefix(tc.wantPrefix) { + t.Errorf("nat64PrefixFromAnswer() = %s, want %s", p, tc.wantPrefix) + } + }) + } +} + +func TestSynthesizeAAAAFromA(t *testing.T) { + req := mkAAAAReq("legacy.example.com") + aReq := req.Copy() + aReq.Question[0].Qtype = dns.TypeA + aAns := new(dns.Msg) + aAns.SetReply(aReq) + aAns.Answer = []dns.RR{ + &dns.CNAME{Hdr: dns.RR_Header{Name: "legacy.example.com.", Rrtype: dns.TypeCNAME, Class: dns.ClassINET, Ttl: 60}, Target: "cdn.example.net."}, + &dns.A{Hdr: dns.RR_Header{Name: "cdn.example.net.", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP("198.51.100.7")}, + } + prefix := netip.MustParsePrefix("64:ff9b::/96") + + out := synthesizeAAAAFromA(req, aAns, prefix) + if out == nil { + t.Fatal("synthesizeAAAAFromA returned nil") + } + var gotAAAA *dns.AAAA + var gotCNAME *dns.CNAME + for _, rr := range out.Answer { + switch v := rr.(type) { + case *dns.AAAA: + gotAAAA = v + case *dns.CNAME: + gotCNAME = v + case *dns.A: + t.Error("synthesized answer still contains an A record") + } + } + if gotCNAME == nil { + t.Error("CNAME chain record not preserved") + } + if gotAAAA == nil { + t.Fatal("no synthesized AAAA record") + } + want := net.ParseIP("64:ff9b::c633:6407") // 198.51.100.7 embedded + if !gotAAAA.AAAA.Equal(want) { + t.Errorf("synthesized AAAA = %s, want %s", gotAAAA.AAAA, want) + } + if gotAAAA.Hdr.Ttl != 60 { + t.Errorf("TTL not preserved: got %d", gotAAAA.Hdr.Ttl) + } + if out.Question[0].Qtype != dns.TypeAAAA { + t.Errorf("reply question type = %d, want AAAA", out.Question[0].Qtype) + } +} + +func TestSynthesizeAAAAFromAIPv4Eligibility(t *testing.T) { + req := mkAAAAReq("blocked.example") + tests := []struct { + name string + prefix netip.Prefix + ip string + want bool + }{ + {"unspecified with well-known prefix", dns64WellKnownPrefix, "0.0.0.0", false}, + {"loopback with well-known prefix", dns64WellKnownPrefix, "127.0.0.1", false}, + {"link-local with well-known prefix", dns64WellKnownPrefix, "169.254.1.1", false}, + {"private with well-known prefix", dns64WellKnownPrefix, "10.0.0.1", false}, + {"private with network-specific prefix", netip.MustParsePrefix("2001:db8:64::/96"), "10.0.0.1", true}, + {"unspecified with network-specific prefix", netip.MustParsePrefix("2001:db8:64::/96"), "0.0.0.0", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + aReq := req.Copy() + aReq.Question[0].Qtype = dns.TypeA + aAns := new(dns.Msg) + aAns.SetReply(aReq) + aAns.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: req.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP(tc.ip)}} + if got := answerHasAAAA(synthesizeAAAAFromA(req, aAns, tc.prefix)); got != tc.want { + t.Fatalf("answerHasAAAA() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestDNS64Eligible(t *testing.T) { + emptyReply := func(req *dns.Msg, rcode int) *dns.Msg { + r := new(dns.Msg) + r.SetReply(req) + r.Rcode = rcode + return r + } + aaaaReq := mkAAAAReq("x.example.") + cdReq := aaaaReq.Copy() + cdReq.CheckingDisabled = true + withAAAA := emptyReply(aaaaReq, dns.RcodeSuccess) + withAAAA.Answer = []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{Name: "x.example.", Rrtype: dns.TypeAAAA, Class: dns.ClassINET}, AAAA: net.ParseIP("2001:db8::1")}} + aReq := new(dns.Msg) + aReq.SetQuestion("x.example.", dns.TypeA) + + tests := []struct { + name string + req *dns.Msg + answer *dns.Msg + want bool + }{ + {"AAAA empty NOERROR -> eligible", aaaaReq, emptyReply(aaaaReq, dns.RcodeSuccess), true}, + {"AAAA with records -> not eligible", aaaaReq, withAAAA, false}, + {"CD query is not synthesized", cdReq, emptyReply(cdReq, dns.RcodeSuccess), false}, + {"NXDOMAIN never synthesized", aaaaReq, emptyReply(aaaaReq, dns.RcodeNameError), false}, + {"SERVFAIL never synthesized", aaaaReq, emptyReply(aaaaReq, dns.RcodeServerFailure), false}, + {"A query not eligible", aReq, emptyReply(aReq, dns.RcodeSuccess), false}, + {"nil answer", aaaaReq, nil, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := dns64Eligible(tc.req, tc.answer); got != tc.want { + t.Errorf("dns64Eligible() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestDNS64ActiveGating(t *testing.T) { + cases := []struct { + name string + hasV4 bool + hasCLAT bool + prefix string + want bool + }{ + {"dual stack, prefix known", true, false, "64:ff9b::/96", false}, + {"clat network, prefix known", false, true, "64:ff9b::/96", false}, + {"v6-only no clat, prefix known", false, false, "64:ff9b::/96", true}, + {"v6-only no clat, no prefix yet", false, false, "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { return tc.hasV4, tc.hasCLAT, nil } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.discovering = true // block background discovery in tests + if tc.prefix != "" { + p.dns64.prefix = netip.MustParsePrefix(tc.prefix) + } + if got := p.dns64Active(); got != tc.want { + t.Errorf("dns64Active() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestStoreDiscoveredNAT64PrefixActivatesImmediately(t *testing.T) { + p := &prog{} + p.dns64.generation = 4 + p.dns64.checkedAt = time.Now() + prefix := netip.MustParsePrefix("64:ff9b::/96") + if !p.storeDiscoveredNAT64Prefix(4, prefix) { + t.Fatal("current discovery result was rejected") + } + if !p.dns64.active || p.dns64.prefix != prefix || p.dns64.checkedAt.IsZero() { + t.Fatalf("discovery did not immediately activate DNS64: active=%v prefix=%s checkedAt=%s", p.dns64.active, p.dns64.prefix, p.dns64.checkedAt) + } + if p.storeDiscoveredNAT64Prefix(3, netip.MustParsePrefix("2001:db8:64::/96")) { + t.Fatal("stale discovery result was accepted") + } + if p.dns64.prefix != prefix { + t.Fatalf("stale discovery replaced prefix: %s", p.dns64.prefix) + } +} + +func TestDNS64NetworkChangeInvalidatesPrefix(t *testing.T) { + p := &prog{} + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + p.dns64.active = true + p.dns64.checkedAt = time.Now() + p.dns64.discovering = true + + delta := &netmon.ChangeDelta{ + Old: &netmon.State{DefaultRouteInterface: "en0", HaveV6: true, InterfaceIPs: map[string][]netip.Prefix{"en0": {netip.MustParsePrefix("2001:db8:1::1/64")}}}, + New: &netmon.State{DefaultRouteInterface: "en0", HaveV6: true, InterfaceIPs: map[string][]netip.Prefix{"en0": {netip.MustParsePrefix("2001:db8:2::1/64")}}}, + } + p.handleDNS64NetworkChange(delta, false) + if p.dns64.active || p.dns64.prefix.IsValid() || !p.dns64.checkedAt.IsZero() || p.dns64.discovering { + t.Fatalf("network change did not invalidate DNS64 state: active=%v prefix=%s checkedAt=%s discovering=%v", p.dns64.active, p.dns64.prefix, p.dns64.checkedAt, p.dns64.discovering) + } + if p.dns64.generation != 1 { + t.Fatalf("generation = %d, want 1", p.dns64.generation) + } +} + +func TestDNS64CacheKeyPartitionsByPrefix(t *testing.T) { + req := mkAAAAReq("legacy.example") + normal := dnscache.NewKey(req, "upstream.0") + wellKnown := dns64CacheKey(req, "upstream.0", netip.MustParsePrefix("64:ff9b::/96")) + carrier := dns64CacheKey(req, "upstream.0", netip.MustParsePrefix("2001:db8:64::/96")) + if normal == wellKnown || wellKnown == carrier { + t.Fatalf("normal and per-prefix synthesized cache keys must be distinct: normal=%+v well-known=%+v carrier=%+v", normal, wellKnown, carrier) + } + cache, err := dnscache.NewLRUCache(4) + if err != nil { + t.Fatal(err) + } + answer := new(dns.Msg) + answer.SetReply(req) + cache.Add(wellKnown, dnscache.NewValue(answer, time.Now().Add(time.Minute))) + if cache.Get(wellKnown) == nil || cache.Get(normal) != nil || cache.Get(carrier) != nil { + t.Fatal("synthesized cache entry crossed the normal or carrier-prefix partition") + } +} + +func TestMaybeDNS64EndToEnd(t *testing.T) { + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { return false, false, nil } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.discovering = true + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + + req := mkAAAAReq("legacy.example.com") + empty := new(dns.Msg) + empty.SetReply(req) + + resolveA := func(aReq *dns.Msg) *dns.Msg { + if aReq.Question[0].Qtype != dns.TypeA { + t.Fatalf("resolveA called with qtype %d", aReq.Question[0].Qtype) + } + r := new(dns.Msg) + r.SetReply(aReq) + r.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: aReq.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 30}, A: net.ParseIP("203.0.113.9")}} + return r + } + out, usedPrefix := p.maybeDNS64(t.Context(), req, empty, resolveA) + if !answerHasAAAA(out) { + t.Fatal("expected synthesized AAAA answer") + } + if usedPrefix != p.dns64.prefix { + t.Fatalf("used prefix = %s, want %s", usedPrefix, p.dns64.prefix) + } + + blocked := new(dns.Msg) + blocked.SetReply(req) + blocked.Rcode = dns.RcodeNameError + if got, _ := p.maybeDNS64(t.Context(), req, blocked, resolveA); got != blocked { + t.Error("NXDOMAIN answer must pass through unsynthesized") + } + + blockedA := func(aReq *dns.Msg) *dns.Msg { + r := new(dns.Msg) + r.SetReply(aReq) + r.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: aReq.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 30}, A: net.IPv4zero}} + return r + } + if got, prefix := p.maybeDNS64(t.Context(), req, empty, blockedA); got != empty || prefix != p.dns64.prefix { + t.Error("NODATA plus 0.0.0.0 block answer must pass through unsynthesized and be cacheable for the current prefix") + } +} + +func TestMaybeDNS64DropsStaleInFlightPrefix(t *testing.T) { + p := &prog{} + p.dns64.active = true + p.dns64.checkedAt = time.Now() + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + req := mkAAAAReq("legacy.example") + empty := new(dns.Msg) + empty.SetReply(req) + + got, prefix := p.maybeDNS64(t.Context(), req, empty, func(aReq *dns.Msg) *dns.Msg { + p.resetDNS64State() + r := new(dns.Msg) + r.SetReply(aReq) + r.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: aReq.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 30}, A: net.ParseIP("203.0.113.9")}} + return r + }) + if got != empty || prefix.IsValid() { + t.Fatal("in-flight synthesis used a prefix invalidated by a network change") + } +} + +// TestDNS64PrefixSnapshotReadsTheStoredStateOnly covers the read that a report +// of the network uses. The active read reclassifies the network and can start +// a discovery query, and a report must change nothing. +func TestDNS64PrefixSnapshotReadsTheStoredStateOnly(t *testing.T) { + classReads := 0 + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { + classReads++ + return false, false, nil + } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + p.dns64.active = true + + prefix, active := p.dns64PrefixSnapshot() + + if !active || prefix != p.dns64.prefix { + t.Fatalf("snapshot = %v, %v, want %v, true", prefix, active, p.dns64.prefix) + } + if classReads != 0 { + t.Fatalf("the snapshot classified the network %d times, want 0", classReads) + } + if p.dns64.discovering { + t.Fatal("the snapshot started a NAT64 prefix discovery") + } +} + +// TestDNS64ResultLinesReachTheJournalAtInfo covers the level of the two NAT64 +// result lines. Both report a change, and a debug line never leaves a host +// that runs at the normal level, so the journal lost the result of a network +// that has no prefix. +func TestDNS64ResultLinesReachTheJournalAtInfo(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + + // The first result of a run reports nothing when the network has no + // prefix, so the prefix has to arrive before it goes away. + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("") + + absent := jsonLogEvents(t, logs, "dns64: no NAT64 prefix present (not a DNS64 network)") + if len(absent) != 1 { + t.Fatalf("got %d no-prefix lines, want 1", len(absent)) + } + wantField(t, absent[0], "level", "info") + wantField(t, absent[0], "journal", true) + + found := jsonLogEvents(t, logs, "dns64: discovered NAT64 prefix 64:ff9b::/96; enabling AAAA synthesis for IPv6-only network without CLAT") + if len(found) != 1 { + t.Fatalf("got %d discovered lines for two identical results, want 1", len(found)) + } + wantField(t, found[0], "level", "info") + wantField(t, found[0], "journal", true) +} diff --git a/cmd/cli/dns_config.go b/cmd/cli/dns_config.go new file mode 100644 index 00000000..22c44ef7 --- /dev/null +++ b/cmd/cli/dns_config.go @@ -0,0 +1,360 @@ +package cli + +import ( + "bufio" + "errors" + "io" + "slices" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // dnsConfigFastInterval is the poll period that follows a network event. + dnsConfigFastInterval = 15 * time.Second + + // dnsConfigSlowInterval is the poll period of a quiet host. + dnsConfigSlowInterval = 5 * time.Minute + + // dnsConfigFastWindow is how long the fast period lasts. A resolver change + // that follows a network event arrives late, after the OS settles. + dnsConfigFastWindow = 10 * time.Minute +) + +// scutilTablePrefix starts a resolver table. macOS prints the table for scoped +// queries after the main one, with the same line. +const scutilTablePrefix = "DNS configuration" + +// scutilResolverPrefix starts one resolver block inside a table. +const scutilResolverPrefix = "resolver #" + +// scutilDNSOutputLimit bounds the text of one read. A resolver table of a host +// is a few kilobytes, so a larger output does not come from a table. +const scutilDNSOutputLimit = 1 << 20 + +// errSCUtilDNSTooLarge ends a read that passed the bound. +var errSCUtilDNSTooLarge = errors.New("scutil --dns output is too large") + +// dnsResolverEntry is one resolver of the DNS configuration of the host. +type dnsResolverEntry struct { + Order int + Nameservers []string + IfIndex int + Interface string + Flags string + SearchDomains []string + Reachable string + // Scoped marks a resolver of the table for scoped queries, which serves one + // interface. A second service keeps its own resolver there while the + // unscoped table stays the same. + Scoped bool + // Action tells what the diff saw: added, changed, or removed. A resolver + // without nameservers looks the same whether it came or went, so the event + // needs the word. + Action string +} + +// The actions of a changed resolver. +const ( + resolverActionAdded = "added" + resolverActionChanged = "changed" + resolverActionRemoved = "removed" +) + +// parseSCUtilDNS reads the resolver tables of "scutil --dns": the unscoped +// table that answers a query of the host, then the table for scoped queries +// with one resolver per interface. Both tables count, because a second +// service can change its resolver while the unscoped table stays the same. +// +// A read that ends early or passes the bound gives an error and no table. A +// part of a table looks like resolvers that went away, and the caller must +// keep the table of the read before instead. +func parseSCUtilDNS(r io.Reader) ([]dnsResolverEntry, error) { + var entries []dnsResolverEntry + scoped := false + var entry *dnsResolverEntry + closeEntry := func() { + if entry == nil { + return + } + entry.Scoped = scoped + entries = append(entries, *entry) + entry = nil + } + read := 0 + scanner := bufio.NewScanner(io.LimitReader(r, scutilDNSOutputLimit+1)) + for scanner.Scan() { + read += len(scanner.Bytes()) + 1 + if read > scutilDNSOutputLimit { + return nil, errSCUtilDNSTooLarge + } + line := strings.TrimSpace(scanner.Text()) + switch { + case strings.HasPrefix(line, scutilTablePrefix): + closeEntry() + scoped = strings.Contains(line, "scoped") + case strings.HasPrefix(line, scutilResolverPrefix): + closeEntry() + entry = &dnsResolverEntry{} + case entry != nil: + applySCUtilField(entry, line) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + closeEntry() + return entries, nil +} + +func applySCUtilField(entry *dnsResolverEntry, line string) { + name, value, ok := strings.Cut(line, ":") + if !ok { + return + } + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + switch { + case strings.HasPrefix(name, "nameserver["): + entry.Nameservers = append(entry.Nameservers, value) + case strings.HasPrefix(name, "search domain["): + entry.SearchDomains = append(entry.SearchDomains, value) + case name == "if_index": + entry.IfIndex, entry.Interface = parseSCUtilIfIndex(value) + case name == "flags": + entry.Flags = value + case name == "reach": + entry.Reachable = scutilBracketText(value) + case name == "order": + entry.Order, _ = strconv.Atoi(value) + } +} + +// parseSCUtilIfIndex reads an "if_index" value of the form "14 (en0)". +func parseSCUtilIfIndex(value string) (int, string) { + number, name, _ := strings.Cut(value, " ") + index, _ := strconv.Atoi(number) + return index, scutilBracketText(name) +} + +// scutilBracketText returns the text in brackets, where scutil puts the +// readable form of a number. +func scutilBracketText(value string) string { + start := strings.Index(value, "(") + end := strings.LastIndex(value, ")") + if start < 0 || end < start { + return "" + } + return value[start+1 : end] +} + +// resolverKey identifies one resolver across two reads. An unscoped resolver +// has an order, and several can share it, so the count of earlier entries +// with that order completes the key. A scoped resolver has no order line and +// serves one interface, so its interface index is its identity. The two +// tables are two key spaces. +type resolverKey struct { + scoped bool + order int + ifIndex int + occurrence int +} + +func resolverKeys(entries []dnsResolverEntry) []resolverKey { + counts := make(map[resolverKey]int, len(entries)) + keys := make([]resolverKey, len(entries)) + for i, entry := range entries { + base := resolverKey{scoped: entry.Scoped, order: entry.Order} + if entry.Scoped { + base = resolverKey{scoped: true, ifIndex: entry.IfIndex} + } + key := base + key.occurrence = counts[base] + keys[i] = key + counts[base]++ + } + return keys +} + +func equalResolverEntry(a, b dnsResolverEntry) bool { + return a.IfIndex == b.IfIndex && + a.Interface == b.Interface && + a.Flags == b.Flags && + a.Reachable == b.Reachable && + slices.Equal(a.Nameservers, b.Nameservers) && + slices.Equal(a.SearchDomains, b.SearchDomains) +} + +// diffResolverTables returns the resolvers that differ between two reads. A +// resolver that vanished comes back without nameservers, so the journal shows +// that the host dropped it. +func diffResolverTables(before, after []dnsResolverEntry) []dnsResolverEntry { + beforeKeys, afterKeys := resolverKeys(before), resolverKeys(after) + previous := make(map[resolverKey]dnsResolverEntry, len(before)) + for i, key := range beforeKeys { + previous[key] = before[i] + } + current := make(map[resolverKey]struct{}, len(after)) + for _, key := range afterKeys { + current[key] = struct{}{} + } + var changed []dnsResolverEntry + for i, key := range afterKeys { + known, ok := previous[key] + if ok && equalResolverEntry(known, after[i]) { + continue + } + entry := after[i] + entry.Action = resolverActionChanged + if !ok { + entry.Action = resolverActionAdded + } + changed = append(changed, entry) + } + for i, key := range beforeKeys { + if _, ok := current[key]; ok { + continue + } + vanished := before[i] + vanished.Nameservers = nil + vanished.Action = resolverActionRemoved + changed = append(changed, vanished) + } + return changed +} + +// dnsConfigPoller reads the DNS configuration of the host on a timer and +// reports the resolvers that changed since the read before. +type dnsConfigPoller struct { + run func() ([]dnsResolverEntry, error) + now func() time.Time + mu sync.Mutex + last []dnsResolverEntry + lastActivity time.Time + started bool + readFailed bool + // kick wakes a loop that waits on the slow delay. One pending signal is + // enough, so a storm of events does not queue polls. + kick chan struct{} +} + +func newDNSConfigPoller(run func() ([]dnsResolverEntry, error)) *dnsConfigPoller { + return &dnsConfigPoller{run: run, now: time.Now, kick: make(chan struct{}, 1)} +} + +// noteActivity keeps the poll fast for a while after the network moved, and +// wakes a loop that waits on the slow delay. +func (p *dnsConfigPoller) noteActivity(now time.Time) { + p.markActivity(now) + select { + case p.kick <- struct{}{}: + default: + } +} + +// markActivity records the time of the last network event. +func (p *dnsConfigPoller) markActivity(now time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastActivity = now +} + +func (p *dnsConfigPoller) nextDelay(now time.Time) time.Duration { + p.mu.Lock() + defer p.mu.Unlock() + if p.lastActivity.IsZero() || now.Sub(p.lastActivity) >= dnsConfigFastWindow { + return dnsConfigSlowInterval + } + return dnsConfigFastInterval +} + +// pollOnce returns the resolvers that changed. The first table is the baseline +// of the process and reports no change. A failed read keeps the table of the +// read before, so a short failure does not report the whole table twice. The +// first failure and the read that ends it log once, so a host whose read +// never works says so. +func (p *dnsConfigPoller) pollOnce(now time.Time) []dnsResolverEntry { + table, err := p.run() + p.mu.Lock() + defer p.mu.Unlock() + if err != nil { + if !p.readFailed { + mainLog.Load().Warn().Err(err).Msg("DNS configuration read failed; the journal keeps the last table") + } + p.readFailed = true + return nil + } + if p.readFailed { + mainLog.Load().Info().Msg("DNS configuration read works again") + p.readFailed = false + } + if !p.started { + p.started = true + p.last = table + return nil + } + changed := diffResolverTables(p.last, table) + p.last = table + return changed +} + +// dnsConfigDelayFn lets a test shorten the wait between two polls. +var dnsConfigDelayFn = (*dnsConfigPoller).nextDelay + +// startLoop runs the poll loop and returns a channel that closes when the loop +// returns. A caller joins that channel before it reads the log. +func (p *dnsConfigPoller) startLoop(stop <-chan struct{}, emit func([]dnsResolverEntry)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + p.loop(stop, emit) + }() + return done +} + +// loop polls until stop closes. The read starts a subprocess on macOS, so the +// delay follows the last activity and a quiet host pays little. The start of a +// run counts as activity, because the table of the host settles after the +// daemon comes up, and the read at entry is the baseline of the process. +func (p *dnsConfigPoller) loop(stop <-chan struct{}, emit func([]dnsResolverEntry)) { + now := p.now() + p.markActivity(now) + p.pollOnce(now) + delay := dnsConfigDelayFn(p, now) + deadline := now.Add(delay) + timer := time.NewTimer(delay) + defer timer.Stop() + for { + select { + case <-stop: + return + case <-p.kick: + // noteActivity set the time just now, so the delay is the fast one. + // A kick only moves a deadline closer; a storm of kicks must not + // push the read back again and again. + now := p.now() + wanted := now.Add(dnsConfigDelayFn(p, now)) + if !wanted.Before(deadline) { + continue + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(wanted.Sub(now)) + deadline = wanted + continue + case <-timer.C: + } + now := p.now() + if changed := p.pollOnce(now); len(changed) > 0 { + emit(changed) + } + delay := dnsConfigDelayFn(p, now) + deadline = now.Add(delay) + timer.Reset(delay) + } +} diff --git a/cmd/cli/dns_config_darwin.go b/cmd/cli/dns_config_darwin.go new file mode 100644 index 00000000..60d0c78e --- /dev/null +++ b/cmd/cli/dns_config_darwin.go @@ -0,0 +1,22 @@ +package cli + +import ( + "bytes" + "context" + "os/exec" + "time" +) + +// scutilDNSTimeout bounds the read. scutil talks to the configuration daemon, +// and that daemon stalls while the network moves. +const scutilDNSTimeout = 5 * time.Second + +func runSCUtilDNS() ([]dnsResolverEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), scutilDNSTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, "scutil", "--dns").Output() + if err != nil { + return nil, err + } + return parseSCUtilDNS(bytes.NewReader(out)) +} diff --git a/cmd/cli/dns_config_others.go b/cmd/cli/dns_config_others.go new file mode 100644 index 00000000..9905dd51 --- /dev/null +++ b/cmd/cli/dns_config_others.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package cli + +// runSCUtilDNS has no source of a resolver table outside macOS, so the poller +// finds no change and logs nothing. +func runSCUtilDNS() ([]dnsResolverEntry, error) { + return nil, nil +} diff --git a/cmd/cli/dns_config_test.go b/cmd/cli/dns_config_test.go new file mode 100644 index 00000000..b389ad0f --- /dev/null +++ b/cmd/cli/dns_config_test.go @@ -0,0 +1,598 @@ +package cli + +import ( + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const scutilDNSFixture = `DNS configuration + +resolver #1 + search domain[0] : corp.example + nameserver[0] : 192.168.50.200 + nameserver[1] : 192.168.50.1 + if_index : 16 (en0) + flags : Request A records, Request AAAA records + reach : 0x00020002 (Reachable,Directly Reachable Address) + order : 200000 + +resolver #2 + nameserver[0] : 100.100.100.100 + if_index : 28 (utun4) + flags : Supplemental, Request A records, Request AAAA records + reach : 0x00000003 (Reachable,Transient Connection) + order : 101600 + +resolver #3 + domain : local + options : mdns + timeout : 5 + flags : Request A records + reach : 0x00000000 (Not Reachable) + order : 300000 + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +const scutilDNSScopedOnlyFixture = `DNS configuration + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +var errStubResolverRun = errors.New("scutil failed") + +func wifiResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + } +} + +func vpnResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 101600, + Nameservers: []string{"100.100.100.100"}, + IfIndex: 28, + Interface: "utun4", + Flags: "Supplemental, Request A records, Request AAAA records", + Reachable: "Reachable,Transient Connection", + } +} + +func mdnsResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 300000, + Flags: "Request A records", + Reachable: "Not Reachable", + } +} + +func Test_parseSCUtilDNS(t *testing.T) { + tests := []struct { + name string + input string + want []dnsResolverEntry + }{ + { + name: "both tables", + input: scutilDNSFixture, + want: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry(), mdnsResolverEntry(), { + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "scoped table when the first is empty", + input: scutilDNSScopedOnlyFixture, + want: []dnsResolverEntry{{ + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "no output", + input: "", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(tc.input)) + if err != nil { + t.Fatalf("parseSCUtilDNS() error = %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("parseSCUtilDNS() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTables(t *testing.T) { + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + vanishedVPN := vpnResolverEntry() + vanishedVPN.Nameservers = nil + vanishedVPN.Action = resolverActionRemoved + appearedVPN := vpnResolverEntry() + appearedVPN.Action = resolverActionAdded + + tests := []struct { + name string + before []dnsResolverEntry + after []dnsResolverEntry + want []dnsResolverEntry + }{ + { + name: "no change", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: nil, + }, + { + name: "one resolver changed", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}, + want: []dnsResolverEntry{changedWiFi}, + }, + { + name: "one resolver vanished", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry()}, + want: []dnsResolverEntry{vanishedVPN}, + }, + { + name: "one resolver appeared", + before: []dnsResolverEntry{wifiResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: []dnsResolverEntry{appearedVPN}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := diffResolverTables(tc.before, tc.after) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("diffResolverTables() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTablesKeepsTheInput(t *testing.T) { + before := []dnsResolverEntry{vpnResolverEntry()} + diffResolverTables(before, nil) + if !reflect.DeepEqual(before[0], vpnResolverEntry()) { + t.Fatalf("before entry = %+v, want %+v", before[0], vpnResolverEntry()) + } +} + +// resolverRunResult is one answer of the stub runner. +type resolverRunResult struct { + table []dnsResolverEntry + err error +} + +// stubResolverRunner answers one result per call and repeats the last one. +type stubResolverRunner struct { + mu sync.Mutex + results []resolverRunResult + calls int +} + +func (s *stubResolverRunner) run() ([]dnsResolverEntry, error) { + s.mu.Lock() + defer s.mu.Unlock() + result := s.results[min(s.calls, len(s.results)-1)] + s.calls++ + return result.table, result.err +} + +func Test_dnsConfigPollerNextDelay(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + activity bool + elapsed time.Duration + want time.Duration + }{ + {name: "nine minutes after activity", activity: true, elapsed: 9 * time.Minute, want: dnsConfigFastInterval}, + {name: "eleven minutes after activity", activity: true, elapsed: 11 * time.Minute, want: dnsConfigSlowInterval}, + {name: "no activity yet", elapsed: time.Minute, want: dnsConfigSlowInterval}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + if tc.activity { + poller.noteActivity(base) + } + if got := poller.nextDelay(base.Add(tc.elapsed)); got != tc.want { + t.Fatalf("nextDelay() = %s, want %s", got, tc.want) + } + }) + } +} + +func Test_dnsConfigPollerPollOnce(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + baseline := []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()} + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: baseline}, + {err: errStubResolverRun}, + {table: baseline}, + {table: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}}, + }} + poller := newDNSConfigPoller(runner.run) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("failed poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after a failure = %+v, want nil", got) + } + want := []dnsResolverEntry{changedWiFi} + if got := poller.pollOnce(now.Add(3 * time.Minute)); !reflect.DeepEqual(got, want) { + t.Fatalf("changed poll = %+v, want %+v", got, want) + } +} + +func Test_dnsConfigPollerLoop(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Millisecond } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop emitted no change") + } + + close(stop) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not stop") + } +} + +// Test_dnsConfigPollerLoopWakesOnActivity proves that a network event does not +// wait for a sleeping timer. The delay is one hour until the event arrives, so +// a loop without the kick would poll after the test deadline. +func Test_dnsConfigPollerLoopWakesOnActivity(t *testing.T) { + var kicked atomic.Bool + parked := make(chan struct{}, 1) + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if kicked.Load() { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + t.Cleanup(func() { + close(stop) + <-done + }) + + // The kick matters only once the loop waits on the long delay. + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not park on the long delay") + } + kicked.Store(true) + poller.noteActivity(time.Now()) + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop polled no table after the activity") + } +} + +// Test_dnsConfigPollerLoopTakesABaseline proves that the loop reads the table +// once at entry. Without that read the first change after the start reports +// the whole table. +func Test_dnsConfigPollerLoopTakesABaseline(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + runner := &stubResolverRunner{results: []resolverRunResult{{table: []dnsResolverEntry{wifiResolverEntry()}}}} + poller := newDNSConfigPoller(runner.run) + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func([]dnsResolverEntry) { t.Error("the baseline read reported a change") }) + }() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + runner.mu.Lock() + calls := runner.calls + runner.mu.Unlock() + if calls > 0 { + break + } + time.Sleep(time.Millisecond) + } + close(stop) + <-done + + runner.mu.Lock() + defer runner.mu.Unlock() + if runner.calls != 1 { + t.Fatalf("reads at the loop entry: got %d, want 1", runner.calls) + } + if !poller.started { + t.Fatal("the loop took no baseline") + } +} + +// oversizedSCUtilDNS is a table with one line that no scanner buffer holds. A +// stalled or hostile process produces output of this shape. +func oversizedSCUtilDNS() string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + strings.Repeat("9", 70*1024) + "\n" +} + +// Test_parseSCUtilDNSRejectsAnOversizedLine proves that a line the scanner +// cannot hold ends the read with an error. A part of a table must not pass for +// the table of the host. +func Test_parseSCUtilDNSRejectsAnOversizedLine(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(oversizedSCUtilDNS())) + if err == nil { + t.Fatal("parseSCUtilDNS accepted a 70 KiB line") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_parseSCUtilDNSRejectsAnOversizedOutput proves that output past the +// bound ends the read with an error instead of a table cut in the middle. +func Test_parseSCUtilDNSRejectsAnOversizedOutput(t *testing.T) { + block := "resolver #1\n nameserver[0] : 192.168.50.200\n order : 200000\n" + repeats := scutilDNSOutputLimit/len(block) + 2 + got, err := parseSCUtilDNS(strings.NewReader("DNS configuration\n\n" + strings.Repeat(block, repeats))) + if err == nil { + t.Fatal("parseSCUtilDNS accepted output past the bound") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead proves that a read the +// parser rejects reports nothing and keeps the table of the read before. +func Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + outputs := []string{scutilDNSFixture, oversizedSCUtilDNS(), scutilDNSFixture} + calls := 0 + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + output := outputs[min(calls, len(outputs)-1)] + calls++ + return parseSCUtilDNS(strings.NewReader(output)) + }) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("oversized poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after the oversized read = %+v, want nil", got) + } +} + +// scutilTwoTables renders a host with one unscoped resolver and one scoped +// resolver, the shape of a Mac with two active services. +func scutilTwoTables(unscoped, scoped string) string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + unscoped + "\n if_index : 14 (en0)\n" + + " flags : Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n" + + "DNS configuration (for scoped queries)\n\nresolver #1\n nameserver[0] : " + scoped + "\n if_index : 15 (en1)\n" + + " flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n" +} + +// Test_diffResolverTablesSeesAScopedChange changes the scoped resolver of a +// second service while the unscoped resolver stays. The diff must report the +// scoped change, because the two tables are not copies of each other. +func Test_diffResolverTablesSeesAScopedChange(t *testing.T) { + before, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "10.0.0.1"))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "9.9.9.9"))) + if err != nil { + t.Fatal(err) + } + changed := diffResolverTables(before, after) + if len(changed) != 1 || len(changed[0].Nameservers) != 1 || changed[0].Nameservers[0] != "9.9.9.9" { + t.Fatalf("changed resolvers = %+v, want the scoped resolver with 9.9.9.9", changed) + } + if !changed[0].Scoped || before[0].Scoped || len(before) != 2 { + t.Fatalf("scope marks: changed %v, before %+v; want the scoped resolver marked and the unscoped one not", changed[0].Scoped, before) + } +} + +// Test_dnsConfigPollerPollsUnderContinuousActivity kicks the poller faster +// than its fast interval. A kick must not push the read back again and again, +// or a host in a storm of transitions never reports a resolver change. +func Test_dnsConfigPollerPollsUnderContinuousActivity(t *testing.T) { + var reads atomic.Int64 + changed := wifiResolverEntry() + changed.Nameservers = []string{"9.9.9.9"} + p := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + if reads.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{changed}, nil + }) + origDelay := dnsConfigDelayFn + t.Cleanup(func() { dnsConfigDelayFn = origDelay }) + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return 50 * time.Millisecond } + + var emits atomic.Int64 + stop := make(chan struct{}) + done := p.startLoop(stop, func([]dnsResolverEntry) { emits.Add(1) }) + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + p.noteActivity(time.Now()) + time.Sleep(5 * time.Millisecond) + } + close(stop) + <-done + + if emits.Load() == 0 { + t.Fatalf("no resolver event during 500 ms of activity (reads %d)", reads.Load()) + } +} + +// scutilScopedTable renders the scoped table the way macOS prints it: one +// resolver per interface and no order line. +func scutilScopedTable(resolvers ...[2]string) string { + var b strings.Builder + b.WriteString("DNS configuration\n\nresolver #1\n nameserver[0] : 100.100.100.100\n if_index : 28 (utun4)\n flags : Request A records, Request AAAA records\n reach : 0x00000003 (Reachable,Transient Connection)\n order : 200000\n\n") + b.WriteString("DNS configuration (for scoped queries)\n\n") + for i, resolver := range resolvers { + fmt.Fprintf(&b, "resolver #%d\n nameserver[0] : %s\n if_index : %s\n flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n", i+1, resolver[1], resolver[0]) + } + return b.String() +} + +// Test_diffResolverTablesKeysScopedResolversByInterface inserts a scoped +// resolver between two others and removes it again. The scoped table has no +// order line, so the interface is the identity, and the diff must name the +// resolver that came and went, not its neighbors. +func Test_diffResolverTablesKeysScopedResolversByInterface(t *testing.T) { + en0 := [2]string{"16 (en0)", "192.168.50.200"} + en7 := [2]string{"30 (en7)", "10.9.9.1"} + utun := [2]string{"28 (utun4)", "100.100.100.100"} + before, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, utun))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, en7, utun))) + if err != nil { + t.Fatal(err) + } + + inserted := diffResolverTables(before, after) + if len(inserted) != 1 || inserted[0].Interface != "en7" || inserted[0].Nameservers == nil { + t.Fatalf("insert of en7 reported %+v, want en7 alone with its nameserver", inserted) + } + removed := diffResolverTables(after, before) + if len(removed) != 1 || removed[0].Interface != "en7" || removed[0].Nameservers != nil { + t.Fatalf("removal of en7 reported %+v, want en7 alone without nameservers", removed) + } +} + +// Test_diffResolverTablesNamesTheAction tells a resolver that came from one +// that went. An mDNS resolver has no nameservers, so without the action the +// two events read the same. +func Test_diffResolverTablesNamesTheAction(t *testing.T) { + mdns := "DNS configuration\n\nresolver #1\n domain : local\n options : mdns\n timeout : 5\n flags : Request A records, Request AAAA records\n reach : 0x00000000 (Not Reachable)\n order : 300000\n" + table, err := parseSCUtilDNS(strings.NewReader(mdns)) + if err != nil { + t.Fatal(err) + } + appeared := diffResolverTables(nil, table) + vanished := diffResolverTables(table, nil) + if len(appeared) != 1 || appeared[0].Action != resolverActionAdded { + t.Fatalf("appeared = %+v, want one resolver with action added", appeared) + } + if len(vanished) != 1 || vanished[0].Action != resolverActionRemoved { + t.Fatalf("vanished = %+v, want one resolver with action removed", vanished) + } + moved := wifiResolverEntry() + moved.Nameservers = []string{"9.9.9.9"} + changed := diffResolverTables([]dnsResolverEntry{wifiResolverEntry()}, []dnsResolverEntry{moved}) + if len(changed) != 1 || changed[0].Action != resolverActionChanged { + t.Fatalf("changed = %+v, want one resolver with action changed", changed) + } +} diff --git a/cmd/cli/dns_intercept_cleanup_windows_test.go b/cmd/cli/dns_intercept_cleanup_windows_test.go new file mode 100644 index 00000000..bdb7fbd1 --- /dev/null +++ b/cmd/cli/dns_intercept_cleanup_windows_test.go @@ -0,0 +1,180 @@ +//go:build windows + +package cli + +import ( + "errors" + "testing" +) + +// TestStaleCleanupRequiresPositiveEvidence pins the guard that protects a live +// pre-session-scoped ctrld service. +// +// ctrldServiceLiveness cannot answer for an unreachable SCM, a caller without rights, or a +// service mid-stop. Folding those into "stopped" would let a hand-run "ctrld run" delete +// the sublayer of a live old-build service - the exact enforcement strip the interactive +// guard exists to prevent - because that build's non-dynamic sublayer looks like an +// orphan. Only a positive stopped answer may unlock the cleanup. +func TestStaleCleanupRequiresPositiveEvidence(t *testing.T) { + tests := []struct { + name string + interactive bool + liveness serviceLiveness + want bool + }{ + // A service start is the deadlock case: nothing of ctrld's is live yet, and this + // is the only path that can break a host locked out by orphaned filters. + {"service start with a running service", false, serviceLivenessRunning, true}, + {"service start with an unknown state", false, serviceLivenessUnknown, true}, + {"service start with a stopped service", false, serviceLivenessStopped, true}, + + // Interactive: only positive evidence of absence unlocks it. + {"ctrld run alongside a live service", true, serviceLivenessRunning, false}, + {"ctrld run when the SCM cannot be queried", true, serviceLivenessUnknown, false}, + {"ctrld run with the service stopped or absent", true, serviceLivenessStopped, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := staleCleanupAllowed(tc.interactive, tc.liveness); got != tc.want { + t.Errorf("staleCleanupAllowed(%v, %v) = %v, want %v", tc.interactive, tc.liveness, got, tc.want) + } + }) + } + + // The zero value must be the safe one: a serviceLiveness that was never assigned - a + // future code path that forgets to set it - must not read as "nothing is live". + if staleCleanupAllowed(true, serviceLiveness(0)) { + t.Error("the zero serviceLiveness must not unlock the cleanup: an unset value is not evidence of absence") + } +} + +// stubStaleCleanup installs fakes for the cleanup's guard inputs and for the deletion +// itself, and returns a pointer to the delete-attempt count. +// +// The delete is a WFP syscall: running it for real would remove live filters from the +// machine running the tests, so the assertion has to be made against a substituted delete. +func stubStaleCleanup(t *testing.T, elevated bool, elevErr error, interactive bool, liveness serviceLiveness) *int { + t.Helper() + oldElev, oldInter, oldLive, oldDel := staleCleanupElevatedFn, staleCleanupInteractiveFn, staleCleanupLivenessFn, deleteStaleWFPSublayerFn + t.Cleanup(func() { + staleCleanupElevatedFn = oldElev + staleCleanupInteractiveFn = oldInter + staleCleanupLivenessFn = oldLive + deleteStaleWFPSublayerFn = oldDel + }) + + deletes := 0 + staleCleanupElevatedFn = func() (bool, error) { return elevated, elevErr } + staleCleanupInteractiveFn = func() bool { return interactive } + staleCleanupLivenessFn = func() serviceLiveness { return liveness } + deleteStaleWFPSublayerFn = func() { deletes++ } + return &deletes +} + +// TestCleanupStaleStateConsultsTheGuardBeforeDeleting is the caller-level half of the +// guard's coverage. +// +// staleCleanupAllowed being correct proves nothing on its own: cleanupStaleDNSInterceptState +// could stop calling it, or call it and delete anyway, and a predicate-only test would stay +// green while a hand-run "ctrld run" stripped a live service's enforcement. This asserts on +// the deletion itself - whether the WFP delete is attempted at all - which is the behaviour +// that matters. +func TestCleanupStaleStateConsultsTheGuardBeforeDeleting(t *testing.T) { + tests := []struct { + name string + elevated bool + elevErr error + interactive bool + liveness serviceLiveness + wantDeletes int + }{ + { + // The deadlock case this function exists for: a service start, where nothing + // of ctrld's is live and the host may be carrying orphaned block-all filters. + name: "service start attempts the delete", + elevated: true, + wantDeletes: 1, + }, + { + // A service start does not consult the SCM at all, so a running service + // reported here must not change the outcome. + name: "service start is not gated on service state", + elevated: true, + liveness: serviceLivenessRunning, + wantDeletes: 1, + }, + { + name: "interactive run with a stopped service attempts the delete", + elevated: true, + interactive: true, + liveness: serviceLivenessStopped, + wantDeletes: 1, + }, + { + // Deleting here would strip a live pre-session-scoped service's enforcement. + name: "interactive run beside a live service does not delete", + elevated: true, + interactive: true, + liveness: serviceLivenessRunning, + }, + { + // The concern this test was added for: "could not tell" is not absence. + name: "interactive run with an unreadable SCM does not delete", + elevated: true, + interactive: true, + liveness: serviceLivenessUnknown, + }, + { + // Elevation is the only barrier between an unprivileged local process and a + // path that opens a WFP engine and deletes ctrld's sublayer. + name: "an unelevated caller does not delete", + elevated: false, + }, + { + name: "an elevation check that fails does not delete", + elevErr: errors.New("cannot determine privilege"), + }, + { + // Elevation reported true alongside an error is not a yes. + name: "an inconclusive elevation check does not delete", + elevated: true, + elevErr: errors.New("cannot determine privilege"), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + deletes := stubStaleCleanup(t, tc.elevated, tc.elevErr, tc.interactive, tc.liveness) + + cleanupStaleDNSInterceptState() + + if *deletes != tc.wantDeletes { + t.Errorf("WFP delete attempts = %d, want %d", *deletes, tc.wantDeletes) + } + }) + } +} + +// TestCleanupStaleStateDoesNotQueryTheSCMOnAServiceStart pins the ordering the deadlock +// recovery depends on. +// +// The cleanup runs before the network-up wait and before API preflight, on the path a +// locked-out host has to take. Consulting the SCM there would make the one case this +// exists for depend on a query that can block or fail - and a failure answers Unknown, +// which refuses the cleanup. A service start must not ask. +func TestCleanupStaleStateDoesNotQueryTheSCMOnAServiceStart(t *testing.T) { + deletes := stubStaleCleanup(t, true, nil, false, serviceLivenessStopped) + queried := false + staleCleanupLivenessFn = func() serviceLiveness { + queried = true + return serviceLivenessRunning + } + + cleanupStaleDNSInterceptState() + + if queried { + t.Error("a service start queried the SCM: an unreadable SCM would then refuse the cleanup the locked-out host needs") + } + if *deletes != 1 { + t.Errorf("WFP delete attempts = %d, want 1", *deletes) + } +} diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go new file mode 100644 index 00000000..fb38f37a --- /dev/null +++ b/cmd/cli/dns_intercept_darwin.go @@ -0,0 +1,2701 @@ +//go:build darwin + +package cli + +import ( + "context" + "crypto/sha256" + "fmt" + "net" + "net/netip" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// skipInitialDNSReset is Windows-only; macOS keeps the normal adapter cleanup +// before installing its pf redirect. +func (p *prog) skipInitialDNSReset() bool { return false } + +const ( + // pfWatchdogInterval is how often the periodic pf watchdog checks + // that our anchor references are still present in the running ruleset. + pfWatchdogInterval = 30 * time.Second + + // pfConsecutiveMissThreshold is the number of consecutive watchdog cycles + // where the anchor was found missing before escalating to ERROR level. + // This indicates something is persistently fighting our pf rules. + pfConsecutiveMissThreshold = 3 + + // pfAnchorRecheckDelay is how long to wait after a network change before + // performing a second pf anchor check. This catches race conditions where + // another program (e.g., Windscribe desktop) clears pf rules slightly + // after our network change handler runs. + pfAnchorRecheckDelay = 2 * time.Second + + // pfAnchorRecheckDelayLong is a second, longer delayed re-check after network + // changes. Some VPNs (e.g., Windscribe) take 3-4s to fully tear down their pf + // rules and DNS settings on disconnect. This catches slower teardowns that the + // 2s re-check misses. + pfAnchorRecheckDelayLong = 4 * time.Second + + // pfExecFailureBackoff suppresses repeated pfctl/scutil checks briefly after + // macOS reports local resource exhaustion. Without this, network-change storms + // can turn a pf ruleset race into a fork/file-descriptor exhaustion loop. + pfExecFailureBackoff = 5 * time.Second + + // pfStabilizationMaxWait bounds the total period ordinary repair can be + // suppressed if PF never becomes stable or pfctl keeps failing. + pfStabilizationMaxWait = 90 * time.Second + + // pfIgnoredChangeReconcileInterval bounds leading-edge pf/VPN-DNS work + // during continuous ignored network-change storms. The 2s/4s delayed checks + // still provide trailing reconciliation once the storm stops. + pfIgnoredChangeReconcileInterval = pfAnchorRecheckDelayLong + + // pfVPNInterfacePrefixes lists interface name prefixes that indicate VPN/tunnel + // interfaces on macOS. Used to add interface-specific DNS intercept rules so that + // VPN software with "pass out quick on " rules cannot bypass our intercept. + // Common prefixes: + // ipsec* - IKEv2/IPsec VPNs (Windscribe, macOS built-in) + // utun* - TUN interfaces (WireGuard, Tailscale, OpenVPN, etc.) + // ppp* - PPTP/L2TP VPNs + // tap* - TAP interfaces (OpenVPN in bridge mode) + // tun* - Legacy TUN interfaces + // lo0 is excluded since our rules already handle loopback. + pfVPNInterfacePrefixes = "ipsec,utun,ppp,tap,tun" +) + +const ( + // pfProbeDomain is the suffix used for pf interception probe queries. + // The full probe domain is "_pf-probe-.". + // These queries are sent by a subprocess WITHOUT the _ctrld group GID, + // so pf should intercept them and redirect to ctrld. If ctrld receives + // the query, interception is working. A sent query without receipt permits + // repair. An unsent or unavailable probe is indeterminate. + // No trailing dot — canonicalName() in the DNS handler strips trailing dots. + pfProbeDomain = "pf-probe.ctrld.test" + + // pfProbeTimeout is how long to wait for a probe query to arrive at ctrld. + pfProbeTimeout = 1 * time.Second + + // pfGroupName is the macOS system group used to scope pf exemption rules. + // Only processes running with this effective GID can bypass the DNS redirect, + // preventing other applications from circumventing ctrld by querying exempted IPs directly. + pfGroupName = "_ctrld" + + // pfAnchorName is the pf anchor name used by ctrld for DNS interception. + // Using reverse-DNS convention to avoid conflicts with other software. + pfAnchorName = "com.controld.ctrld" + + // pfAnchorDir is the directory where pf anchor files are stored on macOS. + pfAnchorDir = "/etc/pf.anchors" + + // pfAnchorFile is the full path to ctrld's pf anchor configuration file. + pfAnchorFile = "/etc/pf.anchors/com.controld.ctrld" +) + +// pfAnchorWriteMu serializes writers of the anchor file. Seven code paths rebuild it +// (startup, tunnel change, watchdog restore, VPN DNS exemptions, forced reload, +// forwarded-source reconcile, firewall shutdown) from timers and network-change +// callbacks in their own goroutines, with no lock between them. +var pfAnchorWriteMu sync.Mutex + +// writePFAnchorFile replaces the anchor file atomically: write a temp file in the +// same directory, then rename over the target. +// +// os.WriteFile truncates and then writes, so a pfctl -f racing that window can read a +// partial ruleset and reject the anchor - taking DNS interception down until the next +// watchdog restore. A rename is atomic, so a concurrent reader sees either the whole +// previous ruleset or the whole new one. +// +// Two writers can still be followed by two loads in either order, but every load now +// sees a complete ruleset, and each writer's load re-applies a full anchor, so the +// worst case is redundant work rather than a broken anchor. +func writePFAnchorFile(rules string) error { + pfAnchorWriteMu.Lock() + defer pfAnchorWriteMu.Unlock() + + tmp, err := os.CreateTemp(filepath.Dir(pfAnchorFile), ".ctrld-anchor-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer func() { + // No-op once the rename succeeded; removes the temp file on any failure path. + _ = os.Remove(tmpName) + }() + + if _, err := tmp.WriteString(rules); err != nil { + tmp.Close() + return err + } + if err := tmp.Chmod(0644); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpName, pfAnchorFile) +} + +// pfState holds the state of the pf DNS interception on macOS. +type pfState struct { + ipv6Diagnostic pfDiagnosticState + + anchorFile string + anchorName string + // Serialized by prog.interceptDNSTargetMu; bounded to one failed decision. + targetDiagnostic dnsTargetDecisionDiagnostic +} + +// ensureCtrldGroup creates the _ctrld system group if it doesn't exist and returns its GID. +// Uses dscl (macOS Directory Services) to manage the group. This function is idempotent — +// safe to call multiple times across restarts. The group is intentionally never removed +// on shutdown to avoid race conditions during rapid restart cycles. +func ensureCtrldGroup() (int, error) { + // Check if the group already exists. + out, err := exec.Command("dscl", ".", "-read", "/Groups/"+pfGroupName, "PrimaryGroupID").CombinedOutput() + if err == nil { + // Group exists — parse and return its GID. + // Output format: "PrimaryGroupID: 350" + line := strings.TrimSpace(string(out)) + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + gid, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return 0, fmt.Errorf("failed to parse existing group GID from %q: %w", line, err) + } + mainLog.Load().Debug().Msgf("DNS intercept: group %s already exists with GID %d", pfGroupName, gid) + return gid, nil + } + return 0, fmt.Errorf("unexpected dscl output for existing group: %q", line) + } + + // Group doesn't exist — find an unused GID in the 350-450 range (system group range on macOS, + // above Apple's reserved range but below typical user groups). + listOut, err := exec.Command("dscl", ".", "-list", "/Groups", "PrimaryGroupID").CombinedOutput() + if err != nil { + return 0, fmt.Errorf("failed to list existing groups: %w (output: %s)", err, strings.TrimSpace(string(listOut))) + } + + usedGIDs := make(map[int]bool) + for _, line := range strings.Split(string(listOut), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 { + if gid, err := strconv.Atoi(fields[len(fields)-1]); err == nil { + usedGIDs[gid] = true + } + } + } + + chosenGID := 0 + for gid := 350; gid <= 450; gid++ { + if !usedGIDs[gid] { + chosenGID = gid + break + } + } + if chosenGID == 0 { + return 0, fmt.Errorf("no unused GID found in range 350-450") + } + + // Create the group record. Handle eDSRecordAlreadyExists gracefully in case of a + // race with another ctrld instance. + createOut, err := exec.Command("dscl", ".", "-create", "/Groups/"+pfGroupName).CombinedOutput() + if err != nil { + outStr := strings.TrimSpace(string(createOut)) + if !strings.Contains(outStr, "eDSRecordAlreadyExists") { + return 0, fmt.Errorf("failed to create group record: %w (output: %s)", err, outStr) + } + } + + // Set the GID. This is idempotent — dscl overwrites the attribute if it already exists. + if out, err := exec.Command("dscl", ".", "-create", "/Groups/"+pfGroupName, "PrimaryGroupID", strconv.Itoa(chosenGID)).CombinedOutput(); err != nil { + return 0, fmt.Errorf("failed to set group GID: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + + if out, err := exec.Command("dscl", ".", "-create", "/Groups/"+pfGroupName, "RealName", "ctrld DNS Intercept Group").CombinedOutput(); err != nil { + return 0, fmt.Errorf("failed to set group RealName: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + + mainLog.Load().Info().Msgf("DNS intercept: created system group %s with GID %d", pfGroupName, chosenGID) + return chosenGID, nil +} + +// setCtrldGroupID sets the process's effective GID to the _ctrld group. +// This must be called before any outbound DNS sockets are created so that +// pf's "group _ctrld" matching applies to ctrld's own DNS queries. +// Only ctrld (running as root with this effective GID) will match the exemption rules, +// preventing other processes from bypassing the DNS redirect. +func setCtrldGroupID(gid int) error { + if err := syscall.Setegid(gid); err != nil { + return fmt.Errorf("syscall.Setegid(%d) failed: %w", gid, err) + } + mainLog.Load().Info().Msgf("DNS intercept: set process effective GID to %d (%s)", gid, pfGroupName) + return nil +} + +// startDNSIntercept activates pf-based DNS interception on macOS. +// It creates a pf anchor that redirects all outbound DNS (port 53) traffic +// to ctrld's local listener at 127.0.0.1:53. This eliminates the race condition +// with VPN software that overwrites interface DNS settings. +// +// The approach: +// 1. Write a pf anchor file with redirect rules for all non-loopback interfaces +// 2. Load the anchor into pf +// 3. Ensure pf is enabled +// +// ctrld's own upstream queries use DoH (port 443), so they are NOT affected +// by the port 53 redirect. If an "os" upstream is configured (which uses port 53), +// we skip the redirect for traffic from the ctrld process itself. +func (p *prog) startDNSIntercept() error { + mainLog.Load().Info().Msg("DNS intercept: initializing macOS packet filter (pf) redirect") + + if err := installPFInterceptFn(p); err != nil { + return err + } + + p.dnsInterceptState = &pfState{ + anchorFile: pfAnchorFile, + anchorName: pfAnchorName, + } + + // Store the initial set of tunnel interfaces so we can detect changes later. + tunnels := p.activeTunnelInterfaces() + p.mu.Lock() + p.lastTunnelIfaces = tunnels + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() + + lc := p.cfg.FirstListener() + if lc != nil { + mainLog.Load().Info().Msgf("DNS intercept: pf redirect active - all outbound DNS (port 53) redirected to %s:%d via anchor %q", lc.IP, lc.Port, pfAnchorName) + } else { + mainLog.Load().Info().Msgf("DNS intercept: pf redirect active - all outbound DNS (port 53) redirected via anchor %q", pfAnchorName) + } + + p.startInterceptBackgroundWork() + + return nil +} + +// installPFInterceptFn is the privileged-install seam. Everything behind it needs root, +// a live pf and a writable /etc/pf.anchors - which is what kept startDNSIntercept, and +// with it the handoff to the background watchers, out of reach of every test. +var installPFInterceptFn = (*prog).installPFIntercept + +// installPFIntercept performs the privileged part of starting interception: validation, +// _ctrld group setup, anchor file write, pfctl load, enable, and post-load verification. +// It returns nil only once pf is authoritatively active, so the caller may publish +// intercept mode; on failure it has already rolled back whatever it applied. +func (p *prog) installPFIntercept() error { + if err := p.validateDNSIntercept(); err != nil { + return err + } + p.pfStabilizing.Store(false) + p.pfLastRestoreTime.Store(0) + p.pfBackoffMultiplier.Store(0) + p.pfExecBackoffUntil.Store(0) + p.pfIgnoredChangeLastReconcile.Store(0) + + // Set up _ctrld group for pf exemption scoping. This ensures that only ctrld's + // own DNS queries (matching "group _ctrld" in pf rules) can bypass the redirect. + // Must happen BEFORE loading pf rules so the effective GID is set when sockets are created. + gid, err := ensureCtrldGroup() + if err != nil { + return fmt.Errorf("dns intercept: failed to create %s group: %w", pfGroupName, err) + } + if err := setCtrldGroupID(gid); err != nil { + return fmt.Errorf("dns intercept: failed to set process GID to %s: %w", pfGroupName, err) + } + + // Clean up any stale state from a previous crash. + if _, err := os.Stat(pfAnchorFile); err == nil { + mainLog.Load().Warn().Msg("DNS intercept: found stale pf anchor file from previous run — cleaning up") + exec.Command("pfctl", "-a", pfAnchorName, "-F", "all").CombinedOutput() + os.Remove(pfAnchorFile) + } + + // Pre-discover VPN DNS configurations before building initial rules. + // Without this, there's a startup gap where the initial anchor has no VPN DNS + // exemptions, causing queries to be intercepted and routed to ctrld. Stale pf + // state entries from the gap persist even after vpnDNS.Refresh() adds exemptions. + var initialExemptions []vpnDNSExemption + if !hardIntercept { + initialConfigs := ctrld.DiscoverVPNDNS(context.Background()) + type key struct{ server, iface string } + seen := make(map[key]bool) + for _, config := range initialConfigs { + for _, server := range config.Servers { + k := key{server, config.InterfaceName} + if !seen[k] { + seen[k] = true + initialExemptions = append(initialExemptions, vpnDNSExemption{ + Server: server, + Interface: config.InterfaceName, + }) + } + } + } + if len(initialExemptions) > 0 { + mainLog.Load().Info().Msgf("DNS intercept: pre-discovered %d VPN DNS exemptions for initial rules", len(initialExemptions)) + } + } + + forwarded := p.currentForwardedSources() + rules := p.buildPFAnchorRulesWith(initialExemptions, forwarded) + + if err := os.MkdirAll(pfAnchorDir, 0755); err != nil { + return fmt.Errorf("dns intercept: failed to create pf anchor directory %s: %w", pfAnchorDir, err) + } + if err := writePFAnchorFile(rules); err != nil { + return fmt.Errorf("dns intercept: failed to write pf anchor file %s: %w", pfAnchorFile, err) + } + mainLog.Load().Debug().Msgf("DNS intercept: wrote pf anchor file: %s", pfAnchorFile) + + out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput() + if err != nil { + p.rollbackDNSInterceptStart() + return fmt.Errorf("dns intercept: failed to load pf anchor: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + // Baseline the reconcile snapshot: the anchor just loaded already contains this + // set, so the first watchdog tick must not treat it as a change. + p.recordAppliedForwardedSources(forwarded) + mainLog.Load().Debug().Msgf("DNS intercept: loaded pf anchor %q from %s", pfAnchorName, pfAnchorFile) + + if err := p.ensurePFAnchorReference(); err != nil { + p.rollbackDNSInterceptStart() + return fmt.Errorf("dns intercept: failed to activate pf anchor references: %w", err) + } + + out, err = exec.Command("pfctl", "-e").CombinedOutput() + if err != nil { + outStr := strings.TrimSpace(string(out)) + if !strings.Contains(outStr, "already enabled") { + p.rollbackDNSInterceptStart() + return fmt.Errorf("dns intercept: failed to enable pf: %w (output: %s)", err, outStr) + } + } + + out, err = exec.Command("pfctl", "-a", pfAnchorName, "-sr").CombinedOutput() + if err != nil { + mainLog.Load().Warn().Msgf("DNS intercept: could not verify anchor rules: %v", err) + } else { + ruleCount := strings.Count(strings.TrimSpace(string(out)), "\n") + 1 + mainLog.Load().Info().Msgf("DNS intercept: pf anchor %q active with %d rules", pfAnchorName, ruleCount) + mainLog.Load().Debug().Msgf("DNS intercept: active pf rules:\n%s", strings.TrimSpace(string(out))) + } + + out, err = exec.Command("pfctl", "-a", pfAnchorName, "-sn").CombinedOutput() + if err == nil && len(strings.TrimSpace(string(out))) > 0 { + mainLog.Load().Debug().Msgf("DNS intercept: active pf NAT/redirect rules:\n%s", strings.TrimSpace(string(out))) + } + + // Post-load verification: do not publish intercept mode unless PF is + // authoritatively active; the caller can then use interface-DNS fallback. + if !p.verifyPFState() { + p.rollbackDNSInterceptStart() + return fmt.Errorf("dns intercept: post-load PF verification failed") + } + + return nil +} + +// pfWatchdogFn and runSuspendWatcherFn are the background-work seams. Tests use them to +// observe what startInterceptBackgroundWork registers without running either loop, which +// would otherwise reach pfctl. +var ( + pfWatchdogFn = (*prog).pfWatchdog + runSuspendWatcherFn = runSuspendWatcher +) + +// startInterceptBackgroundWork launches the goroutines that guard an active intercept. +// +// Split out of startDNSIntercept so the registration is reachable from a test: +// startDNSIntercept needs root, a live pf and a writable /etc/pf.anchors, so nothing +// covers this wiring if it lives inline there. +func (p *prog) startInterceptBackgroundWork() { + // Detect and restore rules if another program (e.g., Windscribe desktop, macOS + // configd) replaces the pf ruleset. + go pfWatchdogFn(p) + + // A host sleep freezes every timer above it, and rule text survives the suspend + // unchanged, so no existing path notices that pf stopped translating while the host + // was asleep. Watch for the resume itself and probe from there. + go runSuspendWatcherFn(p.stopCh, p.verifyInterceptAfterWake) +} + +func (p *prog) rollbackDNSInterceptStart() { + if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-F", "all").CombinedOutput(); err != nil { + mainLog.Load().Warn().Err(err).Msgf("DNS intercept: startup rollback could not flush anchor (output: %s)", strings.TrimSpace(string(out))) + } + if err := os.Remove(pfAnchorFile); err != nil && !os.IsNotExist(err) { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: startup rollback could not remove anchor file") + } + if err := p.removePFAnchorReference(); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: startup rollback could not remove anchor references") + } +} + +// ensurePFAnchorReference ensures the running pf ruleset includes our anchor +// declarations. We dump the RUNNING ruleset via "pfctl -sr" (filter+scrub rules) +// and "pfctl -sn" (NAT/rdr rules), check if our references exist, and if not, +// inject them and reload the combined ruleset via stdin. +// +// pf enforces strict rule ordering: +// +// options → normalization (scrub) → queueing → translation (nat/rdr) → filtering (pass/block/anchor) +// +// "pfctl -sr" returns BOTH scrub-anchor (normalization) AND anchor/pass/block (filter) rules. +// "pfctl -sn" returns rdr-anchor (translation) rules. +// Both commands emit "No ALTQ support in kernel" warnings on stderr. +// +// We must reassemble in correct order: scrub → nat/rdr → filter. +// +// The anchor reference does not survive a reboot, but ctrld re-adds it on every start. +func (p *prog) ensurePFAnchorReference() error { + rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) + anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) + + // Dump running rules. Use CombinedOutput but filter out stderr warnings. + natOut, err := exec.Command("pfctl", "-sn").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to dump running NAT rules: %w (output: %s)", err, strings.TrimSpace(string(natOut))) + } + + filterOut, err := exec.Command("pfctl", "-sr").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to dump running filter rules: %w (output: %s)", err, strings.TrimSpace(string(filterOut))) + } + + // Filter pfctl output into actual pf rules, stripping stderr warnings. + natLines := pfFilterRuleLines(string(natOut)) + filterLines := pfFilterRuleLines(string(filterOut)) + + hasRdrAnchor := pfContainsRule(natLines, rdrAnchorRef) + hasAnchor := pfContainsRule(filterLines, anchorRef) + + if hasRdrAnchor && hasAnchor { + // Verify anchor ordering: our anchor should appear before other anchors + // for reliable DNS interception priority. Log a warning if out of order, + // but don't force a reload (the interface-specific rules in our anchor + // provide a secondary safety net even if ordering is suboptimal). + p.checkAnchorOrdering(filterLines, anchorRef) + mainLog.Load().Debug().Msg("DNS intercept: anchor references already present in running ruleset") + return nil + } + + mainLog.Load().Info().Msg("DNS intercept: injecting anchor references into running pf ruleset") + + // Separate scrub rules from filter rules (pfctl -sr returns both). + // scrub/scrub-anchor = normalization, must come BEFORE translation. + var scrubLines, pureFilterLines []string + for _, line := range filterLines { + if strings.HasPrefix(line, "scrub") { + scrubLines = append(scrubLines, line) + } else { + pureFilterLines = append(pureFilterLines, line) + } + } + + // Inject our references if missing. PREPEND both references to ensure our + // anchor is evaluated BEFORE any other anchors (e.g., Windscribe's + // "windscribe_vpn_traffic"). pf evaluates rules top-to-bottom, so "quick" + // rules in whichever anchor appears first win. By prepending, our DNS + // intercept rules match port 53 traffic before a VPN app's broader + // "pass out quick on all" rules in their anchor. + if !hasRdrAnchor { + natLines = append([]string{rdrAnchorRef}, natLines...) + } + if !hasAnchor { + pureFilterLines = append([]string{anchorRef}, pureFilterLines...) + } + + // Reassemble in pf's required order: scrub → translation → filtering. + // + // KNOWN GAP: no options section is emitted, so this reload drops the running pf + // options - timeouts, limits and any third-party "set skip" directives - which by + // pf semantics revert to defaults. + // + // They cannot simply be carried across, because Apple's pfctl offers no way to read + // them: pfctl(8) accepts -s nat, queue, rules, Anchors, states, Sources, info, + // References, labels, timeouts, memory, Tables, osfp, Interfaces and all, with no + // options modifier. A fix therefore likely means not rebuilding the main ruleset + // from scratch at all. What macOS pf does to system options on a reload like this + // needs confirming on a host - tracked as follow-up to #573. + var combined strings.Builder + for _, line := range scrubLines { + combined.WriteString(line + "\n") + } + for _, line := range natLines { + combined.WriteString(line + "\n") + } + for _, line := range pureFilterLines { + combined.WriteString(line + "\n") + } + + cmd := exec.Command("pfctl", "-f", "-") + cmd.Stdin = strings.NewReader(combined.String()) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to load pf ruleset with anchor references: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + + mainLog.Load().Info().Msg("DNS intercept: anchor references active in running pf ruleset") + return nil +} + +// checkAnchorOrdering logs a warning if our anchor reference is not the first +// anchor in the filter ruleset. When another anchor (e.g., Windscribe's +// "windscribe_vpn_traffic") appears before ours, its "quick" rules may match +// DNS traffic first. The interface-specific tunnel rules in our anchor provide +// a secondary defense, but first position is still preferred. +func (p *prog) checkAnchorOrdering(filterLines []string, ourAnchorRef string) { + for _, line := range filterLines { + if strings.HasPrefix(line, "anchor ") { + if strings.Contains(line, ourAnchorRef) { + // Our anchor is first — ideal ordering. + return + } + // Another anchor appears before ours. + mainLog.Load().Warn().Msgf("DNS intercept: anchor ordering suboptimal — %q appears before our anchor %q. "+ + "Interface-specific rules provide fallback protection, but prepending is preferred.", line, pfAnchorName) + return + } + } +} + +// stopDNSIntercept removes all pf rules and cleans up the DNS interception. +func (p *prog) stopDNSIntercept() error { + // Remove a loopback DNS target set for a DNS-less network (issue #533) + // before tearing down pf, so the service is returned to its saved or + // empty DNS state. + p.removeInterceptDNSTarget("intercept shutdown") + + state, ok := p.dnsInterceptState.(*pfState) + if !ok || state == nil { + mainLog.Load().Debug().Msg("DNS intercept: no pf state to clean up") + return nil + } + + // Revoke lifecycle state and cancel future owners before waiting for any + // in-flight PF mutation. A mutation that already owns the lane may finish, + // but final cleanup always runs after it and no new owner can start. + p.dnsInterceptState = nil + pfShutdownStateRevokedForTest() + p.mu.Lock() + if p.pfStabilizeCancel != nil { + p.pfStabilizeCancel() + p.pfStabilizeCancel = nil + } + p.lastTunnelIfaces = nil + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() + p.pfStabilizing.Store(false) + + p.pfDelayedRecheckMu.Lock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + p.pfDelayedRecheckTimers = nil + if p.pfSettleFollowupTimer != nil { + p.pfSettleFollowupTimer.Stop() + p.pfSettleFollowupTimer = nil + } + p.pfDelayedRecheckMu.Unlock() + + for !p.pfEnsureRunning.CompareAndSwap(false, true) { + time.Sleep(10 * time.Millisecond) + } + defer p.pfEnsureRunning.Store(false) + + mainLog.Load().Info().Msg("DNS intercept: shutting down pf redirect") + out, err := exec.Command("pfctl", "-a", state.anchorName, "-F", "all").CombinedOutput() + if err != nil { + mainLog.Load().Warn().Msgf("DNS intercept: failed to flush pf anchor %q: %v (output: %s)", + state.anchorName, err, strings.TrimSpace(string(out))) + } else { + mainLog.Load().Debug().Msgf("DNS intercept: flushed pf anchor %q", state.anchorName) + } + + if err := os.Remove(state.anchorFile); err != nil && !os.IsNotExist(err) { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove anchor file %s: %v", state.anchorFile, err) + } else { + mainLog.Load().Debug().Msgf("DNS intercept: removed anchor file %s", state.anchorFile) + } + + if err := p.removePFAnchorReference(); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to remove anchor references from running pf ruleset") + } + + p.pfLastRestoreTime.Store(0) + p.pfBackoffMultiplier.Store(0) + p.pfExecBackoffUntil.Store(0) + p.pfIgnoredChangeLastReconcile.Store(0) + mainLog.Load().Info().Msg("DNS intercept: pf shutdown complete") + return nil +} + +// removePFAnchorReference removes our anchor references from the running pf ruleset. +// Uses the same dump → filter → reassemble approach as ensurePFAnchorReference. +// The anchor itself is already flushed by stopDNSIntercept, so even if removal +// fails, the empty anchor is a no-op. +func (p *prog) removePFAnchorReference() error { + rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) + anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) + + natOut, err := exec.Command("pfctl", "-sn").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to dump running NAT rules: %w (output: %s)", err, strings.TrimSpace(string(natOut))) + } + filterOut, err := exec.Command("pfctl", "-sr").CombinedOutput() + if err != nil { + return fmt.Errorf("failed to dump running filter rules: %w (output: %s)", err, strings.TrimSpace(string(filterOut))) + } + + // Nothing of ours in the running ruleset means nothing to remove - and the reload + // below would reset system-wide pf options on the way past. Startup rollback reaches + // here after failures that happen before the references are ever added. + if !pfAnchorReferencesPresent(string(natOut), string(filterOut), pfAnchorName) { + mainLog.Load().Debug().Msg("DNS intercept: no anchor references in the running pf ruleset — nothing to remove") + return nil + } + + // Filter and remove our lines. + natLines := pfFilterRuleLines(string(natOut)) + filterLines := pfFilterRuleLines(string(filterOut)) + + var cleanNat []string + for _, line := range natLines { + if !strings.Contains(line, rdrAnchorRef) { + cleanNat = append(cleanNat, line) + } + } + + // Separate scrub from filter, remove our anchor ref. + var scrubLines, cleanFilter []string + for _, line := range filterLines { + if strings.Contains(line, anchorRef) { + continue + } + if strings.HasPrefix(line, "scrub") { + scrubLines = append(scrubLines, line) + } else { + cleanFilter = append(cleanFilter, line) + } + } + + // Reassemble in correct order: scrub → translation → filtering. + var combined strings.Builder + for _, line := range scrubLines { + combined.WriteString(line + "\n") + } + for _, line := range cleanNat { + combined.WriteString(line + "\n") + } + for _, line := range cleanFilter { + combined.WriteString(line + "\n") + } + + cmd := exec.Command("pfctl", "-f", "-") + cmd.Stdin = strings.NewReader(combined.String()) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to reload pf ruleset without anchor references: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + + mainLog.Load().Debug().Msg("DNS intercept: removed anchor references from running pf ruleset") + return nil +} + +// pfAddressFamily returns "inet" for IPv4 addresses and "inet6" for IPv6 addresses. +// Used to generate pf rules with the correct address family for each IP. +// flushPFStates flushes ALL pf state entries after anchor reloads. +// pf checks state table BEFORE rules — stale entries from old rules keep routing +// packets via route-to even after interface-scoped exemptions are added. +func flushPFStates() { + if out, err := exec.Command("pfctl", "-F", "states").CombinedOutput(); err != nil { + mainLog.Load().Warn().Err(err).Msgf("DNS intercept: failed to flush pf states (output: %s)", strings.TrimSpace(string(out))) + } else { + mainLog.Load().Debug().Msg("DNS intercept: flushed pf states after anchor reload") + } +} + +// pfNameserverAddress keeps socket zones out of group-scoped PF addresses. +// ctrld already has a blanket group exemption in the same anchor. +func pfNameserverAddress(server string) (string, string, bool) { + host, _, err := net.SplitHostPort(server) + if err != nil { + host = server + } + addr, err := netip.ParseAddr(host) + if err != nil { + return "", "", false + } + host = addr.WithZone("").Unmap().String() + return host, pfAddressFamily(host), true +} + +func pfAddressFamily(ip string) string { + if addr := net.ParseIP(ip); addr != nil && addr.To4() == nil { + return "inet6" + } + return "inet" +} + +// discoverTunnelInterfaces returns the names of active VPN/tunnel network interfaces. +// These interfaces may have pf rules from VPN software (e.g., Windscribe's "pass out quick +// on ipsec0") that would match DNS traffic before our anchor rules. By discovering them, +// we can add interface-specific intercept rules that take priority. +func discoverTunnelInterfaces() []string { + ifaces, err := net.Interfaces() + if err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to enumerate network interfaces") + return nil + } + + prefixes := strings.Split(pfVPNInterfacePrefixes, ",") + var tunnels []string + + for _, iface := range ifaces { + // Only consider interfaces that are up — down interfaces can't carry DNS traffic. + if iface.Flags&net.FlagUp == 0 { + continue + } + for _, prefix := range prefixes { + if strings.HasPrefix(iface.Name, strings.TrimSpace(prefix)) { + tunnels = append(tunnels, iface.Name) + break + } + } + } + + return tunnels +} + +var discoverTunnelInterfacesForReconcile = discoverTunnelInterfaces + +// tunnelInterfacesKey holds the last discovered tunnel set, so the discovery +// line reports a change and not every reconcile. +const tunnelInterfacesKey = "tunnel_interfaces" + +// activeTunnelInterfaces returns the live tunnel interfaces and names them on +// change. Every pf reconcile discovers them again, and the set is the same in +// almost every one of those reads. +func (p *prog) activeTunnelInterfaces() []string { + tunnels := discoverTunnelInterfacesForReconcile() + changed, repeats := p.repeats.changed(tunnelInterfacesKey, fmt.Sprintf("%v", tunnels)) + if !changed || len(tunnels) == 0 { + return tunnels + } + mainLog.Load().Debug().Uint64("repeats", repeats). + Msgf("DNS intercept: discovered active tunnel interfaces: %v", tunnels) + return tunnels +} + +// dnsInterceptSupported reports whether DNS intercept mode is supported on this platform. +func dnsInterceptSupported() bool { + _, err := exec.LookPath("pfctl") + return err == nil +} + +// validateDNSIntercept checks that the system meets requirements for DNS intercept mode. +func (p *prog) validateDNSIntercept() error { + if !dnsInterceptSupported() { + return fmt.Errorf("dns intercept: pfctl not found — pf is required for DNS intercept on macOS") + } + + if os.Geteuid() != 0 { + return fmt.Errorf("dns intercept: root privileges required for pf filter management") + } + + if err := os.MkdirAll(filepath.Dir(pfAnchorFile), 0755); err != nil { + return fmt.Errorf("dns intercept: cannot create anchor directory: %w", err) + } + + return interceptConfigBlocker(p.cfg) +} + +// buildPFAnchorRules generates the pf anchor rules for DNS interception. +// vpnExemptions are VPN DNS server+interface pairs to exempt from interception. +// +// macOS pf "rdr" rules only apply to forwarded traffic, NOT locally-originated +// packets. To intercept DNS from the machine itself, we use a two-step approach: +// 1. "pass out route-to lo0" forces outbound DNS through the loopback interface +// 2. "rdr on lo0" catches it on loopback and redirects to our listener +// +// STATE AND ROUTING (critical for VPN firewall coexistence): +// - route-to rules: keep state (default). State is floating (matches on any interface), +// but "pass out on lo0 no state" ensures no state exists on the lo0 outbound path, +// so rdr still fires on the lo0 inbound pass. +// - pass out on lo0: NO STATE — prevents state from being created on lo0 outbound, +// which would match inbound and bypass rdr. +// - rdr: no "pass" keyword — packet goes through filter so "pass in" creates state. +// - pass in on lo0: keep state + REPLY-TO lo0 — creates state for response routing +// AND forces the response back through lo0. Without reply-to, the response to a +// VPN client IP gets routed through the VPN tunnel and is lost. +// +// ctrld's own OS resolver nameservers (used for bootstrap DNS) must be exempted +// from the redirect to prevent ctrld from querying itself in a loop. +// +// pf requires strict rule ordering: translation (rdr) BEFORE filtering (pass). +// buildPFAnchorRules detects the current forwarded-source set itself. Callers that +// need to know which set the anchor was built from - so they can record it as applied +// - should use buildPFAnchorRulesWith instead. +func (p *prog) buildPFAnchorRules(vpnExemptions []vpnDNSExemption) string { + return p.buildPFAnchorRulesWith(vpnExemptions, p.currentForwardedSources()) +} + +// buildPFAnchorRulesWith is buildPFAnchorRules over an explicit forwarded-source set. +// +// Every rebuild path used to re-detect internally, which meant the caller could not +// tell what it had just installed. The reconcile snapshot therefore stayed at the +// older set, and the next reconcile "discovered" the same change again: another +// rebuild, another round of killed states, and a transition logged for something that +// had already taken effect. Passing the set in lets each path record exactly what pf +// accepted. +func (p *prog) buildPFAnchorRulesWith(vpnExemptions []vpnDNSExemption, forwardedSources []forwardedSource) string { + return p.buildPFAnchorRulesForState(vpnExemptions, p.activeTunnelInterfaces(), forwardedSources) +} + +func (p *prog) buildPFAnchorRulesForTunnels(vpnExemptions []vpnDNSExemption, tunnelIfaces []string) string { + return p.buildPFAnchorRulesForState(vpnExemptions, tunnelIfaces, p.currentForwardedSources()) +} + +func (p *prog) buildPFAnchorRulesForState(vpnExemptions []vpnDNSExemption, tunnelIfaces []string, forwardedSources []forwardedSource) string { + // Read the actual listener address from config. In intercept mode, ctrld may + // be on a non-standard port (e.g., 127.0.0.1:5354) if mDNSResponder holds *:53. + // The pf rdr rules must redirect to wherever ctrld is actually listening. + listenerIP := "127.0.0.1" + listenerPort := 53 + if lc := p.cfg.FirstListener(); lc != nil { + if lc.IP != "" && lc.IP != "0.0.0.0" && lc.IP != "::" { + listenerIP = lc.IP + } else if lc.IP == "0.0.0.0" || lc.IP == "::" { + mainLog.Load().Warn().Str("configured_ip", lc.IP). + Msg("DNS intercept: listener configured with wildcard IP, using 127.0.0.1 for pf rules") + } + if lc.Port != 0 { + listenerPort = lc.Port + } + } + listenerAddr := fmt.Sprintf("%s port %d", listenerIP, listenerPort) + + var rules strings.Builder + rules.WriteString("# ctrld DNS Intercept Mode\n") + rules.WriteString("# Intercepts locally-originated DNS (port 53) via route-to + rdr on lo0.\n") + rules.WriteString("#\n") + rules.WriteString("# How it works:\n") + rules.WriteString("# 1. \"pass out route-to lo0\" forces outbound DNS through the loopback interface\n") + rules.WriteString(fmt.Sprintf("# 2. \"rdr on lo0\" catches it on loopback and redirects to ctrld at %s\n", listenerAddr)) + rules.WriteString("#\n") + rules.WriteString("# All ctrld traffic is blanket-exempted via \"pass out quick group " + pfGroupName + "\",\n") + rules.WriteString("# ensuring ctrld's DoH/DoT upstream connections and DNS queries are never\n") + rules.WriteString("# blocked by VPN firewalls (e.g., Windscribe's \"block drop all\").\n") + rules.WriteString("#\n") + rules.WriteString("# pf requires strict rule ordering: translation (rdr) BEFORE filtering (pass).\n\n") + + // --- Translation rules (must come first per pf ordering) --- + // Uses "rdr" without "pass" so the redirected packet continues to filter evaluation. + // The filter rule "pass in on lo0 ... to 127.0.0.1 port 53 keep state" then creates + // a stateful entry that handles response routing. Using "rdr pass" would skip filter + // evaluation, and its implicit state alone is insufficient for response delivery — + // proven by commit 51cf029 where responses were silently dropped. + rules.WriteString("# --- Translation rules (rdr) ---\n") + rules.WriteString("# Redirect DNS on loopback to ctrld's listener.\n") + rules.WriteString(fmt.Sprintf("rdr on lo0 inet proto udp from any to ! %s port 53 -> %s\n", listenerIP, listenerAddr)) + rules.WriteString(fmt.Sprintf("rdr on lo0 inet proto tcp from any to ! %s port 53 -> %s\n\n", listenerIP, listenerAddr)) + + // --- Filtering rules --- + rules.WriteString("# --- Filtering rules (pass) ---\n\n") + + // Blanket exemption: allow ALL outbound traffic from ctrld (group _ctrld) through + // without any pf filtering or redirection. This is critical for VPN coexistence — + // VPN apps like Windscribe load "block drop all" rulesets that would otherwise block + // ctrld's DoH connections (TCP 443) to upstream DNS servers (e.g., 76.76.2.22). + // Because our anchor is prepended before other anchors, this rule evaluates first, + // ensuring ctrld's traffic is never blocked by downstream firewall rules. + // + // The per-IP exemptions below (OS resolver, VPN DNS) remain as defense-in-depth: + // they prevent DNS redirect loops for ctrld's own port-53 queries specifically, + // while this rule handles everything else (DoH, DoT, health checks, etc.). + rules.WriteString("# Blanket exemption: let all ctrld traffic through regardless of other pf rules.\n") + rules.WriteString("# VPN firewalls (e.g., Windscribe's \"block drop all\") would otherwise block\n") + rules.WriteString("# ctrld's DoH (TCP 443) connections to upstream DNS servers.\n") + rules.WriteString(fmt.Sprintf("pass out quick group %s\n\n", pfGroupName)) + + // Exempt OS resolver nameservers (read live from the global OS resolver) + // so ctrld's bootstrap DNS queries don't get redirected back to itself. + // IPv4 addresses use "inet", IPv6 addresses use "inet6". + osNS := ctrld.OsResolverNameservers() + if len(osNS) > 0 { + rules.WriteString("# Exempt OS resolver nameservers (ctrld bootstrap DNS) from redirect.\n") + rules.WriteString("# Scoped to group " + pfGroupName + " so only ctrld's own queries are exempted,\n") + rules.WriteString("# preventing other processes from bypassing the redirect by querying these IPs.\n") + for _, ns := range osNS { + host, af, ok := pfNameserverAddress(ns) + if !ok { + continue + } + rules.WriteString(fmt.Sprintf("pass out quick on ! lo0 %s proto { udp, tcp } from any to %s port 53 group %s\n", af, host, pfGroupName)) + } + rules.WriteString("\n") + } + + // Build sets of VPN DNS interfaces and server IPs for exclusion from intercept rules. + // + // EXIT MODE EXCEPTION: When a VPN is in exit/full-tunnel mode (VPN DNS server is + // also the system default resolver), we do NOT exempt the interface. In exit mode, + // all traffic routes through the VPN, so exempting the interface would bypass ctrld + // for ALL DNS — losing profile enforcement (blocking, filtering). Instead, we keep + // intercepting and let ctrld's VPN DNS split routing + group exemption handle it. + vpnDNSIfaces := make(map[string]bool) // non-exit interfaces to skip in tunnel intercept + vpnDNSIfacePassthrough := make(map[string]bool) // non-exit interfaces needing passthrough rules + vpnDNSServerIPs := make(map[string]bool) // IPs for group exemptions and table + for _, ex := range vpnExemptions { + if ex.Interface != "" && !ex.IsExitMode { + vpnDNSIfaces[ex.Interface] = true + vpnDNSIfacePassthrough[ex.Interface] = true + } + vpnDNSServerIPs[ex.Server] = true + } + + // Group-scoped exemptions for ctrld's own VPN DNS queries. + // When ctrld's proxy() VPN DNS split routing sends queries to VPN DNS servers, + // these rules let ctrld's traffic through without being intercepted by the + // generic route-to rule. Scoped to group _ctrld so only ctrld benefits. + if len(vpnExemptions) > 0 { + rules.WriteString("# Exempt VPN DNS servers: ctrld's own queries (group-scoped).\n") + seen := make(map[string]bool) + for _, ex := range vpnExemptions { + host, af, ok := pfNameserverAddress(ex.Server) + if ok && !seen[host] { + seen[host] = true + rules.WriteString(fmt.Sprintf("pass out quick on ! lo0 %s proto { udp, tcp } from any to %s port 53 group %s\n", af, host, pfGroupName)) + } + } + rules.WriteString("\n") + } + + // Block all outbound IPv6 DNS. ctrld only listens on 0.0.0.0:53 (IPv4), so we cannot + // redirect IPv6 DNS to our listener. Without this rule, macOS may use IPv6 link-local + // DNS servers (e.g., fe80::...%en0) assigned by the router, completely bypassing the + // IPv4 pf intercept. Blocking forces macOS to fall back to IPv4 DNS, which is intercepted. + // This rule must come BEFORE the IPv4 route-to rules (pf evaluates last match by default, + // but "quick" makes first-match — and exemptions above are already "quick"). + rules.WriteString("# Block outbound IPv6 DNS — ctrld listens on IPv4 only (0.0.0.0:53).\n") + rules.WriteString("# Without this, macOS may use IPv6 link-local DNS servers from the router,\n") + rules.WriteString("# bypassing the IPv4 intercept entirely.\n") + rules.WriteString("block out quick on ! lo0 inet6 proto { udp, tcp } from any to any port 53\n\n") + + // --- VPN DNS interface passthrough (split DNS mode only) --- + // + // In split DNS mode, the VPN's DNS handler (e.g., Tailscale MagicDNS) runs as a + // Network Extension that intercepts packets on its tunnel interface. MagicDNS then + // forwards queries to its own upstream nameservers (e.g., 10.3.112.11) — IPs we + // can't know in advance. Without these rules, pf's generic "on !lo0" intercept + // catches MagicDNS's upstream queries, routing them back to ctrld in a loop. + // + // These "pass" rules (no route-to) let MagicDNS's upstream queries pass through. + // Traffic TO the VPN DNS server (e.g., 100.100.100.100) is excluded via + // so those queries get intercepted → ctrld enforces its profile on non-search-domain queries. + // + // NOT applied in exit mode — in exit mode, all traffic routes through the VPN + // interface, so exempting it would bypass ctrld's profile enforcement entirely. + if len(vpnDNSIfacePassthrough) > 0 { + // Build table of VPN DNS server IPs to exclude from passthrough. + var vpnDNSTableMembers []string + for ip := range vpnDNSServerIPs { + if net.ParseIP(ip) != nil && net.ParseIP(ip).To4() != nil { + vpnDNSTableMembers = append(vpnDNSTableMembers, ip) + } + } + if len(vpnDNSTableMembers) > 0 { + rules.WriteString("# Table of VPN DNS server IPs — queries to these must be intercepted.\n") + rules.WriteString(fmt.Sprintf("table { %s }\n", strings.Join(vpnDNSTableMembers, ", "))) + } + rules.WriteString("# --- VPN DNS interface passthrough (split DNS mode) ---\n") + rules.WriteString("# Pass MagicDNS upstream queries; intercept queries TO MagicDNS itself.\n") + for iface := range vpnDNSIfacePassthrough { + if len(vpnDNSTableMembers) > 0 { + rules.WriteString(fmt.Sprintf("pass out quick on %s inet proto udp from any to ! port 53\n", iface)) + rules.WriteString(fmt.Sprintf("pass out quick on %s inet proto tcp from any to ! port 53\n", iface)) + } else { + rules.WriteString(fmt.Sprintf("pass out quick on %s inet proto udp from any to any port 53\n", iface)) + rules.WriteString(fmt.Sprintf("pass out quick on %s inet proto tcp from any to any port 53\n", iface)) + } + } + rules.WriteString("\n") + } + + // --- Interface-specific VPN/tunnel intercept rules --- + // VPN apps (e.g., Windscribe, Cisco AnyConnect) often add pf rules like: + // pass out quick on ipsec0 inet all flags S/SA keep state + // inside their own anchors. If their anchor is evaluated before ours, their + // "quick" match on the VPN interface captures DNS traffic before our generic + // "on ! lo0" rule can intercept it. To counter this, we add explicit intercept + // rules for each active tunnel interface. These use "quick" and match port 53 + // specifically, so they take priority over the VPN app's broader "all" rules + // regardless of anchor ordering. + // + // NOTE: If anchor ordering alone proves insufficient in the future, a "nuclear + // option" is available: inject DNS intercept rules directly into the MAIN pf + // ruleset (not inside our anchor). Main ruleset rules are evaluated before ALL + // anchors, making them impossible for another app to override without explicitly + // removing them. See docs/dns-intercept-mode.md for details. + if len(tunnelIfaces) > 0 { + rules.WriteString("# --- VPN/tunnel interface intercept rules ---\n") + rules.WriteString("# Explicit intercept on tunnel interfaces prevents VPN apps from capturing\n") + rules.WriteString("# DNS traffic with their own broad \"pass out quick on \" rules.\n") + rules.WriteString("# VPN DNS interfaces (split DNS mode) are excluded — passthrough rules above handle them.\n") + for _, iface := range tunnelIfaces { + if vpnDNSIfaces[iface] { + rules.WriteString(fmt.Sprintf("# Skipped %s — VPN DNS interface (passthrough rules handle this)\n", iface)) + continue + } + rules.WriteString(fmt.Sprintf("pass out quick on %s route-to lo0 inet proto udp from any to ! %s port 53\n", iface, listenerIP)) + rules.WriteString(fmt.Sprintf("pass out quick on %s route-to lo0 inet proto tcp from any to ! %s port 53\n", iface, listenerIP)) + } + rules.WriteString("\n") + } + + // Force all remaining outbound IPv4 DNS through loopback for interception. + // route-to rules use stateful tracking (keep state, the default). State is floating + // (matches on any interface), but "pass out on lo0 no state" below ensures no state + // is created on the lo0 outbound path, allowing rdr to fire on lo0 inbound. + rules.WriteString("# Force remaining outbound IPv4 DNS through loopback for interception.\n") + rules.WriteString(fmt.Sprintf("pass out quick on ! lo0 route-to lo0 inet proto udp from any to ! %s port 53\n", listenerIP)) + rules.WriteString(fmt.Sprintf("pass out quick on ! lo0 route-to lo0 inet proto tcp from any to ! %s port 53\n\n", listenerIP)) + + // Allow route-to'd DNS packets to pass outbound on lo0. + // Without this, VPN firewalls with "block drop all" (e.g., Windscribe) drop the packet + // after route-to redirects it to lo0 but before it can reflect inbound for rdr processing. + // + // CRITICAL: This rule MUST use "no state". If it created state, that state would match + // the packet when it reflects inbound on lo0, causing pf to fast-path it and bypass + // rdr entirely. With "no state", the inbound packet gets fresh evaluation and rdr fires. + rules.WriteString("# Pass route-to'd DNS outbound on lo0 — no state to avoid bypassing rdr inbound.\n") + rules.WriteString(fmt.Sprintf("pass out quick on lo0 inet proto udp from any to ! %s port 53 no state\n", listenerIP)) + rules.WriteString(fmt.Sprintf("pass out quick on lo0 inet proto tcp from any to ! %s port 53 no state\n\n", listenerIP)) + + // Allow the redirected traffic through on loopback (inbound after rdr). + // + // "reply-to lo0" is CRITICAL for VPN coexistence. Without it, ctrld's response to a + // VPN client IP (e.g., 100.94.163.168) gets routed via the VPN tunnel interface + // (utun420) by the kernel routing table — the response enters the tunnel and is lost. + // "reply-to lo0" forces pf to route the response back through lo0 regardless of the + // kernel routing table, ensuring it stays local and reaches the client process. + // + // "keep state" (the default) creates the stateful entry used by reply-to to route + // the response. The rdr NAT state handles the address rewrite on the response + // (source 127.0.0.1 → original DNS server IP, e.g., 10.255.255.3). + rules.WriteString("# Accept redirected DNS — reply-to lo0 forces response through loopback.\n") + rules.WriteString(fmt.Sprintf("pass in quick on lo0 reply-to lo0 inet proto { udp, tcp } from any to %s\n", listenerAddr)) + + // Firewall mode: append IP allowlist enforcement rules AFTER DNS intercept rules. + // DNS intercept rules must evaluate first so that DNS queries work (they're how + // IPs get into the allowlist in the first place). + if p.firewallModeEnabled() { + // Make declared VM/container source subnets first-class firewall + // clients (DNS forced through ctrld) before the blanket allowlist block. + rules.WriteString(buildPFForwardedSourceRulesFor(forwardedSources, listenerIP)) + rules.WriteString(buildPFFirewallRules()) + } + + return rules.String() +} + +// verifyPFState checks that the pf ruleset is correctly configured after loading. +// It verifies both the anchor references in the main ruleset and the rules within +// our anchor. Failures are logged at ERROR level to make them impossible to miss. +func (p *prog) verifyPFState() bool { + rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) + anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) + verified := true + + // Check main ruleset for anchor references. + natOut, err := exec.Command("pfctl", "-sn").CombinedOutput() + if err != nil { + p.pfBackoffResourceExhaustion(err, natOut, "verify NAT rules") + mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump NAT rules") + verified = false + } else if !strings.Contains(string(natOut), rdrAnchorRef) { + mainLog.Load().Error().Msg("DNS intercept: VERIFICATION FAILED — rdr-anchor reference missing from running NAT rules") + verified = false + } + + filterOut, err := exec.Command("pfctl", "-sr").CombinedOutput() + if err != nil { + p.pfBackoffResourceExhaustion(err, filterOut, "verify filter rules") + mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump filter rules") + verified = false + } else if !strings.Contains(string(filterOut), anchorRef) { + mainLog.Load().Error().Msg("DNS intercept: VERIFICATION FAILED — anchor reference missing from running filter rules") + verified = false + } + + // Check our anchor has rules loaded. + anchorFilter, err := exec.Command("pfctl", "-a", pfAnchorName, "-sr").CombinedOutput() + if err != nil { + p.pfBackoffResourceExhaustion(err, anchorFilter, "verify anchor filter rules") + mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump anchor filter rules") + verified = false + } else if pfRulesetEmpty(string(anchorFilter)) { + mainLog.Load().Error().Msg("DNS intercept: VERIFICATION FAILED — anchor has no filter rules loaded") + verified = false + } + + anchorNat, err := exec.Command("pfctl", "-a", pfAnchorName, "-sn").CombinedOutput() + if err != nil { + p.pfBackoffResourceExhaustion(err, anchorNat, "verify anchor NAT rules") + mainLog.Load().Error().Err(err).Msg("DNS intercept: VERIFICATION FAILED — could not dump anchor NAT rules") + verified = false + } else if pfRulesetEmpty(string(anchorNat)) { + mainLog.Load().Error().Msg("DNS intercept: VERIFICATION FAILED — anchor has no NAT/redirect rules loaded") + verified = false + } + + // There is deliberately no "set skip on lo0" check here, even though such a skip + // would make our rdr rules on loopback dead. + // + // macOS offers no way to query it as text: pfctl(8) accepts -s nat, queue, rules, + // Anchors, states, Sources, info, References, labels, timeouts, memory, Tables, + // osfp, Interfaces and all - there is no options or skip modifier. Since this + // verification gates whether intercept mode may publish, a check that cannot + // succeed here would roll back every start. + // + // probePFIntercept() detects the condition functionally instead: it sends a real + // query from outside the _ctrld group and confirms the listener received the + // redirect, which cannot succeed if pf is bypassing loopback. An explicit check is + // follow-up work - pfctl(8) documents "-s Interfaces -v" as listing which + // interfaces have skip rules activated, but its output shape needs confirming on a + // host that actually has a skip configured before anything parses it. + + if verified { + mainLog.Load().Info().Msg("DNS intercept: post-load verification passed — all pf rules confirmed active") + } + return verified +} + +// resetUpstreamTransports forces all DoH/DoT/DoQ upstreams to re-bootstrap their +// network transports. This is called when the pf watchdog detects that the pf state +// table was flushed (e.g., by Windscribe running "pfctl -f"), which kills all existing +// TCP connections including ctrld's DoH connections to upstream DNS servers. +// +// Without this, Go's http.Transport keeps trying to use dead connections until each +// request hits its 5s context deadline — causing a ~5s DNS blackout. +// +// ForceReBootstrap() immediately creates a new transport (closing old idle +// connections), so new queries use fresh connections without waiting for the +// lazy re-bootstrap flag. This reduces the blackout from ~5s to ~100ms. +func (p *prog) resetUpstreamTransports() { + if p.cfg == nil { + return + } + count := 0 + for _, uc := range p.cfg.Upstream { + if uc == nil { + continue + } + uc.ForceReBootstrap(ctrld.LoggerCtx(context.Background(), p.logger.Load())) + count++ + } + if count > 0 { + mainLog.Load().Info().Msgf("DNS intercept watchdog: force-reset %d upstream transport(s) — pf state flush likely killed existing DoH connections", count) + } +} + +var ( + ensurePFAnchorReferenceForRestore = func(p *prog) error { + return p.ensurePFAnchorReference() + } + rebuildPFAnchorRulesForReconcile = func(p *prog, exemptions []vpnDNSExemption) ([]string, error) { + return p.rebuildPFAnchorRules(exemptions) + } + restorePFAnchorForReconcile = func(p *prog, reason string) pfAnchorCheckResult { + return p.restorePFAnchor(reason) + } + verifyPFStateFn = (*prog).verifyPFState + pfShutdownStateRevokedForTest = func() {} +) + +func (p *prog) rebuildPFAnchorRules(vpnExemptions []vpnDNSExemption) ([]string, error) { + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) + forwarded := p.currentForwardedSources() + rulesStr := p.buildPFAnchorRulesForState(vpnExemptions, tunnelIfaces, forwarded) + if err := writePFAnchorFile(rulesStr); err != nil { + return nil, fmt.Errorf("write anchor file: %w", err) + } + out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("load rebuilt anchor: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + p.recordAppliedForwardedSources(forwarded) + flushPFStates() + return tunnelIfaces, nil +} + +func (p *prog) restorePFAnchor(reason string) pfAnchorCheckResult { + return p.restorePFAnchorWithTransportReset(reason, true) +} + +func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports bool) pfAnchorCheckResult { + if p.dnsInterceptState == nil { + return pfAnchorCheckSkipped + } + var vpnExemptions []vpnDNSExemption + if p.vpnDNS != nil { + vpnExemptions = p.vpnDNS.CurrentExemptions() + } + mainLog.Load().Info().Str("reason", reason).Msg("DNS intercept: restoring pf anchor") + if err := ensurePFAnchorReferenceForRestore(p); err != nil { + p.pfBackoffResourceExhaustion(err, nil, "restore anchor references") + if resetTransports { + p.resetUpstreamTransports() + } + mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to restore anchor references") + return pfAnchorCheckFailed + } + + tunnelIfaces, err := rebuildPFAnchorRulesForReconcile(p, vpnExemptions) + if err != nil { + p.pfBackoffResourceExhaustion(err, nil, "rebuild anchor") + if resetTransports { + p.resetUpstreamTransports() + } + mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to rebuild anchor") + return pfAnchorCheckFailed + } + + // rebuildPFAnchorRules flushed PF states after the successful load. Reset + // transports even if the authoritative verification below fails, otherwise + // DoH can remain pinned to connections invalidated by that flush. + if resetTransports { + p.resetUpstreamTransports() + } + if !verifyPFStateFn(p) { + mainLog.Load().Error().Str("reason", reason).Msg("DNS intercept: rebuilt anchor failed post-load verification") + return pfAnchorCheckFailed + } + if p.vpnDNS != nil { + p.vpnDNS.markInterceptExemptionsApplied(vpnExemptions) + } + p.commitPFReconcileState(tunnelIfaces) + p.pfLastRestoreTime.Store(time.Now().UnixMilli()) + mainLog.Load().Info().Str("reason", reason).Msg("DNS intercept: pf anchor restored successfully") + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + return pfAnchorCheckRestored +} + +func (p *prog) commitPFReconcileState(tunnelIfaces []string) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastTunnelIfaces = append([]string(nil), tunnelIfaces...) + if p.hasPendingTunnelIfaces && stringSlicesEqual(p.pendingTunnelIfaces, tunnelIfaces) { + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + } +} + +func (p *prog) clearPendingTunnelReconcile() { + p.mu.Lock() + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() +} + +func (p *prog) hasPendingTunnelReconcile() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.hasPendingTunnelIfaces +} + +func (p *prog) reconcilePFAnchorForTunnelChange() pfAnchorCheckResult { + if p.pfExecBackoffActive() { + return pfAnchorCheckSkipped + } + if !p.pfEnsureRunning.CompareAndSwap(false, true) { + return pfAnchorCheckSkipped + } + defer p.pfEnsureRunning.Store(false) + return restorePFAnchorForReconcile(p, "tunnel interface change") +} + +// checkTunnelInterfaceChanges compares the current set of active tunnel interfaces +// against the last known set. If they differ (e.g., a VPN connected and created utun420), +// it rebuilds and reloads the pf anchor rules to include interface-specific intercept +// rules for the new interface. +// +// Returns true when a new tunnel state is observed or a pending state is +// successfully applied. Coalesced/deferred retries return false so ignored-event +// storms do not bypass the macOS VPN-DNS refresh limiter on every notification. +func (p *prog) checkTunnelInterfaceChanges() bool { + if p.dnsInterceptState == nil { + return false + } + + current := append([]string(nil), p.activeTunnelInterfaces()...) + p.noteTunnelInterfaceSet(current) + + p.mu.Lock() + prev := append([]string(nil), p.lastTunnelIfaces...) + stabilizing := p.pfStabilizing.Load() + newObservation := false + if p.hasPendingTunnelIfaces && stringSlicesEqual(p.pendingTunnelIfaces, current) { + if stabilizing { + p.mu.Unlock() + return false + } + if stringSlicesEqual(prev, current) { + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() + return false + } + // The same pending state still differs from the applied baseline. Retry it + // after stabilization/failure instead of treating it as already applied. + } else { + if !p.hasPendingTunnelIfaces && stringSlicesEqual(prev, current) { + p.mu.Unlock() + return false + } + newObservation = true + p.pendingTunnelIfaces = append([]string(nil), current...) + p.hasPendingTunnelIfaces = true + } + p.mu.Unlock() + + if stringSlicesEqual(prev, current) { + p.clearPendingTunnelReconcile() + return true + } + + added, _ := tunnelSetDiff(prev, current) + if len(added) > 0 { + mainLog.Load().Info().Msgf("DNS intercept: new tunnel interface detected: %s", added[0]) + } + + if stabilizing { + mainLog.Load().Debug().Msgf("DNS intercept: tunnel state changed during stabilization (was %v, now %v) — deferring anchor rebuild", prev, current) + return newObservation + } + + if len(added) > 0 { + // A new VPN tunnel appeared. The dedicated post-stabilization repair path + // rebuilds the anchor if no successful exemption update applies it first. + p.pfStartStabilization() + return newObservation + } + + result := p.reconcilePFAnchorForTunnelChange() + if result != pfAnchorCheckRestored { + // Keep the desired tunnel set pending. The next event, delayed check, or + // watchdog tick retries it without confusing pending with applied state. + mainLog.Load().Debug().Msgf("DNS intercept: tunnel rule rebuild deferred/failed (result: %d)", result) + } + return newObservation || result == pfAnchorCheckRestored +} + +// tunnelChangedMessage names the tunnel event of the journal. +const tunnelChangedMessage = "Tunnel interface changed" + +// noteTunnelInterfaceSet logs one event for each change of the tunnel set. The +// reconcile retries a set until it succeeds, and a set that returns to the one +// before it is news as well, so the diff runs against the set of the last +// event and not against the applied set. +func (p *prog) noteTunnelInterfaceSet(current []string) { + p.mu.Lock() + // Before the first event the applied set is the baseline, because no event + // named a set yet. + previous := p.lastLoggedTunnelIfaces + if previous == nil { + previous = p.lastTunnelIfaces + } + previous = append([]string(nil), previous...) + if stringSlicesEqual(previous, current) { + p.mu.Unlock() + return + } + p.lastLoggedTunnelIfaces = append([]string{}, current...) + p.mu.Unlock() + + added, removed := tunnelSetDiff(previous, current) + p.logTunnelInterfaceChange(added, removed) +} + +// logTunnelInterfaceChange names the tunnels that appeared and the ones that +// went away. The owner names the program behind a new tunnel, because its pf +// rules compete with the ctrld anchor for the same DNS packets. +func (p *prog) logTunnelInterfaceChange(added, removed []string) { + owner := "" + if len(added) > 0 { + owner = tunnelOwner(added[0]) + } + journal(mainLog.Load().Info()). + Strs("added", added). + Strs("removed", removed). + Str("owner", owner). + Msg(tunnelChangedMessage) +} + +// tunnelSetDiff reports the tunnels that appeared and the ones that went away, +// each in the order of the set it comes from. +func tunnelSetDiff(prev, current []string) (added, removed []string) { + before := make(map[string]bool, len(prev)) + for _, iface := range prev { + before[iface] = true + } + live := make(map[string]bool, len(current)) + for _, iface := range current { + live[iface] = true + if !before[iface] { + added = append(added, iface) + } + } + for _, iface := range prev { + if !live[iface] { + removed = append(removed, iface) + } + } + return added, removed +} + +// stringSlicesEqual reports whether two string slices have the same elements in the same order. +func stringSlicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// pfStartStabilization enters stabilization mode, suppressing all pf restores +// until the VPN's ruleset stops changing. This prevents a death spiral where +// ctrld and the VPN repeatedly overwrite each other's pf rules. +func (p *prog) pfStartStabilization() { + if !p.pfStabilizing.CompareAndSwap(false, true) { + return + } + + multiplier := max(int(p.pfBackoffMultiplier.Load()), 1) + baseStableTime := 6000 * time.Millisecond // 4 polls at 1.5s + stableRequired := time.Duration(multiplier) * baseStableTime + if stableRequired > 45*time.Second { + stableRequired = 45 * time.Second + } + + journal(mainLog.Load().Info()).Msgf("DNS intercept: VPN connecting — entering stabilization mode (waiting %s for pf to settle)", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationStart, mainLog.Load().Info(), pfRuleDump{}) + + ctx, cancel := context.WithCancel(context.Background()) + p.mu.Lock() + if p.pfStabilizeCancel != nil { + p.pfStabilizeCancel() // Cancel any previous stabilization + } + p.pfStabilizeCancel = cancel + p.mu.Unlock() + + go p.pfStabilizationLoop(ctx, stableRequired) +} + +// pfStabilizationLoop polls pfctl -sr hash until the ruleset is stable for the +// required duration, then restores our anchor rules. +func (p *prog) pfStabilizationLoop(ctx context.Context, stableRequired time.Duration) { + p.pfStabilizationLoopWithMaxWait(ctx, stableRequired, pfStabilizationMaxWait) +} + +func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequired, maxWaitDuration time.Duration) { + defer p.pfStabilizing.Store(false) + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + + pollInterval := 1500 * time.Millisecond + pollTicker := time.NewTicker(pollInterval) + defer pollTicker.Stop() + maxWait := time.NewTimer(maxWaitDuration) + defer maxWait.Stop() + var lastHash string + stableSince := time.Time{} + + for { + select { + case <-ctx.Done(): + mainLog.Load().Debug().Msg("DNS intercept: stabilization cancelled") + return + case <-p.stopCh: + return + case <-maxWait.C: + mainLog.Load().Warn().Msgf("DNS intercept: stabilization timed out after %s — returning ownership to delayed/watchdog recovery", maxWaitDuration) + p.scheduleDelayedRechecks() + return + case <-pollTicker.C: + } + + if p.networkActivityClosed() { + return + } + if p.pfExecBackoffActive() { + continue + } + // Hash the current filter ruleset. + out, err := exec.Command("pfctl", "-sr").CombinedOutput() + if err != nil { + p.pfBackoffResourceExhaustion(err, out, "poll stabilization filter rules") + continue + } + hash := fmt.Sprintf("%x", sha256.Sum256(out)) + + if hash != lastHash { + // Rules changed — reset stability timer + lastHash = hash + stableSince = time.Now() + mainLog.Load().Debug().Msg("DNS intercept: pf rules changed during stabilization — resetting timer") + continue + } + + if stableSince.IsZero() { + stableSince = time.Now() + continue + } + + if time.Since(stableSince) >= stableRequired { + p.finishPFStabilization(stableRequired) + return + } + } +} + +// finishPFStabilization runs the work stabilization exists to do, once the ruleset has +// held still for the required window. The active loop retains ownership throughout: it +// never re-enters stabilization recursively. +func (p *prog) finishPFStabilization(stableRequired time.Duration) { + journal(mainLog.Load().Info()).Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationEnd, mainLog.Load().Info(), pfRuleDump{}) + result := p.reconcilePFAnchorAfterStabilization() + if result != pfAnchorCheckRestored && result != pfAnchorCheckIntact { + p.scheduleDelayedRechecks() + } + routes, domainlessServers, exemptions := p.refreshDNSAfterVPNSettle("pf_stabilized") + if routes == 0 && domainlessServers == 0 && exemptions == 0 { + p.scheduleDNSAfterVPNSettleRefresh("pf_stabilized_followup", pfAnchorRecheckDelayLong) + } + if p.hasPendingTunnelReconcile() { + p.scheduleDelayedRechecks() + } + p.verifyInterceptAfterStabilization() +} + +// probePFInterceptFn and forceReloadPFInterceptFn are the functional verification seams +// shared by all four probe callers. +var ( + probePFInterceptFn = (*prog).probePFIntercept + forceReloadPFInterceptFn = (*prog).forceReloadPFMainRuleset + pfProbeNameservers = ctrld.OsResolverNameservers + newPFProbeCommand = func(ctx context.Context, host, packet string) *exec.Cmd { + cmd := exec.CommandContext(ctx, os.Args[0], "pf-probe-send", host, packet) + // wheel, not _ctrld: the probe must traverse interception. + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 0, Gid: 0}, + } + return cmd + } +) + +// pfFunctionalProbeOwnerWait bounds how long the post-stabilization verifier waits for +// another prober to release functional-probe ownership. A probe monitor that is only +// standing down releases it at once; one that is genuinely probing holds it for its whole +// window, and then the verifier steps aside. A var so tests can shorten the wait. +var pfFunctionalProbeOwnerWait = 2 * time.Second + +// pfFunctionalProbeOwnerPoll is how often that wait re-tries the claim. +const pfFunctionalProbeOwnerPoll = 25 * time.Millisecond + +// interceptProbeMonitorAllowed reports whether the probe monitor may run at all. +// +// The monitor must consult this before claiming ownership. Claiming first and checking +// second means a monitor that is about to stand down still takes the flag, and the +// post-stabilization verifier - which sees a set flag as "somebody else is probing" - +// skips. Neither probes, and the outage lasts until the next watchdog tick. +func (p *prog) interceptProbeMonitorAllowed() bool { + return p.dnsInterceptState != nil && !p.pfStabilizing.Load() +} + +// claimFunctionalProbeOwner takes ownership of functional probing, waiting up to wait for +// a current owner to release it. It reports whether ownership was acquired; the caller +// releases with pfMonitorRunning.Store(false). +func (p *prog) claimFunctionalProbeOwner(wait time.Duration) bool { + deadline := time.Now().Add(wait) + for { + if p.pfMonitorRunning.CompareAndSwap(false, true) { + return true + } + if !time.Now().Before(deadline) { + return false + } + time.Sleep(pfFunctionalProbeOwnerPoll) + } +} + +// verifyInterceptAfterStabilization proves pf is actually translating once stabilization +// has finished, and repairs it once if it is not. +// +// The reconcile above verifies rule text. That cannot distinguish a live redirect from an +// anchor pf has stopped evaluating, and after sleep/wake with a VPN reconnect those come +// apart: rules present, references present, post-load verification passed, and every query +// through the system resolver timing out. Nothing else notices until the periodic watchdog +// runs its own probe - the interception probe monitor stands down while stabilization owns +// pf and is not re-armed afterwards - so recovery waits up to a full watchdog interval on a +// host whose link and default route are already back. +// +// One probe, then at most one forced reload and one confirming probe. Deliberately not the +// probe monitor: that keeps probing for ~7.5s and can force a reload per failed probe, +// where this path needs a single bounded repair and then hands back to the watchdog. +func (p *prog) verifyInterceptAfterStabilization() { + if p.dnsInterceptState == nil || p.pfExecBackoffActive() { + return + } + // The probe monitor is the other functional prober, so only one of us may run - but + // "somebody holds the flag" is not the same as "somebody is probing". A monitor that + // started while stabilization owns pf stands down immediately, and skipping on sight + // left nobody probing at all. Wait briefly for the holder to release instead. + if !p.claimFunctionalProbeOwner(pfFunctionalProbeOwnerWait) { + mainLog.Load().Warn().Msgf("DNS intercept: post-stabilization probe skipped — another prober held ownership for %s; leaving recovery to the watchdog", pfFunctionalProbeOwnerWait) + return + } + defer p.pfMonitorRunning.Store(false) + + // Ownership can take a moment to arrive; make sure there is still an intercept to check. + if p.dnsInterceptState == nil { + return + } + + result := probePFInterceptFn(p) + if result.result == pfProbeIndeterminate { + mainLog.Load().Debug().Msg("DNS intercept: post-stabilization probe indeterminate; repair deferred") + return + } + if result.result == pfProbeIntercepted { + mainLog.Load().Debug().Msg("DNS intercept: post-stabilization probe passed — interception is translating") + return + } + + mainLog.Load().Warn().Msg("DNS intercept: post-stabilization rules are intact but the probe FAILED — forcing one reload") + if !p.repairPFIntercept(result, "post_stabilization") { + mainLog.Load().Error().Msg("DNS intercept: post-stabilization forced reload did not run — leaving recovery to the watchdog") + return + } + cause := result + result = probePFInterceptFn(p) + logPFRepair(cause, "post_stabilization", result.result.String(), true, result) + if result.result == pfProbeIndeterminate { + mainLog.Load().Warn().Msg("DNS intercept: post-stabilization reload completed; verification indeterminate") + return + } + if result.result == pfProbeIntercepted { + mainLog.Load().Info().Msg("DNS intercept: interception restored by the post-stabilization reload") + return + } + mainLog.Load().Error().Msg("DNS intercept: interception still not translating after the post-stabilization reload — the watchdog will retry") +} + +// pfWakeProbeDelays is the bounded retry schedule for the post-resume functional probe. +// Each value is the wait before its own attempt, not an offset from the resume, so the +// attempts land at +0s, +2s, +6s and +14s after the suspend is noticed. A resumed host +// brings its link, default route, resolver list and pf translation state back over +// several seconds, so a single probe at +0 decides too early. A var so tests can shorten +// it. +var pfWakeProbeDelays = []time.Duration{0, 2 * time.Second, 4 * time.Second, 8 * time.Second} + +// pfWakeMaxRepairs bounds forced reloads across the whole schedule. QA on the post-wake +// outage saw one forced reload fail to restore translation and a later one succeed, so a +// single attempt is not enough - but this stays a bounded repair rather than a reload +// loop, because every reload flushes pf state and kills in-flight DoH connections. +const pfWakeMaxRepairs = 2 + +// pfWakeProbeOwnerWait bounds how long one attempt waits for another prober to release +// functional-probe ownership. Short on purpose: a holder that is genuinely probing is +// doing this path's job, and there are further attempts left in the schedule. +var pfWakeProbeOwnerWait = 250 * time.Millisecond + +// verifyInterceptAfterWake proves pf is actually translating after the host resumes from +// sleep, and repairs it a bounded number of times when it is not. +// +// Nothing else covers this window. The periodic watchdog compares rule text, which +// survives a suspend unchanged, so it reports the anchor intact while every query through +// the system resolver times out. The interception probe monitor is only armed by +// interface changes and its backoff schedule is frozen through the suspend, so whether it +// probes after a resume depends on where its timers happened to be when the host slept. +// QA measured 18s of no public DNS on a host whose link and default route were back +// within the first second, recovered only because an unrelated VPN reconnect happened to +// drive the post-stabilization verifier; without that reconnect nothing would have probed. +// +// The OS resolver list is refreshed before every probe. probePFIntercept aims at the +// first usable IPv4 LAN target, or the original usable first public target. +// No target is indeterminate. Refreshing avoids probing a stale VPN resolver. +func (p *prog) verifyInterceptAfterWake(gap time.Duration) { + if p.dnsInterceptState == nil { + return + } + mainLog.Load().Info().Msgf("DNS intercept: host resumed after a %s gap - verifying interception is still translating", gap.Round(time.Second)) + p.noteHostWoke("detector", gap, freshNetworkState()) + // A resume moves the resolvers of the host, so the poll goes fast again. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } + + repairs := 0 + probed := 0 + for attempt, delay := range pfWakeProbeDelays { + if !p.waitBeforeWakeProbe(delay) { + return + } + if p.dnsInterceptState == nil { + return + } + // Stabilization owns pf while a VPN's ruleset settles, and runs its own + // functional verification when it finishes. Probing or reloading underneath it + // is the mutual overwriting stabilization exists to prevent. + if p.pfStabilizing.Load() { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe standing down - stabilization owns pf and verifies on completion") + return + } + if p.pfExecBackoffActive() { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe deferred this attempt - pf exec backoff active") + continue + } + // Eligibility first, ownership second, and release before the next delay: a + // prober that holds the flag while sleeping starves the post-stabilization + // verifier, which reads a held flag as "somebody is probing". + if !p.claimFunctionalProbeOwner(pfWakeProbeOwnerWait) { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe skipped this attempt - another prober holds ownership") + continue + } + restored, repaired := p.runWakeProbeAttempt(attempt, repairs) + probed++ + p.pfMonitorRunning.Store(false) + if repaired { + repairs++ + } + if restored { + return + } + } + // Report what actually ran: an attempt deferred by pf exec backoff or skipped for + // want of probe ownership never probes, so the schedule length would claim + // verification that did not happen - and with every attempt deferred, nothing was + // measured at all. + if probed == 0 { + mainLog.Load().Warn().Msg("DNS intercept: post-wake verification never probed - every attempt was deferred or skipped - the watchdog will retry") + return + } + mainLog.Load().Error().Msgf("DNS intercept: interception not verified after resume (%d probe attempts, %d forced reloads) - the watchdog will retry", + probed, repairs) +} + +// waitBeforeWakeProbe waits out one attempt's delay, reporting false when the service +// stopped while waiting. +func (p *prog) waitBeforeWakeProbe(delay time.Duration) bool { + if delay <= 0 { + return true + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-p.stopCh: + return false + case <-timer.C: + return true + } +} + +// runWakeProbeAttempt runs one probe of the post-resume schedule, forcing a reload when +// the probe fails and repairs so far leave budget for one. It reports whether +// interception is translating and whether this attempt spent a repair. A reload that +// could not run does not count: nothing was changed, so the next attempt may try again. +// The caller holds functional-probe ownership for the call. +func (p *prog) runWakeProbeAttempt(attempt, repairs int) (restored, repaired bool) { + initializeOsResolver(ctrld.LoggerCtx(context.Background(), mainLog.Load()), true, osResolverReasonWakeProbe) + result := probePFInterceptFn(p) + if result.result == pfProbeIndeterminate { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe indeterminate; repair deferred") + return false, false + } + if result.result == pfProbeIntercepted { + if attempt > 0 || repairs > 0 { + mainLog.Load().Info().Msgf("DNS intercept: post-wake probe %d/%d passed - interception is translating", attempt+1, len(pfWakeProbeDelays)) + } else { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe passed - interception is translating") + } + return true, false + } + + if repairs >= pfWakeMaxRepairs { + mainLog.Load().Warn().Msgf("DNS intercept: post-wake probe %d/%d FAILED - repair budget of %d forced reloads is spent, not reloading again", + attempt+1, len(pfWakeProbeDelays), pfWakeMaxRepairs) + logPFRepair(result, "post_wake", "budget_exhausted", false, pfProbeObservation{}) + return false, false + } + + mainLog.Load().Warn().Msgf("DNS intercept: post-wake probe %d/%d FAILED - rules survived the suspend but pf is not translating, forcing a reload", + attempt+1, len(pfWakeProbeDelays)) + if !p.repairPFIntercept(result, "post_wake") { + mainLog.Load().Error().Msg("DNS intercept: post-wake forced reload did not run") + return false, false + } + confirmation := probePFInterceptFn(p) + logPFRepair(result, "post_wake", confirmation.result.String(), true, confirmation) + if confirmation.result == pfProbeIntercepted { + mainLog.Load().Info().Msg("DNS intercept: interception restored by the post-wake reload") + return true, true + } + return false, true +} + +var runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + return exec.Command("pfctl", args...).CombinedOutput() +} + +const ( + // pfAnchorListMessage names the anchor state event of the journal. + pfAnchorListMessage = "PF anchor list changed" + + pfAnchorReasonStabilizationStart = "stabilization_start" + pfAnchorReasonStabilizationEnd = "stabilization_end" + pfAnchorReasonMissing = "missing" + pfAnchorReasonRestored = "restored" + + // pfAnchorStateKey holds the last anchor state, so an anchor that stays + // missing for a whole storm reaches the journal once. + pfAnchorStateKey = "pf_anchor_state" + + // pfAnchorWatchdogKey holds the last state the watchdog saw, so the healthy + // line reports the recovery and not every tick between two of them. + pfAnchorWatchdogKey = "pf_anchor_intact" + + pfAnchorIntactState = "pf anchor intact" + pfAnchorMissingState = "pf anchor missing" + + pfAnchorIntactMessage = "DNS intercept watchdog: " + pfAnchorIntactState +) + +// logPFAnchorList reports which anchors the running ruleset refers to. A capture +// of an outage has to tell whether ctrld or another program owned pf, and the +// anchor names are the only evidence of that. +func (p *prog) logPFAnchorList(reason string, level *ctrld.LogEvent, dump pfRuleDump) { + anchors := p.pfAnchorNames(dump) + enabled, since := p.pfStatus() + if changed, _ := p.repeats.changed(pfAnchorStateKey, pfAnchorStateValue(reason, anchors, enabled)); !changed { + return + } + event := journal(level). + Str("reason", reason). + Bool("pf_enabled", enabled). + Str("pf_since", since) + if anchors == nil { + event.Bool("anchors_known", false).Msg(pfAnchorListMessage) + return + } + event.Strs("anchors", anchors).Msg(pfAnchorListMessage) +} + +// pfAnchorStateValue leaves the pf uptime out, because it grows between two +// reads of the same state. +func pfAnchorStateValue(reason string, anchors []string, enabled bool) string { + if anchors == nil { + return fmt.Sprintf("%s|%t|unknown", reason, enabled) + } + return fmt.Sprintf("%s|%t|%s", reason, enabled, strings.Join(anchors, ",")) +} + +// logPFAnchorIntact reports a healthy anchor. The watchdog finds it healthy on +// almost every tick for the life of the daemon, so only a change belongs in the +// log. +func (p *prog) logPFAnchorIntact() { + changed, repeats := p.repeats.changed(pfAnchorWatchdogKey, pfAnchorIntactState) + if !changed { + return + } + mainLog.Load().Debug().Uint64("repeats", repeats).Msg(pfAnchorIntactMessage) +} + +// notePFAnchorMissing reports the wiped anchor with the anchors that replaced +// it. The record of the missing state makes the next healthy check reach the +// log, so a capture holds both ends of the outage. +func (p *prog) notePFAnchorMissing(dump pfRuleDump) { + p.repeats.changed(pfAnchorWatchdogKey, pfAnchorMissingState) + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), dump) +} + +// ensurePFAnchorActive checks live PF state and returns an explicit outcome for +// callers that must distinguish intact, restored, deferred, skipped, and failed. +func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { + return p.ensurePFAnchorActiveWithPolicy(true, false) +} + +// reconcilePFAnchorAfterStabilization is owned by the active stabilization loop. +// It bypasses ordinary recent-wipe deferral and forces a rebuild only when tunnel +// state is still pending. Otherwise an intact anchor is left alone, avoiding a +// redundant global PF state flush after an earlier successful exemption update. +func (p *prog) reconcilePFAnchorAfterStabilization() pfAnchorCheckResult { + return p.ensurePFAnchorActiveWithPolicy(false, p.hasPendingTunnelReconcile()) +} + +func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebuild bool) pfAnchorCheckResult { + if p.dnsInterceptState == nil { + return pfAnchorCheckSkipped + } + // Ordinary callers must not compete with the loop that owns stabilization. + // Check before acquiring the singleflight guard so the post-stable owner is + // not blocked by a watchdog callback that would immediately skip. + if allowRecentWipeDeferral && p.pfStabilizing.Load() { + mainLog.Load().Debug().Msg("DNS intercept watchdog: suppressed — VPN stabilization in progress") + return pfAnchorCheckSkipped + } + if !p.pfEnsureRunning.CompareAndSwap(false, true) { + mainLog.Load().Debug().Msg("DNS intercept watchdog: check already running, skipping duplicate") + return pfAnchorCheckSkipped + } + defer p.pfEnsureRunning.Store(false) + + if p.pfExecBackoffActive() { + return pfAnchorCheckSkipped + } + if allowRecentWipeDeferral && p.pfStabilizing.Load() { + return pfAnchorCheckSkipped + } + + rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) + anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) + needsRestore := false + // The anchor event parses the rules this check reads, so it starts no + // second pair of pfctl processes. + var dump pfRuleDump + + // Check 1: anchor references in the main ruleset. + natOut, err := runPFAnchorCheckCommand("-sn") + if err != nil { + p.pfBackoffResourceExhaustion(err, natOut, "dump NAT rules") + mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump NAT rules") + return pfAnchorCheckFailed + } + dump.nat = natOut + natStr := string(natOut) + if !strings.Contains(natStr, rdrAnchorRef) { + mainLog.Load().Warn().Msg("DNS intercept watchdog: rdr-anchor reference missing from running ruleset") + needsRestore = true + } + + if !needsRestore { + filterOut, err := runPFAnchorCheckCommand("-sr") + if err != nil { + p.pfBackoffResourceExhaustion(err, filterOut, "dump filter rules") + mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump filter rules") + return pfAnchorCheckFailed + } + dump.rules = filterOut + if !strings.Contains(string(filterOut), anchorRef) { + mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor reference missing from running filter rules") + needsRestore = true + } + } + + // Check 2: anchor content (rules inside our anchor). + // Verify BOTH filter rules (-sr) AND rdr/NAT rules (-sn). Programs like Parallels' + // internet-sharing can flush our anchor's rdr rules while leaving filter rules intact. + // Without rdr, route-to sends packets to lo0 but they never get redirected to 127.0.0.1:53, + // causing an infinite packet loop on lo0 and complete DNS failure. + if !needsRestore { + anchorFilter, err := runPFAnchorCheckCommand("-a", pfAnchorName, "-sr") + if err != nil { + p.pfBackoffResourceExhaustion(err, anchorFilter, "dump anchor filter rules") + mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump anchor filter rules") + return pfAnchorCheckFailed + } + if pfRulesetEmpty(string(anchorFilter)) { + mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor has no filter rules — content was flushed") + needsRestore = true + } + } + if !needsRestore { + anchorNat, err := runPFAnchorCheckCommand("-a", pfAnchorName, "-sn") + if err != nil { + p.pfBackoffResourceExhaustion(err, anchorNat, "dump anchor NAT rules") + mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump anchor NAT rules") + return pfAnchorCheckFailed + } + if pfRulesetEmpty(string(anchorNat)) { + mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor has no rdr rules — translation was flushed (will cause packet loop on lo0)") + needsRestore = true + } + } + + // There is deliberately no "set skip on lo0" check here - see the note in + // verifyPFState for why macOS cannot be asked for skip state. + // + // It matters more here than a missing check might suggest: anything that cannot + // conclude has to return pfAnchorCheckFailed, and the caller only runs + // probePFIntercept() and the forced-reload recovery on an intact result. A check + // that can never conclude would therefore keep this function from ever reporting + // intact, disabling the functional probe and the self-heal it drives. + // + // So the skip case is left to probePFIntercept(). An explicit check can return via + // "pfctl -s Interfaces -v" once its output shape is confirmed on a host that has a + // skip configured. + + // Only classify a recent event as a repeated wipe after the live ruleset + // actually proves that a reference or anchor rule is missing. Previously this + // timestamp check ran first, so every healthy callback for ten seconds after + // stabilization was mislabeled as another wipe. + if lastRestore := p.pfLastRestoreTime.Load(); lastRestore > 0 { + elapsed := time.Since(time.UnixMilli(lastRestore)) + if needsRestore && allowRecentWipeDeferral && elapsed < 10*time.Second { + p.pfBackoffMultiplier.Add(1) + mainLog.Load().Warn().Msgf("DNS intercept: rules wiped %s after restore — entering stabilization (backoff multiplier: %d)", + elapsed, p.pfBackoffMultiplier.Load()) + p.notePFAnchorMissing(dump) + p.pfStartStabilization() + return pfAnchorCheckDeferred + } + if elapsed >= 10*time.Second && p.pfBackoffMultiplier.Load() > 0 { + p.pfBackoffMultiplier.Store(0) + } + } + + if !needsRestore && !forceRebuild { + p.logPFAnchorIntact() + return pfAnchorCheckIntact + } + + reason := "missing pf anchor state" + if forceRebuild && !needsRestore { + reason = "post-stabilization rebuild" + } + if needsRestore { + p.notePFAnchorMissing(dump) + } + return restorePFAnchorForReconcile(p, reason) +} + +// dnsInterceptIgnoredChangeReconcileDue reports whether an ignored macOS +// network delta should run the expensive leading-edge pf/VPN-DNS reconciliation. +// Tunnel changes bypass this decision in the caller. The periodic watchdog and +// coalesced delayed checks remain independent safety nets. +func (p *prog) dnsInterceptIgnoredChangeReconcileDue(now time.Time) bool { + nowMillis := now.UnixMilli() + for { + last := p.pfIgnoredChangeLastReconcile.Load() + if last > 0 && now.Sub(time.UnixMilli(last)) < pfIgnoredChangeReconcileInterval { + return false + } + if p.pfIgnoredChangeLastReconcile.CompareAndSwap(last, nowMillis) { + return true + } + } +} + +// scheduleDNSAfterVPNSettleRefresh queues one follow-up VPN DNS refresh. +// +// The timer is tracked rather than fired and forgotten. Teardown has to be able to +// cancel it, and VPN churn can finish stabilization several times in a row, where +// untracked timers stack up into repeats of the same scutil/VPN-DNS work. +func (p *prog) scheduleDNSAfterVPNSettleRefresh(reason string, delay time.Duration) { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + if p.pfSettleFollowupTimer != nil { + p.pfSettleFollowupTimer.Stop() + } + p.pfSettleFollowupTimer = time.AfterFunc(delay, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + if p.dnsInterceptState == nil { + return + } + p.refreshDNSAfterVPNSettle(reason) + }) +} + +// stopPFSettleFollowup cancels a pending post-settle refresh. A callback that already +// fired is fenced and joined by closeNetMonitor during program shutdown. +func (p *prog) stopPFSettleFollowup() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + if p.pfSettleFollowupTimer != nil { + p.pfSettleFollowupTimer.Stop() + p.pfSettleFollowupTimer = nil + } +} + +func (p *prog) pfExecBackoffActive() bool { + until := p.pfExecBackoffUntil.Load() + if until == 0 { + return false + } + remaining := time.Until(time.UnixMilli(until)) + if remaining <= 0 { + p.pfExecBackoffUntil.CompareAndSwap(until, 0) + return false + } + mainLog.Load().Debug().Msgf("DNS intercept watchdog: suppressed during pf exec backoff (remaining: %s)", remaining) + return true +} + +func (p *prog) pfBackoffResourceExhaustion(err error, output []byte, operation string) bool { + if !isResourceExhaustion(err, output) { + return false + } + until := time.Now().Add(pfExecFailureBackoff) + p.pfExecBackoffUntil.Store(until.UnixMilli()) + mainLog.Load().Warn().Err(err).Msgf("DNS intercept watchdog: backing off after local exec resource exhaustion (operation: %s, backoff: %s)", operation, pfExecFailureBackoff) + return true +} + +func isResourceExhaustion(err error, output []byte) bool { + if err == nil && len(output) == 0 { + return false + } + var msg string + if err != nil { + msg = err.Error() + } + if len(output) > 0 { + msg += "\n" + string(output) + } + msg = strings.ToLower(msg) + return strings.Contains(msg, "resource temporarily unavailable") || + strings.Contains(msg, "too many open files") || + strings.Contains(msg, "too many processes") || + strings.Contains(msg, "cannot allocate memory") +} + +// pfWatchdog periodically checks that our pf anchor is still active. +// Other programs (e.g., Windscribe desktop app, macOS configd) can replace +// scheduleDelayedRechecks schedules delayed re-checks after a network change event. +// VPN apps often modify pf rules and DNS settings asynchronously after the network +// change that triggered our handler. These delayed checks catch: +// - pf anchor wipes by VPN disconnect (Windscribe's firewallOff) +// - Stale OS resolver nameservers (VPN DNS not yet cleaned from scutil) +// - Stale VPN DNS routes in vpnDNSManager +// - Tunnel interface additions/removals not yet visible +// +// Two delays (2s and 4s) cover both fast and slow VPN teardowns. +func (p *prog) scheduleDelayedRechecks() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + p.pfDelayedRecheckTimers = p.pfDelayedRecheckTimers[:0] + + for _, delay := range []time.Duration{pfAnchorRecheckDelay, pfAnchorRecheckDelayLong} { + delay := delay + timer := time.AfterFunc(delay, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + if p.dnsInterceptState == nil || p.pfStabilizing.Load() { + return + } + p.ensurePFAnchorActive() + p.checkTunnelInterfaceChanges() + // Refresh OS resolver — VPN may have finished DNS cleanup since the + // immediate handler ran. This clears stale LAN nameservers (e.g., + // a VPN's DNS IP (e.g., 10.255.255.3) lingering in scutil --dns). + ctx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + ctrld.InitializeOsResolverWithReason(ctx, true, osResolverReasonDelayedRecheck) + if p.vpnDNS != nil { + p.vpnDNS.Refresh(ctx, true) + } + // A VM/container network often gets its address slightly after the + // interface appears, so the immediate handler can see it without a subnet. + // Re-check here (no-op when the forwarded-source set is unchanged). + p.reconcileForwardedSources() + }) + p.pfDelayedRecheckTimers = append(p.pfDelayedRecheckTimers, timer) + } +} + +// the entire pf ruleset with pfctl -f, which wipes our anchor references. +// This watchdog detects and restores them. +func (p *prog) pfWatchdog() { + mainLog.Load().Info().Msgf("DNS intercept: starting pf watchdog (interval: %s)", pfWatchdogInterval) + + var consecutiveMisses atomic.Int32 + ticker := time.NewTicker(pfWatchdogInterval) + defer ticker.Stop() + + for { + select { + case <-p.stopCh: + mainLog.Load().Debug().Msg("DNS intercept: pf watchdog stopped") + return + case <-ticker.C: + if p.dnsInterceptState == nil { + mainLog.Load().Debug().Msg("DNS intercept: pf watchdog exiting — intercept state is nil") + return + } + + // Reconcile the temporary service DNS target even when macOS emits no + // major network delta. This converges both DHCP-return cleanup and a + // later return to a DNS-less network. + ensureInterceptDNSTargetFn(p, []string{}) + + result := p.ensurePFAnchorActive() + result = p.checkPFWatchdogProbe(result) + + if result == pfAnchorCheckIntact { + // Check if backoff should be reset only after an authoritative healthy check. + if p.pfBackoffMultiplier.Load() > 0 && p.pfLastRestoreTime.Load() > 0 { + elapsed := time.Since(time.UnixMilli(p.pfLastRestoreTime.Load())) + if elapsed > 60*time.Second { + p.pfBackoffMultiplier.Store(0) + mainLog.Load().Info().Msg("DNS intercept watchdog: rules stable for >60s — reset backoff") + } + } + } + + switch result { + case pfAnchorCheckRestored: + misses := consecutiveMisses.Add(1) + if misses >= pfConsecutiveMissThreshold { + mainLog.Load().Error().Msgf("DNS intercept watchdog: pf anchor has been missing for %d consecutive checks — something is persistently overwriting pf rules", misses) + } else { + mainLog.Load().Warn().Msgf("DNS intercept watchdog: pf anchor was missing and restored (consecutive misses: %d)", misses) + } + case pfAnchorCheckIntact: + if old := consecutiveMisses.Swap(0); old > 0 { + mainLog.Load().Info().Msgf("DNS intercept watchdog: pf anchor stable again after %d consecutive restores", old) + } + case pfAnchorCheckSkipped, pfAnchorCheckDeferred, pfAnchorCheckFailed: + // Preserve the previous authoritative state. Do not probe, count a + // repair, or reset misses from an indeterminate/deferred result. + } + + // Tunnel transitions and failed PF/WFP exemption updates are normally + // event-driven, but the final delayed attempt must remain retryable even + // if macOS emits no further event. + tunnelChanged := p.checkTunnelInterfaceChanges() + pendingExemptions := p.vpnDNS != nil && p.vpnDNS.interceptExemptionsPending() + // Stabilization owns pf while a VPN's ruleset is still settling, and a + // refresh rebuilds and reloads the anchor - exactly the mutual overwriting + // that stabilization exists to stop. Nothing is lost by deferring: the + // pending tunnel state and pending exemptions both survive, and the next + // tick retries once stabilization has finished. + if (tunnelChanged || pendingExemptions) && p.vpnDNS != nil && + !p.pfExecBackoffActive() && !p.pfStabilizing.Load() { + p.vpnDNS.Refresh(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true) + } + + // VM/container networks appear and disappear without always + // producing a network-change event, and an intact anchor passes every + // check above. Reconciling here bounds how long a started guest can go + // untrusted (or a stopped one stay trusted) to one watchdog interval. + // No-op when the effective forwarded-source set is unchanged. + p.reconcileForwardedSources() + } + } +} + +// A successful forced reload following a probe miss is not evidence that an +// anchor was missing. Keep that repair out of the missing-anchor counter. +func (p *prog) checkPFWatchdogProbe(result pfAnchorCheckResult) pfAnchorCheckResult { + if result != pfAnchorCheckIntact || p.pfMonitorRunning.Load() { + return result + } + probe := probePFInterceptFn(p) + switch probe.result { + case pfProbeIntercepted: + return result + case pfProbeNotIntercepted: + mainLog.Load().Warn().Msg("DNS intercept watchdog: sent probe was not intercepted; forcing full reload") + if !p.repairPFIntercept(probe, "watchdog") { + mainLog.Load().Warn().Msg("DNS intercept watchdog: probe repair did not complete; will retry") + } + case pfProbeIndeterminate: + mainLog.Load().Debug().Msg("DNS intercept watchdog: probe indeterminate; repair deferred") + } + return pfAnchorCheckDeferred +} + +// exemptVPNDNSServers updates the pf anchor rules with interface-scoped exemptions +// for VPN DNS servers, allowing VPN local DNS handlers (e.g., Tailscale MagicDNS +// via Network Extension) to receive DNS queries from all processes on their interface. +// +// Called by vpnDNSManager.Refresh() whenever VPN DNS servers change. +func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { + if p.dnsInterceptState == nil { + return fmt.Errorf("pf state not available") + } + if p.pfExecBackoffActive() { + return fmt.Errorf("pf exemption update deferred during exec backoff") + } + // Defence in depth for the callers guarded at their own call sites: this function + // rewrites and reloads the whole anchor, which is the mutual overwriting that + // stabilization suppresses while a VPN's ruleset settles. + // + // This does defer handleRecovery's DHCP-nameserver exemption, which has no retry of + // its own - the trade is deliberate. Those exemptions are ad hoc: they are never + // registered with the vpnDNSManager, so the post-stabilization reconcile rebuilds + // the anchor from CurrentExemptions and drops them regardless. Letting them through + // mid-settle would buy a few seconds of captive-portal access at the cost of the + // collision stabilization exists to prevent. The error text surfaces in recovery's + // warning so the reason is visible rather than silent. + // + // Manager-driven callers lose nothing: appliedExemptions only advances on success, + // so interceptExemptionsPending stays true and the next tick retries. + if p.pfStabilizing.Load() { + return fmt.Errorf("pf exemption update deferred during stabilization") + } + if !p.pfEnsureRunning.CompareAndSwap(false, true) { + return fmt.Errorf("pf reconciliation already running") + } + defer p.pfEnsureRunning.Store(false) + + forwarded := p.currentForwardedSources() + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) + rulesStr := p.buildPFAnchorRulesForState(exemptions, tunnelIfaces, forwarded) + + if err := writePFAnchorFile(rulesStr); err != nil { + return fmt.Errorf("dns intercept: failed to rewrite pf anchor: %w", err) + } + + out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput() + if err != nil { + p.pfBackoffResourceExhaustion(err, out, "load VPN DNS anchor") + return fmt.Errorf("dns intercept: failed to reload pf anchor: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + p.recordAppliedForwardedSources(forwarded) + + // Flush stale pf states so packets are re-evaluated against new rules. + flushPFStates() + + // Ensure the anchor reference still exists in the main ruleset. + // Another program may have replaced the ruleset since we last checked. + if err := p.ensurePFAnchorReference(); err != nil { + p.pfBackoffResourceExhaustion(err, nil, "restore VPN DNS anchor reference") + p.resetUpstreamTransports() + return fmt.Errorf("dns intercept: failed to restore anchor reference during VPN DNS update: %w", err) + } + p.resetUpstreamTransports() + p.commitPFReconcileState(tunnelIfaces) + + mainLog.Load().Info().Msgf("DNS intercept: updated pf rules — exempted %d VPN DNS + %d OS resolver servers", + len(exemptions), len(ctrld.OsResolverNameservers())) + return nil +} + +// Keep warning coalescing available throughout intercept startup and recovery. +var pfProbeLogs pfProbeLogState + +// probePFIntercept distinguishes a meaningful interception failure from a probe +// that could not send. Only pfProbeNotIntercepted permits destructive repair. +func (p *prog) probePFIntercept() pfProbeObservation { + observed := pfProbeObservation{ + probeID: fmt.Sprintf("_pf-probe-%x.%s", time.Now().UnixNano(), pfProbeDomain), + recoveryGeneration: p.recoveryGen.Load(), + stage: "target", code: "unavailable", + } + if p.dnsInterceptState == nil { + return observed + } + started := time.Now() + servers := pfProbeNameservers() + host := pfProbeTarget(servers) + family := "none" + if len(servers) > 0 { + if first, _, err := net.SplitHostPort(servers[0]); err == nil { + if addr, err := netip.ParseAddr(first); err == nil { + observed.target = net.JoinHostPort(addr.String(), "53") + family = "ipv6" + if addr.Is4() { + family = "ipv4" + } + } + } + } + if host != "" { + observed.target = net.JoinHostPort(host, "53") + family = "ipv4" + probeCh, deregister := p.registerInterceptProbe(observed.probeID) + defer deregister() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + go func() { + select { + case <-p.stopCh: + cancel() + case <-ctx.Done(): + } + }() + cmd := newPFProbeCommand(ctx, host, fmt.Sprintf("%x", buildDNSQueryPacket(observed.probeID))) + result := runPFProbe(ctx, cmd, probeCh, pfProbeTimeout) + observed.result, observed.stage, observed.code = result.result, result.stage, result.code + } + // Cancellation is a shutdown diagnostic, not a change in PF health. + retain, repeats := false, uint64(0) + if observed.code != "canceled" { + retain, repeats = pfProbeLogs.change(observed) + } + event := mainLog.Load().Debug() + if retain { + event = mainLog.Load().Warn() + } + event.Str("probe_id", observed.probeID).Uint64("recovery_generation", observed.recoveryGeneration). + Str("resolver_target", observed.target).Str("resolver_family", family). + Str("stage", observed.stage).Str("error_code", observed.code). + Str("outcome", observed.result.String()).Uint64("repeated_results", repeats). + Bool("repair_eligible", observed.result == pfProbeNotIntercepted). + Int64("elapsed_ms", time.Since(started).Milliseconds()).Msg("DNS intercept probe result") + p.logPFIPv6Diagnostic(observed, family) + return observed +} + +// Carry the cause by value. A concurrent probe cannot replace its correlation fields. +func (p *prog) repairPFIntercept(cause pfProbeObservation, caller string) bool { + logPFRepair(cause, caller, "started", false, pfProbeObservation{}) + performed := forceReloadPFInterceptFn(p) + outcome := "not_run" + if performed { + outcome = "reload_completed" + } + logPFRepair(cause, caller, outcome, performed, pfProbeObservation{}) + return performed +} + +func logPFRepair(cause pfProbeObservation, caller, outcome string, performed bool, confirmation pfProbeObservation) { + mainLog.Load().Warn().Str("probe_id", cause.probeID). + Uint64("recovery_generation", cause.recoveryGeneration). + Str("caller", caller).Str("cause", cause.result.String()). + Str("stage", cause.stage).Str("error_code", cause.code). + Str("resolver_target", cause.target).Str("outcome", outcome). + Bool("reload_performed", performed).Str("confirmation_probe_id", confirmation.probeID). + Msg("PF repair") +} + +// buildDNSQueryPacket constructs a minimal DNS query packet (wire format) for the given domain. +func buildDNSQueryPacket(domain string) []byte { + // DNS header: ID=0x1234, QR=0, OPCODE=0, RD=1, QDCOUNT=1 + header := []byte{ + 0x12, 0x34, // ID + 0x01, 0x00, // Flags: RD=1 + 0x00, 0x01, // QDCOUNT=1 + 0x00, 0x00, // ANCOUNT=0 + 0x00, 0x00, // NSCOUNT=0 + 0x00, 0x00, // ARCOUNT=0 + } + + // Encode domain name in DNS wire format (label-length encoding) + // Remove trailing dot if present + d := strings.TrimSuffix(domain, ".") + var qname []byte + for _, label := range strings.Split(d, ".") { + qname = append(qname, byte(len(label))) + qname = append(qname, []byte(label)...) + } + qname = append(qname, 0x00) // root label + + // QTYPE=A (1), QCLASS=IN (1) + question := append(qname, 0x00, 0x01, 0x00, 0x01) + + return append(header, question...) +} + +// cleanupStaleDNSInterceptState is a startup hook for enforcement that can outlive +// the process. macOS needs no work here: startDNSIntercept flushes the anchor and +// removes a stale anchor file before loading rules, and the pf anchor alone does not +// block traffic until ctrld loads rules into it. +func cleanupStaleDNSInterceptState() {} + +var pfInterceptMonitorDelays = []time.Duration{0, 500 * time.Millisecond, time.Second, 2 * time.Second, 4 * time.Second} + +// pfInterceptMonitor runs asynchronously after interface changes are detected. +// It probes pf interception with exponential backoff and forces a full pf reload +// if the probe fails. Only one instance runs at a time (singleton via atomic.Bool). +// +// The backoff schedule provides both fast detection (immediate + 500ms) and extended +// coverage (up to ~8s) to win the race against async pf reloads by hypervisors. +func (p *prog) pfInterceptMonitor() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + // Eligibility first, ownership second. A monitor that is about to stand down must not + // take the flag on its way out: the post-stabilization verifier reads that flag as + // "another prober is working" and would step aside for a prober that never probes. + if !p.interceptProbeMonitorAllowed() { + mainLog.Load().Debug().Msg("DNS intercept monitor: not starting — intercept disabled or stabilizing") + return + } + if !p.pfMonitorRunning.CompareAndSwap(false, true) { + mainLog.Load().Debug().Msg("DNS intercept monitor: already running, skipping") + return + } + defer p.pfMonitorRunning.Store(false) + + mainLog.Load().Info().Msg("DNS intercept monitor: starting interception probe sequence") + + // Backoff schedule: probe quickly first, then space out. + // Total monitoring window: ~0 + 0.5 + 1 + 2 + 4 = ~7.5s + delays := pfInterceptMonitorDelays + + for i, delay := range delays { + if delay > 0 { + time.Sleep(delay) + } + if p.networkActivityClosed() || !p.interceptProbeMonitorAllowed() { + mainLog.Load().Debug().Msg("DNS intercept monitor: aborting — intercept disabled or stabilizing") + return + } + + result := probePFInterceptFn(p) + if result.result == pfProbeIndeterminate { + mainLog.Load().Debug().Msg("DNS intercept monitor: probe indeterminate; repair deferred") + continue + } + if result.result == pfProbeIntercepted { + mainLog.Load().Debug().Msgf("DNS intercept monitor: probe %d/%d passed", i+1, len(delays)) + continue // working now — keep monitoring in case it breaks later in the window + } + + // Probe failed — pf translation is broken. Force full reload. + mainLog.Load().Warn().Msgf("DNS intercept monitor: probe %d/%d FAILED — pf translation broken, forcing full ruleset reload", i+1, len(delays)) + if !p.repairPFIntercept(result, "monitor") { + continue + } + + // Verify the reload fixed it + time.Sleep(200 * time.Millisecond) + confirmation := probePFInterceptFn(p) + logPFRepair(result, "monitor", confirmation.result.String(), true, confirmation) + switch confirmation.result { + case pfProbeIntercepted: + mainLog.Load().Info().Msg("DNS intercept monitor: probe passed after reload - interception restored") + case pfProbeNotIntercepted: + mainLog.Load().Warn().Msg("DNS intercept monitor: probe still failing after reload - pf may need manual intervention") + case pfProbeIndeterminate: + mainLog.Load().Warn().Msg("DNS intercept monitor: reload completed; verification indeterminate") + } + } + + mainLog.Load().Info().Msg("DNS intercept monitor: probe sequence completed") +} + +// forceReloadPFMainRuleset unconditionally reloads the entire pf ruleset via +// "pfctl -f -". This resets pf's internal translation engine, fixing cases where +// rdr rules exist in text form but aren't being evaluated (e.g., after a hypervisor +// like Parallels reloads /etc/pf.conf as a side effect of creating/destroying +// virtual network interfaces). +// +// Unlike ensurePFAnchorReference() which returns early when anchor references are +// already present, this function always performs the full reload. +// +// The reload is safe for VPN interop because it reassembles from the current running +// ruleset (pfctl -sr/-sn), preserving all existing anchors and rules. +func (p *prog) forceReloadPFMainRuleset() bool { + if p.dnsInterceptState == nil || p.pfExecBackoffActive() { + return false + } + if !p.pfEnsureRunning.CompareAndSwap(false, true) { + return false + } + defer p.pfEnsureRunning.Store(false) + + rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) + anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) + + // Dump running rules. + natOut, err := exec.Command("pfctl", "-sn").CombinedOutput() + if err != nil { + mainLog.Load().Error().Err(err).Msg("DNS intercept: force reload — failed to dump NAT rules") + return false + } + + filterOut, err := exec.Command("pfctl", "-sr").CombinedOutput() + if err != nil { + mainLog.Load().Error().Err(err).Msg("DNS intercept: force reload — failed to dump filter rules") + return false + } + + natLines := pfFilterRuleLines(string(natOut)) + filterLines := pfFilterRuleLines(string(filterOut)) + + // Separate scrub rules from filter rules. + var scrubLines, pureFilterLines []string + for _, line := range filterLines { + if strings.HasPrefix(line, "scrub") { + scrubLines = append(scrubLines, line) + } else { + pureFilterLines = append(pureFilterLines, line) + } + } + + // Ensure our anchor references are present (they may have been wiped). + if !pfContainsRule(natLines, rdrAnchorRef) { + natLines = append([]string{rdrAnchorRef}, natLines...) + } + if !pfContainsRule(pureFilterLines, anchorRef) { + pureFilterLines = append([]string{anchorRef}, pureFilterLines...) + } + + // Reassemble in pf's required order: scrub → translation → filtering. As in + // ensurePFAnchorReference, no options section is emitted; see the KNOWN GAP note + // there. + var combined strings.Builder + for _, line := range scrubLines { + combined.WriteString(line + "\n") + } + for _, line := range natLines { + combined.WriteString(line + "\n") + } + for _, line := range pureFilterLines { + combined.WriteString(line + "\n") + } + + cmd := exec.Command("pfctl", "-f", "-") + cmd.Stdin = strings.NewReader(combined.String()) + out, err := cmd.CombinedOutput() + if err != nil { + mainLog.Load().Error().Err(err).Msgf("DNS intercept: force reload — pfctl -f - failed (output: %s)", strings.TrimSpace(string(out))) + return false + } + // A successful main ruleset reload invalidates PF state even if the + // subsequent anchor rebuild fails, so retire stale DoH transports now. + p.resetUpstreamTransports() + + if result := p.restorePFAnchorWithTransportReset("forced main ruleset reload", false); result != pfAnchorCheckRestored { + mainLog.Load().Error().Msg("DNS intercept: force reload — anchor rebuild failed") + return false + } + + mainLog.Load().Info().Msg("DNS intercept: force reload — pf ruleset and anchor reloaded successfully") + return true +} + +// osHealthcheckSuppressed always returns false on darwin — WFP loopback +// protect (the trigger for suppression) is Windows-only. +func (p *prog) osHealthcheckSuppressed() bool { return false } diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go new file mode 100644 index 00000000..a01e669e --- /dev/null +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -0,0 +1,1966 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +// ============================================================================= +// buildPFAnchorRules tests +// ============================================================================= + +func TestPFBuildAnchorRules_Basic(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}} + rules := p.buildPFAnchorRules(nil) + + // rdr (translation) must come before pass (filtering) + rdrIdx := strings.Index(rules, "rdr on lo0 inet proto udp") + passRouteIdx := strings.Index(rules, "pass out quick on ! lo0 route-to lo0 inet proto udp") + passInIdx := strings.Index(rules, "pass in quick on lo0 reply-to lo0") + + if rdrIdx < 0 { + t.Fatal("missing rdr rule") + } + if passRouteIdx < 0 { + t.Fatal("missing pass out route-to rule") + } + if passInIdx < 0 { + t.Fatal("missing pass in on lo0 rule") + } + if rdrIdx >= passRouteIdx { + t.Error("rdr rules must come before pass out route-to rules") + } + if passRouteIdx >= passInIdx { + t.Error("pass out route-to must come before pass in on lo0") + } + + // Both UDP and TCP rdr rules + if !strings.Contains(rules, "proto udp") || !strings.Contains(rules, "proto tcp") { + t.Error("must have both UDP and TCP rdr rules") + } +} + +func TestPFBuildAnchorRules_WithVPNServers(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}} + vpnServers := []vpnDNSExemption{ + {Server: "10.8.0.1"}, + {Server: "10.8.0.2"}, + } + rules := p.buildPFAnchorRules(vpnServers) + + // VPN exemption rules must appear + for _, s := range vpnServers { + if !strings.Contains(rules, s.Server) { + t.Errorf("missing VPN exemption for %s", s.Server) + } + } + + // VPN exemptions must come before route-to + exemptIdx := strings.Index(rules, "10.8.0.1 port 53 group") + routeIdx := strings.Index(rules, "pass out quick on ! lo0 route-to lo0 inet proto udp") + if exemptIdx < 0 { + t.Fatal("missing VPN exemption rule for 10.8.0.1") + } + if routeIdx < 0 { + t.Fatal("missing route-to rule") + } + if exemptIdx >= routeIdx { + t.Error("VPN exemptions must come before route-to rules") + } +} + +func TestPFBuildAnchorRules_IPv4AndIPv6VPN(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}} + vpnServers := []vpnDNSExemption{ + {Server: "10.8.0.1"}, + {Server: "fd00::1"}, + } + rules := p.buildPFAnchorRules(vpnServers) + + // IPv4 server should use "inet" + lines := strings.Split(rules, "\n") + for _, line := range lines { + if strings.Contains(line, "10.8.0.1") && strings.HasPrefix(line, "pass") { + if !strings.Contains(line, "inet ") { + t.Error("IPv4 VPN server rule should contain 'inet'") + } + if strings.Contains(line, "inet6") { + t.Error("IPv4 VPN server rule should not contain 'inet6'") + } + } + if strings.Contains(line, "fd00::1") && strings.HasPrefix(line, "pass") { + if !strings.Contains(line, "inet6") { + t.Error("IPv6 VPN server rule should contain 'inet6'") + } + } + } +} + +func TestPFBuildAnchorRules_Ordering(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}} + vpnServers := []vpnDNSExemption{ + {Server: "10.8.0.1"}, + } + rules := p.buildPFAnchorRules(vpnServers) + + // Verify ordering: rdr → exemptions → route-to → pass in on lo0 + rdrIdx := strings.Index(rules, "rdr on lo0 inet proto udp") + exemptIdx := strings.Index(rules, "pass out quick on ! lo0 inet proto { udp, tcp } from any to 10.8.0.1 port 53 group _ctrld") + routeIdx := strings.Index(rules, "pass out quick on ! lo0 route-to lo0 inet proto udp") + passInIdx := strings.Index(rules, "pass in quick on lo0 reply-to lo0") + + if rdrIdx < 0 || exemptIdx < 0 || routeIdx < 0 || passInIdx < 0 { + t.Fatalf("missing expected rules: rdr=%d exempt=%d route=%d passIn=%d", rdrIdx, exemptIdx, routeIdx, passInIdx) + } + + if !(rdrIdx < exemptIdx && exemptIdx < routeIdx && routeIdx < passInIdx) { + t.Errorf("incorrect rule ordering: rdr(%d) < exempt(%d) < route(%d) < passIn(%d)", rdrIdx, exemptIdx, routeIdx, passInIdx) + } +} + +// TestPFBuildAnchorRules_FallbackPort verifies that when the listener falls back +// to an alternate local port (e.g. 5354 because mDNSResponder owns *:53), the pf +// rdr rules redirect DNS to the ACTUAL bound port, not the configured default 53. +// Regression test for #551: pf redirected to a dead port after listener fallback. +func TestPFBuildAnchorRules_FallbackPort(t *testing.T) { + // Configured/generated listener is 127.0.0.1:53, but the runtime bound port is 5354. + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}}} + rules := p.buildPFAnchorRules(nil) + + // rdr must redirect to the actual bound port 5354. + if !strings.Contains(rules, "rdr on lo0 inet proto udp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 5354") { + t.Errorf("UDP rdr must redirect to bound port 5354, got:\n%s", rules) + } + if !strings.Contains(rules, "rdr on lo0 inet proto tcp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 5354") { + t.Errorf("TCP rdr must redirect to bound port 5354, got:\n%s", rules) + } + + // The rdr redirect target must NOT point at the dead default port 53. + // Match the exact port at line end so "port 5354" is not a false positive. + if strings.Contains(rules, "-> 127.0.0.1 port 53\n") { + t.Errorf("rdr must not redirect to dead port 53 after fallback, got:\n%s", rules) + } + + // The inbound accept rule must also target the actual bound port. + if !strings.Contains(rules, "127.0.0.1 port 5354") { + t.Errorf("pass in rule must reference bound port 5354, got:\n%s", rules) + } +} + +// TestPFAddressFamily tests the pfAddressFamily helper. +func TestPFAddressFamily(t *testing.T) { + tests := []struct { + ip string + want string + }{ + {"10.0.0.1", "inet"}, + {"192.168.1.1", "inet"}, + {"127.0.0.1", "inet"}, + {"::1", "inet6"}, + {"fd00::1", "inet6"}, + {"2001:db8::1", "inet6"}, + } + for _, tt := range tests { + if got := pfAddressFamily(tt.ip); got != tt.want { + t.Errorf("pfAddressFamily(%q) = %q, want %q", tt.ip, got, tt.want) + } + } +} + +func TestIsResourceExhaustion(t *testing.T) { + tests := []struct { + name string + err error + output []byte + want bool + }{ + { + name: "exec start failure", + err: errors.New("fork/exec /sbin/pfctl: resource temporarily unavailable"), + want: true, + }, + { + name: "fd exhaustion from stderr output", + err: errors.New("exit status 1"), + output: []byte("pfctl: Pipe: Too many open files"), + want: true, + }, + { + name: "process exhaustion from wrapped restore error", + err: errors.New("failed to dump running filter rules: exit status 1 (output: too many processes)"), + want: true, + }, + { + name: "ordinary pf syntax failure", + err: errors.New("exit status 1"), + output: []byte("pfctl: syntax error"), + want: false, + }, + { + name: "nil error and empty output", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isResourceExhaustion(tt.err, tt.output); got != tt.want { + t.Fatalf("isResourceExhaustion() = %v, want %v", got, tt.want) + } + }) + } +} + +func stubPFAnchorCheckCommand(t *testing.T, outputs map[string]string) { + t.Helper() + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + key := strings.Join(args, " ") + output, ok := outputs[key] + if !ok { + t.Fatalf("unexpected pf anchor check command: pfctl %s", key) + } + return []byte(output), nil + } + t.Cleanup(func() { + runPFAnchorCheckCommand = original + }) +} + +func TestEnsurePFAnchorActiveRecentRestoreWithIntactRulesDoesNotStabilize(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + + p := &prog{ + dnsInterceptState: &pfState{}, + stopCh: make(chan struct{}), + } + restoredAt := time.Now().Add(-time.Second).UnixMilli() + p.pfLastRestoreTime.Store(restoredAt) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("intact rules result = %v, want intact", result) + } + if p.pfBackoffMultiplier.Load() != 0 { + t.Fatalf("intact rules incremented backoff to %d", p.pfBackoffMultiplier.Load()) + } + if p.pfStabilizing.Load() { + t.Fatal("intact rules must not enter stabilization") + } + if got := p.pfLastRestoreTime.Load(); got != restoredAt { + t.Fatalf("intact check changed restore timestamp: got %d, want %d", got, restoredAt) + } +} + +func TestEnsurePFAnchorActiveCheckFailureIsNotIntact(t *testing.T) { + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(...string) ([]byte, error) { + return nil, errors.New("pfctl unavailable") + } + t.Cleanup(func() { runPFAnchorCheckCommand = original }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckFailed { + t.Fatalf("failed PF inspection result = %v, want failed", result) + } +} + +func TestEnsurePFAnchorActiveRecentActualWipeStartsStabilization(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": "", + }) + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{ + dnsInterceptState: &pfState{}, + stopCh: stopCh, + } + restoredAt := time.Now().Add(-time.Second).UnixMilli() + p.pfLastRestoreTime.Store(restoredAt) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckDeferred { + t.Fatalf("recent repeated wipe result = %v, want deferred", result) + } + if got := p.pfBackoffMultiplier.Load(); got != 1 { + t.Fatalf("recent repeated wipe backoff = %d, want 1", got) + } + if got := p.pfLastRestoreTime.Load(); got != restoredAt { + t.Fatalf("deferred restore changed restore timestamp: got %d, want %d", got, restoredAt) + } + deadline := time.Now().Add(time.Second) + for p.pfStabilizing.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if p.pfStabilizing.Load() { + t.Fatal("stabilization goroutine did not observe closed stop channel") + } +} + +func TestDNSInterceptIgnoredChangeReconcileDue(t *testing.T) { + p := &prog{} + start := time.Unix(1_000_000, 0) + + if !p.dnsInterceptIgnoredChangeReconcileDue(start) { + t.Fatal("first ignored change must reconcile immediately") + } + if p.dnsInterceptIgnoredChangeReconcileDue(start.Add(pfIgnoredChangeReconcileInterval - time.Millisecond)) { + t.Fatal("ignored changes inside the interval must be coalesced") + } + if !p.dnsInterceptIgnoredChangeReconcileDue(start.Add(pfIgnoredChangeReconcileInterval)) { + t.Fatal("continuous ignored changes must reconcile again at the interval boundary") + } +} + +func TestIgnoredNetworkChangeCallbackBoundsWorkWithoutBurningStabilizedSlot(t *testing.T) { + outputs := map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + } + originalCheck := runPFAnchorCheckCommand + pfChecks := 0 + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + key := strings.Join(args, " ") + output, ok := outputs[key] + if !ok { + t.Fatalf("unexpected pf anchor check command: pfctl %s", key) + } + if key == "-sn" { + pfChecks++ + } + return []byte(output), nil + } + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { + runPFAnchorCheckCommand = originalCheck + discoverTunnelInterfacesForReconcile = originalDiscover + }) + + refreshes := 0 + vpnDNS := newVPNDNSManager(&mainLog, nil) + vpnDNS.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + refreshes++ + return nil + } + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: vpnDNS} + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + + delta := &netmon.ChangeDelta{ + Old: &netmon.State{Interface: map[string]netmon.Interface{}}, + New: &netmon.State{Interface: map[string]netmon.Interface{}}, + } + start := time.Unix(1_000_000, 0) + p.handleDNSInterceptIgnoredNetworkChange(delta, start) + if pfChecks != 1 || refreshes != 1 { + t.Fatalf("first ignored delta work: pf checks=%d refreshes=%d, want 1 each", pfChecks, refreshes) + } + + p.pfStabilizing.Store(true) + p.handleDNSInterceptIgnoredNetworkChange(delta, start.Add(pfIgnoredChangeReconcileInterval)) + if pfChecks != 1 || refreshes != 1 { + t.Fatalf("stabilized delta ran leading reconciliation: pf checks=%d refreshes=%d", pfChecks, refreshes) + } + + p.pfStabilizing.Store(false) + resumeAt := start.Add(pfIgnoredChangeReconcileInterval + time.Millisecond) + p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt) + if pfChecks != 2 || refreshes != 2 { + t.Fatalf("first post-stabilization delta did not reconcile immediately: pf checks=%d refreshes=%d", pfChecks, refreshes) + } + + for i := 1; i <= 8; i++ { + p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt.Add(time.Duration(i)*100*time.Millisecond)) + } + if pfChecks != 2 || refreshes != 2 { + t.Fatalf("ignored delta burst was not coalesced: pf checks=%d refreshes=%d", pfChecks, refreshes) + } + + p.handleDNSInterceptIgnoredNetworkChange(delta, resumeAt.Add(pfIgnoredChangeReconcileInterval)) + if pfChecks != 3 || refreshes != 3 { + t.Fatalf("interval boundary did not reconcile: pf checks=%d refreshes=%d, want 3 each", pfChecks, refreshes) + } +} + +func TestRestorePFAnchorFailureIsNotReportedOrTimestamped(t *testing.T) { + originalReference := ensurePFAnchorReferenceForRestore + originalRebuild := rebuildPFAnchorRulesForReconcile + ensurePFAnchorReferenceForRestore = func(*prog) error { return nil } + rebuildPFAnchorRulesForReconcile = func(*prog, []vpnDNSExemption) ([]string, error) { + return nil, errors.New("pf load failed") + } + t.Cleanup(func() { + ensurePFAnchorReferenceForRestore = originalReference + rebuildPFAnchorRulesForReconcile = originalRebuild + }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.restorePFAnchor("test"); result != pfAnchorCheckFailed { + t.Fatalf("failed restore result = %v, want failed", result) + } + if got := p.pfLastRestoreTime.Load(); got != 0 { + t.Fatalf("failed restore changed timestamp to %d", got) + } + if len(p.lastTunnelIfaces) != 0 { + t.Fatalf("failed restore committed tunnel state: %v", p.lastTunnelIfaces) + } +} + +func TestPFStabilizationTimeoutReturnsOwnershipToDelayedRecovery(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + p.pfStabilizationLoopWithMaxWait(t.Context(), time.Hour, 25*time.Millisecond) + + if p.pfStabilizing.Load() { + t.Fatal("stabilization retained ownership after the maximum wait") + } + p.pfDelayedRecheckMu.Lock() + timers := append([]*time.Timer(nil), p.pfDelayedRecheckTimers...) + p.pfDelayedRecheckTimers = nil + p.pfDelayedRecheckMu.Unlock() + if len(timers) != 2 { + t.Fatalf("expected bounded timeout to schedule delayed recovery, got %d timers", len(timers)) + } + for _, timer := range timers { + timer.Stop() + } +} + +func TestStopDNSInterceptWaitsForInFlightPFMutation(t *testing.T) { + binDir := t.TempDir() + pfctlPath := filepath.Join(binDir, "pfctl") + if err := os.WriteFile(pfctlPath, []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+":"+os.Getenv("PATH")) + + anchorFile := filepath.Join(t.TempDir(), "anchor") + if err := os.WriteFile(anchorFile, []byte("rules"), 0600); err != nil { + t.Fatal(err) + } + p := &prog{dnsInterceptState: &pfState{anchorName: pfAnchorName, anchorFile: anchorFile}} + p.pfEnsureRunning.Store(true) + + revoked := make(chan struct{}) + originalRevokedHook := pfShutdownStateRevokedForTest + pfShutdownStateRevokedForTest = func() { close(revoked) } + t.Cleanup(func() { pfShutdownStateRevokedForTest = originalRevokedHook }) + + done := make(chan error, 1) + go func() { done <- p.stopDNSIntercept() }() + + select { + case <-revoked: + case <-time.After(time.Second): + t.Fatal("shutdown did not revoke PF lifecycle state before waiting") + } + select { + case err := <-done: + t.Fatalf("shutdown completed before in-flight PF owner released: %v", err) + case <-time.After(25 * time.Millisecond): + } + + p.pfEnsureRunning.Store(false) + if err := <-done; err != nil { + t.Fatalf("stopDNSIntercept() error: %v", err) + } + if _, err := os.Stat(anchorFile); !os.IsNotExist(err) { + t.Fatalf("anchor file remained after serialized shutdown: %v", err) + } +} + +func TestPostStabilizationReconcileRetainsOwnershipAndForcesRebuild(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + originalRestore := restorePFAnchorForReconcile + calls := 0 + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { + calls++ + return pfAnchorCheckRestored + } + t.Cleanup(func() { restorePFAnchorForReconcile = originalRestore }) + + p := &prog{ + dnsInterceptState: &pfState{}, + pendingTunnelIfaces: []string{"utun9"}, + hasPendingTunnelIfaces: true, + } + p.pfStabilizing.Store(true) + if result := p.reconcilePFAnchorAfterStabilization(); result != pfAnchorCheckRestored { + t.Fatalf("post-stabilization result = %v, want restored", result) + } + if calls != 1 { + t.Fatalf("post-stabilization restore calls = %d, want 1", calls) + } + if !p.pfStabilizing.Load() { + t.Fatal("post-stabilization reconcile released loop ownership") + } + if p.pfBackoffMultiplier.Load() != 0 { + t.Fatalf("post-stabilization reconcile changed backoff to %d", p.pfBackoffMultiplier.Load()) + } +} + +func TestPostStabilizationIntactWithoutPendingAvoidsRebuild(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + originalRestore := restorePFAnchorForReconcile + calls := 0 + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { + calls++ + return pfAnchorCheckRestored + } + t.Cleanup(func() { restorePFAnchorForReconcile = originalRestore }) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + if result := p.reconcilePFAnchorAfterStabilization(); result != pfAnchorCheckIntact { + t.Fatalf("post-stabilization result = %v, want intact", result) + } + if calls != 0 { + t.Fatalf("intact post-stabilization anchor rebuilt %d times", calls) + } + if !p.pfStabilizing.Load() { + t.Fatal("intact post-stabilization reconcile released loop ownership") + } +} + +func TestTunnelRemovalFailureRetriesBeforeCommittingBaseline(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + current := []string{} + discoverTunnelInterfacesForReconcile = func() []string { + return append([]string(nil), current...) + } + calls := 0 + restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult { + calls++ + if calls == 1 { + return pfAnchorCheckFailed + } + p.commitPFReconcileState(current) + return pfAnchorCheckRestored + } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{ + dnsInterceptState: &pfState{}, + lastTunnelIfaces: []string{"utun7"}, + } + if !p.checkTunnelInterfaceChanges() { + t.Fatal("first tunnel removal was not detected") + } + if !stringSlicesEqual(p.lastTunnelIfaces, []string{"utun7"}) { + t.Fatalf("failed removal committed baseline: %v", p.lastTunnelIfaces) + } + if !p.hasPendingTunnelReconcile() { + t.Fatal("failed removal did not retain desired tunnel state for retry") + } + if !p.checkTunnelInterfaceChanges() { + t.Fatal("failed tunnel removal was not retried") + } + if len(p.lastTunnelIfaces) != 0 { + t.Fatalf("successful retry did not commit empty tunnel baseline: %v", p.lastTunnelIfaces) + } + if calls != 2 { + t.Fatalf("restore calls = %d, want 2", calls) + } +} + +func TestPendingTunnelStateRetriesAfterStabilization(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + current := []string{} + discoverTunnelInterfacesForReconcile = func() []string { return nil } + calls := 0 + restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult { + calls++ + p.commitPFReconcileState(current) + return pfAnchorCheckRestored + } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{ + dnsInterceptState: &pfState{}, + lastTunnelIfaces: []string{"utun7"}, + pendingTunnelIfaces: current, + hasPendingTunnelIfaces: true, + } + if !p.checkTunnelInterfaceChanges() { + t.Fatal("pending tunnel removal was not retried after stabilization") + } + if calls != 1 || len(p.lastTunnelIfaces) != 0 || p.hasPendingTunnelReconcile() { + t.Fatalf("pending retry result: calls=%d baseline=%v pending=%v", calls, p.lastTunnelIfaces, p.hasPendingTunnelReconcile()) + } +} + +func TestTunnelReconcileHonorsPFExecBackoff(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + current := []string{} + discoverTunnelInterfacesForReconcile = func() []string { return nil } + calls := 0 + restorePFAnchorForReconcile = func(p *prog, _ string) pfAnchorCheckResult { + calls++ + p.commitPFReconcileState(current) + return pfAnchorCheckRestored + } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7"}} + p.pfExecBackoffUntil.Store(time.Now().Add(time.Minute).UnixMilli()) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("tunnel removal was not detected during PF exec backoff") + } + if calls != 0 || !stringSlicesEqual(p.lastTunnelIfaces, []string{"utun7"}) { + t.Fatalf("PF restore ran during exec backoff: calls=%d baseline=%v", calls, p.lastTunnelIfaces) + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("identical deferred tunnel retry bypassed the ignored-event limiter") + } + p.pfExecBackoffUntil.Store(0) + if !p.checkTunnelInterfaceChanges() || calls != 1 || len(p.lastTunnelIfaces) != 0 { + t.Fatalf("tunnel removal did not retry after backoff: calls=%d baseline=%v", calls, p.lastTunnelIfaces) + } +} + +func TestTunnelRapidReversalClearsUnappliedPendingState(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{ + dnsInterceptState: &pfState{}, + pendingTunnelIfaces: []string{"utun9"}, + hasPendingTunnelIfaces: true, + } + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("rapid tunnel reversal was not observed") + } + if p.hasPendingTunnelReconcile() || len(p.lastTunnelIfaces) != 0 { + t.Fatalf("rapid reversal left unapplied tunnel state: baseline=%v pending=%v", p.lastTunnelIfaces, p.hasPendingTunnelReconcile()) + } +} + +func TestTunnelAdditionIsCoalescedUntilSuccessfulRebuild(t *testing.T) { + originalDiscover := discoverTunnelInterfacesForReconcile + current := []string{"utun9"} + discoverTunnelInterfacesForReconcile = func() []string { + return append([]string(nil), current...) + } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("new tunnel was not detected") + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("identical pending tunnel state was not coalesced") + } + if len(p.lastTunnelIfaces) != 0 { + t.Fatalf("pending tunnel was committed before PF rebuild: %v", p.lastTunnelIfaces) + } + if !p.hasPendingTunnelReconcile() { + t.Fatal("new tunnel was not retained as pending") + } + + p.commitPFReconcileState(current) + if !stringSlicesEqual(p.lastTunnelIfaces, current) { + t.Fatalf("successful rebuild baseline = %v, want %v", p.lastTunnelIfaces, current) + } + if p.hasPendingTunnelReconcile() { + t.Fatal("successful rebuild did not clear pending tunnel state") + } +} + +// TestVPNDNSRefreshDeferredWhileStabilizing covers the ignored network-change path, +// which can trigger a VPN DNS refresh from outside stabilization. +// +// A refresh rebuilds and reloads the pf anchor. Stabilization owns pf while a VPN's +// ruleset is still settling, so refreshing then is the mutual-overwrite collision +// stabilization exists to prevent - and these deltas arrive exactly when a VPN is +// coming up. Deferring is safe: checkTunnelInterfaceChanges keeps the observation +// pending, so the transition is retried afterwards. +// +// The watchdog tick carries the same guard for the same reason; it is not driven here +// because that would mean running its 30s loop. +func TestVPNDNSRefreshDeferredWhileStabilizing(t *testing.T) { + newProg := func(t *testing.T, refreshes *int, tunnels []string) *prog { + t.Helper() + outputs := map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + } + originalCheck := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + output, ok := outputs[strings.Join(args, " ")] + if !ok { + return nil, fmt.Errorf("unexpected pf anchor check command") + } + return []byte(output), nil + } + // Discovery reports no tunnels. With a seeded baseline that is a removal, which + // checkTunnelInterfaceChanges reports as a change without touching pf while + // stabilizing - so this fixture never reaches a real pfctl write. + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { + runPFAnchorCheckCommand = originalCheck + discoverTunnelInterfacesForReconcile = originalDiscover + }) + + vpnDNS := newVPNDNSManager(&mainLog, nil) + vpnDNS.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + *refreshes++ + return nil + } + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: vpnDNS, lastTunnelIfaces: tunnels} + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + return p + } + delta := func() *netmon.ChangeDelta { + return &netmon.ChangeDelta{ + Old: &netmon.State{Interface: map[string]netmon.Interface{}}, + New: &netmon.State{Interface: map[string]netmon.Interface{}}, + } + } + + t.Run("tunnel change during stabilization does not refresh", func(t *testing.T) { + refreshes := 0 + // Seeded baseline plus empty discovery = a tunnel transition to report, so the + // refresh is eligible on everything except the stabilization guard. + p := newProg(t, &refreshes, []string{"utun9"}) + p.pfStabilizing.Store(true) + + p.handleDNSInterceptIgnoredNetworkChange(delta(), time.Unix(1_000_000, 0)) + + if refreshes != 0 { + t.Errorf("refreshed %d time(s) while stabilizing — that rebuilds the anchor under a settling VPN ruleset", refreshes) + } + }) + + t.Run("refresh still happens outside stabilization", func(t *testing.T) { + refreshes := 0 + p := newProg(t, &refreshes, nil) + + p.handleDNSInterceptIgnoredNetworkChange(delta(), time.Unix(1_000_000, 0)) + + if refreshes == 0 { + t.Error("no refresh outside stabilization — the guard must defer, not disable") + } + }) +} + +// TestExemptVPNDNSServersDeferredWhileStabilizing checks the mutation point itself, +// not just the call sites: any future caller reaching it during stabilization is +// refused before the anchor is rewritten. +// +// It returns before pfEnsureRunning is taken and before any pfctl work, so this drives +// the real function without touching the host's pf state. +func TestExemptVPNDNSServersDeferredWhileStabilizing(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + + err := p.exemptVPNDNSServers([]vpnDNSExemption{{Server: "192.168.1.1"}}) + if err == nil { + t.Fatal("exemption applied while stabilizing — that rewrites the anchor under a settling VPN ruleset") + } + if !strings.Contains(err.Error(), "stabilization") { + t.Errorf("error does not name the reason: %v", err) + } + // The refusal must happen before the reconcile latch is claimed, or a deferral + // would lock out the reconcile that runs once stabilization ends. + if p.pfEnsureRunning.Load() { + t.Error("pfEnsureRunning was left held by a deferred exemption") + } +} + +// stubStabilizationProbe replaces the post-stabilization verification seams and returns +// counters for probe and forced-reload calls. +func stubStabilizationProbe(t *testing.T, probeResults []bool, reloadOK bool) (probes, reloads *int) { + t.Helper() + originalProbe, originalReload := probePFInterceptFn, forceReloadPFInterceptFn + t.Cleanup(func() { + probePFInterceptFn, forceReloadPFInterceptFn = originalProbe, originalReload + }) + probeCalls, reloadCalls := 0, 0 + probePFInterceptFn = func(*prog) pfProbeObservation { + result := false + if probeCalls < len(probeResults) { + result = probeResults[probeCalls] + } + probeCalls++ + if result { + return pfProbeObservation{result: pfProbeIntercepted} + } + return pfProbeObservation{result: pfProbeNotIntercepted} + } + forceReloadPFInterceptFn = func(*prog) bool { + reloadCalls++ + return reloadOK + } + return &probeCalls, &reloadCalls +} + +// TestPostStabilizationVerifiesInterceptionFunctionally is the post-wake continuity +// boundary: the reconcile above it only proves rule text, and QA saw rules intact, +// references intact and post-load verification passed while every query through the system +// resolver timed out. Nothing else probes until the periodic watchdog, because the probe +// monitor stands down while stabilization owns pf, so recovery waited for that tick. +func TestPostStabilizationVerifiesInterceptionFunctionally(t *testing.T) { + // Probe fails once, then passes after the reload. + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterStabilization() + + if *probes != 2 { + t.Errorf("probe calls = %d, want 2: one to detect and one to confirm the repair", *probes) + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want exactly 1 bounded repair", *reloads) + } +} + +// TestPostStabilizationProbePassSkipsReload keeps the healthy path free of a pf reload, +// which would flush states and kill in-flight DoH connections for nothing. +func TestPostStabilizationProbePassSkipsReload(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterStabilization() + + if *probes != 1 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 1/0", *probes, *reloads) + } +} + +// TestPostStabilizationRepairIsBounded pins the "one bounded recovery" contract: a probe +// that never passes must not turn into a reload loop here - the watchdog owns retries. +func TestPostStabilizationRepairIsBounded(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{false, false, false}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterStabilization() + + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1: the repair must not loop", *reloads) + } + if *probes != 2 { + t.Errorf("probe calls = %d, want 2", *probes) + } +} + +// TestPostStabilizationWaitsForAProberThatStandsDown is the interleaving that made +// "skip when the flag is set" wrong. A probe monitor started by an ignored network change +// claims functional-probe ownership and then aborts, because stabilization still owns pf. +// If the verifier treats the claimed flag as "somebody is probing", neither path probes and +// the outage lasts until the next watchdog tick - the exact window this is meant to close. +func TestPostStabilizationWaitsForAProberThatStandsDown(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + // Model the monitor's claim-then-abort: ownership is held, then released. + p.pfMonitorRunning.Store(true) + released := make(chan struct{}) + go func() { + time.Sleep(50 * time.Millisecond) + p.pfMonitorRunning.Store(false) + close(released) + }() + + p.verifyInterceptAfterStabilization() + <-released + + if *probes != 2 { + t.Errorf("probe calls = %d, want 2: the verifier must wait out a prober that stands down", *probes) + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } +} + +// TestPostStabilizationYieldsToAProberThatKeepsProbing is the other half of the handoff: +// when the holder is genuinely working through its probe sequence, the verifier must step +// aside rather than run a second prober against the same pf state. +func TestPostStabilizationYieldsToAProberThatKeepsProbing(t *testing.T) { + originalWait := pfFunctionalProbeOwnerWait + pfFunctionalProbeOwnerWait = 30 * time.Millisecond + t.Cleanup(func() { pfFunctionalProbeOwnerWait = originalWait }) + + probes, reloads := stubStabilizationProbe(t, []bool{false}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfMonitorRunning.Store(true) // held for the whole wait + p.verifyInterceptAfterStabilization() + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 while another prober is working", *probes, *reloads) + } +} + +// TestInterceptMonitorDoesNotClaimOwnershipWhileStabilizing pins the source of that race: +// a monitor which cannot do useful work must not take functional-probe ownership on its way +// out, or it starves the post-stabilization verifier. +func TestInterceptMonitorDoesNotClaimOwnershipWhileStabilizing(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, []bool{false}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + if p.interceptProbeMonitorAllowed() { + t.Fatal("the probe monitor considers itself eligible while stabilization owns pf") + } + + p.pfInterceptMonitor() + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 from a monitor that cannot run", *probes, *reloads) + } + if !p.claimFunctionalProbeOwner(0) { + t.Error("the aborted monitor left functional-probe ownership taken; the verifier would skip") + } + p.pfMonitorRunning.Store(false) +} + +// TestFinishPFStabilizationRunsFunctionalVerification wires the fix to the production +// completion path: deleting the verification call, or reordering it before the reconcile, +// makes this fail. +func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) { + stubPFAnchorCheckCommand(t, map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", + }) + originalResolver := initializeOsResolver + initializeOsResolver = func(context.Context, bool, string) []string { return nil } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + p.finishPFStabilization(time.Millisecond) + + if *probes == 0 { + t.Fatal("stabilization completed without probing functional interception; recovery would wait for the watchdog") + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } + + stopPFTestTimers(p) +} + +// stopPFTestTimers cancels every timer a stabilization pass can leave pending. +// +// The whole test binary shares one unsynchronised log sink, so a timer that outlives its +// test logs concurrently with whichever test is running next and trips the race detector +// there - reported against the innocent test. The post-settle follow-up is a 4s timer, so +// the collision lands far from its origin. +func stopPFTestTimers(p *prog) { + p.pfDelayedRecheckMu.Lock() + timers := append([]*time.Timer(nil), p.pfDelayedRecheckTimers...) + p.pfDelayedRecheckTimers = nil + p.pfDelayedRecheckMu.Unlock() + for _, timer := range timers { + timer.Stop() + } + p.stopPFSettleFollowup() +} + +// TestScheduleDNSAfterVPNSettleRefreshIsCancellable pins the tracking itself. An +// untracked follow-up cannot be cancelled by teardown or by this helper, which is how a +// 4s timer escaped its test and raced the log sink of a later one. +func TestScheduleDNSAfterVPNSettleRefreshIsCancellable(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.scheduleDNSAfterVPNSettleRefresh("test", time.Hour) + + p.pfDelayedRecheckMu.Lock() + tracked := p.pfSettleFollowupTimer != nil + p.pfDelayedRecheckMu.Unlock() + if !tracked { + t.Fatal("the follow-up refresh was scheduled without being tracked; nothing can cancel it") + } + + p.stopPFSettleFollowup() + p.pfDelayedRecheckMu.Lock() + cleared := p.pfSettleFollowupTimer == nil + p.pfDelayedRecheckMu.Unlock() + if !cleared { + t.Error("stopPFSettleFollowup left the timer in place") + } +} + +// TestScheduleDNSAfterVPNSettleRefreshReplacesPending keeps VPN churn from stacking +// repeats of the same scutil/VPN-DNS refresh. +func TestScheduleDNSAfterVPNSettleRefreshReplacesPending(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.scheduleDNSAfterVPNSettleRefresh("first", time.Hour) + p.pfDelayedRecheckMu.Lock() + first := p.pfSettleFollowupTimer + p.pfDelayedRecheckMu.Unlock() + + p.scheduleDNSAfterVPNSettleRefresh("second", time.Hour) + p.pfDelayedRecheckMu.Lock() + second := p.pfSettleFollowupTimer + p.pfDelayedRecheckMu.Unlock() + defer p.stopPFSettleFollowup() + + if first == second { + t.Fatal("the second schedule reused the first timer") + } + if first.Stop() { + t.Error("the superseded timer was still armed; stabilization churn would stack refreshes") + } +} + +// ============================================================================= +// post-wake functional verification tests +// ============================================================================= + +// stubWakeProbeSchedule shortens the post-resume retry schedule and neutralises the OS +// resolver refresh, so the bounded-repair contract can be tested without real delays or +// a real scutil call. It returns the number of resolver refreshes performed. +func stubWakeProbeSchedule(t *testing.T, attempts int) (refreshes *int) { + t.Helper() + originalDelays, originalWait := pfWakeProbeDelays, pfWakeProbeOwnerWait + originalResolver := initializeOsResolver + t.Cleanup(func() { + pfWakeProbeDelays, pfWakeProbeOwnerWait = originalDelays, originalWait + initializeOsResolver = originalResolver + }) + + pfWakeProbeDelays = make([]time.Duration, attempts) + for i := range pfWakeProbeDelays { + pfWakeProbeDelays[i] = time.Millisecond + } + pfWakeProbeOwnerWait = 10 * time.Millisecond + + calls := 0 + initializeOsResolver = func(context.Context, bool, string) []string { + calls++ + return []string{"10.0.0.1:53"} + } + return &calls +} + +// TestWakeProbeDetectsInterceptionBrokenBySuspend is the hole this closes. Rule text +// survives a suspend unchanged, so the watchdog reports the anchor intact, and the probe +// monitor is only armed by interface changes with a schedule frozen through the sleep. +// Nothing probed after the resume, and QA measured 18s of no public DNS on a host whose +// link and default route were already back. +func TestWakeProbeDetectsInterceptionBrokenBySuspend(t *testing.T) { + stubWakeProbeSchedule(t, 4) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 2 { + t.Errorf("probe calls = %d, want 2: one to detect and one to confirm the repair", *probes) + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } +} + +// TestWakeProbePassStopsImmediately keeps a healthy resume free of pf work: a reload +// flushes pf state and kills in-flight DoH connections for nothing. +func TestWakeProbePassStopsImmediately(t *testing.T) { + stubWakeProbeSchedule(t, 4) + probes, reloads := stubStabilizationProbe(t, []bool{true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 1 { + t.Errorf("probe calls = %d, want 1: a passing probe must end the schedule", *probes) + } + if *reloads != 0 { + t.Errorf("forced reloads = %d, want 0 on a healthy resume", *reloads) + } +} + +// TestWakeProbeRetriesAcrossTheSchedule covers the observed shape directly: the first +// forced reload did not restore translation and a later one did, so a single attempt is +// not enough. The second repair must still happen, and the confirming pass must end it. +func TestWakeProbeRetriesAcrossTheSchedule(t *testing.T) { + stubWakeProbeSchedule(t, 4) + // attempt 1: probe fails, reload, confirm fails. + // attempt 2: probe fails, reload, confirm passes. + probes, reloads := stubStabilizationProbe(t, []bool{false, false, false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *reloads != 2 { + t.Errorf("forced reloads = %d, want 2: one failed repair must not end the schedule", *reloads) + } + if *probes != 4 { + t.Errorf("probe calls = %d, want 4", *probes) + } +} + +// TestWakeProbeRepairsAreBounded pins the bound. A probe that never passes must exhaust +// the repair budget and then keep probing without reloading, so this can never become the +// unbounded reload loop the issue rules out. +func TestWakeProbeRepairsAreBounded(t *testing.T) { + stubWakeProbeSchedule(t, 6) + probes, reloads := stubStabilizationProbe(t, nil, true) // every probe fails + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *reloads != pfWakeMaxRepairs { + t.Errorf("forced reloads = %d, want the budget of %d", *reloads, pfWakeMaxRepairs) + } + // Two repaired attempts probe twice each; the remaining four probe once each. + if want := 2*pfWakeMaxRepairs + (6 - pfWakeMaxRepairs); *probes != want { + t.Errorf("probe calls = %d, want %d", *probes, want) + } + if p.pfMonitorRunning.Load() { + t.Error("functional-probe ownership was left held after the schedule finished") + } +} + +// TestWakeProbeRefreshesResolverBeforeEveryProbe is why a stale list matters: the probe +// aims at the first OS nameserver and reports success when it has none, so a list still +// holding a pre-sleep VPN resolver turns the probe into a meaningless pass or a false +// failure. The captured run showed the resolver set changing after the resume. +func TestWakeProbeRefreshesResolverBeforeEveryProbe(t *testing.T) { + refreshes := stubWakeProbeSchedule(t, 3) + stubStabilizationProbe(t, nil, false) // probes fail, reload refuses to run + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *refreshes != 3 { + t.Errorf("OS resolver refreshes = %d, want one per attempt (3)", *refreshes) + } +} + +// TestWakeProbeStandsDownForStabilization keeps the two repair paths from fighting over +// pf. Stabilization owns the ruleset while a VPN settles and runs its own functional +// verification when it finishes; reloading underneath it is the mutual overwriting +// stabilization exists to prevent. +func TestWakeProbeStandsDownForStabilization(t *testing.T) { + stubWakeProbeSchedule(t, 4) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 while stabilization owns pf", *probes, *reloads) + } + if !p.claimFunctionalProbeOwner(0) { + t.Error("standing down took functional-probe ownership; the post-stabilization verifier would skip") + } + p.pfMonitorRunning.Store(false) +} + +// TestWakeProbeYieldsToAnotherProberThenRetries is the ownership handoff. A probe monitor +// that is genuinely working owns this attempt, but the schedule must not be spent waiting +// on it: the next attempt still has to probe once ownership is free. +func TestWakeProbeYieldsToAnotherProberThenRetries(t *testing.T) { + stubWakeProbeSchedule(t, 3) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfMonitorRunning.Store(true) + go func() { + time.Sleep(30 * time.Millisecond) + p.pfMonitorRunning.Store(false) + }() + + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes == 0 { + t.Fatal("the schedule gave up entirely because another prober held ownership at first") + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } +} + +// TestWakeProbeSkippedDuringExecBackoff protects the resource-exhaustion guard: each +// probe forks a helper, which is exactly what backoff exists to stop. +func TestWakeProbeSkippedDuringExecBackoff(t *testing.T) { + stubWakeProbeSchedule(t, 3) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfExecBackoffUntil.Store(time.Now().Add(time.Minute).UnixMilli()) + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 during pf exec backoff", *probes, *reloads) + } +} + +// TestWakeProbeSkippedWhenInterceptDisabled keeps the watcher harmless when intercept +// mode is off: the detector runs for the process, the repair is intercept-only. +func TestWakeProbeSkippedWhenInterceptDisabled(t *testing.T) { + stubWakeProbeSchedule(t, 3) + probes, reloads := stubStabilizationProbe(t, []bool{false}, true) + + p := &prog{} + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 with intercept disabled", *probes, *reloads) + } +} + +// TestWakeProbeStopsWhenServiceStops keeps a resume during shutdown from holding the +// schedule open past the stop request. +func TestWakeProbeStopsWhenServiceStops(t *testing.T) { + originalDelays := pfWakeProbeDelays + pfWakeProbeDelays = []time.Duration{0, time.Hour} + t.Cleanup(func() { pfWakeProbeDelays = originalDelays }) + originalResolver := initializeOsResolver + initializeOsResolver = func(context.Context, bool, string) []string { return []string{"10.0.0.1:53"} } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + + probes, _ := stubStabilizationProbe(t, nil, false) // first probe fails, reload refuses + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + + done := make(chan struct{}) + go func() { + p.verifyInterceptAfterWake(18 * time.Second) + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("verifyInterceptAfterWake ignored the stop request and waited out its schedule") + } + if *probes != 1 { + t.Errorf("probe calls = %d, want 1 before the stop was observed", *probes) + } +} + +// ============================================================================= +// post-wake watcher registration tests +// ============================================================================= + +// TestStartInterceptBackgroundWorkRegistersTheWakeVerifier covers the caller wiring. The +// detector and the wake-repair path can both be correct while nothing connects them, and +// then a host sleep is never noticed in production. +func TestStartInterceptBackgroundWorkRegistersTheWakeVerifier(t *testing.T) { + type registration struct { + stopCh <-chan struct{} + onResume func(time.Duration) + } + registered := make(chan registration, 1) + watchdogStarted := make(chan struct{}, 1) + originalWatcher, originalWatchdog := runSuspendWatcherFn, pfWatchdogFn + runSuspendWatcherFn = func(stopCh <-chan struct{}, onResume func(time.Duration)) { + registered <- registration{stopCh: stopCh, onResume: onResume} + } + // Both loops are stubbed rather than run: each would reach pfctl, and stopping the + // real watchdog means closing stopCh, which the wake verifier reads as "service + // stopping" and would abandon its probe schedule before the first probe. + pfWatchdogFn = func(*prog) { watchdogStarted <- struct{}{} } + t.Cleanup(func() { runSuspendWatcherFn, pfWatchdogFn = originalWatcher, originalWatchdog }) + + stubWakeProbeSchedule(t, 4) + probes, _ := stubStabilizationProbe(t, []bool{false, true}, true) + + // Open for the whole test: an active intercept is not stopping. + stopCh := make(chan struct{}) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.startInterceptBackgroundWork() + + select { + case <-watchdogStarted: + case <-time.After(2 * time.Second): + t.Error("no pf watchdog was started; a replaced pf ruleset would never be restored") + } + + var reg registration + select { + case reg = <-registered: + case <-time.After(2 * time.Second): + t.Fatal("no suspend watcher was started; a host sleep would go unnoticed") + } + if reg.stopCh != stopCh { + t.Error("the watcher was not wired to the service stop channel; it would outlive the intercept") + } + + // Prove the callback is the post-wake verifier and not some other function: a + // reported resume has to probe interception. How many probes and repairs that + // schedule is worth belongs to TestWakeProbeDetectsInterceptionBrokenBySuspend; + // pinning it again here would only couple this test to that tuning. + reg.onResume(18 * time.Second) + if *probes == 0 { + t.Error("the resume callback probed nothing; it is not wired to verifyInterceptAfterWake") + } +} + +// TestStartDNSInterceptStartsBackgroundWork closes the caller-wiring gap. Everything the +// privileged install does sits behind installPFInterceptFn, so this drives the real +// startDNSIntercept and checks what a successful start hands off: without the handoff, an +// active intercept runs with no pf watchdog and no post-wake verification, and every +// detector, recovery and registration test still passes. +func TestStartDNSInterceptStartsBackgroundWork(t *testing.T) { + installed := 0 + originalInstall := installPFInterceptFn + installPFInterceptFn = func(*prog) error { installed++; return nil } + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + + watchdogStarted := make(chan struct{}, 1) + watcherStarted := make(chan struct{}, 1) + originalWatchdog, originalWatcher := pfWatchdogFn, runSuspendWatcherFn + pfWatchdogFn = func(*prog) { watchdogStarted <- struct{}{} } + runSuspendWatcherFn = func(<-chan struct{}, func(time.Duration)) { watcherStarted <- struct{}{} } + t.Cleanup(func() { + installPFInterceptFn = originalInstall + discoverTunnelInterfacesForReconcile = originalDiscover + pfWatchdogFn, runSuspendWatcherFn = originalWatchdog, originalWatcher + }) + + p := &prog{ + cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}, + stopCh: make(chan struct{}), + } + if err := p.startDNSIntercept(); err != nil { + t.Fatalf("startDNSIntercept() = %v, want nil", err) + } + if installed != 1 { + t.Errorf("privileged install ran %d times, want 1", installed) + } + if p.dnsInterceptState == nil { + t.Fatal("a successful start did not publish intercept state") + } + + select { + case <-watchdogStarted: + case <-time.After(2 * time.Second): + t.Error("startDNSIntercept did not start the pf watchdog; a replaced pf ruleset would never be restored") + } + select { + case <-watcherStarted: + case <-time.After(2 * time.Second): + t.Error("startDNSIntercept did not start the suspend watcher; a host sleep would go unnoticed") + } +} + +// TestStartDNSInterceptSkipsBackgroundWorkWhenInstallFails keeps the handoff conditional: +// watchers over an intercept that was never installed would repair state nothing owns. +func TestStartDNSInterceptSkipsBackgroundWorkWhenInstallFails(t *testing.T) { + originalInstall := installPFInterceptFn + installPFInterceptFn = func(*prog) error { return errors.New("pf unavailable") } + originalWatchdog, originalWatcher := pfWatchdogFn, runSuspendWatcherFn + pfWatchdogFn = func(*prog) { t.Error("the pf watchdog started after a failed install") } + runSuspendWatcherFn = func(<-chan struct{}, func(time.Duration)) { + t.Error("the suspend watcher started after a failed install") + } + t.Cleanup(func() { + installPFInterceptFn = originalInstall + pfWatchdogFn, runSuspendWatcherFn = originalWatchdog, originalWatcher + }) + + p := &prog{stopCh: make(chan struct{})} + if err := p.startDNSIntercept(); err == nil { + t.Fatal("startDNSIntercept() = nil after the privileged install failed") + } + if p.dnsInterceptState != nil { + t.Error("a failed start published intercept state; the caller would skip its DNS fallback") + } + // The goroutines above are started before this returns if at all, but give a failing + // wiring a moment to report rather than racing the test's end. + time.Sleep(50 * time.Millisecond) +} + +// ============================================================================= +// pf anchor state, tunnel, and wake event tests +// ============================================================================= + +// pfStateCommandsMu guards the answer of the pf and tunnel state reads, because +// a watchdog or stabilization goroutine can read while a test replaces it. +var ( + pfStateCommandsMu sync.Mutex + pfStateCommandFn func(string, ...string) ([]byte, error) +) + +// init keeps the whole test binary away from pfctl and ifconfig. A test that +// needs values stubs them with stubPFStateCommands. +func init() { + pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + pfStateCommandsMu.Lock() + run := pfStateCommandFn + pfStateCommandsMu.Unlock() + if run == nil { + return nil, errors.New("pf state read is not stubbed") + } + return run(name, args...) + } +} + +// stubPFStateCommands answers the pf and tunnel state reads from fixture text, +// keyed by the whole command line. +func stubPFStateCommands(t *testing.T, outputs map[string]string) { + t.Helper() + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + output, ok := outputs[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) +} + +// stubPFStateCommandFn answers the pf and tunnel state reads from run. +func stubPFStateCommandFn(t *testing.T, run func(string, ...string) ([]byte, error)) { + t.Helper() + pfStateCommandsMu.Lock() + original := pfStateCommandFn + pfStateCommandFn = run + pfStateCommandsMu.Unlock() + t.Cleanup(func() { + pfStateCommandsMu.Lock() + pfStateCommandFn = original + pfStateCommandsMu.Unlock() + }) +} + +// pfIntactAnchorRules answers the anchor check of a healthy ruleset. +var pfIntactAnchorRules = map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", +} + +// pfStateFixtures answers the anchor list and status reads with a real ruleset. +var pfStateFixtures = map[string]string{ + "pfctl -sr": pfShowRules, + "pfctl -sn": pfShowNAT, + "pfctl -si": pfStatusEnabled, +} + +// waitForStabilizationExit waits for the stabilization loop to release pf. +func waitForStabilizationExit(t *testing.T, p *prog) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for p.pfStabilizing.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if p.pfStabilizing.Load() { + t.Fatal("the stabilization loop did not observe the closed stop channel") + } +} + +// Test_logPFAnchorListLogsStabilizationStartAndEnd covers the anchor list at the +// two stabilization events. A capture of a VPN connect has to name the anchors +// that were loaded before ctrld waited, and the ones that were loaded after. +func Test_logPFAnchorListLogsStabilizationStartAndEnd(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, pfIntactAnchorRules) + originalResolver := initializeOsResolver + initializeOsResolver = func(context.Context, bool, string) []string { return nil } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + stubStabilizationProbe(t, []bool{true}, true) + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfStartStabilization() + waitForStabilizationExit(t, p) + p.pfStabilizing.Store(true) + p.finishPFStabilization(time.Millisecond) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one stabilization, want 2", len(events)) + } + wantField(t, events[0], "reason", "stabilization_start") + wantField(t, events[0], "journal", true) + wantField(t, events[0], "pf_enabled", true) + wantField(t, events[0], "pf_since", "0 days 02:11:05") + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[1], "reason", "stabilization_end") + wantField(t, events[1], "journal", true) + + for _, prefix := range []string{ + "DNS intercept: VPN connecting", + "DNS intercept: pf stable for", + } { + lines := eventsWithPrefix(t, logs, prefix) + if len(lines) != 1 { + t.Fatalf("got %d lines with prefix %q, want 1", len(lines), prefix) + } + wantField(t, lines[0], "journal", true) + } +} + +// Test_logPFAnchorListReportsAMissingAnchorOnce covers the watchdog state. The +// anchor stays missing for as long as another program holds pf, and one line per +// tick would fill the journal with the same fact. +func Test_logPFAnchorListReportsAMissingAnchorOnce(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + outputs := pfIntactAnchorRules + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + return []byte(outputs[strings.Join(args, " ")]), nil + } + originalRestore := restorePFAnchorForReconcile + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckRestored } + t.Cleanup(func() { + runPFAnchorCheckCommand = original + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}} + for range 3 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("healthy check result = %v, want intact", result) + } + } + intact := eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 1 { + t.Fatalf("got %d intact lines for three healthy checks, want 1", len(intact)) + } + wantField(t, intact[0], "repeats", float64(0)) + + outputs = map[string]string{} + for range 2 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckRestored { + t.Fatalf("wiped check result = %v, want restored", result) + } + } + missing := jsonLogEvents(t, logs, "PF anchor list changed") + if len(missing) != 1 { + t.Fatalf("got %d anchor list events for two wiped checks, want 1", len(missing)) + } + wantField(t, missing[0], "reason", "missing") + wantField(t, missing[0], "level", "warn") + wantField(t, missing[0], "journal", true) + + outputs = pfIntactAnchorRules + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("recovered check result = %v, want intact", result) + } + intact = eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 2 { + t.Fatalf("got %d intact lines after the anchor came back, want 2", len(intact)) + } + // The second wiped check repeated the state of the first one, and that held + // line is the count the recovered line carries. + wantField(t, intact[1], "repeats", float64(1)) +} + +// Test_logPFAnchorListReportsARestoredAnchor covers the restore report. The +// missing event alone leaves a capture without the anchors ctrld ended with. +func Test_logPFAnchorListReportsARestoredAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalReference := ensurePFAnchorReferenceForRestore + originalRebuild := rebuildPFAnchorRulesForReconcile + originalVerify := verifyPFStateFn + ensurePFAnchorReferenceForRestore = func(*prog) error { return nil } + rebuildPFAnchorRulesForReconcile = func(*prog, []vpnDNSExemption) ([]string, error) { return nil, nil } + verifyPFStateFn = func(*prog) bool { return true } + t.Cleanup(func() { + ensurePFAnchorReferenceForRestore = originalReference + rebuildPFAnchorRulesForReconcile = originalRebuild + verifyPFStateFn = originalVerify + }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.restorePFAnchorWithTransportReset("test", false); result != pfAnchorCheckRestored { + t.Fatalf("restore result = %v, want restored", result) + } + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events for one restore, want 1", len(events)) + } + wantField(t, events[0], "reason", "restored") + wantField(t, events[0], "level", "info") + wantField(t, events[0], "journal", true) + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) +} + +// Test_logPFAnchorListMarksAnUnknownAnchorList covers a failed read. pfctl fails +// on a host that revoked the privileges of ctrld, and an empty list there reads +// as a ruleset that holds no anchor at all. +func Test_logPFAnchorListMarksAnUnknownAnchorList(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, nil) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a failed read reported an anchor list: %v", events[0]) + } +} + +// TestTunnelInterfaceChangedNamesTheOwner covers the tunnel event. The #611 +// capture could not tell a VPN tunnel from a mesh client, and the owner is the +// field that names the program whose pf rules compete with the ctrld anchor. +func TestTunnelInterfaceChangedNamesTheOwner(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, map[string]string{"ifconfig -v utun9": ifconfigWindscribeTunnel}) + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun9"} } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7"}} + // Stabilization owns the rebuild, so the event is all this drives. + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("the tunnel set change was not observed") + } + + events := jsonLogEvents(t, logs, "Tunnel interface changed") + if len(events) != 1 { + t.Fatalf("got %d tunnel events for one set change, want 1", len(events)) + } + wantField(t, events[0], "owner", "Windscribe VPN") + wantField(t, events[0], "journal", true) + wantStrings(t, events[0], "added", []string{"utun9"}) + wantStrings(t, events[0], "removed", []string{"utun7"}) + + discovered := eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for one tunnel set, want 1", len(discovered)) + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("the identical pending tunnel set was not coalesced") + } + discovered = eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for an unchanged tunnel set, want 1", len(discovered)) + } +} + +// TestVerifyInterceptAfterWakeReportsTheWake covers the detector source of the +// wake event. The 2026-09-14 capture has no wake line at all, and the resolver +// table after a resume is the state the investigation needs next. +func TestVerifyInterceptAfterWakeReportsTheWake(t *testing.T) { + logs := captureDebugMainLog(t) + stubWakeProbeSchedule(t, 4) + stubStabilizationProbe(t, []bool{true}, true) + now := time.Date(2026, 9, 14, 7, 14, 0, 0, time.UTC) + originalNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = originalNow }) + + p := &prog{ + dnsInterceptState: &pfState{}, + dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }), + } + p.verifyInterceptAfterWake(18 * time.Second) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events for one resume, want 1", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(18000)) + wantField(t, events[0], "journal", true) + if got := p.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } +} + +func wantAnchors(t *testing.T, event map[string]any, want []string) { + t.Helper() + wantStrings(t, event, "anchors", want) +} + +func wantStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + } +} + +// TestIgnoredNetworkChangeJournalsTheInterfaceChange proves that an interface +// that appears or disappears on the ignored path enters the journal with its +// class and its action, so a hypervisor adapter storm is readable after the fact. +func TestIgnoredNetworkChangeJournalsTheInterfaceChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t, "bridge101") + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: newVPNDNSManager(&mainLog, nil)} + p.pfStabilizing.Store(true) + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + stateWith := func(names ...string) *netmon.State { + state := &netmon.State{Interface: map[string]netmon.Interface{}} + for _, name := range names { + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: net.FlagUp}} + } + return state + } + now := time.Unix(1_000_000, 0) + + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0"), New: stateWith("en0", "bridge101")}, now) + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0", "bridge101"), New: stateWith("en0")}, now.Add(time.Second)) + + events := jsonLogEvents(t, logs, "DNS intercept: interface appeared/disappeared — starting interception probe monitor") + if len(events) != 2 { + t.Fatalf("interface events: got %d, want 2: %s", len(events), logs.String()) + } + for i, action := range []string{"added", "removed"} { + wantField(t, events[i], "journal", true) + wantField(t, events[i], "interface", "bridge101") + wantField(t, events[i], "class", "virtual") + wantField(t, events[i], "action", action) + } +} + +// TestTunnelInterfaceChangedLogsOncePerChange covers the retry path and the +// flap back. The reconcile keeps a new tunnel set pending until it succeeds, +// so every retry repeated the same event, and a set that returned to the one +// before it reported nothing at all. +func TestTunnelInterfaceChangedLogsOncePerChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7"} } + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckFailed } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7", "utun9"}} + for range 3 { + p.checkTunnelInterfaceChanges() + } + + events := jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d tunnel events for three retries of one change, want 1", len(events)) + } + wantStrings(t, events[0], "removed", []string{"utun9"}) + + // The tunnel comes back before a reconcile applied its removal. + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7", "utun9"} } + p.checkTunnelInterfaceChanges() + + events = jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 2 { + t.Fatalf("got %d tunnel events after the tunnel came back, want 2", len(events)) + } + wantStrings(t, events[1], "added", []string{"utun9"}) +} + +// TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor covers the wipe that +// follows a restore within ten seconds. That path enters stabilization instead +// of a rebuild, and it left the journal without the wipe that caused it. +func TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, map[string]string{"-sn": ""}) + stubStabilizationProbe(t, []bool{true}, true) + stopCh := make(chan struct{}) + close(stopCh) + + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfLastRestoreTime.Store(time.Now().UnixMilli()) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckDeferred { + t.Fatalf("wipe after restore result = %v, want deferred", result) + } + waitForStabilizationExit(t, p) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one wipe, want 2", len(events)) + } + wantField(t, events[0], "reason", pfAnchorReasonMissing) + wantField(t, events[0], "level", "warn") + wantField(t, events[1], "reason", pfAnchorReasonStabilizationStart) +} + +// TestPFAnchorListParsesTheRulesTheCallerHolds covers the anchor read of the +// watchdog. The watchdog already read both rule sets, and one more pair of +// pfctl processes on the network change path is work the host does not need. +func TestPFAnchorListParsesTheRulesTheCallerHolds(t *testing.T) { + logs := captureDebugMainLog(t) + var reads []string + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + reads = append(reads, key) + output, ok := pfStateFixtures[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), pfRuleDump{ + rules: []byte(pfShowRules), + nat: []byte(pfShowNAT), + }) + + if want := []string{"pfctl -si"}; !slices.Equal(reads, want) { + t.Fatalf("pf state reads = %v, want %v", reads, want) + } + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[0], "pf_enabled", true) +} + +// TestPFAnchorListBacksOffOnExhaustedResources covers a failed anchor read. +// Every other pfctl read feeds the backoff, so the anchor read kept starting +// processes on a host that had none left. +func TestPFAnchorListBacksOffOnExhaustedResources(t *testing.T) { + captureDebugMainLog(t) + stubPFStateCommandFn(t, func(string, ...string) ([]byte, error) { + return nil, errors.New("fork/exec /sbin/pfctl: resource temporarily unavailable") + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + if !p.pfExecBackoffActive() { + t.Fatal("a failed anchor read did not start the pfctl backoff") + } +} + +// TestPFAnchorListMarksATruncatedRuleListUnknown covers an oversized read. A +// reader stops at a line it cannot hold, and the anchors it collected before +// that line read as a ruleset that lost the rest of them. +func TestPFAnchorListMarksATruncatedRuleListUnknown(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{ + rules: []byte(pfShowRules + strings.Repeat("a", 70<<10) + "\n"), + nat: []byte(pfShowNAT), + }) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a truncated read reported an anchor list: %v", events[0]) + } +} + +// TestPFStateOutputIsBounded covers the memory a state read can take. A pfctl +// that prints without end must not grow the daemon. +func TestPFStateOutputIsBounded(t *testing.T) { + out := &boundedBuffer{limit: 8} + if _, err := out.Write([]byte("12345")); err != nil { + t.Fatalf("write below the limit: %v", err) + } + if _, err := out.Write([]byte("67890")); err != nil { + t.Fatalf("write above the limit: %v", err) + } + if !out.exceeded { + t.Fatal("the writer did not report the dropped bytes") + } + if got := out.Len(); got > 8 { + t.Fatalf("the writer kept %d bytes, want at most 8", got) + } +} diff --git a/cmd/cli/dns_intercept_ignored_change_windows_test.go b/cmd/cli/dns_intercept_ignored_change_windows_test.go new file mode 100644 index 00000000..8ea5d5e3 --- /dev/null +++ b/cmd/cli/dns_intercept_ignored_change_windows_test.go @@ -0,0 +1,20 @@ +//go:build windows + +package cli + +import ( + "testing" + "time" +) + +func TestDNSInterceptIgnoredChangeReconcileDueWindowsPreservesImmediateBehavior(t *testing.T) { + p := &prog{} + now := time.Now() + + if !p.dnsInterceptIgnoredChangeReconcileDue(now) { + t.Fatal("first ignored Windows change must reconcile immediately") + } + if !p.dnsInterceptIgnoredChangeReconcileDue(now) { + t.Fatal("Windows ignored changes must not inherit the macOS pf rate limit") + } +} diff --git a/cmd/cli/dns_intercept_lifecycle_windows_test.go b/cmd/cli/dns_intercept_lifecycle_windows_test.go new file mode 100644 index 00000000..55799cd3 --- /dev/null +++ b/cmd/cli/dns_intercept_lifecycle_windows_test.go @@ -0,0 +1,320 @@ +//go:build windows + +package cli + +import ( + "runtime" + "testing" + "time" +) + +// newInterceptTestProg returns a prog with a published intercept state, fake NRPT +// operations already installed, and no WFP engine (engineHandle 0). +// +// The fake is installed here, before anything can inspect registry state, and it is the +// safety boundary - not the empty wfpState. A zero-valued state has owner None, and +// shutdown's None branch sweeps orphaned ctrld rules, so an unfaked stopDNSIntercept would +// reach the production nrptCatchAllRuleExists / removeNRPTCatchAllRule / signalNRPTChange. +// On a host that has ctrld's deterministic key - a developer box, or a CI runner where +// ctrld is installed - that deletes live policy and forces a Group Policy refresh, a +// Dnscache paramchange and a cache flush. A green run on a clean runner proves nothing +// about that. +func newInterceptTestProg(t *testing.T) (*prog, *wfpState, *fakeNRPTOps) { + t.Helper() + f := fakeNRPTOpsForTest(t) + // Prove the fake is in effect before anything can inspect registry state. Asserting + // zero side effects afterwards cannot do that: an uninstalled fake reports zero + // whether it was consulted or bypassed. + requireFakeNRPTOpsInstalled(t, f) + state := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + p := &prog{} + p.dnsInterceptState = state + return p, state, f +} + +// assertNoNRPTSideEffects fails when a lifecycle path wrote NRPT policy or signalled the +// DNS Client. Every test in this file exercises a guard that is supposed to stand down, so +// any registry write or signal here means the guard did not hold - and, without the fake, +// would have hit the host's real policy. +func assertNoNRPTSideEffects(t *testing.T, f *fakeNRPTOps) { + t.Helper() + add, restoreLocal := f.writes() + _, remove, signal, _ := f.counts() + if add != 0 || restoreLocal != 0 || remove != 0 || signal != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, signal = %d, want 0/0/0/0: this path must not write NRPT policy", + add, restoreLocal, remove, signal) + } + if flush := f.flushCount(); flush != 0 { + t.Errorf("flush calls = %d, want 0: this path must not flush the resolver cache", flush) + } +} + +// TestStopDNSInterceptRevokesBeforeTeardown pins the ordering the shutdown/monitor race +// depends on. Teardown deletes our WFP sublayer, and a missing sublayer is precisely what +// the health monitor treats as "our filters were wiped, rebuild everything". Were the +// state revoked only after teardown, a monitor tick inside that window would rebuild the +// intercept during shutdown. +func TestStopDNSInterceptRevokesBeforeTeardown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + + if p.interceptStateRevoked(state) { + t.Fatal("a freshly published intercept state must not read as retired") + } + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + if !p.interceptStateRevoked(state) { + t.Error("state still reads live after shutdown: the monitor and heal flows would keep writing host DNS state") + } + if p.dnsInterceptState != nil { + t.Error("dnsInterceptState survived shutdown") + } + if p.dnsInterceptStopRequested.Load() { + t.Error("stop-requested flag was left set; a later start would see a phantom shutdown") + } +} + +// TestRebuildDNSInterceptRefusedAfterShutdown is the regression test for the reported +// race: SCM stop runs resetDNS -> stopDNSIntercept while the health monitor is mid-tick, +// and the monitor then reaches the rebuild path before the process exits. The rebuild +// must refuse - completing it would re-add the NRPT catch-all and the WFP filters moments +// before ctrld disappears, leaving Windows resolving through a listener that is gone. +// +// That refusal is also what keeps this test safe on a real Windows host: a rebuild that +// did not refuse would run startDNSIntercept and write NRPT policy to the machine +// running the tests. +func TestRebuildDNSInterceptRefusedAfterShutdown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + if got := p.rebuildDNSIntercept(state, "WFP sublayer missing during health check"); got != interceptRebuildRetired { + t.Fatalf("rebuildDNSIntercept() = %v, want interceptRebuildRetired - a post-shutdown rebuild resurrects DNS interception", got) + } + if p.dnsInterceptState != nil { + t.Error("rebuild published new intercept state after shutdown") + } +} + +// TestRebuildDNSInterceptRefusedForReplacedState covers the other stale-owner case: an +// earlier rebuild already replaced the state, so a goroutine still holding the old one +// must not tear down its successor. +func TestRebuildDNSInterceptRefusedForReplacedState(t *testing.T) { + p, old, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + current := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + p.dnsInterceptState = current + + if got := p.rebuildDNSIntercept(old, "WFP sublayer missing during health check"); got != interceptRebuildRetired { + t.Fatalf("rebuildDNSIntercept() = %v, want interceptRebuildRetired for a superseded state", got) + } + if p.dnsInterceptState != any(current) { + t.Error("a superseded state's rebuild replaced the live intercept") + } + if p.interceptStateRevoked(current) { + t.Error("the live state was revoked by a superseded rebuild") + } +} + +// TestRepairMissingWFPStandsDownAfterShutdown checks the monitor's entry point. It must +// not even query WFP for a retired state - the sublayer it looks for is what teardown +// just deleted - and it must tell the monitor goroutine to exit. +func TestRepairMissingWFPStandsDownAfterShutdown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + // Set the handle only after teardown. A fake handle proves the revocation check + // comes first, but must never reach the real WFP calls in cleanupWFPFilters. + state.engineHandle = 1 + + if !p.repairMissingWFP(state) { + t.Error("repairMissingWFP() = false after shutdown; the health monitor would keep running for a dead intercept") + } + if p.dnsInterceptState != nil { + t.Error("repairMissingWFP rebuilt the intercept after shutdown") + } +} + +// TestPendingStopSignalsRevocation covers how a stop avoids waiting: while it is blocked +// on the lifecycle lock it must already read as revoked, so an in-flight NRPT heal +// abandons its probe backoff instead of making the service stop wait it out. A stop that +// waits too long is killed by the Service Control Manager, which cleans up nothing. +func TestPendingStopSignalsRevocation(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + + p.dnsInterceptMu.Lock() + stopped := make(chan struct{}) + go func() { + defer close(stopped) + _ = p.stopDNSIntercept() + }() + + // Wait for the stop to announce itself while it is blocked on the lock. + deadline := time.Now().Add(5 * time.Second) + for !p.dnsInterceptStopRequested.Load() { + if time.Now().After(deadline) { + p.dnsInterceptMu.Unlock() + <-stopped + t.Fatal("stop never announced itself before waiting for the lifecycle lock") + } + runtime.Gosched() + } + if !p.interceptStateRevoked(state) { + t.Error("a pending stop does not read as revoked; the heal flows would keep it waiting") + } + p.dnsInterceptMu.Unlock() + <-stopped + + if p.dnsInterceptState != nil { + t.Error("the pending stop did not tear down the intercept once it acquired the lock") + } +} + +// TestInterceptWaitAbandonsPromptlyOnPendingStop is the bound on how long a stop can be +// delayed by a recovery flow: the heal sequence's waits add up to tens of seconds, and +// each one must end as soon as a stop is pending. +func TestInterceptWaitAbandonsPromptlyOnPendingStop(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + p.dnsInterceptStopRequested.Store(true) + + start := time.Now() + if p.interceptWait(state, 30*time.Second) { + t.Fatal("interceptWait() = true with a stop pending; the caller would carry on writing host DNS state") + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Errorf("interceptWait took %v to notice a pending stop; shutdown would inherit that delay", elapsed) + } +} + +// TestInterceptWaitRunsToCompletionWhileLive guards the other direction: the cancellable +// wait must still actually wait, or the recovery flows lose their backoff. +func TestInterceptWaitRunsToCompletionWhileLive(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + + start := time.Now() + if !p.interceptWait(state, 250*time.Millisecond) { + t.Fatal("interceptWait() = false for a live intercept") + } + if elapsed := time.Since(start); elapsed < 250*time.Millisecond { + t.Errorf("interceptWait returned after %v, want at least 250ms", elapsed) + } +} + +// TestNRPTNeedsCtrldActivation covers the recovery gap that left a machine unfiltered +// until restart: a failed NRPT write clears ownership, and an owner-None tick used to do +// nothing at all, so nothing ever retried the write. +func TestNRPTNeedsCtrldActivation(t *testing.T) { + tests := []struct { + name string + owner nrptRuleOwner + ruleExists bool + want bool + }{ + { + // The reported hole: activation failed, ownership was cleared, and no + // other path re-arms it. In hard mode WFP keeps blocking DNS meanwhile. + name: "no owner retries the failed write", + owner: nrptRuleOwnerNone, + want: true, + }, + { + name: "no owner retries even if a rule is somehow present", + owner: nrptRuleOwnerNone, + ruleExists: true, + want: true, + }, + { + name: "ctrld-owned rule removed externally is re-added", + owner: nrptRuleOwnerCtrld, + want: true, + }, + { + name: "healthy ctrld-owned rule is left alone", + owner: nrptRuleOwnerCtrld, + ruleExists: true, + want: false, + }, + { + // Writing beside external policy would be ambiguous policy, not recovery. + name: "external policy is never overwritten", + owner: nrptRuleOwnerGroupPolicy, + want: false, + }, + { + name: "external policy is never overwritten even with a ctrld rule present", + owner: nrptRuleOwnerGroupPolicy, + ruleExists: true, + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := nrptNeedsCtrldActivation(tc.owner, tc.ruleExists); got != tc.want { + t.Errorf("nrptNeedsCtrldActivation(%v, %v) = %v, want %v", tc.owner, tc.ruleExists, got, tc.want) + } + }) + } +} + +// TestActivateCtrldNRPTFallbackRefusedAfterShutdown guards the worst leftover. A +// catch-all re-added after shutdown points every DNS query on the machine at a listener +// that no longer exists, so nothing resolves at all. Refusing early also keeps this test +// from writing NRPT policy on the machine running it. +func TestActivateCtrldNRPTFallbackRefusedAfterShutdown(t *testing.T) { + p, state, f := newInterceptTestProg(t) + defer assertNoNRPTSideEffects(t, f) + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + if p.activateCtrldNRPTFallback(state, "ctrld-owned rule missing during health check") { + t.Error("activateCtrldNRPTFallback() = true after shutdown: the catch-all would outlive ctrld") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("NRPT owner = %v after a refused fallback, want nrptRuleOwnerNone", owner) + } +} + +// TestAllowHandbackAttemptRateLimits covers the throttle on testing an external +// catch-all. Each attempt takes ctrld's rule out of the way for a probe, so a rule that +// never routes would cost a brief DNS outage on every 30s health tick without this - in +// hard mode a window where WFP blocks DNS and nothing redirects it. +func TestHandbackThrottleIsPerRule(t *testing.T) { + state := &wfpState{stopCh: make(chan struct{})} + now := time.Now() + + if !state.handbackAllowed(now, "{GP-RULE}", nrptHandbackRetryInterval) { + t.Fatal("first handback attempt must be allowed") + } + // Checking alone must not spend the budget: a pre-probe can still abort the attempt + // without disturbing NRPT, and that must not cost the rule its next window. + if !state.handbackAllowed(now, "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("handbackAllowed must not consume the budget by itself") + } + + state.recordHandbackAttempt(now, "{GP-RULE}", nrptHandbackRetryInterval) + if state.handbackAllowed(now.Add(nrptHandbackRetryInterval-time.Second), "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("re-testing the same rule inside the interval must be suppressed") + } + // Group Policy alternating between two names must not erase either one's memory: + // with a single slot every swap costs another removal of the live rule. + if !state.handbackAllowed(now.Add(time.Second), "{OTHER-RULE}", nrptHandbackRetryInterval) { + t.Error("a different rule name means the administrator changed policy: test it now") + } + state.recordHandbackAttempt(now.Add(time.Second), "{OTHER-RULE}", nrptHandbackRetryInterval) + if state.handbackAllowed(now.Add(2*time.Second), "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("testing another rule must not clear the first rule's throttle") + } + + if !state.handbackAllowed(now.Add(2*nrptHandbackRetryInterval), "{GP-RULE}", nrptHandbackRetryInterval) { + t.Error("the same rule must be testable again after the interval") + } +} diff --git a/cmd/cli/dns_intercept_others.go b/cmd/cli/dns_intercept_others.go new file mode 100644 index 00000000..f425bf3b --- /dev/null +++ b/cmd/cli/dns_intercept_others.go @@ -0,0 +1,58 @@ +//go:build !windows && !darwin + +package cli + +import ( + "fmt" + "time" +) + +// startDNSIntercept is not supported on this platform. +// DNS intercept mode is only available on Windows (via WFP) and macOS (via pf). +func (p *prog) startDNSIntercept() error { + return fmt.Errorf("dns intercept: not supported on this platform (only Windows and macOS)") +} + +// stopDNSIntercept is a no-op on unsupported platforms. +func (p *prog) stopDNSIntercept() error { + return nil +} + +// skipInitialDNSReset is Windows-only; other platforms keep the normal reset. +func (p *prog) skipInitialDNSReset() bool { return false } + +// exemptVPNDNSServers is a no-op on unsupported platforms. +func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { + return nil +} + +// ensurePFAnchorActive is a no-op on unsupported platforms. +func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { + return pfAnchorCheckSkipped +} + +// checkTunnelInterfaceChanges is a no-op on unsupported platforms. +func (p *prog) checkTunnelInterfaceChanges() bool { + return false +} + +func (p *prog) dnsInterceptIgnoredChangeReconcileDue(time.Time) bool { + return false +} + +// scheduleDelayedRechecks is a no-op on unsupported platforms. +func (p *prog) scheduleDelayedRechecks() {} + +// pfInterceptMonitor is a no-op on unsupported platforms. +func (p *prog) pfInterceptMonitor() {} + +// reconcileForwardedSources is a no-op on unsupported platforms (macOS-only). +func (p *prog) reconcileForwardedSources() {} + +// cleanupStaleDNSInterceptState is a no-op on unsupported platforms — there is no +// intercept state that can outlive the process here. +func cleanupStaleDNSInterceptState() {} + +// osHealthcheckSuppressed always returns false on non-Windows platforms — +// WFP loopback protect (the trigger for suppression) is Windows-only. +func (p *prog) osHealthcheckSuppressed() bool { return false } diff --git a/cmd/cli/dns_intercept_settle.go b/cmd/cli/dns_intercept_settle.go new file mode 100644 index 00000000..25a8022b --- /dev/null +++ b/cmd/cli/dns_intercept_settle.go @@ -0,0 +1,52 @@ +package cli + +import ( + "context" + + "github.com/Control-D-Inc/ctrld" +) + +var initializeOsResolver = ctrld.InitializeOsResolverWithReason + +// Reasons that the journal reports when the resolver set changes. Each one +// names the code path that read the resolvers of the host. +const ( + osResolverReasonStart = "start" + osResolverReasonWakeProbe = "wake_probe" + osResolverReasonVPNSettle = "vpn_settle" + osResolverReasonDelayedRecheck = "delayed_recheck" +) + +func (p *prog) refreshDNSAfterVPNSettle(reason string) (routes, domainlessServers, exemptions int) { + mainLog.Load().Info().Msgf("DNS intercept: refreshing OS/VPN DNS route state after VPN settle (%s)", reason) + ctx := ctrld.LoggerCtx(context.Background(), mainLog.Load()) + ns := initializeOsResolver(ctx, true, osResolverReasonVPNSettle) + mainLog.Load().Debug().Msgf("DNS intercept: post-settle OS resolver nameservers: %v", ns) + + if p.vpnDNS == nil { + mainLog.Load().Debug().Msg("DNS intercept: post-settle VPN DNS route refresh skipped — manager unavailable") + return 0, 0, 0 + } + + routes, domainlessServers, exemptions = p.vpnDNS.RefreshRoutesOnly() + mainLog.Load().Info().Msgf("DNS intercept: post-settle VPN DNS route refresh completed — %d routes, %d domainless servers, %d exemptions", + routes, domainlessServers, exemptions) + return routes, domainlessServers, exemptions +} + +func vpnDNSExemptionsEqual(a, b []vpnDNSExemption) bool { + if len(a) != len(b) { + return false + } + seen := make(map[vpnDNSExemption]int, len(a)) + for _, ex := range a { + seen[ex]++ + } + for _, ex := range b { + if seen[ex] == 0 { + return false + } + seen[ex]-- + } + return true +} diff --git a/cmd/cli/dns_intercept_settle_test.go b/cmd/cli/dns_intercept_settle_test.go new file mode 100644 index 00000000..ccc9504c --- /dev/null +++ b/cmd/cli/dns_intercept_settle_test.go @@ -0,0 +1,59 @@ +package cli + +import ( + "context" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { + oldInitialize := initializeOsResolver + defer func() { initializeOsResolver = oldInitialize }() + + var initialized []bool + var reasons []string + initializeOsResolver = func(ctx context.Context, force bool, reason string) []string { + initialized = append(initialized, force) + reasons = append(reasons, reason) + return []string{"10.102.26.10:53"} + } + + var exemptionUpdates [][]vpnDNSExemption + p := &prog{} + p.vpnDNS = newVPNDNSManager(&mainLog, func(exemptions []vpnDNSExemption) error { + exemptionUpdates = append(exemptionUpdates, append([]vpnDNSExemption{}, exemptions...)) + return nil + }) + p.vpnDNS.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun4", + Servers: []string{"10.102.26.10"}, + Domains: []string{"bmwgroup.net"}, + }} + } + + routes, domainlessServers, exemptions := p.refreshDNSAfterVPNSettle("test") + + if routes != 1 || domainlessServers != 0 || exemptions != 1 { + t.Fatalf("expected 1 route, 0 domainless servers, 1 exemption, got routes=%d domainless=%d exemptions=%d", + routes, domainlessServers, exemptions) + } + if len(initialized) != 1 || !initialized[0] { + t.Fatalf("expected forced OS resolver refresh once, got %v", initialized) + } + if len(reasons) != 1 || reasons[0] != osResolverReasonVPNSettle { + t.Fatalf("OS resolver reasons = %v, want [%s]", reasons, osResolverReasonVPNSettle) + } + if got := p.vpnDNS.UpstreamForDomain("jira.cc.bmwgroup.net."); len(got) != 1 || got[0] != "10.102.26.10" { + t.Fatalf("expected refreshed VPN DNS route, got %v", got) + } + if len(exemptionUpdates) != 1 || len(exemptionUpdates[0]) != 1 || exemptionUpdates[0][0].Server != "10.102.26.10" { + t.Fatalf("expected one serialized pf exemption update for the late VPN DNS server, got %+v", exemptionUpdates) + } + + p.refreshDNSAfterVPNSettle("test-repeat") + if len(exemptionUpdates) != 1 { + t.Fatalf("unchanged post-settle VPN DNS state rewrote pf: %+v", exemptionUpdates) + } +} diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go new file mode 100644 index 00000000..48db2527 --- /dev/null +++ b/cmd/cli/dns_intercept_windows.go @@ -0,0 +1,3654 @@ +//go:build windows + +package cli + +import ( + "context" + "fmt" + "math/rand" + "net" + "os/exec" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/kardianos/service" + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/registry" + + "github.com/Control-D-Inc/ctrld" +) + +// DNS Intercept Mode — Windows Implementation (WFP) +// +// This file implements DNS interception using Windows Filtering Platform (WFP). +// WFP is a kernel-level network filtering framework that allows applications to +// inspect and modify network traffic at various layers of the TCP/IP stack. +// +// Strategy: +// - Create a WFP sublayer at maximum priority (weight 0xFFFF) +// - Add PERMIT filters (weight 10) for DNS to localhost (ctrld's listener) +// - Add BLOCK filters (weight 1) for all other outbound DNS +// - Dynamically add/remove PERMIT filters for VPN DNS server exemptions +// +// This means even if VPN software overwrites adapter DNS settings, the OS +// cannot reach those DNS servers on port 53 — all DNS must flow through ctrld. +// +// Key advantages over macOS pf: +// - WFP filters are per-process kernel objects — other apps can't wipe them +// - No watchdog or stabilization needed +// - Connection-level filtering — no packet state/return-path complications +// - Full IPv4 + IPv6 support +// +// See docs/wfp-dns-intercept.md for architecture diagrams and debugging tips. + +// WFP GUIDs and constants for DNS interception. +// These are defined by Microsoft's Windows Filtering Platform API. +var ( + // ctrldSubLayerGUID is a unique GUID for ctrld's WFP sublayer. + // Generated specifically for ctrld DNS intercept mode. + ctrldSubLayerGUID = windows.GUID{ + Data1: 0x7a4e5b6c, + Data2: 0x3d2f, + Data3: 0x4a1e, + Data4: [8]byte{0x9b, 0x8c, 0x1d, 0x2e, 0x3f, 0x4a, 0x5b, 0x6c}, + } + + // Well-known WFP layer GUIDs from Microsoft documentation. + // FWPM_LAYER_ALE_AUTH_CONNECT_V4: filters outbound IPv4 connection attempts. + fwpmLayerALEAuthConnectV4 = windows.GUID{ + Data1: 0xc38d57d1, + Data2: 0x05a7, + Data3: 0x4c33, + Data4: [8]byte{0x90, 0x4f, 0x7f, 0xbc, 0xee, 0xe6, 0x0e, 0x82}, + } + // FWPM_LAYER_ALE_AUTH_CONNECT_V6: filters outbound IPv6 connection attempts. + fwpmLayerALEAuthConnectV6 = windows.GUID{ + Data1: 0x4a72393b, + Data2: 0x319f, + Data3: 0x44bc, + Data4: [8]byte{0x84, 0xc3, 0xba, 0x54, 0xdc, 0xb3, 0xb6, 0xb4}, + } + + // FWPM_CONDITION_IP_REMOTE_PORT: condition matching on remote port. + fwpmConditionIPRemotePort = windows.GUID{ + Data1: 0xc35a604d, + Data2: 0xd22b, + Data3: 0x4e1a, + Data4: [8]byte{0x91, 0xb4, 0x68, 0xf6, 0x74, 0xee, 0x67, 0x4b}, + } + // FWPM_CONDITION_IP_REMOTE_ADDRESS: condition matching on remote address. + fwpmConditionIPRemoteAddress = windows.GUID{ + Data1: 0xb235ae9a, + Data2: 0x1d64, + Data3: 0x49b8, + Data4: [8]byte{0xa4, 0x4c, 0x5f, 0xf3, 0xd9, 0x09, 0x50, 0x45}, + } + // FWPM_CONDITION_IP_PROTOCOL: condition matching on IP protocol. + fwpmConditionIPProtocol = windows.GUID{ + Data1: 0x3971ef2b, + Data2: 0x623e, + Data3: 0x4f9a, + Data4: [8]byte{0x8c, 0xb1, 0x6e, 0x79, 0xb8, 0x06, 0xb9, 0xa7}, + } +) + +const ( + // WFP action constants. These combine a base action with the TERMINATING flag. + // See: https://docs.microsoft.com/en-us/windows/win32/api/fwptypes/ne-fwptypes-fwp_action_type + fwpActionFlagTerminating uint32 = 0x00001000 + fwpActionBlock uint32 = 0x00000001 | fwpActionFlagTerminating // 0x00001001 + fwpActionPermit uint32 = 0x00000002 | fwpActionFlagTerminating // 0x00001002 + + // FWP_MATCH_EQUAL is the match type for exact value comparison. + fwpMatchEqual uint32 = 0 // FWP_MATCH_EQUAL + + // FWP_DATA_TYPE constants for condition values. + // Enum starts at FWP_EMPTY=0, so FWP_UINT8=1, etc. + // See: https://learn.microsoft.com/en-us/windows/win32/api/fwptypes/ne-fwptypes-fwp_data_type + fwpUint8 uint32 = 1 // FWP_UINT8 + fwpUint16 uint32 = 2 // FWP_UINT16 + fwpUint32 uint32 = 3 // FWP_UINT32 + fwpByteArray16Type uint32 = 11 // FWP_BYTE_ARRAY16_TYPE + fwpV4AddrMask uint32 = 0x100 // FWP_V4_ADDR_MASK (after FWP_SINGLE_DATA_TYPE_MAX=0xff) + fwpV6AddrMask uint32 = 0x101 // FWP_V6_ADDR_MASK + + // IP protocol numbers. + ipprotoUDP uint8 = 17 + ipprotoTCP uint8 = 6 + + // DNS port. + dnsPort uint16 = 53 + + // FWPM_FILTER_FLAG constants from fwpmtypes.h. + // See: https://learn.microsoft.com/en-us/windows/win32/api/fwpmtypes/ns-fwpmtypes-fwpm_filter0 + // + // FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT (0x08) prevents lower-weight sublayers + // from overriding this filter's PERMIT action ("hard permit"). Used in DNS + // mode to override third-party WFP blocks (e.g., OpenVPN's block-outside-dns). + fwpmFilterFlagClearActionRight uint32 = 0x00000008 + + // WFP error codes from winerror.h, needed to tell "there was nothing to clean + // up" apart from "there was, and we could not remove it" - which is the lockout + // condition itself and must not be logged as success. + // See: https://learn.microsoft.com/en-us/windows/win32/fwp/wfp-error-codes + // + // FWP_E_SUBLAYER_NOT_FOUND: the sub-layer does not exist. + fwpESubLayerNotFound uintptr = 0x80320007 + // FWP_E_IN_USE: the object is referenced by other objects, so it cannot be + // deleted. Whether a sublayer's own filters count as such references is not + // documented - see the UNRESOLVED note on cleanupStaleDNSInterceptState. It is + // handled because if it does occur, deleting the sublayer alone did not clear the + // orphaned enforcement. + fwpEInUse uintptr = 0x8032000A + // FWP_E_DYNAMIC_SESSION_IN_PROGRESS: the call is not allowed from within a + // dynamic session. Returned when deleting an object that was NOT added in a + // dynamic session - i.e. exactly the stale objects this cleanup targets. + fwpEDynamicSessionInProgress uintptr = 0x8032000B + // FWP_E_WRONG_SESSION: the call was made from the wrong session. Returned when + // deleting an object added by a *different* dynamic session, i.e. one owned by + // another live ctrld. + fwpEWrongSession uintptr = 0x8032000C + + // fwpmSessionFlagDynamic is FWPM_SESSION_FLAG_DYNAMIC from fwpmtypes.h. + // + // Every WFP object added through a dynamic session is owned by that session and + // is deleted by the OS when the engine handle closes - including when the process + // exits, crashes, or is killed. ctrld relies on this so its filters can never + // outlive the process that installed them. + // + // This matters most in Firewall Mode: its block-all filters are machine-wide, so + // an orphaned set silently denies outbound traffic for every process on the host + // (browsers, other users, even a replacement ctrld's own API bootstrap) until a + // reboot. Session-scoped ownership makes the OS clean that up for us instead of + // depending on ctrld reaching its own shutdown or startup cleanup path. + // See: https://learn.microsoft.com/en-us/windows/win32/api/fwpmtypes/ns-fwpmtypes-fwpm_session0 + fwpmSessionFlagDynamic uint32 = 0x00000001 +) + +// WFP API structures. These mirror the C structures from fwpmtypes.h and fwptypes.h. +// We define them here because golang.org/x/sys/windows doesn't include WFP types. +// +// IMPORTANT: These struct layouts must match the C ABI exactly (64-bit Windows). +// Field alignment and padding are critical. Any mismatch will cause access violations +// or silent corruption. The layouts below are for AMD64 only. +// If issues arise, verify against the Windows SDK headers with offsetof() checks. + +// fwpmSession0 represents FWPM_SESSION0 for opening a WFP engine handle. +type fwpmSession0 struct { + sessionKey windows.GUID + displayData fwpmDisplayData0 + flags uint32 + txnWaitTimeoutInMSec uint32 + processId uint32 + sid *windows.SID + username *uint16 + kernelMode int32 // Windows BOOL is int32, not Go bool + _ [4]byte // padding to next 8-byte boundary +} + +// fwpmDisplayData0 represents FWPM_DISPLAY_DATA0 for naming WFP objects. +type fwpmDisplayData0 struct { + name *uint16 + description *uint16 +} + +// fwpmSublayer0 represents FWPM_SUBLAYER0 for creating a WFP sublayer. +type fwpmSublayer0 struct { + subLayerKey windows.GUID + displayData fwpmDisplayData0 + flags uint32 + _ [4]byte // padding + providerKey *windows.GUID + providerData fwpByteBlob + weight uint16 + _ [6]byte // padding +} + +// fwpByteBlob represents FWP_BYTE_BLOB for raw data blobs. +type fwpByteBlob struct { + size uint32 + _ [4]byte // padding + data *byte +} + +// fwpmFilter0 represents FWPM_FILTER0 for adding WFP filters. +type fwpmFilter0 struct { + filterKey windows.GUID + displayData fwpmDisplayData0 + flags uint32 + _ [4]byte // padding + providerKey *windows.GUID + providerData fwpByteBlob + layerKey windows.GUID + subLayerKey windows.GUID + weight fwpValue0 + numFilterConds uint32 + _ [4]byte // padding + filterCondition *fwpmFilterCondition0 + action fwpmAction0 + // After action is a union of UINT64 (rawContext) and GUID (providerContextKey). + // GUID is 16 bytes, UINT64 is 8 bytes. Union size = 16 bytes. + rawContext uint64 // first 8 bytes of the union + _rawContextPad uint64 // remaining 8 bytes (unused, for GUID alignment) + reserved *windows.GUID + filterId uint64 + effectiveWeight fwpValue0 +} + +// fwpValue0 represents FWP_VALUE0, a tagged union for filter weights and values. +type fwpValue0 struct { + valueType uint32 + _ [4]byte // padding + value uint64 // union: uint8/uint16/uint32/uint64/pointer +} + +// fwpmFilterCondition0 represents FWPM_FILTER_CONDITION0 for filter match conditions. +type fwpmFilterCondition0 struct { + fieldKey windows.GUID + matchType uint32 + _ [4]byte // padding + condValue fwpConditionValue0 +} + +// fwpConditionValue0 represents FWP_CONDITION_VALUE0, the value to match against. +type fwpConditionValue0 struct { + valueType uint32 + _ [4]byte // padding + value uint64 // union +} + +// fwpV4AddrAndMask represents FWP_V4_ADDR_AND_MASK for subnet matching. +// Both addr and mask are in host byte order. +type fwpV4AddrAndMask struct { + addr uint32 + mask uint32 +} + +// fwpV6AddrAndMask represents FWP_V6_ADDR_AND_MASK for IPv6 subnet matching. +type fwpV6AddrAndMask struct { + addr [16]byte + prefixLength uint8 +} + +// fwpmAction0 represents FWPM_ACTION0 for specifying what happens on match. +// Size: 20 bytes (uint32 + GUID). No padding needed — GUID has 4-byte alignment. +type fwpmAction0 struct { + actionType uint32 + filterType windows.GUID // union: filterType or calloutKey +} + +type nrptRuleOwner uint8 + +const ( + nrptRuleOwnerNone nrptRuleOwner = iota + nrptRuleOwnerCtrld + nrptRuleOwnerGroupPolicy +) + +// wfpState holds the state of the WFP DNS interception filters. +// It tracks the engine handle and all filter IDs for cleanup on shutdown. +// All filter IDs are stored so we can remove them individually without +// needing to enumerate the sublayer's filters via WFP API. +// +// In "dns" mode, engineHandle is 0 (no WFP filters) and only NRPT is active. +// In "hard" mode, both NRPT and WFP filters are active. +// +// The engine handle is opened once at startup and kept for the lifetime +// of the ctrld process. Filter additions/removals happen through this handle. +type wfpState struct { + engineHandle uintptr + filterIDv4UDP uint64 + filterIDv4TCP uint64 + filterIDv6UDP uint64 + filterIDv6TCP uint64 + // Permit filter IDs for localhost traffic (prevent blocking ctrld's own listener). + permitIDv4UDP uint64 + permitIDv4TCP uint64 + permitIDv6UDP uint64 + permitIDv6TCP uint64 + // Dynamic permit filter IDs for VPN DNS server IPs. + vpnPermitFilterIDs []uint64 + // Static permit filter IDs for RFC1918/CGNAT subnet ranges. + // These allow VPN DNS servers on private IPs to work without dynamic exemptions. + subnetPermitFilterIDs []uint64 + // nrptOwner distinguishes a ctrld-created rule from a GP rule that ctrld is + // only observing. Shutdown and recovery must never delete the latter. + nrptOwner nrptRuleOwner + // externalGPRuleName is the GP child key currently routing the catch-all to + // listenerIP. It is diagnostic identity, not an ownership claim. + externalGPRuleName string + // listenerIP is the actual IP address ctrld is listening on (e.g., "127.0.0.1" + // or "127.0.0.2" on AD DC). Used by NRPT rule creation and health monitor to + // ensure NRPT points to the correct address. + listenerIP string + // stopCh is used to shut down the NRPT health monitor goroutine. + stopCh chan struct{} + // mu protects NRPT ownership, externalGPRuleName, loopbackProtectActive, + // loopbackPermitIDs, and engineHandle from concurrent monitor/recovery/stop use. + mu sync.Mutex + // loopbackProtectActive is true when DNS mode has activated a minimal WFP + // session to permit loopback DNS. This counters third-party WFP block filters + // (e.g., OpenVPN's block-outside-dns) that prevent NRPT from routing queries + // to ctrld's listener on 127.0.0.1. See issue #526. + loopbackProtectActive bool + // loopbackPermitIDs stores the filter IDs for the loopback protect permits. + loopbackPermitIDs []uint64 + // nrptRecoveryLimiter prevents repeated Windows policy/Dnscache signaling + // when another agent keeps putting NRPT back into a broken state. + nrptRecoveryLimiter nrptRecoveryLimiter + // handbackAttempts records, per external rule name, when ctrld last removed its own + // catch-all to test whether that rule routes on its own. Protected by mu. + // + // It is a map rather than one (rule, time) pair because Group Policy can alternate + // between two rule names: with a single slot each swap erases the memory of the other + // one, and every swap costs another removal of the live rule. + handbackAttempts map[string]time.Time +} + +// handbackAllowed reports whether ctrld may test external rule ruleName again, without +// recording anything. +// +// Each attempt takes ctrld's rule out of the way for a probe, so a rule that never routes +// would otherwise cost a brief DNS outage on every health tick - in hard mode a window +// where WFP blocks DNS with nothing redirecting it. A different rule name means the +// administrator changed policy, which is worth testing immediately; the same name is held +// off for minInterval. +func (s *wfpState) handbackAllowed(now time.Time, ruleName string, minInterval time.Duration) bool { + s.mu.Lock() + defer s.mu.Unlock() + last, ok := s.handbackAttempts[ruleName] + return !ok || now.Sub(last) >= minInterval +} + +// recordHandbackAttempt spends ruleName's budget. Callers record only once an attempt is +// actually about to disturb NRPT, so a cheap abort - a pre-probe that shows nothing is +// routing - does not cost the rule its next 15 minutes. +func (s *wfpState) recordHandbackAttempt(now time.Time, ruleName string, minInterval time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if s.handbackAttempts == nil { + s.handbackAttempts = make(map[string]time.Time, 2) + } + // Drop entries whose window has passed, so a churning GP store cannot grow this map. + for rule, at := range s.handbackAttempts { + if now.Sub(at) >= minInterval { + delete(s.handbackAttempts, rule) + } + } + s.handbackAttempts[ruleName] = now +} + +func (s *wfpState) nrptPolicyOwner() (nrptRuleOwner, string) { + s.mu.Lock() + defer s.mu.Unlock() + return s.nrptOwner, s.externalGPRuleName +} + +func (s *wfpState) setNRPTPolicyOwner(owner nrptRuleOwner, externalGPRuleName string) { + s.mu.Lock() + defer s.mu.Unlock() + s.nrptOwner = owner + s.externalGPRuleName = externalGPRuleName +} + +// Lazy-loaded WFP DLL procedures. +var ( + fwpuclntDLL = windows.NewLazySystemDLL("fwpuclnt.dll") + procFwpmEngineOpen0 = fwpuclntDLL.NewProc("FwpmEngineOpen0") + procFwpmEngineClose0 = fwpuclntDLL.NewProc("FwpmEngineClose0") + procFwpmSubLayerAdd0 = fwpuclntDLL.NewProc("FwpmSubLayerAdd0") + procFwpmSubLayerDeleteByKey0 = fwpuclntDLL.NewProc("FwpmSubLayerDeleteByKey0") + procFwpmFilterAdd0 = fwpuclntDLL.NewProc("FwpmFilterAdd0") + procFwpmFilterDeleteById0 = fwpuclntDLL.NewProc("FwpmFilterDeleteById0") + procFwpmSubLayerGetByKey0 = fwpuclntDLL.NewProc("FwpmSubLayerGetByKey0") + procFwpmFreeMemory0 = fwpuclntDLL.NewProc("FwpmFreeMemory0") +) + +// Lazy-loaded dnsapi.dll for flushing the DNS Client cache after NRPT changes. +var ( + dnsapiDLL = windows.NewLazySystemDLL("dnsapi.dll") + procDnsFlushResolverCache = dnsapiDLL.NewProc("DnsFlushResolverCache") +) + +// Lazy-loaded userenv.dll for triggering Group Policy refresh so DNS Client +// picks up new NRPT registry entries without waiting for the next GP cycle. +var ( + userenvDLL = windows.NewLazySystemDLL("userenv.dll") + procRefreshPolicyEx = userenvDLL.NewProc("RefreshPolicyEx") +) + +// NRPT (Name Resolution Policy Table) Registry Constants +// +// NRPT tells the Windows DNS Client service where to send queries for specific +// namespaces. We add a catch-all rule ("." matches everything) that directs all +// DNS queries to ctrld's listener (typically 127.0.0.1, but may be 127.0.0.x on AD DC). +// +// This complements the WFP block filters: +// - NRPT: tells Windows DNS Client to send queries to ctrld (positive routing) +// - WFP: blocks any DNS that somehow bypasses NRPT (enforcement backstop) +// +// Without NRPT, WFP blocks outbound DNS but doesn't redirect it — applications +// would just see DNS failures instead of getting answers from ctrld. +const ( + // nrptBaseKey is the GP registry path where Windows stores NRPT policy rules. + nrptBaseKey = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig` + // nrptDirectKey is the local service store path. The DNS Client reads NRPT + // from both locations, but on some machines (including stock Win11) it only + // honors the direct path. This is the same path Add-DnsClientNrptRule uses. + nrptDirectKey = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig` + // nrptDirectRuleName is the key name for the direct service store path. + // The DNS Client requires direct-path rules to use GUID-in-braces format. + // Using a plain name like "CtrldCatchAll" makes the rule visible in + // Get-DnsClientNrptRule but DNS Client won't apply it for resolution + // (Get-DnsClientNrptPolicy returns empty). This is a deterministic GUID + // so we can reliably find and clean up our own rule. + nrptDirectRuleName = `{B2E9A3C1-7F4D-4A8E-9D6B-5C1E0F3A2B8D}` + + // nrptDirectRulePath is the full path of ctrld's local-store catch-all: the + // primary key, written on every start and the only one a failed handback puts + // back. Named once so the writers, the reader and the remover cannot drift. + nrptDirectRulePath = nrptDirectKey + `\` + nrptDirectRuleName +) + +func (p *prog) nrptListenerIP() string { + listenerIP := "127.0.0.1" + if lc := p.cfg.FirstListener(); lc != nil && lc.IP != "" && lc.IP != "0.0.0.0" && lc.IP != "::" { + listenerIP = lc.IP + } + return listenerIP +} + +// skipInitialDNSReset is the first half of GP-rule adoption. postRun normally +// resets adapter DNS before setDNS starts intercept mode; doing that first would +// violate externally managed policy even if startDNSIntercept adopted the GP rule +// a few milliseconds later. This is only a read-only candidate check. The rule is +// not trusted until a DNS Client probe reaches the listener and the same child is +// re-read by startDNSIntercept. +func (p *prog) skipInitialDNSReset() bool { + mode := p.configuredInterceptMode() + if mode != "dns" && mode != "hard" { + return false + } + if ruleName := findMatchingGPNRPTRule(p.nrptListenerIP()); ruleName != "" { + mainLog.Load().Info().Str("rule", ruleName). + Msg("DNS intercept: matching GP-managed NRPT candidate found - preserving adapter DNS until functional verification") + return true + } + return false +} + +// findMatchingGPNRPTRule returns the first non-ctrld GP child that is exactly a +// catch-all for listenerIP. Multiple namespaces or nameservers are deliberately +// rejected: ctrld must not infer exclusive routing from a broader policy shape. +func findMatchingGPNRPTRule(listenerIP string) string { + parent, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return "" + } + names, err := parent.ReadSubKeyNames(-1) + parent.Close() + if err != nil { + return "" + } + for _, name := range names { + if gpNRPTRuleMatches(name, listenerIP) { + return name + } + } + return "" +} + +func gpNRPTRuleMatches(ruleName, listenerIP string) bool { + namespaces, dnsServers, ok := readGPNRPTRule(ruleName) + return ok && isMatchingGPNRPTRule(ruleName, namespaces, dnsServers, listenerIP) +} + +func readGPNRPTRule(ruleName string) ([]string, string, bool) { + if ruleName == "" || strings.EqualFold(ruleName, nrptRuleName) { + return nil, "", false + } + key, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey+`\`+ruleName, registry.QUERY_VALUE) + if err != nil { + return nil, "", false + } + defer key.Close() + namespaces, _, err := key.GetStringsValue("Name") + if err != nil { + return nil, "", false + } + dnsServers, _, err := key.GetStringValue("GenericDNSServers") + if err != nil { + // A malformed external catch-all is still authoritative enough to block + // ctrld from creating a second catch-all; it simply cannot be adopted. + dnsServers = "" + } + return namespaces, dnsServers, true +} + +// findConflictingGPCatchAll reports an administrator-owned catch-all that no +// longer targets ctrld. Adding another GP catch-all beside it would create the +// same ambiguous policy class as a competing local-store rule, so callers leave +// policy untouched and wait for the administrator to restore/remove it. +func findConflictingGPCatchAll(listenerIP string) (string, string) { + parent, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return "", "" + } + names, err := parent.ReadSubKeyNames(-1) + parent.Close() + if err != nil { + return "", "" + } + for _, name := range names { + namespaces, dnsServers, ok := readGPNRPTRule(name) + if !ok || !isExternalGPCatchAll(name, namespaces) { + continue + } + if !isMatchingGPNRPTRule(name, namespaces, dnsServers, listenerIP) { + return name, dnsServers + } + } + return "", "" +} + +// addNRPTCatchAllRule creates an NRPT catch-all rule that directs all DNS queries +// to the specified listener IP. +// +// Windows NRPT has two registry paths with all-or-nothing precedence: +// - GP path: SOFTWARE\Policies\...\DnsPolicyConfig (Group Policy) +// - Local path: SYSTEM\CurrentControlSet\...\DnsPolicyConfig (service store) +// +// If the GP path contains real rules (from IT policy, VPN, MDM, etc.), DNS +// Client enters "GP mode" and ignores ALL local-path rules entirely. An empty GP +// parent key is worse: it still puts DNS Client in GP mode, but contributes no +// usable rule, so our local catch-all is hidden until that empty parent is gone. +// +// Strategy (matching Tailscale's approach): +// - Always write to the local path (baseline for non-domain machines). +// - Check if OTHER software has GP rules. If yes, also write to the GP path +// so our rule isn't invisible. If no, clean our stale GP rules and delete the +// empty GP key to stay in "local mode". +// - After GP writes, call RefreshPolicyEx to activate. +func addNRPTCatchAllRule(listenerIP string) error { + // Always write to local/direct service store path. + if err := writeNRPTRule(nrptDirectRulePath, listenerIP); err != nil { + return fmt.Errorf("failed to write NRPT local path rule: %w", err) + } + + // Check if other software has GP NRPT rules. If so, we must also write + // to the GP path — otherwise DNS Client's "GP mode" hides our local rule. + if otherGPRulesExist() { + mainLog.Load().Info().Msg("DNS intercept: other GP NRPT rules detected — also writing to GP path") + if err := writeNRPTRule(nrptBaseKey+`\`+nrptRuleName, listenerIP); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to write NRPT GP rule (local rule still active if GP clears)") + } + } else { + // No other GP rules — clean our stale GP entry and delete the empty + // GP parent key so DNS Client stays in "local mode". + cleanGPPath() + } + return nil +} + +// restoreNRPTLocalCatchAllRule writes ctrld's catch-all to the local/direct store and +// touches nothing else. It is the only writer ctrld may use while an administrator's +// catch-all is on disk: the handback whose probe found no route, and startup when it must +// install ctrld's own rule beside external policy. +// +// addNRPTCatchAllRule cannot be used there. It writes ctrld's GP-path rule whenever any +// other GP rule exists, and the administrator's catch-all under test is exactly such a +// rule - so "restoring" through it plants CtrldCatchAll beside that child: the second GP +// catch-all that an administrator did not write. +// +// The contract this file keeps, which is narrower than a restore: +// +// - ctrld does not write its GP rule next to an administrator catch-all. +// - This writes back the local GUID key, and only that key. +// - Every other key removeNRPTCatchAllRule deleted stays deleted: the legacy local name +// and ctrld's GP CtrldCatchAll. A GP sibling left behind by an earlier run does not +// come back, so the store after this write is not the store the transition started +// from. +// - What comes back is ownership, not delivery. The branch is reached because the +// post-removal probe failed, and while GP policy is in effect the DNS Client can +// ignore the local key entirely. Nor is the withheld sibling knowably useless: +// isMatchingGPNRPTRule reads only Name and GenericDNSServers, so a child it accepts +// can still be one the DNS Client skips for want of Comment, DisplayName or +// IPSECCARestriction - in which case ctrld's GP rule was the route. +// +// The cost of that contract is that there may be no DNS until the administrator's rule is +// corrected. ctrld logs a warning, keeps ownership, and re-tests the external child after +// nrptHandbackRetryInterval. +func restoreNRPTLocalCatchAllRule(listenerIP string) error { + if err := writeNRPTRule(nrptDirectRulePath, listenerIP); err != nil { + return fmt.Errorf("failed to restore NRPT local path rule: %w", err) + } + return nil +} + +// otherGPRulesExist checks if non-ctrld NRPT rules exist in the GP path. +// When other software (IT policy, VPN, MDM) has GP rules, DNS Client enters +// "GP mode" and ignores ALL local-path rules. +func otherGPRulesExist() bool { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return false // GP key doesn't exist — no GP rules. + } + names, err := k.ReadSubKeyNames(-1) + k.Close() + if err != nil { + return false + } + for _, name := range names { + if name != nrptRuleName { // Not our CtrldCatchAll + return true + } + } + return false +} + +// cleanGPPath removes only ctrld's GP-path rule and deletes the GP parent when +// no rules remain. The return value tells callers whether the parent key was +// actually deleted, which means DNS Client should be signaled once. +// +// Do not leave an empty GP parent behind: Windows treats the parent key itself +// as the policy store boundary, so an empty key can still hide local-path rules. +// removeCtrldGPRule deletes ctrld's GP-store catch-all and nothing else: not the parent +// key, which an administrator rule still occupies, and not the local store. +func removeCtrldGPRule() { + registry.DeleteKey(registry.LOCAL_MACHINE, nrptBaseKey+`\`+nrptRuleName) +} + +func cleanGPPath() bool { + // Delete our specific rule. + registry.DeleteKey(registry.LOCAL_MACHINE, nrptBaseKey+`\`+nrptRuleName) + + // If the GP parent key is now empty, delete it entirely to exit "GP mode". + k, err := registry.OpenKey(registry.LOCAL_MACHINE, nrptBaseKey, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return false // Key doesn't exist — clean state. + } + names, err := k.ReadSubKeyNames(-1) + k.Close() + if err != nil || len(names) > 0 { + if len(names) > 0 { + mainLog.Load().Debug().Strs("remaining", names).Msg("DNS intercept: GP path has other rules, leaving parent key") + } + return false + } + // Empty — delete it to exit "GP mode". + if err := registry.DeleteKey(registry.LOCAL_MACHINE, nrptBaseKey); err == nil { + mainLog.Load().Info().Msg("DNS intercept: deleted empty GP DnsPolicyConfig key (exits GP mode)") + return true + } + return false +} + +// writeNRPTRule writes a single NRPT catch-all rule at the given registry keyPath. +func writeNRPTRule(keyPath, listenerIP string) error { + k, _, err := registry.CreateKey(registry.LOCAL_MACHINE, keyPath, registry.SET_VALUE) + if err != nil { + return fmt.Errorf("failed to create NRPT registry key %q: %w", keyPath, err) + } + defer k.Close() + + // Name (REG_MULTI_SZ): namespace patterns to match. "." = catch-all. + if err := k.SetStringsValue("Name", []string{"."}); err != nil { + return fmt.Errorf("failed to set NRPT Name value: %w", err) + } + // GenericDNSServers (REG_SZ): DNS server(s) to use for matching queries. + if err := k.SetStringValue("GenericDNSServers", listenerIP); err != nil { + return fmt.Errorf("failed to set NRPT GenericDNSServers value: %w", err) + } + // ConfigOptions (REG_DWORD): 0x8 = use standard DNS resolution (no DirectAccess). + if err := k.SetDWordValue("ConfigOptions", 0x8); err != nil { + return fmt.Errorf("failed to set NRPT ConfigOptions value: %w", err) + } + // Version (REG_DWORD): 0x2 = NRPT rule version 2. + if err := k.SetDWordValue("Version", 0x2); err != nil { + return fmt.Errorf("failed to set NRPT Version value: %w", err) + } + // Match the exact fields Add-DnsClientNrptRule creates. The DNS Client CIM + // provider writes these as empty strings; their absence may cause the service + // to skip the rule on some Windows builds. + k.SetStringValue("Comment", "") + k.SetStringValue("DisplayName", "") + k.SetStringValue("IPSECCARestriction", "") + return nil +} + +// removeNRPTCatchAllRule deletes the ctrld NRPT catch-all registry key and +// cleans up the empty parent key if no other NRPT rules remain. +// +// The empty parent cleanup is critical: an empty DnsPolicyConfig key causes +// DNS Client to cache a "no rules" state. On next start, DNS Client ignores +// newly written rules because it still has the cached empty state. By deleting +// the empty parent on stop, we ensure a clean slate for the next start. +func removeNRPTCatchAllRule() error { + // Remove our GUID-named rule from local/direct path. + if err := registry.DeleteKey(registry.LOCAL_MACHINE, nrptDirectRulePath); err != nil { + if err != registry.ErrNotExist { + return fmt.Errorf("failed to delete NRPT local rule: %w", err) + } + } + deleteEmptyParentKey(nrptDirectKey) + // Clean up legacy rules from earlier builds (plain name in direct path, GP path rules). + registry.DeleteKey(registry.LOCAL_MACHINE, nrptDirectKey+`\`+nrptRuleName) + cleanGPPath() + return nil +} + +// deleteEmptyParentKey removes a registry key if it exists but has no subkeys. +func deleteEmptyParentKey(keyPath string) { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, keyPath, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return + } + names, err := k.ReadSubKeyNames(-1) + k.Close() + if err != nil || len(names) > 0 { + return + } + registry.DeleteKey(registry.LOCAL_MACHINE, keyPath) +} + +// nrptCatchAllRuleExists checks whether our NRPT catch-all rule exists +// in either the local or GP path. +func nrptCatchAllRuleExists() bool { + for _, path := range []string{ + nrptDirectRulePath, + nrptBaseKey + `\` + nrptRuleName, + } { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, path, registry.QUERY_VALUE) + if err == nil { + k.Close() + return true + } + } + return false +} + +// refreshNRPTPolicy triggers a machine Group Policy refresh so the DNS Client +// service picks up new/changed NRPT registry entries immediately. Without this, +// NRPT changes only take effect on the next GP cycle (default: 90 minutes). +// +// Uses RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE=1) from userenv.dll. +// See: https://learn.microsoft.com/en-us/windows/win32/api/userenv/nf-userenv-refreshpolicyex +// nrptSignalExecTimeout bounds every helper process the NRPT signalling path shells out +// to. These run while a transition holds nrptTransitionMu, and a service stop takes that +// same lock: "gpupdate /force" against a slow or unreachable domain controller can take +// tens of seconds, which is exactly the Service Control Manager timeout the locking exists +// to avoid. A signal that cannot finish in this window has already failed as a nudge. +const nrptSignalExecTimeout = 10 * time.Second + +// runBoundedNRPTExec runs one signalling helper with a hard deadline and reports its +// combined output. +func runBoundedNRPTExec(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), nrptSignalExecTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if ctx.Err() != nil { + mainLog.Load().Warn().Str("command", name).Str("timeout", nrptSignalExecTimeout.String()). + Msg("DNS intercept: NRPT signalling helper timed out and was killed") + } + return out, err +} + +func runGPUpdate() { + if out, err := runBoundedNRPTExec("gpupdate", "/target:computer", "/force"); err != nil { + mainLog.Load().Debug().Msgf("DNS intercept: gpupdate failed: %v: %s", err, string(out)) + } else { + mainLog.Load().Debug().Msg("DNS intercept: triggered GP refresh via gpupdate") + } +} + +func refreshNRPTPolicy() { + if err := userenvDLL.Load(); err != nil { + mainLog.Load().Debug().Err(err).Msg("DNS intercept: userenv.dll not available, falling back to gpupdate") + runGPUpdate() + return + } + if err := procRefreshPolicyEx.Find(); err != nil { + mainLog.Load().Debug().Err(err).Msg("DNS intercept: RefreshPolicyEx not found, falling back to gpupdate") + runGPUpdate() + return + } + // RefreshPolicyEx(BOOL bMachine, DWORD dwOptions) + // bMachine=1 (TRUE) = refresh computer policy, dwOptions=1 (RP_FORCE) = force refresh. + // This one only asks the policy engine to refresh and returns; it does not wait for a + // domain controller, which is why it is preferred over gpupdate. + ret, _, _ := procRefreshPolicyEx.Call(1, 1) + if ret != 0 { + mainLog.Load().Debug().Msg("DNS intercept: triggered machine GP refresh via RefreshPolicyEx") + } else { + mainLog.Load().Debug().Msg("DNS intercept: RefreshPolicyEx returned FALSE, falling back to gpupdate") + runGPUpdate() + } +} + +// flushDNSCache flushes the Windows DNS Client resolver cache and triggers a +// Group Policy refresh so NRPT changes take effect immediately. +func flushDNSCache() { + refreshNRPTPolicy() + flushDNSCacheOnly() +} + +func flushDNSCacheOnly() { + if err := dnsapiDLL.Load(); err == nil { + if err := procDnsFlushResolverCache.Find(); err == nil { + ret, _, _ := procDnsFlushResolverCache.Call() + if ret != 0 { + mainLog.Load().Debug().Msg("DNS intercept: flushed DNS resolver cache via DnsFlushResolverCache") + return + } + } + } + if out, err := runBoundedNRPTExec("ipconfig", "/flushdns"); err != nil { + mainLog.Load().Debug().Msgf("DNS intercept: ipconfig /flushdns failed: %v: %s", err, string(out)) + } else { + mainLog.Load().Debug().Msg("DNS intercept: flushed DNS resolver cache via ipconfig /flushdns") + } +} + +func signalNRPTChange() { + refreshNRPTPolicy() + sendParamChange() + flushDNSCacheOnly() +} + +// sendParamChange sends SERVICE_CONTROL_PARAMCHANGE to the DNS Client (Dnscache) +// service, signaling it to re-read its configuration including NRPT rules from +// the registry. This is the standard mechanism used by FortiClient, Tailscale, +// and other DNS-aware software — it's reliable and non-disruptive unlike +// restarting the Dnscache service (which always fails on modern Windows because +// Dnscache is a protected shared svchost service). +func sendParamChange() { + if out, err := runBoundedNRPTExec("sc", "control", "dnscache", "paramchange"); err != nil { + mainLog.Load().Debug().Err(err).Str("output", string(out)).Msg("DNS intercept: sc control dnscache paramchange failed") + } else { + mainLog.Load().Debug().Msg("DNS intercept: sent paramchange to Dnscache service") + } +} + +// cleanEmptyNRPTParent removes empty NRPT parent keys that block activation. +// Empty GP and local parents have different failure shapes: +// - empty GP parent: DNS Client is in GP mode and ignores local-path rules; +// - empty local parent: DNS Client can cache an empty local policy store. +// +// This helper only changes registry state. The caller sends the single +// RefreshPolicyEx/paramchange/flush signal after it knows cleanup occurred. +// +// Returns true if cleanup was performed (caller should signal DNS Client). +func cleanEmptyNRPTParent() bool { + // Always clean the GP path — its existence blocks local path activation. + cleaned := cleanGPPath() + + // Clean empty local/direct path parent key. + if !nrptParentKeyEmpty(nrptDirectKey) { + return cleaned + } + + mainLog.Load().Warn().Msg("DNS intercept: found empty NRPT local parent key (blocks activation) — removing") + if err := registry.DeleteKey(registry.LOCAL_MACHINE, nrptDirectKey); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to delete empty NRPT local parent key") + return cleaned + } + return true +} + +func nrptParentKeyEmpty(keyPath string) bool { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, keyPath, registry.ENUMERATE_SUB_KEYS) + if err != nil { + return false + } + names, err := k.ReadSubKeyNames(-1) + k.Close() + return err == nil && len(names) == 0 +} + +// logNRPTParentKeyState logs the state of both NRPT registry paths for diagnostics. +func logNRPTParentKeyState(context string) { + for _, path := range []struct { + name string + key string + }{ + {"GP", nrptBaseKey}, + {"local", nrptDirectKey}, + } { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, path.key, registry.ENUMERATE_SUB_KEYS) + if err != nil { + mainLog.Load().Debug().Str("context", context).Str("path", path.name). + Msg("DNS intercept: NRPT parent key does not exist") + continue + } + names, err := k.ReadSubKeyNames(-1) + k.Close() + if err != nil { + continue + } + if len(names) == 0 { + mainLog.Load().Warn().Str("context", context).Str("path", path.name). + Msg("DNS intercept: NRPT parent key exists but is EMPTY — blocks activation") + } else { + mainLog.Load().Debug().Str("context", context).Str("path", path.name). + Int("subkeys", len(names)).Strs("names", names). + Msg("DNS intercept: NRPT parent key state") + } + } +} + +// startDNSIntercept activates WFP-based DNS interception on Windows. +// It creates a WFP sublayer and adds filters that block all outbound DNS (port 53) +// traffic except to localhost (127.0.0.1/::1), ensuring all DNS queries must go +// through ctrld's local listener. This eliminates the race condition with VPN +// software that overwrites interface DNS settings. +// +// The approach: +// 1. Permit outbound DNS to 127.0.0.1/::1 (ctrld's listener) +// 2. Block all other outbound DNS (port 53 UDP+TCP) +// +// This means even if a VPN overwrites DNS settings to its own servers, +// the OS cannot reach those servers on port 53 — queries fail and fall back +// to ctrld via the loopback address. +func (p *prog) startDNSIntercept() error { + p.dnsInterceptMu.Lock() + defer p.dnsInterceptMu.Unlock() + return p.startDNSInterceptLocked() +} + +// startDNSInterceptLocked is startDNSIntercept with p.dnsInterceptMu already held, so +// the rebuild path can make teardown and re-create one atomic transition. +// +// Nothing it calls may take p.dnsInterceptMu. It runs nrptProbeAndHeal synchronously, +// and that whole family - nrptProbeAndHeal, activateCtrldNRPTFallback, +// tryAdoptMatchingGPNRPT - stays lock-free on purpose and uses interceptStateRevoked +// instead: those flows wait seconds between probes, and holding the lifecycle lock +// across them would make a service stop wait just as long. +func (p *prog) startDNSInterceptLocked() error { + ops := p.nrptOps() + listenerIP := p.nrptListenerIP() + if lc := p.cfg.FirstListener(); lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "::") { + mainLog.Load().Warn().Str("configured_ip", lc.IP). + Msg("DNS intercept: listener configured with wildcard IP, using 127.0.0.1 for NRPT rules") + } + + state := &wfpState{ + stopCh: make(chan struct{}), + listenerIP: listenerIP, + } + // The probe and heal flows take state as an argument rather than reading the + // published field, so nothing is published until startup has fully succeeded. + // Publishing a provisional state would expose a half-built wfpState - no engine + // handle yet, filter IDs still being assigned - to exemptVPNDNSServers, which the + // VPN DNS manager can call at any time. + + mainLog.Load().Info().Msgf("DNS intercept: initializing (mode: %s)", interceptMode) + + logNRPTParentKeyState("pre-write") + + // GP adoption is a two-part proof. Registry shape establishes ownership; the + // DNS Client probe establishes that the policy actually routes to this listener. + // Re-reading the same child after the probe prevents adopting a rule replaced + // during a concurrent Group Policy refresh. + externalProbeOK := false + if ruleName := ops.findGPRule(listenerIP); ruleName != "" { + // Adoption goes through the same handback transition the running service uses. + // A rule left behind by an earlier unclean exit points at this very listener, so + // a probe taken with it still installed proves nothing about the GP rule; the + // transition removes ctrld's keys first, and puts them back if the GP rule + // cannot carry DNS on its own. + switch p.nrptHandbackToExternal(state, ruleName, "startup GP-managed catch-all candidate") { + case nrptHandbackVerified: + externalProbeOK = true + mainLog.Load().Info().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: adopted working GP-managed NRPT catch-all; ctrld will not modify NRPT policy") + case nrptHandbackUnverified: + mainLog.Load().Warn().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT catch-all is present but the probe did not reach ctrld; leaving external policy untouched") + case nrptHandbackKeptCtrld: + mainLog.Load().Warn().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT catch-all could not carry DNS alone; keeping the ctrld-owned rule from the previous run") + case nrptHandbackConflict: + // The candidate turned out to target another resolver. Startup then refuses to + // write a competing rule below, which is a hard startup failure by design. + mainLog.Load().Error().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP catch-all does not target ctrld; refusing to write a competing NRPT rule") + case nrptHandbackAborted: + // Undecided, not decided: nothing was proved about the candidate and no + // ownership was recorded. Say so, because the fall-through below writes + // ctrld's rule, and an unlogged fall-through here is indistinguishable from + // "no external policy exists". + mainLog.Load().Warn().Str("rule", ruleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT candidate could not be tested at startup; continuing without external ownership") + } + } + + owner, _ := state.nrptPolicyOwner() + if owner == nrptRuleOwnerNone { + // No working external ownership contract exists. Preserve the current ctrld + // path unless another GP catch-all already owns the namespace; writing a + // second catch-all would create an ambiguous policy rather than recovery. + if gpCatchAllConflictBlocksFallback(state, "startup GP catch-all does not target ctrld") { + return fmt.Errorf("dns intercept: conflicting GP NRPT catch-all targets another resolver") + } + // A matching candidate that is still there means the handback above came back + // undecided - typically because the pre-probe ran while the DNS Client was still + // settling at boot. Give it one more pass before writing anything: the DNS Client + // has had the startup work since, and a decision here avoids writing beside an + // administrator rule that no probe has tested. + if ruleName := ops.findGPRule(listenerIP); ruleName != "" { + switch p.nrptHandbackToExternal(state, ruleName, "startup retry of an undecided GP candidate") { + case nrptHandbackVerified: + externalProbeOK = true + case nrptHandbackUnverified, nrptHandbackConflict, nrptHandbackKeptCtrld: + // Ownership is recorded by the transition (or ctrld's own rule was put + // back), so the write below is neither needed nor safe. + } + } + } + + owner, _ = state.nrptPolicyOwner() + if owner == nrptRuleOwnerNone { + exists := ops.ruleExists() + if exists && ops.findGPRule(listenerIP) != "" { + // An earlier run left ctrld's GP-store catch-all beside an administrator + // catch-all, and adopting would inherit it: nothing downstream removes it. + // Every handback from here stops at the pre-probe while that rule is + // unproven, the monitor calls removeRule only inside a handback, and the + // orphan sweep runs at stop - so the second catch-all would stand for the + // whole process lifetime, through a DNS outage. Drop it before adopting, + // then re-read: ruleExists is also true for the sibling alone, and a store + // holding nothing else must fall through to the writer below. + ops.dropGPRule() + exists = ops.ruleExists() + } + if exists { + // A rule from an earlier run already points at this listener. Adopt it rather + // than writing a second time: with a GP child present, addNRPTCatchAllRule + // would also write ctrld's GP-path sibling. + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + mainLog.Load().Info().Str("listener", listenerIP). + Msg("DNS intercept: adopting the ctrld NRPT catch-all already present from an earlier run") + } else { + // Which writer is allowed here is decided the same way the handback decides + // it: with an administrator catch-all on disk, addNRPTCatchAllRule would + // place ctrld's GP rule beside it, so only the local key may be written. + // + // This is not a hypothetical branch. The sibling cleanup above falls into it + // whenever the store held nothing but ctrld's GP key - having just deleted + // that key, the full writer would put it straight back, next to the very + // rule it was removed from. + write := ops.addRule + if ops.findGPRule(listenerIP) != "" { + write = ops.restoreLocalRule + } + if ops.cleanParent() { + ops.signal() + } + if err := write(listenerIP); err != nil { + return fmt.Errorf("dns intercept: failed to add NRPT catch-all rule: %w", err) + } + logNRPTParentKeyState("post-write") + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + ops.signal() + mainLog.Load().Info().Msgf("DNS intercept: NRPT catch-all rule active - all DNS queries directed to %s", listenerIP) + } + } + + // In hard mode, also set up WFP filters to block non-local DNS. + if hardIntercept { + if err := ops.startWFP(state); err != nil { + owner, _ := state.nrptPolicyOwner() + if owner == nrptRuleOwnerGroupPolicy && externalProbeOK { + // A GP rule the probe proved is routing keeps DNS flowing through ctrld + // even with no WFP filters, and rewriting adapter DNS would violate that + // external policy - so this is not a fall-back-to-adapter-DNS failure. + // + // It is still a hard-mode enforcement gap: with no block filters, raw DNS + // to a public resolver, DoH clients and apps with their own resolver are + // not filtered at all. Publish the state and start the health monitor so + // repairMissingWFP keeps retrying WFP instead of the process running + // unenforced for its whole life on one error line. Ownership stays with + // Group Policy so the monitor never writes NRPT policy here. + mainLog.Load().Error().Err(err). + Msg("DNS intercept: WFP setup failed while GP-managed NRPT is verified routing - DNS resolves through ctrld but hard-mode enforcement is OFF; retrying WFP in the background") + p.dnsInterceptState = state + go p.nrptHealthMonitor(state) + // The service start is still reported as failed (setDnsOK stays false in + // setDNS): a hard-mode process with no block filters must not read as a + // healthy start, even though name resolution works. + return fmt.Errorf("dns intercept: WFP setup failed: %w: %w", err, errGPNRPTVerified) + } + if owner == nrptRuleOwnerCtrld { + mainLog.Load().Error().Err(err).Msg("DNS intercept: WFP setup failed, rolling back ctrld-owned NRPT") + _ = ops.removeRule() + ops.flush() + } else { + mainLog.Load().Error().Err(err).Msg("DNS intercept: WFP setup failed; leaving GP-managed NRPT untouched") + } + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + return fmt.Errorf("dns intercept: WFP setup failed: %w", err) + } + } else { + mainLog.Load().Info().Msg("DNS intercept: dns mode — NRPT only, no WFP filters (graceful)") + // Proactively add loopback WFP permit filters to protect the NRPT + // → 127.0.0.1 path from third-party DNS block filters (e.g., OpenVPN's + // block-outside-dns). These are narrowly scoped (port 53 to localhost + // only) and use CLEAR_ACTION_RIGHT to override any block from other + // sublayers. Adding them at startup eliminates the DNS outage window + // that would otherwise occur between VPN connect and reactive activation. + if err := ops.loopback(state); err != nil { + // Non-fatal: loopback protect is a defense-in-depth measure. + // NRPT still works when no third-party WFP blocks are present. + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to activate proactive loopback WFP protect — will retry on probe failure") + } + } + + owner, externalRuleName := state.nrptPolicyOwner() + if owner == nrptRuleOwnerGroupPolicy && !externalProbeOK { + // The first probe ran before loopback WFP protection existed. Verify once + // more synchronously after WFP setup so service readiness does not race an + // async proof; this path is ownership-aware and never mutates GP NRPT. + externalProbeOK = p.nrptProbeAndHeal(state) + } + + // Everything the host needs is in place: publish, then start the goroutines that + // keep it that way. They receive state directly, so this ordering is only about + // when the rest of ctrld may observe intercept mode as active. + p.dnsInterceptState = state + go p.nrptHealthMonitor(state) + + owner, _ = state.nrptPolicyOwner() + if owner == nrptRuleOwnerCtrld { + // ctrld-owned policy keeps the existing asynchronous activation/heal path. + go p.nrptProbeAndHeal(state) + } + + if owner == nrptRuleOwnerGroupPolicy && !externalProbeOK { + // External policy owns the namespace and neither synchronous proof reached ctrld. + // The recovery state and monitor stay up - the rule may start routing once the DNS + // Client settles, and only external policy may fix it - but this start is not + // ready: the DNS Client is not delivering queries to ctrld, adapter DNS was + // deliberately preserved, and no owned fallback may be written beside an + // administrator's catch-all. Reporting success here would publish readiness while + // nothing is filtering, and in hard mode WFP is simultaneously blocking every + // other resolver, which is an outage rather than degraded health. + mainLog.Load().Error().Str("rule", externalRuleName).Str("listener", listenerIP). + Msg("DNS intercept: GP-managed NRPT owns the namespace but no probe reached ctrld; leaving adapter DNS untouched and reporting a failed start until a probe succeeds") + return fmt.Errorf("dns intercept: %w (rule %q)", errGPNRPTIneffective, externalRuleName) + } + + return nil +} + +// removeOrphanedCtrldNRPTRule deletes a ctrld-owned NRPT catch-all that no running +// ctrld is backing. It is safe against external policy: the rule is found by ctrld's +// own deterministic GUID, never by shape. +// +// Without this, one unclean exit can strand a rule that later takes the whole machine +// off DNS. ctrld dies without a stop while it owns policy, so the GUID rule stays in +// the local store pointing at 127.0.0.1. The org then deploys a GP catch-all: from that +// point every start adopts the GP rule and every stop takes the GP branch, so nothing +// ever looks at the local store - including the stop during uninstall. The orphan stays +// invisible, because any rule in the GP store puts the DNS Client in GP mode where the +// local store is ignored entirely. When the admin eventually removes the GP rule - +// most plausibly while cleaning up after uninstalling ctrld - the DNS Client leaves GP +// mode, reads the local store again, and every query on the machine goes to a listener +// that has not existed for months. It is also miserable to diagnose: local-store rules +// do not appear in Get-DnsClientNrptPolicy, so the standard tooling reports no policy +// at all while nothing resolves. +func (p *prog) removeOrphanedCtrldNRPTRule(reason string) { + ops := p.nrptOps() + if !ops.ruleExists() { + return + } + mainLog.Load().Warn().Str("reason", reason). + Msg("DNS intercept: removing orphaned ctrld NRPT catch-all left by an earlier run") + if err := ops.removeRule(); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to remove orphaned ctrld NRPT catch-all") + return + } + ops.signal() +} + +// cleanupStaleDNSInterceptState removes ctrld-owned WFP objects left behind by a +// previous process. It runs at startup before anything that needs network access. +// +// Filters installed by a ctrld that predates session-scoped ownership (or by any +// process whose session was not dynamic) survive that process's death. In Firewall +// Mode those include machine-wide block-all filters, so the whole host - browsers, +// other users, and a replacement ctrld's own API bootstrap - stays denied outbound +// traffic until a reboot. A replacement that cannot reach the API never finishes +// preflight, so it never reaches the cleanup inside startWFPFilters and the host +// stays locked out. Deleting the sublayer here is the attempt to break that deadlock +// before the first API call. +// +// RELEASE GATE: whether deleting the sublayer is sufficient must be answered on a live +// host that carries real pre-fix orphaned filters - run this binary there and record the +// HRESULT. If it is FWP_E_IN_USE, the delete did not remove the child filters, the block-all +// stays active, and this needs the enumerate-and-delete fallback described below before the +// self-heal can be claimed to work. +// +// UNRESOLVED until then: whether deleting the sublayer is sufficient. It is sufficient only if +// the delete also removes the filters inside it. FwpmSubLayerDeleteByKey0's Remarks +// say nothing about child filters either way, while Object Management states the +// general rule that "an object cannot be deleted until all objects that reference it +// have first been deleted" - and FWP_E_IN_USE exists for exactly that. Whether a +// filter's subLayerKey counts as such a reference is not documented, and this is +// Windows-only syscall code that cannot be exercised off-Windows, so it is not +// asserted here in either direction. What the code does is safe under both readings: +// it attempts the delete and, on FWP_E_IN_USE, says so instead of reporting success, +// so a support log distinguishes "cleared it" from "could not clear it". If a live +// Windows check shows FWP_E_IN_USE against orphaned filters, this needs to enumerate +// and delete them first - which is deliberately not written blind, because +// FWPM_FILTER_ENUM_TEMPLATE0 has no sublayer field, so selecting ctrld's own filters +// means reading subLayerKey at a computed offset in FWPM_FILTER0, and getting that +// offset wrong would delete other software's filters. +// +// Deleting objects owned by a live session is expected to fail; those are cleaned up +// by the OS when that session closes. Every failure is therefore debug-level: no +// stale state is the normal case. +// +// Two guards decide whether this may run at all. Elevation, because opening a WFP engine +// and deleting ctrld's sublayer must not be reachable from an unprivileged local process: +// FwpmEngineOpen0 is expected to fail without elevation, but that is a documented +// property of the API rather than something this code checks, and it is the only barrier. +// And interactive invocation, because a live ctrld built before session-scoped ownership +// holds a *non-dynamic* sublayer - exactly what this targets - so the HRESULTs cannot +// tell that service apart from an orphan. A service start is not interactive, so the +// deadlock case still gets cleaned; a hand-run "ctrld run" alongside a live service does +// not strip its enforcement. That second guard requires positive evidence that nothing is +// live: an SCM that cannot be queried is not an absent service, so an unknown answer skips +// the cleanup exactly as a running one does. +func cleanupStaleDNSInterceptState() { + if elevated, err := staleCleanupElevatedFn(); err != nil || !elevated { + mainLog.Load().Debug().Err(err).Msg("DNS intercept: skipping stale WFP state cleanup - not elevated") + return + } + interactive := staleCleanupInteractiveFn() + liveness := serviceLivenessStopped + if interactive { + liveness = staleCleanupLivenessFn() + } + if !staleCleanupAllowed(interactive, liveness) { + reason := "the ctrld service is running" + if liveness == serviceLivenessUnknown { + reason = "the ctrld service's state could not be determined" + } + mainLog.Load().Info().Msgf("DNS intercept: skipping stale WFP state cleanup - %s and a pre-session-scoped build's sublayer is indistinguishable from an orphan", reason) + return + } + deleteStaleWFPSublayerFn() +} + +// Seams for the guard decision and the deletion it protects. The delete is a WFP syscall +// that cannot be run in a test even on Windows - it would remove real filters - so the +// only way to assert that the guard is consulted *before* anything is deleted is to +// substitute both ends. +var ( + staleCleanupElevatedFn = hasElevatedPrivilege + staleCleanupInteractiveFn = service.Interactive + staleCleanupLivenessFn = ctrldServiceLiveness + deleteStaleWFPSublayerFn = deleteStaleWFPSublayer +) + +// deleteStaleWFPSublayer opens a WFP engine and deletes ctrld's sublayer. Callers must +// have established that no live ctrld owns it; see cleanupStaleDNSInterceptState. +func deleteStaleWFPSublayer() { + var engineHandle uintptr + session := fwpmSession0{} + sessionName, _ := windows.UTF16PtrFromString("ctrld Stale State Cleanup") + session.displayData.name = sessionName + // Deliberately NOT a dynamic session. FwpmSubLayerDeleteByKey0 is documented to + // fail with FWP_E_DYNAMIC_SESSION_IN_PROGRESS when called from a dynamic session + // for an object that was not added in one - and the only orphans that can exist + // are exactly those: a ctrld built before session-scoped ownership added its + // sublayer statically. Anything a *newer* ctrld leaves behind is removed by the OS + // when its session ends, so there is nothing for this to clean there. Opening this + // session dynamically would therefore make the whole cleanup a no-op in the one + // case it exists for. + // + // What protects a live ctrld is documented ownership, not the child-filter question + // above: a session-scoped ctrld's sublayer belongs to a different dynamic session, so + // the delete fails with FWP_E_WRONG_SESSION. A ctrld that predates session scoping has + // no such protection here - it holds a non-dynamic sublayer, indistinguishable from an + // orphan - which is why the caller refuses to run this from an interactive invocation + // unless the service is provably not live. FWP_E_IN_USE may refuse the delete as well, + // but whether it does is the open question, so nothing relies on it. + // + // This session adds no objects, so it needs no automatic teardown of its own. + + const rpcCAuthnDefault = 0xFFFFFFFF + r1, _, _ := procFwpmEngineOpen0.Call( + 0, + uintptr(rpcCAuthnDefault), + 0, + uintptr(unsafe.Pointer(&session)), + uintptr(unsafe.Pointer(&engineHandle)), + ) + if r1 != 0 { + mainLog.Load().Debug().Msgf("DNS intercept: could not open WFP engine for stale state cleanup (HRESULT 0x%x)", r1) + return + } + defer procFwpmEngineClose0.Call(engineHandle) + + r1, _, _ = procFwpmSubLayerDeleteByKey0.Call( + engineHandle, + uintptr(unsafe.Pointer(&ctrldSubLayerGUID)), + ) + switch r1 { + case 0: + // Warn, not info: this means a previous ctrld left machine-wide enforcement + // installed, which is worth seeing in a support log. + mainLog.Load().Warn().Msg("DNS intercept: removed WFP filters left by a previous ctrld process (including any Firewall Mode block-all) before startup") + case fwpESubLayerNotFound: + // The normal case: nothing was left behind. + mainLog.Load().Debug().Msg("DNS intercept: no stale WFP state from a previous process") + default: + // Something is installed under our GUID and we could not remove it. Never + // report this as "nothing to clean up": if it is orphaned enforcement, this is + // the lockout condition, and the operator needs the code to act on. + // FWP_E_WRONG_SESSION means another live ctrld owns it, which is benign. + // FWP_E_IN_USE is the answer the release gate above is waiting for: the delete was + // refused while something still references the sublayer, which would mean the + // enumerate-and-delete fallback is required before this self-heal works at all. + mainLog.Load().Warn().Msgf("DNS intercept: found WFP state under ctrld's sublayer but could not remove it (%s); if ctrld is not already running, enforcement from a previous process may still be active", + wfpDeleteErrString(r1)) + } +} + +// staleCleanupAllowed reports whether the stale-state cleanup may run. +// +// A service start (not interactive) always may: that is the deadlock this exists to break, +// and nothing else of ctrld's is live at that point in startup. +// +// A hand-run "ctrld run" may only proceed on positive evidence that nothing is live. A +// pre-session-scoped build's sublayer is indistinguishable from an orphan, so cleaning +// while that service runs would strip its enforcement. Unknown is therefore treated like +// running, not like stopped - an SCM that cannot be queried is not an absent service. +func staleCleanupAllowed(interactive bool, liveness serviceLiveness) bool { + if !interactive { + return true + } + return liveness == serviceLivenessStopped +} + +// wfpDeleteErrString names the delete failures that carry a specific meaning for stale +// state, so a support log says which case was hit rather than only a raw HRESULT. +func wfpDeleteErrString(r1 uintptr) string { + switch r1 { + case fwpEInUse: + return "FWP_E_IN_USE: refused while the sublayer is still referenced (see the release gate at cleanupStaleDNSInterceptState)" + case fwpEDynamicSessionInProgress: + return "FWP_E_DYNAMIC_SESSION_IN_PROGRESS: cannot delete a non-dynamic object from a dynamic session" + case fwpEWrongSession: + return "FWP_E_WRONG_SESSION: owned by another live session" + default: + return fmt.Sprintf("HRESULT 0x%x", r1) + } +} + +// startWFPFilters opens the WFP engine and adds all block/permit filters. +// Called only in hard intercept mode. +func (p *prog) startWFPFilters(state *wfpState) error { + mainLog.Load().Info().Msg("DNS intercept: initializing Windows Filtering Platform (WFP)") + + var engineHandle uintptr + session := fwpmSession0{} + sessionName, _ := windows.UTF16PtrFromString("ctrld DNS Intercept") + session.displayData.name = sessionName + // Session-scoped ownership: if this process dies without running its shutdown + // path, Windows removes our filters (including Firewall Mode's machine-wide + // block-all) instead of leaving the host enforced by a dead ctrld. + session.flags = fwpmSessionFlagDynamic + + // RPC_C_AUTHN_DEFAULT (0xFFFFFFFF) lets the system pick the appropriate + // authentication service. RPC_C_AUTHN_NONE (0) returns ERROR_NOT_SUPPORTED + // on some Windows configurations (e.g., Parallels VMs). + const rpcCAuthnDefault = 0xFFFFFFFF + r1, _, _ := procFwpmEngineOpen0.Call( + 0, + uintptr(rpcCAuthnDefault), + 0, + uintptr(unsafe.Pointer(&session)), + uintptr(unsafe.Pointer(&engineHandle)), + ) + if r1 != 0 { + return fmt.Errorf("FwpmEngineOpen0 failed: HRESULT 0x%x", r1) + } + mainLog.Load().Info().Msgf("DNS intercept: WFP engine opened (handle: 0x%x, session-scoped)", engineHandle) + + // Clean up a sublayer left over from an earlier session. + // + // Note this runs on the dynamic session opened above, and + // FwpmSubLayerDeleteByKey0 is documented to fail with + // FWP_E_DYNAMIC_SESSION_IN_PROGRESS from a dynamic session when the object was not + // added in one. So this cannot clear state left by a build that predates + // session-scoped ownership - cleanupStaleDNSInterceptState(), which opens a + // non-dynamic session at startup, is what handles that. What remains for this call + // is a leftover from a previous dynamic session whose teardown had not completed + // when we opened ours. + r1, _, _ = procFwpmSubLayerDeleteByKey0.Call( + engineHandle, + uintptr(unsafe.Pointer(&ctrldSubLayerGUID)), + ) + if r1 == 0 { + mainLog.Load().Info().Msg("DNS intercept: cleaned up stale WFP sublayer from previous session") + } + // A non-zero r1 is not necessarily "nothing to clean up": it is also + // FWP_E_SUBLAYER_NOT_FOUND (the normal case), FWP_E_WRONG_SESSION for a sublayer a + // live ctrld owns, or FWP_E_DYNAMIC_SESSION_IN_PROGRESS for a non-dynamic one this + // dynamic session may not delete. None of them need handling here: the add below + // fails cleanly if the sublayer really is still present, and the non-dynamic case is + // what cleanupStaleDNSInterceptState handles at startup. + + sublayer := fwpmSublayer0{ + subLayerKey: ctrldSubLayerGUID, + weight: 0xFFFF, + } + sublayerName, _ := windows.UTF16PtrFromString("ctrld DNS Intercept Sublayer") + sublayerDesc, _ := windows.UTF16PtrFromString("Blocks outbound DNS except to ctrld listener. Prevents VPN DNS conflicts.") + sublayer.displayData.name = sublayerName + sublayer.displayData.description = sublayerDesc + + r1, _, _ = procFwpmSubLayerAdd0.Call( + engineHandle, + uintptr(unsafe.Pointer(&sublayer)), + 0, + ) + if r1 != 0 { + procFwpmEngineClose0.Call(engineHandle) + return fmt.Errorf("FwpmSubLayerAdd0 failed: HRESULT 0x%x", r1) + } + mainLog.Load().Info().Msg("DNS intercept: WFP sublayer created (weight: 0xFFFF — maximum priority)") + + state.engineHandle = engineHandle + + permitFilters := []struct { + name string + layer windows.GUID + proto uint8 + idField *uint64 + }{ + {"Permit DNS to localhost (IPv4/UDP)", fwpmLayerALEAuthConnectV4, ipprotoUDP, &state.permitIDv4UDP}, + {"Permit DNS to localhost (IPv4/TCP)", fwpmLayerALEAuthConnectV4, ipprotoTCP, &state.permitIDv4TCP}, + {"Permit DNS to localhost (IPv6/UDP)", fwpmLayerALEAuthConnectV6, ipprotoUDP, &state.permitIDv6UDP}, + {"Permit DNS to localhost (IPv6/TCP)", fwpmLayerALEAuthConnectV6, ipprotoTCP, &state.permitIDv6TCP}, + } + + for _, pf := range permitFilters { + filterID, err := p.addWFPPermitLocalhostFilter(engineHandle, pf.name, pf.layer, pf.proto) + if err != nil { + p.cleanupWFPFilters(state) + return fmt.Errorf("failed to add permit filter %q: %w", pf.name, err) + } + *pf.idField = filterID + mainLog.Load().Debug().Msgf("DNS intercept: added permit filter %q (ID: %d)", pf.name, filterID) + } + + blockFilters := []struct { + name string + layer windows.GUID + proto uint8 + idField *uint64 + }{ + {"Block outbound DNS (IPv4/UDP)", fwpmLayerALEAuthConnectV4, ipprotoUDP, &state.filterIDv4UDP}, + {"Block outbound DNS (IPv4/TCP)", fwpmLayerALEAuthConnectV4, ipprotoTCP, &state.filterIDv4TCP}, + {"Block outbound DNS (IPv6/UDP)", fwpmLayerALEAuthConnectV6, ipprotoUDP, &state.filterIDv6UDP}, + {"Block outbound DNS (IPv6/TCP)", fwpmLayerALEAuthConnectV6, ipprotoTCP, &state.filterIDv6TCP}, + } + + for _, bf := range blockFilters { + filterID, err := p.addWFPBlockDNSFilter(engineHandle, bf.name, bf.layer, bf.proto) + if err != nil { + p.cleanupWFPFilters(state) + return fmt.Errorf("failed to add block filter %q: %w", bf.name, err) + } + *bf.idField = filterID + mainLog.Load().Debug().Msgf("DNS intercept: added block filter %q (ID: %d)", bf.name, filterID) + } + + // Add static permit filters for RFC1918 + CGNAT ranges (UDP + TCP). + // This allows VPN DNS servers on private IPs (MagicDNS upstreams, F5, Windscribe, etc.) + // to work without dynamic per-server exemptions. + privateRanges := []struct { + name string + addr uint32 + mask uint32 + }{ + {"10.0.0.0/8", 0x0A000000, 0xFF000000}, + {"172.16.0.0/12", 0xAC100000, 0xFFF00000}, + {"192.168.0.0/16", 0xC0A80000, 0xFFFF0000}, + {"100.64.0.0/10", 0x64400000, 0xFFC00000}, + } + for _, r := range privateRanges { + for _, proto := range []struct { + num uint8 + name string + }{{ipprotoUDP, "UDP"}, {ipprotoTCP, "TCP"}} { + filterName := fmt.Sprintf("Permit DNS to %s (%s)", r.name, proto.name) + filterID, err := p.addWFPPermitSubnetFilter(engineHandle, filterName, proto.num, r.addr, r.mask) + if err != nil { + mainLog.Load().Warn().Err(err).Msgf("DNS intercept: failed to add subnet permit for %s/%s", r.name, proto.name) + continue + } + state.subnetPermitFilterIDs = append(state.subnetPermitFilterIDs, filterID) + mainLog.Load().Debug().Msgf("DNS intercept: added subnet permit %q (ID: %d)", filterName, filterID) + } + } + mainLog.Load().Info().Msgf("DNS intercept: %d subnet permit filters active (RFC1918 + CGNAT)", len(state.subnetPermitFilterIDs)) + + mainLog.Load().Info().Msgf("DNS intercept: WFP filters active — all outbound DNS (port 53) blocked except to localhost and private ranges. "+ + "Filter IDs: v4UDP=%d, v4TCP=%d, v6UDP=%d, v6TCP=%d (block), "+ + "v4UDP=%d, v4TCP=%d, v6UDP=%d, v6TCP=%d (permit localhost)", + state.filterIDv4UDP, state.filterIDv4TCP, state.filterIDv6UDP, state.filterIDv6TCP, + state.permitIDv4UDP, state.permitIDv4TCP, state.permitIDv6UDP, state.permitIDv6TCP) + + return nil +} + +// addWFPBlockDNSFilter adds a WFP filter that blocks outbound DNS traffic (port 53) +// for the given protocol (UDP or TCP) on the specified layer (V4 or V6). +func (p *prog) addWFPBlockDNSFilter(engineHandle uintptr, name string, layerKey windows.GUID, proto uint8) (uint64, error) { + filterName, _ := windows.UTF16PtrFromString("ctrld: " + name) + + conditions := make([]fwpmFilterCondition0, 2) + + conditions[0] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPProtocol, + matchType: fwpMatchEqual, + } + conditions[0].condValue.valueType = fwpUint8 + conditions[0].condValue.value = uint64(proto) + + conditions[1] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemotePort, + matchType: fwpMatchEqual, + } + conditions[1].condValue.valueType = fwpUint16 + conditions[1].condValue.value = uint64(dnsPort) + + filter := fwpmFilter0{ + layerKey: layerKey, + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 2, + filterCondition: &conditions[0], + } + filter.displayData.name = filterName + filter.weight.valueType = fwpUint8 + filter.weight.value = 1 + filter.action.actionType = fwpActionBlock + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(conditions) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 failed: HRESULT 0x%x", r1) + } + return filterID, nil +} + +// addWFPPermitLocalhostFilter adds a WFP filter that permits outbound DNS to localhost. +// This ensures ctrld's listener at 127.0.0.1/::1 can receive DNS queries. +// +// TODO: On AD DC where ctrld listens on 127.0.0.x, this filter should match +// the actual listener IP instead of hardcoded 127.0.0.1. Currently hard mode +// is unlikely on AD DC (NRPT dns mode is preferred), but if needed, this must +// be parameterized like addNRPTCatchAllRule. +// These filters have higher weight than block filters so they're matched first. +func (p *prog) addWFPPermitLocalhostFilter(engineHandle uintptr, name string, layerKey windows.GUID, proto uint8) (uint64, error) { + filterName, _ := windows.UTF16PtrFromString("ctrld: " + name) + + ipv6Loopback := [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + + conditions := make([]fwpmFilterCondition0, 3) + + conditions[0] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPProtocol, + matchType: fwpMatchEqual, + } + conditions[0].condValue.valueType = fwpUint8 + conditions[0].condValue.value = uint64(proto) + + conditions[1] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemotePort, + matchType: fwpMatchEqual, + } + conditions[1].condValue.valueType = fwpUint16 + conditions[1].condValue.value = uint64(dnsPort) + + conditions[2] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + if layerKey == fwpmLayerALEAuthConnectV4 { + conditions[2].condValue.valueType = fwpUint32 + conditions[2].condValue.value = 0x7F000001 + } else { + conditions[2].condValue.valueType = fwpByteArray16Type + conditions[2].condValue.value = uint64(uintptr(unsafe.Pointer(&ipv6Loopback))) + } + + filter := fwpmFilter0{ + layerKey: layerKey, + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 3, + filterCondition: &conditions[0], + } + filter.displayData.name = filterName + filter.weight.valueType = fwpUint8 + filter.weight.value = 10 + filter.action.actionType = fwpActionPermit + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(&ipv6Loopback) + runtime.KeepAlive(conditions) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 failed: HRESULT 0x%x", r1) + } + return filterID, nil +} + +// addWFPPermitDNSFilter is the unified helper for adding a WFP permit filter for +// outbound DNS (port 53) with caller-specified address condition, flags, and weight. +// Both subnet permits (RFC1918/CGNAT, flags=0, weight=10) and hard loopback permits +// (CLEAR_ACTION_RIGHT, weight=15) use this to avoid code drift. +func (p *prog) addWFPPermitDNSFilter(engineHandle uintptr, name string, layerKey windows.GUID, proto uint8, addrCond fwpmFilterCondition0, flags uint32, weight uint8) (uint64, error) { + filterName, _ := windows.UTF16PtrFromString("ctrld: " + name) + + conditions := make([]fwpmFilterCondition0, 3) + + conditions[0] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPProtocol, + matchType: fwpMatchEqual, + } + conditions[0].condValue.valueType = fwpUint8 + conditions[0].condValue.value = uint64(proto) + + conditions[1] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemotePort, + matchType: fwpMatchEqual, + } + conditions[1].condValue.valueType = fwpUint16 + conditions[1].condValue.value = uint64(dnsPort) + + conditions[2] = addrCond + + filter := fwpmFilter0{ + flags: flags, + layerKey: layerKey, + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 3, + filterCondition: &conditions[0], + } + filter.displayData.name = filterName + filter.weight.valueType = fwpUint8 + filter.weight.value = uint64(weight) + filter.action.actionType = fwpActionPermit + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(conditions) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 failed: HRESULT 0x%x", r1) + } + return filterID, nil +} + +// addWFPPermitSubnetFilter adds a WFP filter that permits outbound DNS to a given +// IPv4 subnet (addr/mask in host byte order). Used to exempt RFC1918 and CGNAT ranges +// so VPN DNS servers on private IPs are not blocked. +func (p *prog) addWFPPermitSubnetFilter(engineHandle uintptr, name string, proto uint8, addr, mask uint32) (uint64, error) { + addrMask := fwpV4AddrAndMask{addr: addr, mask: mask} + + addrCond := fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + addrCond.condValue.valueType = fwpV4AddrMask + addrCond.condValue.value = uint64(uintptr(unsafe.Pointer(&addrMask))) + + filterID, err := p.addWFPPermitDNSFilter(engineHandle, name, fwpmLayerALEAuthConnectV4, proto, addrCond, 0, 10) + runtime.KeepAlive(&addrMask) + return filterID, err +} + +// wfpSublayerExists checks whether our WFP sublayer still exists in the engine. +// Used by the watchdog to detect if another program removed our filters. +func wfpSublayerExists(engineHandle uintptr) bool { + var sublayerPtr uintptr + r1, _, _ := procFwpmSubLayerGetByKey0.Call( + engineHandle, + uintptr(unsafe.Pointer(&ctrldSubLayerGUID)), + uintptr(unsafe.Pointer(&sublayerPtr)), + ) + if r1 != 0 { + return false + } + if sublayerPtr != 0 { + procFwpmFreeMemory0.Call(uintptr(unsafe.Pointer(&sublayerPtr))) + } + return true +} + +// cleanupWFPFilters removes all WFP filters and the sublayer, then closes the engine. +// It logs each step and continues cleanup even if individual removals fail, +// to ensure maximum cleanup on shutdown. +func (p *prog) cleanupWFPFilters(state *wfpState) { + if state == nil || state.engineHandle == 0 { + return + } + + // Hold state.mu across the whole teardown: engineHandle and every filter ID slice + // below is shared with the VPN DNS exemption path and the recovery flows. + state.mu.Lock() + defer state.mu.Unlock() + + // Clean up loopback protect filters (DNS mode VPN workaround). + loopbackIDs := state.loopbackPermitIDs + state.loopbackPermitIDs = nil + state.loopbackProtectActive = false + for _, filterID := range loopbackIDs { + r1, _, _ := procFwpmFilterDeleteById0.Call(state.engineHandle, uintptr(filterID)) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove loopback protect filter (ID: %d, code: 0x%x)", filterID, r1) + } else { + mainLog.Load().Debug().Msgf("DNS intercept: removed loopback protect filter (ID: %d)", filterID) + } + } + + for _, filterID := range state.vpnPermitFilterIDs { + r1, _, _ := procFwpmFilterDeleteById0.Call(state.engineHandle, uintptr(filterID)) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove VPN permit filter (ID: %d, code: 0x%x)", filterID, r1) + } else { + mainLog.Load().Debug().Msgf("DNS intercept: removed VPN permit filter (ID: %d)", filterID) + } + } + + for _, filterID := range state.subnetPermitFilterIDs { + r1, _, _ := procFwpmFilterDeleteById0.Call(state.engineHandle, uintptr(filterID)) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove subnet permit filter (ID: %d, code: 0x%x)", filterID, r1) + } else { + mainLog.Load().Debug().Msgf("DNS intercept: removed subnet permit filter (ID: %d)", filterID) + } + } + + filterIDs := []struct { + name string + id uint64 + }{ + {"permit v4 UDP", state.permitIDv4UDP}, + {"permit v4 TCP", state.permitIDv4TCP}, + {"permit v6 UDP", state.permitIDv6UDP}, + {"permit v6 TCP", state.permitIDv6TCP}, + {"block v4 UDP", state.filterIDv4UDP}, + {"block v4 TCP", state.filterIDv4TCP}, + {"block v6 UDP", state.filterIDv6UDP}, + {"block v6 TCP", state.filterIDv6TCP}, + } + + for _, f := range filterIDs { + if f.id == 0 { + continue + } + r1, _, _ := procFwpmFilterDeleteById0.Call(state.engineHandle, uintptr(f.id)) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove WFP filter %q (ID: %d, code: 0x%x)", f.name, f.id, r1) + } else { + mainLog.Load().Debug().Msgf("DNS intercept: removed WFP filter %q (ID: %d)", f.name, f.id) + } + } + + r1, _, _ := procFwpmSubLayerDeleteByKey0.Call( + state.engineHandle, + uintptr(unsafe.Pointer(&ctrldSubLayerGUID)), + ) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove WFP sublayer (code: 0x%x)", r1) + } else { + mainLog.Load().Debug().Msg("DNS intercept: removed WFP sublayer") + } + + r1, _, _ = procFwpmEngineClose0.Call(state.engineHandle) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to close WFP engine (code: 0x%x)", r1) + } else { + mainLog.Load().Debug().Msg("DNS intercept: WFP engine closed") + } +} + +// activateLoopbackWFPProtect opens a minimal WFP session and adds "hard permit" +// filters for DNS to localhost. This is used in DNS mode when NRPT probe failures +// are detected, typically caused by third-party VPN software (e.g., OpenVPN) that +// installs WFP block filters via block-outside-dns. The hard permit (with +// FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT) in a max-weight sublayer overrides the +// third-party blocks without affecting their protection for non-loopback DNS. +func (p *prog) activateLoopbackWFPProtect(state *wfpState) error { + state.mu.Lock() + defer state.mu.Unlock() + + if state.loopbackProtectActive { + mainLog.Load().Debug().Msg("DNS intercept: loopback WFP protect already active") + return nil + } + // Only activate in DNS mode. Hard mode manages its own full WFP state + // (block + permit filters in the same sublayer). Activating loopback + // protect would delete the hard mode sublayer and all its filters. + if hardIntercept { + mainLog.Load().Debug().Msg("DNS intercept: skipping loopback WFP protect in hard mode") + return nil + } + + mainLog.Load().Info().Msg("DNS intercept: activating loopback WFP protect (countering third-party DNS block filters)") + + // Open WFP engine if not already open (DNS mode doesn't open it normally). + if state.engineHandle == 0 { + var engineHandle uintptr + session := fwpmSession0{} + sessionName, _ := windows.UTF16PtrFromString("ctrld DNS Loopback Protect") + session.displayData.name = sessionName + // Session-scoped, like the hard-intercept engine: no ctrld filter should + // outlive the process that installed it. + session.flags = fwpmSessionFlagDynamic + + const rpcCAuthnDefault = 0xFFFFFFFF + r1, _, _ := procFwpmEngineOpen0.Call( + 0, + uintptr(rpcCAuthnDefault), + 0, + uintptr(unsafe.Pointer(&session)), + uintptr(unsafe.Pointer(&engineHandle)), + ) + if r1 != 0 { + return fmt.Errorf("FwpmEngineOpen0 failed: HRESULT 0x%x", r1) + } + mainLog.Load().Info().Msgf("DNS intercept: WFP engine opened for loopback protect (handle: 0x%x)", engineHandle) + state.engineHandle = engineHandle + } + + // Clean up any stale sublayer from a previous session. + procFwpmSubLayerDeleteByKey0.Call( + state.engineHandle, + uintptr(unsafe.Pointer(&ctrldSubLayerGUID)), + ) + + // Create sublayer at maximum priority. + sublayer := fwpmSublayer0{ + subLayerKey: ctrldSubLayerGUID, + weight: 0xFFFF, + } + sublayerName, _ := windows.UTF16PtrFromString("ctrld DNS Loopback Protect Sublayer") + sublayerDesc, _ := windows.UTF16PtrFromString("Permits DNS to localhost, overriding third-party VPN block filters") + sublayer.displayData.name = sublayerName + sublayer.displayData.description = sublayerDesc + + r1, _, _ := procFwpmSubLayerAdd0.Call( + state.engineHandle, + uintptr(unsafe.Pointer(&sublayer)), + 0, + ) + if r1 != 0 { + return fmt.Errorf("FwpmSubLayerAdd0 failed: HRESULT 0x%x", r1) + } + + // Add hard permit filters for loopback DNS (v4+v6, UDP+TCP). + permitFilters := []struct { + name string + layer windows.GUID + proto uint8 + }{ + {"Loopback Protect: Permit DNS to localhost (IPv4/UDP)", fwpmLayerALEAuthConnectV4, ipprotoUDP}, + {"Loopback Protect: Permit DNS to localhost (IPv4/TCP)", fwpmLayerALEAuthConnectV4, ipprotoTCP}, + {"Loopback Protect: Permit DNS to localhost (IPv6/UDP)", fwpmLayerALEAuthConnectV6, ipprotoUDP}, + {"Loopback Protect: Permit DNS to localhost (IPv6/TCP)", fwpmLayerALEAuthConnectV6, ipprotoTCP}, + } + + for _, pf := range permitFilters { + filterID, err := p.addWFPHardPermitLocalhostFilter(state.engineHandle, pf.name, pf.layer, pf.proto, state.listenerIP) + if err != nil { + // Partial failure — clean up what we added (already holding mu). + p.deactivateLoopbackWFPProtectLocked(state) + return fmt.Errorf("failed to add loopback protect filter %q: %w", pf.name, err) + } + state.loopbackPermitIDs = append(state.loopbackPermitIDs, filterID) + mainLog.Load().Debug().Str("filter", pf.name).Uint64("id", filterID).Msg("DNS intercept: added loopback protect filter") + } + + state.loopbackProtectActive = true + mainLog.Load().Info().Int("filters", len(state.loopbackPermitIDs)). + Msg("DNS intercept: loopback WFP protect activated — localhost DNS permitted with CLEAR_ACTION_RIGHT") + return nil +} + +// osHealthcheckSuppressed reports whether the upstream.os healthcheck should +// be skipped because DNS intercept mode is active and the WFP loopback protect +// has been engaged. Loopback protect is only activated when an external WFP +// block filter (e.g. OpenVPN's block-outside-dns) is interfering with DNS, +// which is the same condition that makes the OS resolver healthcheck fail +// every 2s with i/o timeout — so suppressing the check avoids the log spam +// described in issue #526. +func (p *prog) osHealthcheckSuppressed() bool { + if !dnsIntercept || p.dnsInterceptState == nil { + return false + } + state, ok := p.dnsInterceptState.(*wfpState) + if !ok || state == nil { + return false + } + state.mu.Lock() + defer state.mu.Unlock() + return state.loopbackProtectActive +} + +// deactivateLoopbackWFPProtectLocked is the lock-free inner implementation. +// Caller must hold state.mu. +func (p *prog) deactivateLoopbackWFPProtectLocked(state *wfpState) { + if !state.loopbackProtectActive && len(state.loopbackPermitIDs) == 0 { + return + } + + for _, filterID := range state.loopbackPermitIDs { + if state.engineHandle != 0 { + r1, _, _ := procFwpmFilterDeleteById0.Call(state.engineHandle, uintptr(filterID)) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove loopback protect filter (ID: %d, code: 0x%x)", filterID, r1) + } + } + } + state.loopbackPermitIDs = nil + state.loopbackProtectActive = false + mainLog.Load().Info().Msg("DNS intercept: loopback WFP protect deactivated") +} + +// addWFPHardPermitLocalhostFilter adds a WFP permit filter for DNS to localhost with +// FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT. This "hard permit" prevents lower-priority +// sublayers (e.g., OpenVPN's block-outside-dns sublayer) from blocking DNS to +// ctrld's loopback listener. Weight is set to 15 (above hard mode's permit=10). +// For IPv4, the address is derived from listenerIP (e.g., 127.0.0.1 or 127.0.0.2). +func (p *prog) addWFPHardPermitLocalhostFilter(engineHandle uintptr, name string, layerKey windows.GUID, proto uint8, listenerIP string) (uint64, error) { + addrCond := fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + + ipv6Loopback := [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + + if layerKey == fwpmLayerALEAuthConnectV4 { + addrCond.condValue.valueType = fwpUint32 + addrCond.condValue.value = uint64(parseIPv4AsUint32(listenerIP)) + } else { + addrCond.condValue.valueType = fwpByteArray16Type + addrCond.condValue.value = uint64(uintptr(unsafe.Pointer(&ipv6Loopback))) + } + + filterID, err := p.addWFPPermitDNSFilter(engineHandle, name, layerKey, proto, addrCond, fwpmFilterFlagClearActionRight, 15) + runtime.KeepAlive(&ipv6Loopback) + return filterID, err +} + +// stopDNSIntercept removes all WFP filters and shuts down the DNS interception. +func (p *prog) stopDNSIntercept() error { + // Announce the stop before waiting for the lifecycle lock. A rebuild that holds it + // runs a full start, whose NRPT verification can sit in probe backoffs for seconds; + // the flag is what lets those flows abandon their work instead of making the stop + // wait. A stop that waits too long is not a delay but a failure mode: the Service + // Control Manager kills ctrld on timeout, and then nothing is cleaned up at all. + p.dnsInterceptStopRequested.Store(true) + p.dnsInterceptMu.Lock() + defer p.dnsInterceptMu.Unlock() + defer p.dnsInterceptStopRequested.Store(false) + return p.stopDNSInterceptLocked() +} + +// stopDNSInterceptLocked is stopDNSIntercept with p.dnsInterceptMu already held. +// +// It revokes the state before removing anything. Teardown deletes the WFP sublayer, and +// a missing sublayer is exactly what the health monitor reads as "our filters were +// wiped, rebuild everything" - so a monitor tick landing inside the shutdown window +// would otherwise re-add the NRPT catch-all and the WFP filters moments before the +// process exits, leaving Windows resolving through a ctrld that is gone. Revoking first +// means every such flow sees a retired state and stands down. +func (p *prog) stopDNSInterceptLocked() error { + if p.dnsInterceptState == nil { + mainLog.Load().Debug().Msg("DNS intercept: no state to clean up") + return nil + } + + state := p.dnsInterceptState.(*wfpState) + + // Revoke first, then remove. Both signals are what the monitor, the delayed + // rechecks and the NRPT heal flows read to decide whether they may still write + // host DNS state. + p.dnsInterceptState = nil + // Stop the health monitor goroutine. + if state.stopCh != nil { + close(state.stopCh) + } + + // Remove only ctrld-owned NRPT state. A GP-owned catch-all is an external + // deployment contract and must survive service stop, restart, and uninstall. + // + // Hold the transition lock across the removal so an in-flight NRPT transition + // finishes first and any later one sees the revoked state under the same lock. The + // stop-requested flag is already set, so an in-flight transition abandons its probes + // rather than making this wait. + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + + ops := p.nrptOps() + owner, externalRuleName := state.nrptPolicyOwner() + switch owner { + case nrptRuleOwnerCtrld: + if err := ops.removeRule(); err != nil { + mainLog.Load().Warn().Err(err).Msg("DNS intercept: failed to remove ctrld-owned NRPT catch-all rule") + } else { + mainLog.Load().Info().Msg("DNS intercept: removed ctrld-owned NRPT catch-all rule") + } + ops.flush() + case nrptRuleOwnerGroupPolicy: + mainLog.Load().Info().Str("rule", externalRuleName). + Msg("DNS intercept: leaving GP-managed NRPT catch-all untouched during shutdown") + // External policy stays, but a ctrld rule from an earlier unclean exit must + // not: while GP mode hides the local store, this stop is the last chance + // anything will look there. See removeOrphanedCtrldNRPTRule. + p.removeOrphanedCtrldNRPTRule("shutdown with externally owned NRPT policy") + case nrptRuleOwnerNone: + // No ownership was ever established this run - an activation that failed, or a + // start that never got that far. A ctrld rule found here is still ours. + p.removeOrphanedCtrldNRPTRule("shutdown with no NRPT owner") + } + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + + // Clean up WFP if the engine was opened (hard mode or loopback protect). + if state.engineHandle != 0 { + mainLog.Load().Info().Msg("DNS intercept: shutting down WFP filters") + p.cleanupWFPFilters(state) + mainLog.Load().Info().Msg("DNS intercept: WFP shutdown complete") + } + + mainLog.Load().Info().Msg("DNS intercept: shutdown complete") + return nil +} + +// interceptRebuildResult reports what rebuildDNSIntercept did. +type interceptRebuildResult int + +const ( + // interceptRebuildRetired means the caller's state was no longer the live one - + // shutdown revoked it, or an earlier rebuild replaced it - so nothing was touched. + interceptRebuildRetired interceptRebuildResult = iota + // interceptRebuildDone means the intercept was torn down and re-created. + interceptRebuildDone + // interceptRebuildFailed means teardown ran but the re-create returned an error. + // dnsInterceptState is nil afterwards, except on the one path that publishes a + // partial intercept on purpose - hard mode with verified GP NRPT but no WFP - which + // leaves a health monitor running to keep retrying. + interceptRebuildFailed +) + +// rebuildDNSIntercept tears the intercept down and creates it again, for callers that +// found our filters wiped from underneath us. +// +// It refuses unless state is still the published intercept state. That check is what +// stops a health monitor tick from resurrecting DNS interception during or after a +// service stop: shutdown revokes the state under this same lock before it removes +// anything, so a monitor arriving late finds its state retired and does nothing. +// Holding the lock across teardown and create also means a stop that arrives +// mid-rebuild waits, then tears down whatever the rebuild published - never a +// half-built intercept. +// +// Whatever the result, the caller's state is dead afterwards: the monitor goroutine +// that owns it must exit. +// rebuildDNSInterceptFn is the rebuild entry point. Indirected so tests can assert which +// conditions ask for a rebuild without running a real teardown and start. Assigned in +// init because the rebuild reaches back here through the health monitor. +var rebuildDNSInterceptFn func(*prog, *wfpState, string) interceptRebuildResult + +func init() { + rebuildDNSInterceptFn = (*prog).rebuildDNSIntercept +} + +func (p *prog) rebuildDNSIntercept(state *wfpState, reason string) interceptRebuildResult { + p.dnsInterceptMu.Lock() + defer p.dnsInterceptMu.Unlock() + + if live, ok := p.dnsInterceptState.(*wfpState); !ok || live != state { + mainLog.Load().Info().Str("reason", reason). + Msg("DNS intercept: not rebuilding - this intercept was already retired by shutdown or an earlier rebuild") + return interceptRebuildRetired + } + + mainLog.Load().Warn().Str("reason", reason).Msg("DNS intercept: rebuilding interception") + _ = p.stopDNSInterceptLocked() + if err := p.startDNSInterceptLocked(); err != nil { + mainLog.Load().Error().Err(err).Str("reason", reason).Msg("DNS intercept: rebuild failed") + return interceptRebuildFailed + } + return interceptRebuildDone +} + +// interceptStateRevoked reports whether state has been retired, meaning nothing may +// write host DNS state on its behalf any more. +// +// stopDNSInterceptLocked closes stopCh before it removes the NRPT rule or the WFP +// filters, and a stop waiting for the lifecycle lock sets dnsInterceptStopRequested +// first. Together they are the cheap shutdown signal for the flows that must not take +// p.dnsInterceptMu: the NRPT probe and heal sequences, which wait seconds between +// probes and also run from inside the locked start path. +func (p *prog) interceptStateRevoked(state *wfpState) bool { + if state == nil || state.stopCh == nil { + return true + } + if p.dnsInterceptStopRequested.Load() { + return true + } + select { + case <-state.stopCh: + return true + default: + return false + } +} + +// interceptRevocationPollInterval is how quickly the recovery flows notice a stop that +// is waiting for the lifecycle lock. A pending stop can only set a flag - it cannot +// close stopCh until it owns the lock - so waits poll instead of selecting on a channel. +const interceptRevocationPollInterval = 100 * time.Millisecond + +// interceptWait waits for d, or until the intercept is retired, whichever comes first. +// It reports whether the caller may keep working. +// +// Every wait in the NRPT recovery flows goes through this. Those flows can be holding +// the lifecycle lock - startDNSInterceptLocked runs one synchronously, and a rebuild +// holds the lock across the whole start - and their backoffs add up to tens of seconds. +// A plain time.Sleep there makes a service stop wait that long for the lock, risking +// the Service Control Manager killing ctrld before it removes the NRPT rule and the WFP +// filters, which is the unclean shutdown the locking exists to prevent. Bounding each +// wait by the shutdown signal keeps a stop's wait to about one poll interval plus +// whatever uncancellable Windows call is in flight. +func (p *prog) interceptWait(state *wfpState, d time.Duration) bool { + deadline := time.Now().Add(d) + for { + if p.interceptStateRevoked(state) { + return false + } + remaining := time.Until(deadline) + if remaining <= 0 { + return true + } + if remaining > interceptRevocationPollInterval { + remaining = interceptRevocationPollInterval + } + timer := time.NewTimer(remaining) + select { + case <-state.stopCh: + timer.Stop() + return false + case <-timer.C: + } + } +} + +// exemptVPNDNSServers updates the WFP filters to permit outbound DNS to the given +// VPN DNS server IPs. This prevents the block filters from intercepting ctrld's own +// forwarded queries to VPN DNS servers (split DNS routing). +// +// The function is idempotent: it first removes ALL existing VPN permit filters, +// then adds new ones for the current server list. When called with nil/empty +// exemptions (VPN disconnected), it just removes the old permits — leaving only +// the localhost permits and block-all filters active. +// +// On Windows, WFP filters are process-scoped (not interface-scoped like macOS pf), +// so we only use the server IPs from the exemptions. +// +// Supports both IPv4 and IPv6 VPN DNS servers. +// +// Called by vpnDNSManager.onServersChanged() whenever VPN DNS servers change. +func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { + state, ok := p.dnsInterceptState.(*wfpState) + if !ok || state == nil { + return fmt.Errorf("DNS intercept state not available") + } + // engineHandle, loopbackProtectActive and vpnPermitFilterIDs are all shared with the + // monitor, the recovery flows and teardown, so this runs under state.mu like every + // other reader and writer of them. + state.mu.Lock() + defer state.mu.Unlock() + + // In dns mode (no WFP) or loopback-protect-only mode, VPN DNS exemptions + // are not needed — there are no ctrld block filters to exempt from. + // Loopback protect only adds hard-permit filters for localhost DNS; + // VPN DNS traffic uses the tunnel interface and is already permitted by + // the VPN's own WFP rules. + if state.engineHandle == 0 || state.loopbackProtectActive { + mainLog.Load().Debug().Msg("DNS intercept: dns mode — skipping VPN DNS exemptions (no WFP block filters)") + return nil + } + + for _, filterID := range state.vpnPermitFilterIDs { + r1, _, _ := procFwpmFilterDeleteById0.Call(state.engineHandle, uintptr(filterID)) + if r1 != 0 { + mainLog.Load().Warn().Msgf("DNS intercept: failed to remove old VPN permit filter (ID: %d, code: 0x%x)", filterID, r1) + } + } + state.vpnPermitFilterIDs = nil + + // Extract unique server IPs from exemptions (WFP doesn't need interface info). + seen := make(map[string]bool) + var servers []string + for _, ex := range exemptions { + if !seen[ex.Server] { + seen[ex.Server] = true + servers = append(servers, ex.Server) + } + } + + for _, server := range servers { + ipv4 := parseIPv4AsUint32(server) + isIPv6 := ipv4 == 0 + + for _, proto := range []uint8{ipprotoUDP, ipprotoTCP} { + protoName := "UDP" + if proto == ipprotoTCP { + protoName = "TCP" + } + filterName := fmt.Sprintf("ctrld: Permit VPN DNS to %s (%s)", server, protoName) + + var filterID uint64 + var err error + if isIPv6 { + ipv6Bytes := parseIPv6AsBytes(server) + if ipv6Bytes == nil { + mainLog.Load().Warn().Msgf("DNS intercept: skipping invalid VPN DNS server: %s", server) + continue + } + filterID, err = p.addWFPPermitIPv6Filter(state.engineHandle, filterName, fwpmLayerALEAuthConnectV6, proto, ipv6Bytes) + } else { + filterID, err = p.addWFPPermitIPFilter(state.engineHandle, filterName, fwpmLayerALEAuthConnectV4, proto, ipv4) + } + if err != nil { + return fmt.Errorf("failed to add VPN DNS permit filter for %s/%s: %w", server, protoName, err) + } + state.vpnPermitFilterIDs = append(state.vpnPermitFilterIDs, filterID) + mainLog.Load().Debug().Msgf("DNS intercept: added VPN DNS permit filter for %s/%s (ID: %d)", server, protoName, filterID) + } + } + + mainLog.Load().Info().Msgf("DNS intercept: exempted %d VPN DNS servers from WFP block (%d filters)", len(servers), len(state.vpnPermitFilterIDs)) + return nil +} + +// addWFPPermitIPFilter adds a WFP permit filter for outbound DNS to a specific IPv4 address. +func (p *prog) addWFPPermitIPFilter(engineHandle uintptr, name string, layerKey windows.GUID, proto uint8, ipAddr uint32) (uint64, error) { + filterName, _ := windows.UTF16PtrFromString(name) + + conditions := make([]fwpmFilterCondition0, 3) + + conditions[0] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPProtocol, + matchType: fwpMatchEqual, + } + conditions[0].condValue.valueType = fwpUint8 + conditions[0].condValue.value = uint64(proto) + + conditions[1] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemotePort, + matchType: fwpMatchEqual, + } + conditions[1].condValue.valueType = fwpUint16 + conditions[1].condValue.value = uint64(dnsPort) + + conditions[2] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + conditions[2].condValue.valueType = fwpUint32 + conditions[2].condValue.value = uint64(ipAddr) + + filter := fwpmFilter0{ + layerKey: layerKey, + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 3, + filterCondition: &conditions[0], + } + filter.displayData.name = filterName + filter.weight.valueType = fwpUint8 + filter.weight.value = 10 + filter.action.actionType = fwpActionPermit + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(conditions) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 failed: HRESULT 0x%x", r1) + } + return filterID, nil +} + +// addWFPPermitIPv6Filter adds a WFP permit filter for outbound DNS to a specific IPv6 address. +func (p *prog) addWFPPermitIPv6Filter(engineHandle uintptr, name string, layerKey windows.GUID, proto uint8, ipAddr *[16]byte) (uint64, error) { + filterName, _ := windows.UTF16PtrFromString(name) + + conditions := make([]fwpmFilterCondition0, 3) + + conditions[0] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPProtocol, + matchType: fwpMatchEqual, + } + conditions[0].condValue.valueType = fwpUint8 + conditions[0].condValue.value = uint64(proto) + + conditions[1] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemotePort, + matchType: fwpMatchEqual, + } + conditions[1].condValue.valueType = fwpUint16 + conditions[1].condValue.value = uint64(dnsPort) + + conditions[2] = fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + conditions[2].condValue.valueType = fwpByteArray16Type + conditions[2].condValue.value = uint64(uintptr(unsafe.Pointer(ipAddr))) + + filter := fwpmFilter0{ + layerKey: layerKey, + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 3, + filterCondition: &conditions[0], + } + filter.displayData.name = filterName + filter.weight.valueType = fwpUint8 + filter.weight.value = 10 + filter.action.actionType = fwpActionPermit + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(ipAddr) + runtime.KeepAlive(conditions) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 failed: HRESULT 0x%x", r1) + } + return filterID, nil +} + +// parseIPv6AsBytes parses an IPv6 address string into a 16-byte array for WFP. +// Returns nil if the string is not a valid IPv6 address. +func parseIPv6AsBytes(ipStr string) *[16]byte { + ip := net.ParseIP(ipStr) + if ip == nil { + return nil + } + ip = ip.To16() + if ip == nil || ip.To4() != nil { + // It's IPv4, not IPv6 + return nil + } + var result [16]byte + copy(result[:], ip) + return &result +} + +// parseIPv4AsUint32 converts an IPv4 string to a uint32 in host byte order for WFP. +func parseIPv4AsUint32(ipStr string) uint32 { + parts := [4]byte{} + n := 0 + val := uint32(0) + for i := 0; i < len(ipStr) && n < 4; i++ { + if ipStr[i] == '.' { + parts[n] = byte(val) + n++ + val = 0 + } else if ipStr[i] >= '0' && ipStr[i] <= '9' { + val = val*10 + uint32(ipStr[i]-'0') + } else { + return 0 + } + } + if n == 3 { + parts[3] = byte(val) + return uint32(parts[0])<<24 | uint32(parts[1])<<16 | uint32(parts[2])<<8 | uint32(parts[3]) + } + return 0 +} + +// ensurePFAnchorActive is a no-op on Windows (WFP handles intercept differently). +func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { + return pfAnchorCheckSkipped +} + +// checkTunnelInterfaceChanges is a no-op on Windows (WFP handles intercept differently). +func (p *prog) checkTunnelInterfaceChanges() bool { + return false +} + +// Windows preserves the existing immediate reconciliation behavior. NRPT/WFP +// and adapter DNS settling have different lifecycle requirements from macOS pf. +func (p *prog) dnsInterceptIgnoredChangeReconcileDue(time.Time) bool { + return true +} + +// pfAnchorRecheckDelay is the delay for deferred pf anchor re-checks. +// Defined here as a stub for Windows (referenced from dns_proxy.go). +const pfAnchorRecheckDelay = 2 * time.Second + +// pfAnchorRecheckDelayLong is the longer delayed re-check for slower VPN teardowns. +const pfAnchorRecheckDelayLong = 4 * time.Second + +func gpCatchAllConflictBlocksFallback(state *wfpState, reason string) bool { + ruleName, dnsServers := findConflictingGPCatchAll(state.listenerIP) + if ruleName == "" { + return false + } + mainLog.Load().Error().Str("rule", ruleName).Str("nameservers", dnsServers).Str("reason", reason). + Msg("DNS intercept: GP catch-all targets another resolver; refusing to create a competing fallback rule") + return true +} + +// nrptOps is the seam between NRPT ownership decisions and the Windows side effects +// they cause. Tests substitute it to drive a transition - probe outcomes, registry +// state, concurrency - without touching the host's registry or DNS Client. +type nrptOps struct { + probe func(state *wfpState) bool + ruleExists func() bool + addRule func(listenerIP string) error + // restoreLocalRule re-writes only ctrld's local-store catch-all, leaving the GP + // store exactly as found. See restoreNRPTLocalCatchAllRule. + restoreLocalRule func(listenerIP string) error + removeRule func() error + signal func() + findGPRule func(listenerIP string) string + // dropGPRule deletes ctrld's GP-store catch-all, leaving the administrator's rule + // and the local store alone. See removeCtrldGPRule. + dropGPRule func() + gpRuleMatches func(ruleName, listenerIP string) bool + gpConflicts func(state *wfpState, reason string) bool + loopback func(state *wfpState) error + wait func(state *wfpState, d time.Duration) bool + flush func() + parentEmpty func(keyPath string) bool + cleanParent func() bool + startWFP func(state *wfpState) error +} + +// nrptOpsForTest overrides the NRPT side effects. Windows tests only. +var nrptOpsForTest *nrptOps + +func (p *prog) nrptOps() nrptOps { + if nrptOpsForTest != nil { + return *nrptOpsForTest + } + return nrptOps{ + probe: p.probeNRPT, + ruleExists: nrptCatchAllRuleExists, + addRule: addNRPTCatchAllRule, + restoreLocalRule: restoreNRPTLocalCatchAllRule, + removeRule: removeNRPTCatchAllRule, + signal: signalNRPTChange, + findGPRule: findMatchingGPNRPTRule, + dropGPRule: removeCtrldGPRule, + gpRuleMatches: gpNRPTRuleMatches, + gpConflicts: p.gpCatchAllConflictBlocksFallbackOps, + loopback: p.activateLoopbackWFPProtect, + wait: p.interceptWait, + flush: flushDNSCache, + parentEmpty: nrptParentKeyEmpty, + cleanParent: cleanEmptyNRPTParent, + startWFP: p.startWFPFilters, + } +} + +func (p *prog) gpCatchAllConflictBlocksFallbackOps(state *wfpState, reason string) bool { + return gpCatchAllConflictBlocksFallback(state, reason) +} + +// nrptHandbackResult reports how a handback attempt ended. +type nrptHandbackResult int + +const ( + // nrptHandbackVerified: external policy owns NRPT and proved, with ctrld's own keys + // gone, that it routes to this listener. + nrptHandbackVerified nrptHandbackResult = iota + // nrptHandbackUnverified: external policy owns the namespace but is not routing. + // Nothing of ctrld's was removed, so there was nothing to lose by recording it. + nrptHandbackUnverified + // nrptHandbackKeptCtrld: the proof failed with ctrld's rule removed, so the rule was + // restored and ctrld keeps ownership. + nrptHandbackKeptCtrld + // nrptHandbackAborted: shutdown landed, a registry step failed, the attempt was + // throttled, or the candidate child is no longer there. Nothing was decided. + nrptHandbackAborted + // nrptHandbackConflict: an administrator-owned catch-all is present that does not + // target ctrld. External policy owns the namespace and ctrld must not write beside it. + nrptHandbackConflict +) + +// nrptHandbackRetryInterval bounds how often ctrld will take its own rule out of the way +// to re-test the same external catch-all. Each attempt briefly removes the only working +// route, so retrying on every 30s health tick would be its own outage. A rule the +// administrator has changed is retested immediately regardless. +const nrptHandbackRetryInterval = 15 * time.Minute + +// nrptHandbackProbePasses bounds how many probes one handback spends chasing a Group +// Policy store that keeps changing under it. Each pass costs a probe timeout, and the +// budget being spent is not an excuse to guess: see externalAfterRemoval. +const nrptHandbackProbePasses = 2 + +// nrptHandbackToExternal is the only path that records external (Group Policy) +// ownership of NRPT. +// +// The proof has to be produced with ctrld's own keys gone. A probe taken while the ctrld +// fallback is still installed can be answered by that fallback, so a present-but- +// ineffective GP child would otherwise let ctrld delete the last working route and then +// declare external ownership. In hard mode that is a machine-wide DNS outage: WFP keeps +// blocking outbound DNS with nothing redirecting it to ctrld. +// +// The transition is: remove only ctrld's own keys, signal, probe again, re-read the same +// GP child - and if that second probe fails, put ctrld's rule back and keep ctrld +// ownership. Everything runs under nrptTransitionMu so a stop cannot interleave with it. +func (p *prog) nrptHandbackToExternal(state *wfpState, ruleName, reason string) nrptHandbackResult { + ops := p.nrptOps() + + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + + if p.interceptStateRevoked(state) { + return nrptHandbackAborted + } + + if !ops.gpRuleMatches(ruleName, state.listenerIP) { + return nrptHandbackAborted + } + + // Nothing of ours in the way: a probe already measures external policy alone, and + // refusing would mean writing a competing catch-all beside an administrator's rule. + if !ops.ruleExists() { + child, class, routes := p.externalAfterRemoval(ops, state, ruleName, reason) + switch { + case class == gpChildSameExact && routes: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + state.nrptRecoveryLimiter.recordStableSuccess() + mainLog.Load().Info().Str("rule", child).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all verified routing to ctrld; external policy owns NRPT") + return nrptHandbackVerified + case class == gpChildSameExact: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + mainLog.Load().Warn().Str("rule", child).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all owns the namespace but is not routing; leaving external policy untouched") + return nrptHandbackUnverified + case class == gpChildConflicting: + // An administrator-owned catch-all now targets another resolver, or is + // malformed. It owns the namespace and ctrld must not write a sibling. + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + mainLog.Load().Error().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP catch-all changed to one that does not target ctrld; refusing to write a competing rule") + return nrptHandbackConflict + default: + // External policy is gone, or the store is still churning. Decide nothing: + // the caller re-reads and retries. + return nrptHandbackAborted + } + } + + // ctrld's rule is installed. Taking it out to test the external one is disruptive, so + // it is throttled: an external rule that never routes would otherwise cost a brief + // outage on every health tick. The check comes before the probe so a throttled tick + // costs nothing, and the budget is only spent below, once the attempt is real. + now := time.Now() + if !state.handbackAllowed(now, ruleName, nrptHandbackRetryInterval) { + return nrptHandbackAborted + } + + // Pre-probe: this measures whatever routes DNS today, ctrld's own rule included, so + // it can never prove anything about external policy - it only says whether there is a + // working route here to risk. If nothing is routing there is nothing to protect and + // nothing to compare against, so leave it to the heal cycle rather than start + // deleting rules. + if !ops.probe(state) { + return nrptHandbackAborted + } + state.recordHandbackAttempt(now, ruleName, nrptHandbackRetryInterval) + + if err := ops.removeRule(); err != nil { + mainLog.Load().Warn().Err(err).Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP catch-all found but ctrld's own rule could not be removed for the handback probe") + return nrptHandbackAborted + } + ops.signal() + if !ops.wait(state, nrptHandbackSettleDelay) { + // Shutdown landed. NRPT is clean, which is the right state to leave behind. + return nrptHandbackAborted + } + + // Post-removal probe and classification, always - not only when the probe succeeded. + // Group Policy can refresh during the probe, and what it changed into decides whether + // restoring ctrld's rule is right or would create a sibling that must never be written. + child, class, routes := p.externalAfterRemoval(ops, state, ruleName, reason) + + switch { + case class == gpChildSameExact && routes: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + state.nrptRecoveryLimiter.recordStableSuccess() + mainLog.Load().Info().Str("rule", child).Str("listener", state.listenerIP).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all carried DNS without ctrld's rule; returned NRPT ownership to Group Policy") + return nrptHandbackVerified + + case class == gpChildConflicting: + // The child changed into a catch-all that does not target ctrld. It owns the + // namespace, so ctrld's rule stays off: restoring it here is exactly the + // competing sibling beside administrator policy that is forbidden elsewhere. + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + mainLog.Load().Error().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP catch-all changed to one that does not target ctrld during the handback probe; leaving NRPT to Group Policy and not restoring the ctrld rule") + return nrptHandbackConflict + + case class == gpChildSameExact && child != ruleName: + // A different administrator catch-all took the namespace while ctrld's keys were + // off, and its own pass says it is not routing yet. ctrld's rule still must not + // come back: addNRPTCatchAllRule writes ctrld's GP catch-all whenever another GP + // rule exists, so restoring would put a sibling beside a catch-all that was not + // even there when this transition started. Record external ownership and let the + // health monitor keep watching the new child. + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, child) + mainLog.Load().Warn().Str("old_rule", ruleName).Str("rule", child).Str("reason", reason). + Msg("DNS intercept: a different GP catch-all took the namespace during the handback probe and is not routing yet; leaving NRPT to Group Policy without restoring the ctrld rule") + return nrptHandbackUnverified + + } + + // ctrld's route has to come back. Class decides the writer: + // + // gpChildGone the namespace is free, so the full writer is correct - it may + // place ctrld's GP rule where unrelated GP rules would otherwise + // hide the local store, or drop an empty parent to leave GP mode. + // anything else an external catch-all is still on disk, so only the local key + // may be written. See restoreNRPTLocalCatchAllRule. + // + // The local writer is the default rather than the exception: reaching the full writer + // with any external catch-all present writes the sibling that must never exist, so a + // class added later must opt in to it deliberately. + restore := ops.restoreLocalRule + if class == gpChildGone { + restore = ops.addRule + } + if err := restore(state.listenerIP); err != nil { + mainLog.Load().Error().Err(err).Str("rule", ruleName). + Msg("DNS intercept: handback probe failed and the ctrld NRPT rule could not be restored; the health monitor will retry") + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + return nrptHandbackAborted + } + ops.signal() + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + if class == gpChildGone { + mainLog.Load().Warn().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all disappeared during the handback probe; restored the ctrld fallback and kept ctrld ownership") + } else { + mainLog.Load().Warn().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all could not be proved to carry DNS without ctrld's rule; wrote back ctrld's local-store key without writing a GP sibling, and kept ctrld ownership") + } + return nrptHandbackKeptCtrld +} + +// gpChildClass is what an external NRPT catch-all looks like at the moment ctrld checks, +// which is not necessarily what it looked like when a probe was sent: Group Policy can +// refresh while the probe is in flight. +type gpChildClass int + +const ( + // gpChildSameExact: the same child still names exactly this listener. + gpChildSameExact gpChildClass = iota + // gpChildReplaced: a different child now matches this listener exactly. It has not + // proved anything yet, so it is not adopted on this pass. + gpChildReplaced + // gpChildConflicting: an administrator-owned catch-all is present that does not target + // ctrld - another resolver, or malformed. ctrld must not write a rule beside it. + gpChildConflicting + // gpChildGone: no external catch-all owns the namespace any more. + gpChildGone +) + +// externalAfterRemoval probes external policy with ctrld's keys already gone, then says +// which child the result belongs to, what state that child is in, and whether it routed. +// It writes nothing: the caller decides what the verdict means. +// +// A probe result can only ever be attributed to the child that was on disk for the whole +// probe. When Group Policy swaps the child mid-probe the old result is void, so the +// replacement gets one pass of its own here rather than inheriting a verdict it never +// earned. Two passes is the limit; a store that keeps changing is reported as still +// churning so the caller can retry instead of guessing. +func (p *prog) externalAfterRemoval(ops nrptOps, state *wfpState, candidate, reason string) (string, gpChildClass, bool) { + for pass := 0; pass < nrptHandbackProbePasses; pass++ { + routes := ops.probe(state) + class := p.classifyGPChild(ops, state, candidate, reason) + if class != gpChildReplaced { + return candidate, class, routes + } + if pass == nrptHandbackProbePasses-1 { + break + } + next := ops.findGPRule(state.listenerIP) + if next == "" { + return candidate, gpChildGone, routes + } + mainLog.Load().Warn().Str("old_rule", candidate).Str("rule", next).Str("reason", reason). + Msg("DNS intercept: a different GP catch-all appeared during the probe; testing that one instead") + candidate = next + } + + // The probe budget is spent and the store is still moving. Report the store as it is + // now, with no route proved, rather than reporting churn: "undecided" would let + // startup and owned recovery fall through to writing ctrld's rule, and + // addNRPTCatchAllRule puts that in the GP path beside whatever exact catch-all is + // there - the sibling that must never exist. Failing safe from the current store + // keeps every outcome terminal for those callers unless the namespace is genuinely + // free. + mainLog.Load().Warn().Str("rule", candidate).Str("reason", reason). + Msg("DNS intercept: GP catch-alls kept changing during the handback probe; classifying the store as it stands with no route proved") + if current := ops.findGPRule(state.listenerIP); current != "" { + return current, gpChildSameExact, false + } + if ops.gpConflicts(state, reason) { + return candidate, gpChildConflicting, false + } + return candidate, gpChildGone, false +} + +// classifyGPChild re-reads the GP store and says what state the external catch-all is in. +// Every post-probe decision goes through it, so a child that changed mid-probe can never +// be treated as the child that was measured. +func (p *prog) classifyGPChild(ops nrptOps, state *wfpState, ruleName, reason string) gpChildClass { + if ops.gpRuleMatches(ruleName, state.listenerIP) { + return gpChildSameExact + } + if other := ops.findGPRule(state.listenerIP); other != "" { + return gpChildReplaced + } + if ops.gpConflicts(state, reason) { + return gpChildConflicting + } + return gpChildGone +} + +// nrptHandbackSettleDelay gives the DNS Client a moment to drop ctrld's removed rule +// before the handback probe decides whether external policy routes on its own. +const nrptHandbackSettleDelay = 1 * time.Second + +// deferToExternalCatchAll stops ctrld-owned recovery when an administrator catch-all owns +// the namespace, and reports whether the caller must stop. +// +// It hands back where a verdict is possible. Where one is not - the handback needs a +// working route to compare against, and during a heal cycle there often is none - the +// presence of an exact external catch-all is itself the ownership signal, so ownership is +// recorded without proof and recovery stops anyway. Continuing would mean signalling the +// DNS Client, or deleting and recreating ctrld's rule, while administrator policy owns the +// namespace: the two things #576 forbids. The health monitor keeps testing the rule and +// can hand back properly once it routes. +func (p *prog) deferToExternalCatchAll(ops nrptOps, state *wfpState, reason string) bool { + ruleName := ops.findGPRule(state.listenerIP) + if ruleName == "" || !ops.gpRuleMatches(ruleName, state.listenerIP) { + return false + } + + switch result := p.nrptHandbackToExternal(state, ruleName, reason); { + case result == nrptHandbackKeptCtrld: + // The external rule was proved unable to carry DNS and ctrld's rule is back, so + // owned recovery is exactly what should continue. + return false + case nrptExternalOwns(result): + if result == nrptHandbackUnverified { + p.healBlockedLoopbackDNS(state, reason) + } + return true + default: + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, ruleName) + mainLog.Load().Warn().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: a GP catch-all owns the namespace but could not be tested; stopping ctrld-owned recovery and leaving external policy untouched") + p.healBlockedLoopbackDNS(state, reason) + return true + } +} + +// nrptExternalOwns reports whether a handback left external policy owning the namespace. +// +// All three outcomes count, not just Verified: an administrator's catch-all owns the +// namespace whether it routes to ctrld (Verified), does not route at all (Unverified), or +// points somewhere else entirely (Conflict). Each is terminal for ctrld-owned recovery, +// because continuing would signal the DNS Client and delete and recreate ctrld's rule +// beside that catch-all - the competing, ambiguous policy #576 exists to avoid. Where the +// external rule is present but ineffective, loopback WFP protect is the only remediation +// left. +func nrptExternalOwns(result nrptHandbackResult) bool { + switch result { + case nrptHandbackVerified, nrptHandbackUnverified, nrptHandbackConflict: + return true + default: + return false + } +} + +// nrptTransition runs one complete NRPT mutation - write or delete, signal, record +// owner - with the transition lock held, and reports whether it ran. +// +// Checking interceptStateRevoked and then mutating is not enough on its own: a stop can +// land in between, revoke the state, remove NRPT and finish, after which the mutation +// would write a catch-all pointing at a listener that no longer exists. The stop takes +// this same lock around its own NRPT removal, so holding it across the whole +// observe-mutate-signal step is what makes the two mutually exclusive. Callers must keep +// probe backoffs outside fn: the lock is for a single transition, not for a heal cycle. +func (p *prog) nrptTransition(state *wfpState, fn func()) bool { + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + if p.interceptStateRevoked(state) { + return false + } + fn() + return true +} + +func (p *prog) activateCtrldNRPTFallback(state *wfpState, reason string) bool { + ops := p.nrptOps() + + // Close the observation-to-write race in both directions. Group Policy can refresh + // between the monitor's missing-rule check and this call, and again between this + // check and the write - so the write path re-checks under the transition lock and + // sends us back here when a matching child has appeared. Two passes is enough: the + // second either hands back or writes with the store checked under the lock. + for attempt := 0; attempt < 2; attempt++ { + if ruleName := ops.findGPRule(state.listenerIP); ruleName != "" && + ops.gpRuleMatches(ruleName, state.listenerIP) { + // A matching child owns the namespace, so hand back rather than create a + // sibling rule. The handback runs its own transition, so it cannot be called + // with the lock held. + if p.nrptHandbackToExternal(state, ruleName, reason) == nrptHandbackKeptCtrld { + // The handback restored ctrld's rule, which is what this call wanted. + return true + } + // Every other outcome means external policy owns the namespace, or that + // nothing could be decided. Either way this must not write a rule beside it. + return false + } + wrote, gpAppeared := p.writeCtrldFallback(ops, state, reason) + if !gpAppeared { + return wrote + } + } + return false +} + +// writeCtrldFallback writes ctrld's own catch-all under the transition lock. It reports +// whether it wrote, and whether it stood down because a matching GP child appeared - in +// which case the caller must take the handback path instead. +func (p *prog) writeCtrldFallback(ops nrptOps, state *wfpState, reason string) (wrote, gpAppeared bool) { + p.nrptTransitionMu.Lock() + defer p.nrptTransitionMu.Unlock() + + // Re-check under the transition lock. A retired state must not write NRPT policy: + // shutdown has already removed the ctrld-owned catch-all, and re-adding it + // afterwards leaves the DNS Client routing every query to a listener that no longer + // exists - a machine-wide resolution outage, not a cosmetic leftover. Checking + // before the lock cannot close that gap, because a stop can land between the check + // and the write; the stop takes this same lock around its own NRPT removal. + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Str("reason", reason). + Msg("DNS intercept: skipping ctrld NRPT fallback - intercept was retired") + return false, false + } + // The same reasoning applies to the GP store, which the caller read before taking + // this lock. gpConflicts alone does not cover it: findConflictingGPCatchAll skips a + // *matching* child by design, so without this re-read a policy refresh inside that + // window would let the write land beside an administrator catch-all that no probe has + // tested. + if ruleName := ops.findGPRule(state.listenerIP); ruleName != "" && + ops.gpRuleMatches(ruleName, state.listenerIP) { + mainLog.Load().Info().Str("rule", ruleName).Str("reason", reason). + Msg("DNS intercept: a matching GP catch-all appeared before the fallback write; handing back instead of writing beside it") + return false, true + } + if ops.gpConflicts(state, reason) { + return false, false + } + // Another health or delayed-recheck path may have written the rule while this one + // waited for the lock; do not duplicate the write and the signalling. + if ops.ruleExists() { + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + mainLog.Load().Debug().Str("reason", reason). + Msg("DNS intercept: ctrld NRPT rule was already restored by a concurrent transition") + return false, false + } + if err := ops.addRule(state.listenerIP); err != nil { + mainLog.Load().Error().Err(err).Str("reason", reason). + Msg("DNS intercept: failed to activate ctrld-owned NRPT fallback; the health monitor will retry") + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + return false, false + } + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + ops.signal() + mainLog.Load().Warn().Str("reason", reason). + Msg("DNS intercept: GP-managed catch-all unavailable - activated ctrld-owned NRPT fallback") + return true, false +} + +// tryAdoptMatchingGPNRPT hands NRPT ownership back to Group Policy when a matching +// external catch-all exists. It reports whether external policy now owns NRPT. +// +// The proof lives in nrptHandbackToExternal: a probe taken while ctrld's fallback is +// still installed proves nothing about the external rule, so the decision is always +// made with ctrld's keys removed. +func (p *prog) tryAdoptMatchingGPNRPT(state *wfpState) bool { + if p.interceptStateRevoked(state) { + return false + } + ruleName := p.nrptOps().findGPRule(state.listenerIP) + if ruleName == "" { + return false + } + switch p.nrptHandbackToExternal(state, ruleName, "matching GP-managed catch-all detected") { + case nrptHandbackVerified: + return true + case nrptHandbackUnverified: + // External policy owns the namespace but is not routing. ctrld must not write a + // competing rule, so the only remediation left is loopback WFP protect. Report + // external ownership: the caller must not fall through to owned recovery. + p.healBlockedLoopbackDNS(state, "GP-managed catch-all owns the namespace but is not routing") + return true + case nrptHandbackConflict: + // External policy now points somewhere other than ctrld. It owns the namespace, + // so report external ownership: owned recovery must not write beside it. + return true + case nrptHandbackKeptCtrld: + // The external rule could not carry DNS alone. A third-party WFP block dropping + // DNS below NRPT looks exactly like this, so try the one remediation that leaves + // external policy untouched; the next handback attempt can then succeed. + p.healBlockedLoopbackDNS(state, "GP-managed catch-all did not carry DNS on its own") + return false + } + return false +} + +// nrptNeedsCtrldActivation reports whether ctrld should write its own NRPT catch-all, +// given who owns policy and whether a ctrld rule is currently present. +// +// Owner None is the interesting case: it means an earlier write failed. Nothing else +// re-arms it - nrptProbeAndHeal returns early without ctrld ownership, and the health +// monitor used to skip the owner-None tick entirely - so without retrying, the machine +// keeps no NRPT rule for the rest of the process lifetime. In hard mode that is a full +// DNS outage rather than degraded interception: WFP goes on blocking outbound DNS while +// nothing redirects it to ctrld, and only a restart recovers. +func nrptNeedsCtrldActivation(owner nrptRuleOwner, ruleExists bool) bool { + switch owner { + case nrptRuleOwnerNone: + return true + case nrptRuleOwnerCtrld: + return !ruleExists + default: + // nrptRuleOwnerGroupPolicy: external policy owns the namespace, and a competing + // ctrld catch-all beside it would be ambiguous policy, not recovery. + return false + } +} + +// loopbackProtectSettleDelay gives WFP a moment to apply the loopback permits before +// the retry probe goes out. +const loopbackProtectSettleDelay = 500 * time.Millisecond + +// healBlockedLoopbackDNS retries the NRPT probe from behind loopback WFP protection and +// reports whether the probe then succeeded. +// +// A failed probe does not always mean the NRPT rule is wrong. Third-party WFP filters - +// OpenVPN's block-outside-dns is the common one - can drop DNS below NRPT, so the +// policy is correct and the packets never arrive. Loopback protect is the one +// remediation that fixes this while leaving externally owned policy untouched. Without +// this attempt, a GP rule blocked that way is never adopted and never healed: the +// monitor keeps finding a "present but not routing" rule for the life of the process. +// It is also the only remediation ctrld may run while external policy owns NRPT. +// signalNRPTChange is not a content-neutral nudge - it forces machine Group Policy via +// RefreshPolicyEx, sends Dnscache paramchange and flushes the resolver cache - so #576's +// contract for a present-but-ineffective GP rule is a warning plus WFP-only retries, with +// no refresh/paramchange/flush loop. +func (p *prog) healBlockedLoopbackDNS(state *wfpState, reason string) bool { + ops := p.nrptOps() + if p.interceptStateRevoked(state) { + return false + } + if hardIntercept { + // Hard mode owns the whole sublayer; loopback protect deliberately does + // nothing there, so a retry probe would only burn the probe timeout. + return false + } + if err := ops.loopback(state); err != nil { + mainLog.Load().Warn().Err(err).Str("reason", reason). + Msg("DNS intercept: could not activate loopback WFP protect while retrying a failed probe") + return false + } + if !ops.wait(state, loopbackProtectSettleDelay) { + return false + } + if !ops.probe(state) { + return false + } + mainLog.Load().Info().Str("reason", reason). + Msg("DNS intercept: probe recovered behind loopback WFP protect - a third-party WFP block was dropping DNS below NRPT") + return true +} + +// scheduleDelayedRechecks schedules delayed OS resolver and VPN DNS refreshes after +// network change events. While WFP filters don't get wiped like pf anchors, the OS +// resolver and VPN DNS state can still be stale after VPN disconnect (same issue as macOS). +func (p *prog) scheduleDelayedRechecks() { + for _, delay := range []time.Duration{pfAnchorRecheckDelay, pfAnchorRecheckDelayLong} { + time.AfterFunc(delay, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + if p.dnsInterceptState == nil { + return + } + // Refresh OS resolver — VPN may have finished DNS cleanup since the + // immediate handler ran. + ctx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + ctrld.InitializeOsResolverWithReason(ctx, true, osResolverReasonDelayedRecheck) + if p.vpnDNS != nil { + p.vpnDNS.Refresh(ctx, true) + } + + // Delayed rechecks must respect NRPT ownership. The old path looked only + // for ctrld's deterministic key, so it recreated that key immediately + // after startup had correctly adopted a GP-managed catch-all. + state, ok := p.dnsInterceptState.(*wfpState) + if ok && !p.interceptStateRevoked(state) { + owner, _ := state.nrptPolicyOwner() + switch owner { + case nrptRuleOwnerGroupPolicy: + if findMatchingGPNRPTRule(state.listenerIP) == "" { + if p.activateCtrldNRPTFallback(state, "matching GP rule disappeared during delayed network recheck") { + go p.nrptProbeAndHeal(state) + } + } + case nrptRuleOwnerNone, nrptRuleOwnerCtrld: + if nrptNeedsCtrldActivation(owner, nrptCatchAllRuleExists()) { + mainLog.Load().Warn().Msg("DNS intercept: no ctrld NRPT catch-all in place - re-adding") + if p.activateCtrldNRPTFallback(state, "ctrld NRPT rule missing during delayed network recheck") { + go p.nrptProbeAndHeal(state) + } + } + } + } + + // WFP watchdog: verify our sublayer still exists. If another program + // or a crash removed it, the block filters are gone too. A timer that + // fires during shutdown must not rebuild what teardown removed, so this + // goes through rebuildDNSIntercept's ownership check. + if ok && !p.interceptStateRevoked(state) && state.engineHandle != 0 && !wfpSublayerExists(state.engineHandle) { + mainLog.Load().Warn().Msg("DNS intercept: WFP sublayer was removed externally — re-creating all filters") + rebuildDNSInterceptFn(p, state, "WFP sublayer removed externally (delayed network recheck)") + } + }) + } +} + +// repairMissingWFP re-creates the intercept when hard mode has no WFP enforcement - +// because the sublayer disappeared, or because the engine never opened in the first place. +// It reports whether the caller's monitor goroutine must stop. +func (p *prog) repairMissingWFP(state *wfpState) bool { + // Never interrogate WFP for a retired state: shutdown deletes our sublayer, so a + // tick landing in the shutdown window would read "missing" and try to rebuild what + // stopDNSIntercept just removed. + if p.interceptStateRevoked(state) { + return true + } + + reason := "" + switch { + case state.engineHandle == 0: + // In dns mode a closed engine is the normal state: loopback protect opens one + // only when it is needed. In hard mode it means startWFPFilters never got the + // engine open, so nothing is being blocked at all. That is the state a startup + // WFP failure leaves behind, and this is what retries it - without this entry + // point the process would run unenforced for its whole life. + if !hardIntercept { + return false + } + reason = "hard mode has no WFP engine - enforcement never started" + case wfpSublayerExists(state.engineHandle): + return false + default: + reason = "WFP sublayer missing during health check" + } + + mainLog.Load().Warn().Str("reason", reason).Msg("DNS intercept: WFP health check - re-initializing all filters") + if rebuildDNSInterceptFn(p, state, reason) == interceptRebuildDone { + mainLog.Load().Info().Msg("DNS intercept: WFP filters restored by health monitor") + } + return true +} + +// nrptHealthMonitor periodically checks that the NRPT catch-all rule is still +// present and re-adds it if removed by VPN software or Group Policy updates. +// In hard mode, it also verifies the WFP sublayer exists and re-initializes +// all filters if they were removed. +// +// One monitor belongs to one intercept state, and it exits when that state is retired: +// nothing here may act on state after stopDNSIntercept or a rebuild has moved on. +func (p *prog) nrptHealthMonitor(state *wfpState) { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-state.stopCh: + return + case <-ticker.C: + // A tick and a shutdown can become ready together and select picks either, + // so re-check before doing any health work for a retired intercept. + if p.interceptStateRevoked(state) { + return + } + owner, externalRuleName := state.nrptPolicyOwner() + switch owner { + case nrptRuleOwnerNone: + // No owner means an earlier NRPT write failed. Nothing else re-arms + // it - nrptProbeAndHeal returns early without ctrld ownership, and the + // missing-rule branch below used to be unreachable from here - so + // without this retry the machine keeps no NRPT rule for the rest of the + // process lifetime. In hard mode that is a full DNS outage: WFP still + // blocks outbound DNS while nothing redirects it to ctrld. Fall through + // to the activation path below. + case nrptRuleOwnerGroupPolicy: + currentRule := p.nrptOps().findGPRule(state.listenerIP) + if currentRule == "" { + if p.activateCtrldNRPTFallback(state, "matching GP rule disappeared during health check") { + go p.nrptProbeAndHeal(state) + } + continue + } + // Confirm through the handback transition so the verdict is always about + // external policy alone, never about a ctrld rule left behind by an + // earlier run that happens to answer the probe. + switch p.nrptHandbackToExternal(state, currentRule, "GP-managed NRPT health check") { + case nrptHandbackVerified: + // Ownership and stable-success are recorded by the transition. + case nrptHandbackKeptCtrld: + // External policy could not carry DNS; ctrld owns the rule again. + // The next tick continues as ctrld-owned. + case nrptHandbackConflict: + // The administrator's catch-all no longer targets ctrld. Nothing to + // remediate: ctrld may not write beside it, and the conflict is + // already logged by the transition. + default: + mainLog.Load().Warn().Str("rule", externalRuleName). + Msg("DNS intercept: GP-managed NRPT rule is present but its probe failed; leaving external policy untouched") + // The only remediation allowed over external policy: a third-party + // WFP block dropping DNS below NRPT looks exactly like an ineffective + // rule. No policy refresh, paramchange or cache flush here - see + // healBlockedLoopbackDNS. + if !p.healBlockedLoopbackDNS(state, "GP-managed NRPT rule present but not routing") { + go p.nrptProbeAndHeal(state) + } + } + if p.repairMissingWFP(state) { + return + } + continue + case nrptRuleOwnerCtrld: + // Group Policy may have returned after ctrld activated its fallback. + // Prefer the externally owned rule once it proves the same route without + // ctrld's rule installed. + if p.tryAdoptMatchingGPNRPT(state) { + continue + } + } + if nrptNeedsCtrldActivation(owner, nrptCatchAllRuleExists()) { + now := time.Now() + if ok, wait := state.nrptRecoveryLimiter.allow(now, p.cfg); !ok { + if state.nrptRecoveryLimiter.shouldLogSkip(now) { + mainLog.Load().Warn().Str("remaining", wait.String()). + Msg("DNS intercept: NRPT rule restore suppressed after repeated recovery flows") + } + continue + } + reason := "ctrld-owned rule missing during health check" + if owner == nrptRuleOwnerNone { + reason = "no NRPT owner during health check - retrying a failed activation" + } + if p.activateCtrldNRPTFallback(state, reason) { + state.nrptRecoveryLimiter.recordRecoveryFlow(time.Now(), p.cfg) + go p.nrptProbeAndHeal(state) + } else if owner == nrptRuleOwnerNone && hardIntercept { + // Worth shouting about: hard mode keeps blocking outbound DNS + // whether or not NRPT redirects it, so a machine stuck here has no + // working resolver until activation succeeds. + mainLog.Load().Error(). + Msg("DNS intercept: hard mode is blocking DNS but no NRPT rule could be activated - DNS will not resolve until this recovers") + } + continue + } + + if !p.nrptOps().probe(state) { + mainLog.Load().Warn().Msg("DNS intercept: ctrld-owned NRPT rule present but probe failed, running heal cycle") + go p.nrptProbeAndHeal(state) + } else { + state.nrptRecoveryLimiter.recordStableSuccess() + } + + if p.repairMissingWFP(state) { + return // our state was retired: shutdown, or a rebuild that started a new monitor + } + } + } +} + +// pfInterceptMonitor is a no-op on Windows — WFP filters are kernel objects +// and don't suffer from the pf translation state corruption that macOS has. +func (p *prog) pfInterceptMonitor() {} + +// reconcileForwardedSources is a no-op on Windows — forwarded-workload DNS +// interception is macOS-pf-only; Windows Firewall Mode is tracked separately. +func (p *prog) reconcileForwardedSources() {} + +const ( + // nrptProbeDomain is the suffix used for NRPT verification probe queries. + // Probes use "_nrpt-probe-." — ctrld recognizes the + // prefix in the DNS handler and responds immediately without upstream forwarding. + nrptProbeDomain = "nrpt-probe.ctrld.test" + + // nrptProbeTimeout is how long to wait for a single probe query to arrive. + nrptProbeTimeout = 2 * time.Second +) + +// nrptProbeRunning ensures only one NRPT probe sequence runs at a time. +// Prevents the health monitor and startup from overlapping. +var nrptProbeRunning atomic.Bool + +// probeNRPT tests whether the NRPT catch-all rule is actually routing DNS queries +// to ctrld's listener. It sends a DNS query for a synthetic probe domain through +// the Windows DNS Client service (via Go's net.Resolver / GetAddrInfoW). If ctrld +// receives the query on its listener, NRPT is working. +// +// Returns true if NRPT is verified working, false if the probe timed out or a shutdown +// arrived first. Reporting an abandoned probe as "not working" is safe: every recovery +// action a failed probe can trigger checks for revocation before it writes anything. +// +// state is passed explicitly rather than read from p.dnsInterceptState so that startup +// can probe before it publishes, and so a probe belongs to exactly one intercept. +func (p *prog) probeNRPT(state *wfpState) bool { + if state == nil { + return true + } + + // Generate unique probe domain to defeat DNS caching. + probeID := fmt.Sprintf("_nrpt-probe-%x.%s", rand.Uint32(), nrptProbeDomain) + + // Register this attempt's own domain so overlapping probes - a health tick, a + // handback and a heal cycle can each have one out - cannot cancel each other. + probeCh, deregister := p.registerInterceptProbe(probeID) + defer deregister() + + mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: sending NRPT verification probe") + + // Use Go's default resolver which calls GetAddrInfoW → DNS Client service → NRPT. + // If NRPT is active, the DNS Client routes this to 127.0.0.1 → ctrld receives it. + // If NRPT isn't loaded, the query goes to interface DNS → times out or NXDOMAIN. + ctx, cancel := context.WithTimeout(context.Background(), nrptProbeTimeout) + defer cancel() + + go func() { + resolver := &net.Resolver{} + // We don't care about the result — only whether ctrld's handler receives it. + _, _ = resolver.LookupHost(ctx, probeID) + }() + + // Poll for a pending stop so a service stop never waits out the probe timeout on + // top of the lifecycle lock. + shutdownPoll := time.NewTicker(interceptRevocationPollInterval) + defer shutdownPoll.Stop() + for { + select { + case <-probeCh: + mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: NRPT probe received - interception verified") + return true + case <-ctx.Done(): + mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: NRPT probe timed out - interception not working") + return false + case <-shutdownPoll.C: + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Str("domain", probeID).Msg("DNS intercept: abandoning NRPT probe - shutdown in progress") + return false + } + } + } +} + +// nrptProbeAndHeal runs the NRPT probe with retries and escalating remediation. +// Called asynchronously after startup and from the health monitor. +// +// Retry sequence: +// 1. Immediate probe. +// 2. If the GP parent is empty, clean it immediately, signal once, then probe. +// This is intentionally before the normal retry loop: policy refresh and +// Dnscache paramchange cannot make local rules visible while GP mode is +// selected by an empty GP parent. +// 3. Otherwise, signal DNS Client with increasing backoff between probes. +// +// state is passed in rather than read from p.dnsInterceptState: this runs from the +// locked start path before anything is published, and it must keep acting on the +// intercept it was launched for and no other. +// It reports whether the cycle ended with a probe that reached ctrld. Callers that need a +// readiness answer - startup's synchronous verification - use that; the asynchronous +// callers ignore it. +func (p *prog) nrptProbeAndHeal(state *wfpState) bool { + if p.interceptStateRevoked(state) { + return false + } + if !nrptProbeRunning.CompareAndSwap(false, true) { + mainLog.Load().Debug().Msg("DNS intercept: NRPT probe already running, skipping") + return false + } + defer nrptProbeRunning.Store(false) + + ops := p.nrptOps() + owner, externalRuleName := state.nrptPolicyOwner() + if owner == nrptRuleOwnerGroupPolicy { + currentRule := ops.findGPRule(state.listenerIP) + if currentRule == "" { + if !p.activateCtrldNRPTFallback(state, "matching GP rule disappeared before verification") { + return false + } + owner = nrptRuleOwnerCtrld + } else { + switch p.nrptHandbackToExternal(state, currentRule, "GP-managed NRPT verification") { + case nrptHandbackVerified: + mainLog.Load().Info().Str("rule", currentRule). + Msg("DNS intercept: GP-managed NRPT verified working") + return true + case nrptHandbackKeptCtrld: + // External policy could not carry DNS without ctrld's rule, which the + // transition has restored. Continue as the ctrld-owned flow below. + owner = nrptRuleOwnerCtrld + case nrptHandbackConflict: + // External policy owns the namespace and points elsewhere. Owned recovery + // must not write beside it, so this heal cycle ends here. + return false + default: + if ops.findGPRule(state.listenerIP) == "" { + // The GP child went away while we were looking at it. + if !p.activateCtrldNRPTFallback(state, "matching GP rule disappeared during verification") { + return false + } + owner = nrptRuleOwnerCtrld + break + } + // A matching external rule owns the GP store, so ctrld may not rewrite + // policy or signal the DNS Client here. Loopback WFP protect is the one + // ctrld-owned remediation left: a third-party WFP block dropping DNS + // below NRPT presents exactly as an ineffective rule. + if p.healBlockedLoopbackDNS(state, "GP-managed NRPT present but not routing") { + mainLog.Load().Info().Str("rule", currentRule). + Msg("DNS intercept: GP-managed NRPT verified after loopback WFP protection") + return true + } + mainLog.Load().Error().Str("rule", externalRuleName). + Msg("DNS intercept: GP-managed NRPT remains present but ineffective; no NRPT recovery actions were taken") + return false + } + } + } + if owner != nrptRuleOwnerCtrld { + return false + } + + now := time.Now() + if ok, wait := state.nrptRecoveryLimiter.allow(now, p.cfg); !ok { + if state.nrptRecoveryLimiter.shouldLogSkip(now) { + mainLog.Load().Warn().Str("remaining", wait.String()). + Msg("DNS intercept: NRPT recovery suppressed after repeated failed recovery flows") + } + return false + } + + remediated := false + defer func() { + if remediated && state != nil { + state.nrptRecoveryLimiter.recordRecoveryFlow(time.Now(), p.cfg) + } + }() + + mainLog.Load().Info().Msg("DNS intercept: starting NRPT verification probe sequence") + + // Log parent key state for diagnostics. + logNRPTParentKeyState("probe-start") + + // Attempt 1: immediate probe + if ops.probe(state) { + mainLog.Load().Info().Msg("DNS intercept: NRPT verified working") + return true + } + // Group Policy can appear while a ctrld-owned heal is running. Once an exact + // administrator catch-all is there, this cycle stops: everything below signals the + // DNS Client or rewrites ctrld's rule, and neither may happen beside external policy. + if p.deferToExternalCatchAll(ops, state, "GP catch-all appeared during ctrld recovery") { + mainLog.Load().Warn().Msg("DNS intercept: matching GP catch-all present during ctrld recovery; stopping NRPT mutations and deferring to Group Policy") + return false + } + remediated = true + + // If the GP parent exists but is empty, do not burn retries on Windows + // signaling. Those retries create SIEM noise but cannot succeed because DNS + // Client is still reading the empty GP store instead of the populated local + // store. Delete the blocker, send one notification, then re-probe. + if ops.parentEmpty(nrptBaseKey) { + mainLog.Load().Warn().Msg("DNS intercept: NRPT probe failed with empty GP parent — cleaning before retry signaling") + if ops.cleanParent() { + ops.signal() + if !ops.wait(state, nrptHandbackSettleDelay) { + return false + } + logNRPTParentKeyState("empty-gp-after-clean") + if ops.probe(state) { + mainLog.Load().Info().Msg("DNS intercept: NRPT verified working after empty GP parent cleanup") + return true + } + } + if ops.parentEmpty(nrptBaseKey) { + mainLog.Load().Warn().Msg("DNS intercept: empty GP NRPT parent still present after cleanup; skipping redundant policy refresh retries") + return false + } + } + + // Attempts 2-4: signal DNS Client with increasing backoff between probes. + delays := []time.Duration{1 * time.Second, 2 * time.Second, 4 * time.Second} + for i, delay := range delays { + attempt := i + 2 + // Each round signals Windows and then waits; a stop must not have the whole + // remaining backoff added to the time it waits for the lifecycle lock. + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Int("attempt", attempt). + Msg("DNS intercept: intercept retired - abandoning NRPT probe retries") + return false + } + mainLog.Load().Info().Int("attempt", attempt).Str("delay", delay.String()). + Msg("DNS intercept: NRPT probe failed, retrying with policy refresh + paramchange") + logNRPTParentKeyState(fmt.Sprintf("probe-attempt-%d", attempt)) + ops.signal() + if !ops.wait(state, delay) { + return false + } + if ops.probe(state) { + mainLog.Load().Info().Int("attempt", attempt). + Msg("DNS intercept: NRPT verified working") + return true + } + } + + // Re-check external ownership before the destructive two-phase recovery. A GP refresh + // can land during the bounded retry waits above, and the delete half of that recovery + // must not run once it has: it would strand the machine mid-recovery under policy + // ctrld does not own. + if p.deferToExternalCatchAll(ops, state, "GP catch-all appeared during ctrld retries") { + mainLog.Load().Warn().Msg("DNS intercept: matching GP catch-all present after the retries; skipping ctrld two-phase recovery") + return false + } + if gpCatchAllConflictBlocksFallback(state, "GP catch-all changed during ctrld recovery") { + return false + } + + // A stop can land during the retry waits above, and its own NRPT removal has + // already run: deleting and re-adding from here would leave the rule behind. + if p.interceptStateRevoked(state) { + mainLog.Load().Debug().Msg("DNS intercept: intercept retired during probe retries - skipping two-phase NRPT recovery") + return false + } + + // Nuclear option: two-phase delete → re-add cycle. + // DNS Client may have cached a stale "no rules" state. Delete our rule, + // signal DNS Client to forget it, wait, then re-add and signal again. + mainLog.Load().Warn().Msg("DNS intercept: all probes failed — attempting two-phase NRPT recovery (delete → signal → re-add)") + listenerIP := state.listenerIP + + // Phase 1: Remove our rule and the parent key if now empty. Each phase is its own + // transition: serialized against a stop and against other NRPT writers, but the lock + // is released across the wait between them. + if !p.nrptTransition(state, func() { + _ = ops.removeRule() + // If parent key is now empty after removing our rule, delete it too. + ops.cleanParent() + ops.signal() + logNRPTParentKeyState("nuclear-after-delete") + }) { + mainLog.Load().Debug().Msg("DNS intercept: intercept retired - skipping two-phase NRPT recovery") + return false + } + + // Wait for DNS Client to process the deletion. Stopping here is the clean outcome: + // phase 1 left no ctrld rule behind. + if !ops.wait(state, nrptHandbackSettleDelay) { + mainLog.Load().Debug().Msg("DNS intercept: intercept retired mid-recovery - leaving NRPT removed") + return false + } + + // Group Policy may refresh while the ctrld-owned rule is absent. Never re-create our + // catch-all beside a newly authoritative GP catch-all. With ctrld's rule already + // gone, this is the one place a probe measures external policy on its own. + if p.deferToExternalCatchAll(ops, state, "GP catch-all appeared during two-phase recovery") { + mainLog.Load().Warn().Msg("DNS intercept: matching GP catch-all appeared during two-phase recovery; skipping ctrld re-add") + return false + } + if ops.gpConflicts(state, "GP catch-all appeared during two-phase recovery") { + return false + } + + // Phase 2: Re-add the rule. Phase 1 left NRPT clean, so if a stop arrived during the + // wait the transition refuses and the host stays clean. + readded := false + if !p.nrptTransition(state, func() { + // Re-validate the ownership picture here, inside the lock. The checks above ran + // before this transition queued for nrptTransitionMu, and another health or + // delayed path can complete a whole handback while this one waits: it may have + // given the namespace to a GP child that arrived meanwhile and recorded + // GroupPolicy. Re-adding on top of that would plant the competing catch-all this + // work exists to prevent, and then stamp ctrld ownership over the administrator's. + if owner, ruleName := state.nrptPolicyOwner(); owner == nrptRuleOwnerGroupPolicy { + mainLog.Load().Warn().Str("rule", ruleName). + Msg("DNS intercept: ownership moved to Group Policy while the two-phase re-add waited for the transition lock; leaving NRPT to external policy") + return + } + if ruleName := ops.findGPRule(state.listenerIP); ruleName != "" && + ops.gpRuleMatches(ruleName, state.listenerIP) { + mainLog.Load().Warn().Str("rule", ruleName). + Msg("DNS intercept: a matching GP catch-all appeared while the two-phase re-add waited for the transition lock; skipping the ctrld re-add") + return + } + if ops.gpConflicts(state, "two-phase re-add revalidation") { + return + } + if err := ops.addRule(listenerIP); err != nil { + mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to re-add NRPT after nuclear recovery") + return + } + ops.signal() + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + logNRPTParentKeyState("nuclear-after-readd") + readded = true + }) || !readded { + mainLog.Load().Debug().Msg("DNS intercept: two-phase recovery did not restore the ctrld NRPT rule") + return false + } + + // Final probe after recovery. + if !ops.wait(state, nrptHandbackSettleDelay) { + // This cycle is retired. Do not clean up from here: the deterministic ctrld key is + // process-global, and by now it can belong to a successor intercept that a rebuild + // started while this goroutine waited - deleting it would take out the successor's + // route and leave hard mode blocking DNS with nothing redirecting it. Teardown of + // the state this cycle belonged to already owns that cleanup. + mainLog.Load().Debug().Msg("DNS intercept: intercept retired after the two-phase re-add - leaving cleanup to the owning teardown") + return false + } + if ops.probe(state) { + mainLog.Load().Info().Msg("DNS intercept: NRPT verified working after two-phase recovery") + return true + } + + logNRPTParentKeyState("probe-failed-final") + mainLog.Load().Warn().Msg("DNS intercept: NRPT verification failed after all retries including two-phase recovery") + + // Last resort: activate WFP loopback protection. + // Third-party VPN software (e.g., OpenVPN with block-outside-dns) may have + // installed WFP filters that block DNS to non-tunnel interfaces, including + // loopback. A high-priority "hard permit" for localhost DNS overrides these + // blocks and restores NRPT routing to ctrld's listener. + // Bail out if shutdown is in progress — avoid racing with cleanupWFPFilters. + if p.interceptStateRevoked(state) { + mainLog.Load().Info().Msg("DNS intercept: shutdown in progress, skipping loopback WFP protect activation") + return false + } + + if err := p.activateLoopbackWFPProtect(state); err != nil { + mainLog.Load().Error().Err(err).Msg("DNS intercept: failed to activate loopback WFP protect — " + + "DNS queries may not be routed through ctrld. A network interface toggle may be needed.") + return false + } + + // Retry NRPT probe now that loopback DNS is explicitly permitted through WFP. + if !ops.wait(state, loopbackProtectSettleDelay) { + return false + } + if ops.probe(state) { + mainLog.Load().Info().Msg("DNS intercept: NRPT verified working after loopback WFP protect activation") + return true + } + mainLog.Load().Error().Msg("DNS intercept: NRPT probe still failing after loopback WFP protect — " + + "DNS queries may not be routed through ctrld. A network interface toggle may be needed.") + return false +} diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 994741b1..f5555d3f 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -25,56 +25,100 @@ import ( "github.com/Control-D-Inc/ctrld/internal/controld" "github.com/Control-D-Inc/ctrld/internal/dnscache" ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" - "github.com/Control-D-Inc/ctrld/internal/router" + "github.com/Control-D-Inc/ctrld/internal/rulematcher" ) +// DNS proxy constants for configuration and behavior control const ( + // staleTTL is the TTL for stale cache entries + // This allows serving cached responses even when upstreams are temporarily unavailable staleTTL = 60 * time.Second + + // localTTL is the TTL for local network responses + // Longer TTL for local queries reduces unnecessary repeated lookups localTTL = 3600 * time.Second + // zpaDNSEchoDomain is Zscaler Client Connector's fixed public health-check + // name. Client Connector must see the answer to it arrive on its own DNS + // path before it will synthesize addresses for Private Access applications. + zpaDNSEchoDomain = "dnsechotest.zscaler.com" + // zpaDNSEchoPolicyName is reported as the matched policy when the built-in + // ZPA DNS echo bypass routes a query, so log lines name the reason. + zpaDNSEchoPolicyName = "Built-in ZPA DNS echo bypass" + // noRuleMatched is the matchedRule placeholder used when no listener domain + // rule matched the query. It is compared, not only logged: the built-in ZPA + // echo bypass yields to an explicit domain rule for the same name. + noRuleMatched = "no rule" + // EDNS0_OPTION_MAC is dnsmasq EDNS0 code for adding mac option. // https://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=blob;f=src/dns-protocol.h;h=76ac66a8c28317e9c121a74ab5fd0e20f6237dc8;hb=HEAD#l81 // This is also dns.EDNS0LOCALSTART, but define our own constant here for clarification. + // This enables MAC address-based client identification for policy routing EDNS0_OPTION_MAC = 0xFDE9 // selfUninstallMaxQueries is number of REFUSED queries seen before checking for self-uninstallation. + // This prevents premature self-uninstallation due to temporary network issues selfUninstallMaxQueries = 32 ) +// zpaDNSEchoBypassEnabled limits the built-in ZPA DNS echo bypass to macOS, +// the only platform where pf interception is known to break Zscaler Private +// Access. Tests override it. +var zpaDNSEchoBypassEnabled = runtime.GOOS == "darwin" + +// zpaDNSEchoBypassActive reports whether the built-in Zscaler DNS echo bypass +// applies. Like every other VPN accommodation it is skipped in +// --intercept-mode hard, where the operator has asked for all DNS to go +// through ctrld with no split routing. +func zpaDNSEchoBypassActive() bool { + return zpaDNSEchoBypassEnabled && dnsIntercept && !hardIntercept +} + +// osUpstreamConfig defines the default OS resolver configuration +// This is used as a fallback when all configured upstreams fail var osUpstreamConfig = &ctrld.UpstreamConfig{ Name: "OS resolver", Type: ctrld.ResolverTypeOS, Timeout: 3000, } +// privateUpstreamConfig defines the default private resolver configuration +// This is used for internal network queries that should not go to public resolvers var privateUpstreamConfig = &ctrld.UpstreamConfig{ Name: "Private resolver", Type: ctrld.ResolverTypePrivate, Timeout: 2000, } -var localUpstreamConfig = &ctrld.UpstreamConfig{ - Name: "Local resolver", - Type: ctrld.ResolverTypeLocal, - Timeout: 2000, -} - // proxyRequest contains data for proxying a DNS query to upstream. +// This structure encapsulates all the information needed to process a DNS request type proxyRequest struct { - msg *dns.Msg - ci *ctrld.ClientInfo - failoverRcodes []int - ufr *upstreamForResult + msg *dns.Msg + ci *ctrld.ClientInfo + failoverRcodes []int + ufr *upstreamForResult + staleAnswer *dns.Msg + isLanOrPtrQuery bool + upstreamConfigs []*ctrld.UpstreamConfig + // internalFallback is set when the upstreams are the configured resolvers + // of an Internal Domain in explicit-with-network-fallback mode. + // internalNegative keeps the best SERVFAIL or NXDOMAIN answer those + // resolvers gave, so a name nobody resolves gets a real negative answer. + internalFallback bool + internalNegative *dns.Msg } // proxyResponse contains data for proxying a DNS response from upstream. +// This structure encapsulates the response and metadata for logging and metrics type proxyResponse struct { answer *dns.Msg + upstream string cached bool clientInfo bool - upstream string + refused bool } // upstreamForResult represents the result of processing rules for a request. +// This contains the matched policy information for logging and debugging type upstreamForResult struct { upstreams []string matchedPolicy string @@ -82,172 +126,359 @@ type upstreamForResult struct { matchedRule string matched bool srcAddr string + // zpaDNSEcho keeps the health check on the OS path and out of the cache. + zpaDNSEcho bool +} + +func (p *prog) addCachedResponse(key dnscache.Key, answer *dns.Msg) { + ttl := ttlFromMsg(answer) + now := time.Now() + expired := now.Add(time.Duration(ttl) * time.Second) + if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 { + expired = now.Add(time.Duration(cachedTTL) * time.Second) + } + setCachedAnswerTTL(answer, now, expired) + p.cache.Add(key, dnscache.NewValue(answer, expired)) } -func (p *prog) serveDNS(listenerNum string) error { +func (p *prog) cachedResponse(req *dns.Msg, upstream string, dns64Prefix netip.Prefix, dns64Active bool, now time.Time) (answer, stale *dns.Msg, hit, dns64Hit, dns64Bypass bool) { + if dns64Active && dns64RequestAllowed(req) { + if cachedValue := p.cache.Get(dns64CacheKey(req, upstream, dns64Prefix)); cachedValue != nil { + answer = cachedValue.Msg.Copy() + ctrld.SetCacheReply(answer, req, answer.Rcode) + if cachedValue.Expire.After(now) { + setCachedAnswerTTL(answer, now, cachedValue.Expire) + return answer, nil, true, true, false + } + stale = answer + } + } + + cachedValue := p.cache.Get(dnscache.NewKey(req, upstream)) + if cachedValue == nil { + return nil, stale, false, false, false + } + answer = cachedValue.Msg.Copy() + ctrld.SetCacheReply(answer, req, answer.Rcode) + if cachedValue.Expire.After(now) { + if dns64Eligible(req, answer) && dns64Active { + if stale == nil { + stale = answer + } + return nil, stale, false, false, true + } + setCachedAnswerTTL(answer, now, cachedValue.Expire) + return answer, stale, true, false, false + } + if stale == nil { + stale = answer + } + return nil, stale, false, false, false +} + +// serveDNS sets up and starts a DNS server on the specified listener, handling DNS queries and network monitoring. +// This is the main entry point for DNS server functionality +func (p *prog) serveDNS(ctx context.Context, listenerNum string) error { + logger := p.logger.Load() + logger.Debug().Msg("DNS server setup started") + listenerConfig := p.cfg.Listener[listenerNum] - // make sure ip is allocated if allocErr := p.allocateIP(listenerConfig.IP); allocErr != nil { - mainLog.Load().Error().Err(allocErr).Str("ip", listenerConfig.IP).Msg("serveUDP: failed to allocate listen ip") + p.Error().Err(allocErr).Str("ip", listenerConfig.IP).Msg("serveUDP: Failed to allocate listen IP") return allocErr } handler := dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { - p.sema.acquire() - defer p.sema.release() - if len(m.Question) == 0 { - answer := new(dns.Msg) - answer.SetRcode(m, dns.RcodeFormatError) - _ = w.WriteMsg(answer) - return - } - listenerConfig := p.cfg.Listener[listenerNum] - reqId := requestID() - ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqId) - if !listenerConfig.AllowWanClients && isWanClient(w.RemoteAddr()) { - ctrld.Log(ctx, mainLog.Load().Debug(), "query refused, listener does not allow WAN clients: %s", w.RemoteAddr().String()) - answer := new(dns.Msg) - answer.SetRcode(m, dns.RcodeRefused) - _ = w.WriteMsg(answer) - return - } - go p.detectLoop(m) - q := m.Question[0] - domain := canonicalName(q.Name) - switch { - case domain == "": - answer := new(dns.Msg) - answer.SetRcode(m, dns.RcodeFormatError) - _ = w.WriteMsg(answer) - return - case domain == selfCheckInternalTestDomain: - answer := resolveInternalDomainTestQuery(ctx, domain, m) - _ = w.WriteMsg(answer) - return - } + p.handleDNSQuery(w, m, listenerNum, listenerConfig) + }) - if _, ok := p.cacheFlushDomainsMap[domain]; ok && p.cache != nil { - p.cache.Purge() - ctrld.Log(ctx, mainLog.Load().Debug(), "received query %q, local cache is purged", domain) - } - remoteIP, _, _ := net.SplitHostPort(w.RemoteAddr().String()) - ci := p.getClientInfo(remoteIP, m) - ci.ClientIDPref = p.cfg.Service.ClientIDPref - stripClientSubnet(m) - remoteAddr := spoofRemoteAddr(w.RemoteAddr(), ci) - fmtSrcToDest := fmtRemoteToLocal(listenerNum, ci.Hostname, remoteAddr.String()) - t := time.Now() - ctrld.Log(ctx, mainLog.Load().Info(), "QUERY: %s: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], domain) - ur := p.upstreamFor(ctx, listenerNum, listenerConfig, remoteAddr, ci.Mac, domain) - - labelValues := make([]string, 0, len(statsQueriesCountLabels)) - labelValues = append(labelValues, net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port))) - labelValues = append(labelValues, ci.IP) - labelValues = append(labelValues, ci.Mac) - labelValues = append(labelValues, ci.Hostname) - - var answer *dns.Msg - if !ur.matched && listenerConfig.Restricted { - ctrld.Log(ctx, mainLog.Load().Info(), "query refused, %s does not match any network policy", remoteAddr.String()) - answer = new(dns.Msg) - answer.SetRcode(m, dns.RcodeRefused) - labelValues = append(labelValues, "") // no upstream - } else { - var failoverRcode []int - if listenerConfig.Policy != nil { - failoverRcode = listenerConfig.Policy.FailoverRcodeNumbers - } - pr := p.proxy(ctx, &proxyRequest{ - msg: m, - ci: ci, - failoverRcodes: failoverRcode, - ufr: ur, - }) - go p.doSelfUninstall(pr.answer) + logger.Debug().Msg("DNS server setup completed") + return p.startListeners(ctx, listenerConfig, handler) +} - answer = pr.answer - rtt := time.Since(t) - ctrld.Log(ctx, mainLog.Load().Debug(), "received response of %d bytes in %s", answer.Len(), rtt) - upstream := pr.upstream - switch { - case pr.cached: - upstream = "cache" - case pr.clientInfo: - upstream = "client_info_table" - } - labelValues = append(labelValues, upstream) - } - labelValues = append(labelValues, dns.TypeToString[q.Qtype]) - labelValues = append(labelValues, dns.RcodeToString[answer.Rcode]) - go func() { - p.WithLabelValuesInc(statsQueriesCount, labelValues...) - p.WithLabelValuesInc(statsClientQueriesCount, []string{ci.IP, ci.Mac, ci.Hostname}...) - p.forceFetchingAPI(domain) - }() - if err := w.WriteMsg(answer); err != nil { - ctrld.Log(ctx, mainLog.Load().Error().Err(err), "serveDNS: failed to send DNS response to client") - } - }) +// startListeners starts DNS listeners on specified configurations, supporting UDP and TCP protocols. +// It handles local IPv6, RFC 1918, and specified IP listeners, reacting to stop signals or errors. +// This function manages the lifecycle of DNS server listeners +func (p *prog) startListeners(ctx context.Context, cfg *ctrld.ListenerConfig, handler dns.Handler) error { + logger := p.logger.Load() + logger.Debug().Msg("Starting DNS listeners") + + g, gctx := errgroup.WithContext(ctx) - g, ctx := errgroup.WithContext(context.Background()) for _, proto := range []string{"udp", "tcp"} { - proto := proto if needLocalIPv6Listener() { + logger.Debug().Str("protocol", proto).Msg("Starting local IPv6 listener") g.Go(func() error { - s, errCh := runDNSServer(net.JoinHostPort("::1", strconv.Itoa(listenerConfig.Port)), proto, handler) + s, errCh := runDNSServer(net.JoinHostPort("::1", strconv.Itoa(cfg.Port)), proto, handler) defer s.Shutdown() select { case <-p.stopCh: - case <-ctx.Done(): + case <-p.runAbortCh: + case <-gctx.Done(): case err := <-errCh: - // Local ipv6 listener should not terminate ctrld. - // It's a workaround for a quirk on Windows. - mainLog.Load().Warn().Err(err).Msg("local ipv6 listener failed") + p.Warn().Err(err).Msg("Local IPv6 listener failed") } return nil }) } + // When we spawn a listener on 127.0.0.1, also spawn listeners on the RFC1918 addresses of the machine // if explicitly set via setting rfc1918 flag, so ctrld could receive queries from LAN clients. - if needRFC1918Listeners(listenerConfig) { + if needRFC1918Listeners(cfg) { + logger.Debug().Str("protocol", proto).Msg("Starting RFC1918 listeners") g.Go(func() error { for _, addr := range ctrld.Rfc1918Addresses() { func() { - listenAddr := net.JoinHostPort(addr, strconv.Itoa(listenerConfig.Port)) + listenAddr := net.JoinHostPort(addr, strconv.Itoa(cfg.Port)) s, errCh := runDNSServer(listenAddr, proto, handler) defer s.Shutdown() select { case <-p.stopCh: - case <-ctx.Done(): + case <-p.runAbortCh: + case <-gctx.Done(): case err := <-errCh: - // RFC1918 listener should not terminate ctrld. - // It's a workaround for a quirk on system with systemd-resolved. - mainLog.Load().Warn().Err(err).Msgf("could not listen on %s: %s", proto, listenAddr) + p.Warn().Err(err).Msgf("Could not listen on %s: %s", proto, listenAddr) } }() } return nil }) } + + logger.Debug().Str("protocol", proto).Str("ip", cfg.IP).Int("port", cfg.Port).Msg("Starting main listener") g.Go(func() error { - addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)) + addr := net.JoinHostPort(cfg.IP, strconv.Itoa(cfg.Port)) s, errCh := runDNSServer(addr, proto, handler) defer s.Shutdown() - - p.started <- struct{}{} - + select { + case p.started <- struct{}{}: + case <-p.stopCh: + return nil + case <-p.runAbortCh: + return nil + case <-gctx.Done(): + return nil + } select { case <-p.stopCh: - case <-ctx.Done(): + case <-p.runAbortCh: + case <-gctx.Done(): case err := <-errCh: return err } return nil }) } + + logger.Debug().Msg("DNS listeners started successfully") return g.Wait() } +// handleDNSQuery processes incoming DNS queries, validates client access, and routes the query to appropriate handlers. +// This is the main entry point for all DNS query processing +func (p *prog) handleDNSQuery(w dns.ResponseWriter, m *dns.Msg, listenerNum string, listenerConfig *ctrld.ListenerConfig) { + p.sema.acquire() + defer p.sema.release() + + if len(m.Question) == 0 { + sendDNSResponse(w, m, dns.RcodeFormatError) + return + } + + reqID := requestID() + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, reqID) + ctx = ctrld.LoggerCtx(ctx, p.logger.Load()) + + ctrld.Log(ctx, p.Debug(), "Processing DNS query from %s", w.RemoteAddr().String()) + + if !listenerConfig.AllowWanClients && isWanClient(w.RemoteAddr()) { + ctrld.Log(ctx, p.Debug(), "Query refused, listener does not allow WAN clients: %s", w.RemoteAddr().String()) + sendDNSResponse(w, m, dns.RcodeRefused) + return + } + + go p.detectLoop(m) + + q := m.Question[0] + domain := canonicalName(q.Name) + + if p.handleSpecialDomains(ctx, w, m, domain) { + ctrld.Log(ctx, p.Debug(), "Special domain query handled") + return + } + + ctrld.Log(ctx, p.Debug(), "Processing standard query for domain: %s", domain) + p.processStandardQuery(&standardQueryRequest{ + ctx: ctx, + writer: w, + msg: m, + listenerNum: listenerNum, + listenerConfig: listenerConfig, + domain: domain, + }) +} + +// handleSpecialDomains processes special domain queries, handles errors, purges cache if necessary, and returns a bool status. +// This handles internal test domains and cache management commands +func (p *prog) handleSpecialDomains(ctx context.Context, w dns.ResponseWriter, m *dns.Msg, domain string) bool { + switch { + case domain == "": + ctrld.Log(ctx, p.Debug(), "Empty domain query, sending format error") + sendDNSResponse(w, m, dns.RcodeFormatError) + return true + case domain == selfCheckInternalTestDomain: + ctrld.Log(ctx, p.Debug(), "Internal test domain query: %s", domain) + answer := resolveInternalDomainTestQuery(ctx, domain, m) + _ = w.WriteMsg(answer) + return true + } + + // Interception probe: if we're expecting a probe query and this matches, + // signal the prober and respond NXDOMAIN. Used by both macOS pf probes + // (_pf-probe-*) and Windows NRPT probes (_nrpt-probe-*) to verify that + // DNS interception is actually routing queries to ctrld's listener. + if p.signalInterceptProbe(domain) { + sendDNSResponse(w, m, dns.RcodeNameError) // NXDOMAIN + return true + } + + if _, ok := p.cacheFlushDomainsMap[domain]; ok && p.cache != nil { + p.cache.Purge() + ctrld.Log(ctx, p.Debug(), "Received query %q, local cache is purged", domain) + } + + return false +} + +// standardQueryRequest represents a standard DNS query request with associated context and configuration. +// This encapsulates all the data needed to process a standard DNS query +type standardQueryRequest struct { + ctx context.Context + writer dns.ResponseWriter + msg *dns.Msg + listenerNum string + listenerConfig *ctrld.ListenerConfig + domain string +} + +// processStandardQuery handles a standard DNS query by routing it through appropriate upstreams and writing a DNS response. +// This is the main processing pipeline for normal DNS queries +func (p *prog) processStandardQuery(req *standardQueryRequest) { + ctrld.Log(req.ctx, p.Debug(), "Processing standard query started") + + remoteIP, _, _ := net.SplitHostPort(req.writer.RemoteAddr().String()) + ci := p.getClientInfo(remoteIP, req.msg) + ci.ClientIDPref = p.cfg.Service.ClientIDPref + + stripClientSubnet(req.msg) + remoteAddr := spoofRemoteAddr(req.writer.RemoteAddr(), ci) + fmtSrcToDest := fmtRemoteToLocal(req.listenerNum, ci.Hostname, remoteAddr.String()) + + startTime := time.Now() + q := req.msg.Question[0] + ctrld.Log(req.ctx, p.Info(), "QUERY: %s: %s %s", fmtSrcToDest, dns.TypeToString[q.Qtype], req.domain) + + ur := p.upstreamFor(req.ctx, req.listenerNum, req.listenerConfig, remoteAddr, ci.Mac, req.domain) + + var answer *dns.Msg + // Handle restricted listener case + if !ur.matched && req.listenerConfig.Restricted { + ctrld.Log(req.ctx, p.Debug(), "Query refused, %s does not match any network policy", remoteAddr.String()) + answer = new(dns.Msg) + answer.SetRcode(req.msg, dns.RcodeRefused) + // The grade of the window must not depend on the goroutine below. + p.health.countQuery() + // Process the refused query + go p.postProcessStandardQuery(ci, req.listenerConfig, q, &proxyResponse{answer: answer, refused: true}) + } else { + // Process a normal query + ctrld.Log(req.ctx, p.Debug(), "Starting proxy query processing") + pr := p.proxy(req.ctx, &proxyRequest{ + msg: req.msg, + ci: ci, + failoverRcodes: p.getFailoverRcodes(req.listenerConfig), + ufr: ur, + }) + + rtt := time.Since(startTime) + ctrld.Log(req.ctx, p.Debug(), "Received response of %d bytes in %s", pr.answer.Len(), rtt) + + // Firewall mode must learn resolved IPs before the DNS response is sent + // back to the client. Otherwise the app can receive the answer and attempt + // the first connection before the platform allowlist has been updated. + if p.firewallModeEnabled() && pr.answer != nil { + p.firewallRecordResolvedIPs(pr.answer, canonicalName(q.Name)) + } + + // The grade of the window must not depend on the goroutine below. + p.health.countQuery() + go p.postProcessStandardQuery(ci, req.listenerConfig, q, pr) + answer = pr.answer + } + + if err := req.writer.WriteMsg(answer); err != nil { + ctrld.Log(req.ctx, p.querySampler.event(sampleClassSendResponseFailed, "").Err(err), "serveDNS: failed to send DNS response to client") + } + + ctrld.Log(req.ctx, p.Debug(), "Standard query processing completed") +} + +// postProcessStandardQuery performs additional actions after processing a standard DNS query, such as metrics recording, +// handling canonical name adjustments, and triggering specific post-query actions like uninstallation procedures. +func (p *prog) postProcessStandardQuery(ci *ctrld.ClientInfo, listenerConfig *ctrld.ListenerConfig, q dns.Question, pr *proxyResponse) { + p.doSelfUninstall(pr) + p.recordMetrics(ci, listenerConfig, q, pr) + p.forceFetchingAPI(canonicalName(q.Name)) + +} + +// getFailoverRcodes retrieves the failover response codes from the provided ListenerConfig. Returns nil if no policy exists. +func (p *prog) getFailoverRcodes(cfg *ctrld.ListenerConfig) []int { + if cfg.Policy != nil { + return cfg.Policy.FailoverRcodeNumbers + } + return nil +} + +// recordMetrics updates Prometheus metrics for DNS queries, including query count and client-specific query statistics. +func (p *prog) recordMetrics(ci *ctrld.ClientInfo, cfg *ctrld.ListenerConfig, q dns.Question, pr *proxyResponse) { + upstream := pr.upstream + switch { + case pr.cached: + upstream = "cache" + case pr.clientInfo: + upstream = "client_info_table" + } + labelValues := []string{ + net.JoinHostPort(cfg.IP, strconv.Itoa(cfg.Port)), + ci.IP, + ci.Mac, + ci.Hostname, + upstream, + dns.TypeToString[q.Qtype], + dns.RcodeToString[pr.answer.Rcode], + } + p.WithLabelValuesInc(statsQueriesCount, labelValues...) + p.WithLabelValuesInc(statsClientQueriesCount, []string{ci.IP, ci.Mac, ci.Hostname}...) +} + +// sendDNSResponse sends a DNS response with the specified RCODE to the client using the provided ResponseWriter. +func sendDNSResponse(w dns.ResponseWriter, m *dns.Msg, rcode int) { + answer := new(dns.Msg) + answer.SetRcode(m, rcode) + _ = w.WriteMsg(answer) +} + +// upstreamForRequest contains all parameters needed for upstream determination +type upstreamForRequest struct { + DefaultUpstreamNum string + ListenerConfig *ctrld.ListenerConfig + Addr net.Addr + SrcMac string + Domain string + MatchingConfig *rulematcher.MatchingConfig +} + // upstreamFor returns the list of upstreams for resolving the given domain, // matching by policies defined in the listener config. The second return value // reports whether the domain matches the policy. @@ -256,94 +487,147 @@ func (p *prog) serveDNS(listenerNum string) error { // processed later, because policy logging want to know whether a network rule // is disregarded in favor of the domain level rule. func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *ctrld.ListenerConfig, addr net.Addr, srcMac, domain string) (res *upstreamForResult) { - upstreams := []string{upstreamPrefix + defaultUpstreamNum} - matchedPolicy := "no policy" - matchedNetwork := "no network" - matchedRule := "no rule" - matched := false - res = &upstreamForResult{srcAddr: addr.String()} + var matchingConfig *rulematcher.MatchingConfig + if lc.Policy != nil && lc.Policy.Matching != nil { + // Convert string-based order to RuleType enum + var order []rulematcher.RuleType + for _, ruleTypeStr := range lc.Policy.Matching.Order { + switch ruleTypeStr { + case "network": + order = append(order, rulematcher.RuleTypeNetwork) + case "mac": + order = append(order, rulematcher.RuleTypeMac) + case "domain": + order = append(order, rulematcher.RuleTypeDomain) + } + } + matchingConfig = &rulematcher.MatchingConfig{ + Order: order, + } + } + + req := &upstreamForRequest{ + DefaultUpstreamNum: defaultUpstreamNum, + ListenerConfig: lc, + Addr: addr, + SrcMac: srcMac, + Domain: domain, + MatchingConfig: matchingConfig, + } + + return p.upstreamForWithConfig(ctx, req) +} + +// upstreamForWithConfig determines upstreams using configurable rule matching +func (p *prog) upstreamForWithConfig(ctx context.Context, req *upstreamForRequest) (res *upstreamForResult) { + // Default upstreams + upstreams := []string{upstreamPrefix + req.DefaultUpstreamNum} + res = &upstreamForResult{srcAddr: req.Addr.String()} defer func() { - res.upstreams = upstreams - res.matched = matched - res.matchedPolicy = matchedPolicy - res.matchedNetwork = matchedNetwork - res.matchedRule = matchedRule + // Zscaler Private Access: under macOS Intercept Mode, pf routes Client + // Connector's health-check query for zpaDNSEchoDomain into ctrld, which + // answers it from the Control D upstream. Client Connector never sees the + // answer on its own DNS path, so it keeps Private Access disabled. + // + // Route only that one exact name the way a Control D "bypass" rule does: + // an empty upstream list, which proxy() resolves through upstream.os — + // the system resolver set, which under Intercept Mode still includes + // Client Connector's own DNS servers. Matching on the name alone covers + // every record type, the same as a bypass rule. + // + // This runs after policy evaluation, and never touches `matched`. That + // flag is also the source authorization bit: serveDNS refuses a query on + // a Restricted listener when it is false, so forcing it true here would + // let an unauthorized client resolve this name through such a listener. + // + // The scope is one fixed public health-check name: every other domain, + // including all Private Access application domains, stays intercepted and + // filtered by Control D. --intercept-mode hard opts out, the same as it + // opts out of VPN DNS split routing. + if zpaDNSEchoBypassActive() && canonicalName(req.Domain) == zpaDNSEchoDomain { + switch { + case res.matchedRule == noRuleMatched: + // The listener policy says nothing about this name, so the + // built-in route decides. Network- and MAC-policy targets route + // every name from a source and so state nothing about this one: + // override them, and the default upstream, and label the result + // so logs name the reason. + res.upstreams = nil + res.matchedPolicy = zpaDNSEchoPolicyName + res.matchedRule = zpaDNSEchoDomain + res.zpaDNSEcho = true + case len(res.upstreams) == 0: + // An explicit domain rule already selects the OS path: a rule + // with empty targets, which is how a Control D profile's bypass + // list reaches us (see the cfg.Listener rules built from + // resolverConfig.Exclude). That is the documented pre-fix + // workaround for this very bug, so keep the operator's own + // labels but still classify it as the health check — otherwise + // anyone who keeps the workaround after upgrading would route + // correctly and then have the answer served from our cache, + // losing the reconnect fix. + res.zpaDNSEcho = true + default: + // Any other explicit domain rule is a deliberate non-OS route + // for this exact name. That is operator intent, and the + // supported way to opt out without leaving Intercept Mode, so + // leave the policy's decision untouched. + } + } }() - if lc.Policy == nil { + // If no policy, return default upstreams + if req.ListenerConfig.Policy == nil { + res.upstreams = upstreams + res.matched = false + res.matchedPolicy = "no policy" + res.matchedNetwork = "no network" + res.matchedRule = noRuleMatched return } - do := func(policyUpstreams []string) { - upstreams = append([]string(nil), policyUpstreams...) - } - - var networkTargets []string + // Extract source IP from address var sourceIP net.IP - switch addr := addr.(type) { + switch addr := req.Addr.(type) { case *net.UDPAddr: sourceIP = addr.IP case *net.TCPAddr: sourceIP = addr.IP } -networkRules: - for _, rule := range lc.Policy.Networks { - for source, targets := range rule { - networkNum := strings.TrimPrefix(source, "network.") - nc := p.cfg.Network[networkNum] - if nc == nil { - continue - } - for _, ipNet := range nc.IPNets { - if ipNet.Contains(sourceIP) { - matchedPolicy = lc.Policy.Name - matchedNetwork = source - networkTargets = targets - matched = true - break networkRules - } - } - } + // Create match request + matchRequest := &rulematcher.MatchRequest{ + SourceIP: sourceIP, + SourceMac: req.SrcMac, + Domain: req.Domain, + Policy: req.ListenerConfig.Policy, + Config: p.cfg, } -macRules: - for _, rule := range lc.Policy.Macs { - for source, targets := range rule { - if source != "" && (strings.EqualFold(source, srcMac) || wildcardMatches(strings.ToLower(source), strings.ToLower(srcMac))) { - matchedPolicy = lc.Policy.Name - matchedNetwork = source - networkTargets = targets - matched = true - break macRules - } - } - } + // Use matching engine to find upstreams + engine := rulematcher.NewMatchingEngine(req.MatchingConfig) + matchResult := engine.FindUpstreams(ctx, matchRequest) - for _, rule := range lc.Policy.Rules { - // There's only one entry per rule, config validation ensures this. - for source, targets := range rule { - if source == domain || wildcardMatches(source, domain) { - matchedPolicy = lc.Policy.Name - if len(networkTargets) > 0 { - matchedNetwork += " (unenforced)" - } - matchedRule = source - do(targets) - matched = true - return - } - } - } + // Convert result to upstreamForResult format + res.upstreams = matchResult.Upstreams + res.matched = matchResult.Matched + res.matchedPolicy = matchResult.MatchedPolicy + res.matchedNetwork = matchResult.MatchedNetwork + res.matchedRule = matchResult.MatchedRule - if matched { - do(networkTargets) + // If no match found, use default upstreams + if !matchResult.Matched { + res.upstreams = upstreams } return } +// proxyPrivatePtrLookup performs a private PTR DNS lookup based on the client info table for the given query. +// It prevents DNS loops by locking the processing of the same domain name simultaneously. +// If a valid IP-to-hostname mapping exists, it creates a PTR DNS record as the response. +// Returns the DNS response if a hostname is found or nil otherwise. func (p *prog) proxyPrivatePtrLookup(ctx context.Context, msg *dns.Msg) *dns.Msg { cDomainName := msg.Question[0].Name locked := p.ptrLoopGuard.TryLock(cDomainName) @@ -364,8 +648,8 @@ func (p *prog) proxyPrivatePtrLookup(ctx context.Context, msg *dns.Msg) *dns.Msg }, Ptr: dns.Fqdn(name), }} - ctrld.Log(ctx, mainLog.Load().Info(), "private PTR lookup, using client info table") - ctrld.Log(ctx, mainLog.Load().Debug(), "client info: %v", ctrld.ClientInfo{ + ctrld.Log(ctx, p.Info(), "Private PTR lookup, using client info table") + ctrld.Log(ctx, p.Debug(), "Client info: %v", ctrld.ClientInfo{ Mac: p.ciTable.LookupMac(ip.String()), IP: ip.String(), Hostname: name, @@ -375,6 +659,10 @@ func (p *prog) proxyPrivatePtrLookup(ctx context.Context, msg *dns.Msg) *dns.Msg return nil } +// proxyLanHostnameQuery resolves LAN hostnames to their corresponding IP addresses based on the dns.Msg request. +// It uses a loop guard mechanism to prevent DNS query loops and ensures a hostname is processed only once at a time. +// This method queries the client info table for the hostname's IP address and logs relevant debug and client info. +// If the hostname matches known IPs in the table, it generates an appropriate dns.Msg response; otherwise, it returns nil. func (p *prog) proxyLanHostnameQuery(ctx context.Context, msg *dns.Msg) *dns.Msg { q := msg.Question[0] hostname := strings.TrimSuffix(q.Name, ".") @@ -409,8 +697,8 @@ func (p *prog) proxyLanHostnameQuery(ctx context.Context, msg *dns.Msg) *dns.Msg AAAA: ip.AsSlice(), }} } - ctrld.Log(ctx, mainLog.Load().Info(), "lan hostname lookup, using client info table") - ctrld.Log(ctx, mainLog.Load().Debug(), "client info: %v", ctrld.ClientInfo{ + ctrld.Log(ctx, p.Info(), "Lan hostname lookup, using client info table") + ctrld.Log(ctx, p.Debug(), "Client info: %v", ctrld.ClientInfo{ Mac: p.ciTable.LookupMac(ip.String()), IP: ip.String(), Hostname: hostname, @@ -420,243 +708,669 @@ func (p *prog) proxyLanHostnameQuery(ctx context.Context, msg *dns.Msg) *dns.Msg return nil } -func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { - var staleAnswer *dns.Msg - upstreams := req.ufr.upstreams - serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale - upstreamConfigs := p.upstreamConfigsFromUpstreamNumbers(upstreams) +// handleSpecialQueryTypes processes specific types of DNS queries such as SRV, PTR, and LAN hostname lookups. +// It modifies upstreams and upstreamConfigs based on the query type and updates the query context accordingly. +// Returns a proxyResponse if the query is resolved locally; otherwise, returns nil to proceed with upstream processing. +func (p *prog) handleSpecialQueryTypes(ctx *context.Context, req *proxyRequest, upstreams *[]string, upstreamConfigs *[]*ctrld.UpstreamConfig) *proxyResponse { + if req.ufr.matched { + ctrld.Log(*ctx, p.Debug(), "%s, %s, %s -> %v", + req.ufr.matchedPolicy, req.ufr.matchedNetwork, req.ufr.matchedRule, *upstreams) + // A rule sending an Active Directory name to the OS resolver (the + // auto-added AD split rule, or an operator's equivalent) is a LAN + // lookup: keep it off Control D's public resolver. + if osUpstreamOnly(*upstreams) && len(req.msg.Question) > 0 && inActiveDirectoryDomain(req.msg.Question[0].Name) { + *ctx = ctrld.LanQueryCtx(*ctx) + ctrld.Log(*ctx, p.Debug(), "Active Directory domain lookup, skipping public DNS") + } + return nil + } - if len(upstreamConfigs) == 0 { - upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} - upstreams = []string{upstreamOS} - // For OS resolver, local addresses are ignored to prevent possible looping. - // However, on Active Directory Domain Controller, where it has local DNS server - // running and listening on local addresses, these local addresses must be used - // as nameservers, so queries for ADDC could be resolved as expected. - if p.isAdDomainQuery(req.msg) { - ctrld.Log(ctx, mainLog.Load().Debug(), - "AD domain query detected for %s in domain %s", - req.msg.Question[0].Name, p.adDomain) - upstreamConfigs = []*ctrld.UpstreamConfig{localUpstreamConfig} - upstreams = []string{upstreamOSLocal} + switch { + case req.ufr.zpaDNSEcho: + ctrld.Log(*ctx, p.Debug(), "%s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedRule, *upstreams) + return nil + case isSrvLanLookup(req.msg): + *upstreams = []string{upstreamOS} + *upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} + *ctx = ctrld.LanQueryCtx(*ctx) + ctrld.Log(*ctx, p.Debug(), "SRV record lookup, using upstreams: %v", *upstreams) + return nil + case isPrivatePtrLookup(req.msg): + req.isLanOrPtrQuery = true + if answer := p.proxyPrivatePtrLookup(*ctx, req.msg); answer != nil { + return &proxyResponse{answer: answer, clientInfo: true} + } + *upstreams, *upstreamConfigs = p.upstreamsAndUpstreamConfigForPtr(*upstreams, *upstreamConfigs) + *ctx = ctrld.LanQueryCtx(*ctx) + ctrld.Log(*ctx, p.Debug(), "Private PTR lookup, using upstreams: %v", *upstreams) + return nil + case isLanHostnameQuery(req.msg): + req.isLanOrPtrQuery = true + if answer := p.proxyLanHostnameQuery(*ctx, req.msg); answer != nil { + return &proxyResponse{answer: answer, clientInfo: true} } + *upstreams = []string{upstreamOS} + *upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} + *ctx = ctrld.LanQueryCtx(*ctx) + ctrld.Log(*ctx, p.Debug(), "Lan hostname lookup, using upstreams: %v", *upstreams) + return nil + default: + ctrld.Log(*ctx, p.Debug(), "No explicit policy matched, using default routing -> %v", *upstreams) + return nil } +} - res := &proxyResponse{} +// proxy handles DNS query proxying by selecting upstreams, attempting cache lookups, and querying configured resolvers. +func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { + ctrld.Log(ctx, p.Debug(), "Proxy query processing started") - // LAN/PTR lookup flow: + // DNS intercept recovery bypass: forward all queries to OS/DHCP resolver. + // This runs when upstreams are unreachable (e.g., captive portal network) + // and allows the network's DNS to handle authentication pages. // - // 1. If there's matching rule, follow it. - // 2. Try from client info table. - // 3. Try private resolver. - // 4. Try remote upstream. - isLanOrPtrQuery := false - if req.ufr.matched { - ctrld.Log(ctx, mainLog.Load().Debug(), "%s, %s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedNetwork, req.ufr.matchedRule, upstreams) - } else { - switch { - case isSrvLanLookup(req.msg): - upstreams = []string{upstreamOS} - upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} - ctx = ctrld.LanQueryCtx(ctx) - ctrld.Log(ctx, mainLog.Load().Debug(), "SRV record lookup, using upstreams: %v", upstreams) - case isPrivatePtrLookup(req.msg): - isLanOrPtrQuery = true - if answer := p.proxyPrivatePtrLookup(ctx, req.msg); answer != nil { - res.answer = answer - res.clientInfo = true - return res + // An Internal Domain with explicit resolvers does not take part. The bypass + // exists because general DNS is broken, which says nothing about the + // administrator's own resolvers: they may well be reachable, and they are + // to be tried first. Such a query continues to the normal flow, where it + // reaches its configured resolvers first. In "explicit resolver only" mode + // it fails if none answer; otherwise it falls back to the VPN resolvers and + // the network's LAN nameservers, never to the public nameservers the bypass + // may use. + if dnsIntercept && p.recoveryBypass.Load() && !p.internalDomainExplicitUpstreams(req.ufr.upstreams) { + ctrld.Log(ctx, p.Debug(), "Recovery bypass active: forwarding to OS resolver") + // The bypass runs before rule handling, so an Active Directory name + // needs its LAN mark here too, or it races the domain controller + // against Control D's public resolver in plaintext. + bypassCtx := ctx + if len(req.msg.Question) > 0 && inActiveDirectoryDomain(req.msg.Question[0].Name) { + bypassCtx = ctrld.LanQueryCtx(ctx) + ctrld.Log(ctx, p.Debug(), "Active Directory domain lookup, skipping public DNS") + } + answer := p.queryUpstream(bypassCtx, req, upstreamOS, osUpstreamConfig) + if answer != nil { + return &proxyResponse{answer: answer, upstream: osUpstreamConfig.Endpoint} + } + ctrld.Log(ctx, p.Debug(), "OS resolver failed during recovery bypass") + // Fall through to normal flow as last resort + } + + upstreams, upstreamConfigs := p.initializeUpstreams(req) + ctrld.Log(ctx, p.Debug(), "Initialized upstreams: %v", upstreams) + + if specialRes := p.handleSpecialQueryTypes(&ctx, req, &upstreams, &upstreamConfigs); specialRes != nil { + ctrld.Log(ctx, p.Debug(), "Special query type handled") + return specialRes + } + + if cachedRes := p.tryCache(ctx, req, upstreams); cachedRes != nil { + ctrld.Log(ctx, p.Debug(), "Cache hit, returning cached response") + p.health.countCacheHit() + return cachedRes + } + + // VPN DNS split routing (only in dns-intercept mode). Explicit Internal + // Domain resolvers take precedence over auto-detected VPN routes: handing + // the query to a VPN DNS server first would override an explicit selection + // with a discovered one. In fallback mode the VPN DNS servers are asked + // later, once the configured resolvers have not resolved the name. OS-mode + // Internal Domains still follow the endpoint's VPN resolver. + if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !p.internalDomainExplicitUpstreams(upstreams) { + domain := req.msg.Question[0].Name + if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 { + ctrld.Log(ctx, p.Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers) + + var gotTransportFailure bool + for _, server := range vpnServers { + upstreamConfig := p.vpnDNS.upstreamConfigFor(server) + ctrld.Log(ctx, p.Debug(), "Querying VPN DNS server: %s", server) + + answer := p.queryUpstream(ctx, req, "vpn-dns", upstreamConfig) + if answer != nil { + p.vpnDNS.VPNDNSReachable() + ctrld.Log(ctx, p.Debug(), "VPN DNS query successful") + + // Update cache if enabled + if p.cache != nil { + p.updateCache(ctx, req, answer, "vpn-dns") + } + + return &proxyResponse{answer: answer, cached: false} + } + gotTransportFailure = true + ctrld.Log(ctx, p.Debug(), "VPN DNS server %s failed", server) } - upstreams, upstreamConfigs = p.upstreamsAndUpstreamConfigForPtr(upstreams, upstreamConfigs) - ctx = ctrld.LanQueryCtx(ctx) - ctrld.Log(ctx, mainLog.Load().Debug(), "private PTR lookup, using upstreams: %v", upstreams) - case isLanHostnameQuery(req.msg): - isLanOrPtrQuery = true - if answer := p.proxyLanHostnameQuery(ctx, req.msg); answer != nil { - res.answer = answer - res.clientInfo = true - return res + + // Explicit VPN DNS routes are authoritative for their suffix. If all + // routed servers fail at the transport layer while Windows is serving + // retained VPN DNS state, fail closed instead of leaking VPN/internal + // names to normal upstreams. + if gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, vpnServers) { + ctrld.Log(ctx, p.Debug(), + "All VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) + answer := new(dns.Msg) + answer.SetRcode(req.msg, dns.RcodeServerFailure) + return &proxyResponse{answer: answer, cached: false} } - upstreams = []string{upstreamOS} - upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} - ctx = ctrld.LanQueryCtx(ctx) - ctrld.Log(ctx, mainLog.Load().Debug(), "lan hostname lookup, using upstreams: %v", upstreams) - default: - ctrld.Log(ctx, mainLog.Load().Debug(), "no explicit policy matched, using default routing -> %v", upstreams) + + ctrld.Log(ctx, p.Debug(), "All VPN DNS servers failed, falling back to normal upstreams") } } - // Inverse query should not be cached: https://www.rfc-editor.org/rfc/rfc1035#section-7.4 - if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR { - for _, upstream := range upstreams { - cachedValue := p.cache.Get(dnscache.NewKey(req.msg, upstream)) - if cachedValue == nil { - continue + // Domain-less VPN DNS fallback: when a query is going to upstream.os via a + // split-rule (matched policy) and we have VPN DNS servers with no associated + // domains, try those servers for this query. This handles cases like F5 VPN + // where the VPN doesn't advertise DNS search domains but its DNS servers + // know the internal zones referenced by split-rules (e.g., *.provisur.local). + // These servers are NOT used for general OS resolver queries to avoid + // polluting captive portal / DHCP flows. + if dnsIntercept && p.vpnDNS != nil && req.ufr.matched && + len(upstreams) > 0 && upstreams[0] == upstreamOS && + len(req.msg.Question) > 0 { + if dlServers := p.vpnDNS.DomainlessServers(); len(dlServers) > 0 { + domain := req.msg.Question[0].Name + ctrld.Log(ctx, p.Debug(), + "Split-rule query %s going to upstream.os, trying %d domain-less VPN DNS servers first: %v", + domain, len(dlServers), dlServers) + + var gotDNSAnswer bool + var gotTransportFailure bool + for _, server := range dlServers { + upstreamConfig := p.vpnDNS.upstreamConfigFor(server) + ctrld.Log(ctx, p.Debug(), "Querying domain-less VPN DNS server: %s", server) + + answer := p.queryUpstream(ctx, req, "vpn-dns", upstreamConfig) + if answer != nil { + gotDNSAnswer = true + p.vpnDNS.VPNDNSReachable() + } + if answer != nil && answer.Rcode == dns.RcodeSuccess { + ctrld.Log(ctx, p.Debug(), + "Domain-less VPN DNS server %s answered %s successfully", server, domain) + return &proxyResponse{answer: answer, cached: false} + } + if answer != nil { + ctrld.Log(ctx, p.Debug(), + "Domain-less VPN DNS server %s returned %s for %s, trying next", + server, dns.RcodeToString[answer.Rcode], domain) + } else { + gotTransportFailure = true + ctrld.Log(ctx, p.Debug(), "Domain-less VPN DNS server %s failed for %s", server, domain) + } } - answer := cachedValue.Msg.Copy() - ctrld.SetCacheReply(answer, req.msg, answer.Rcode) - now := time.Now() - if cachedValue.Expire.After(now) { - ctrld.Log(ctx, mainLog.Load().Debug(), "hit cached response") - setCachedAnswerTTL(answer, now, cachedValue.Expire) - res.answer = answer - res.cached = true - return res + + // If every domainless VPN DNS attempt failed before receiving a DNS + // packet while Windows is serving retained VPN DNS state, fail closed + // instead of asking LAN/public DNS about internal split-rule names and + // caching false negatives. Reachable negative DNS responses still fall + // through to the old OS fallback behavior below. + if !gotDNSAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, dlServers) { + ctrld.Log(ctx, p.Debug(), + "All domain-less VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) + answer := new(dns.Msg) + answer.SetRcode(req.msg, dns.RcodeServerFailure) + return &proxyResponse{answer: answer, cached: false} } - staleAnswer = answer + + ctrld.Log(ctx, p.Debug(), + "All domain-less VPN DNS servers failed for %s, falling back to OS resolver", domain) } } - resolve1 := func(upstream string, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) (*dns.Msg, error) { - ctrld.Log(ctx, mainLog.Load().Debug(), "sending query to %s: %s", upstream, upstreamConfig.Name) - dnsResolver, err := ctrld.NewResolver(upstreamConfig) - if err != nil { - ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to create resolver") - return nil, err - } - resolveCtx, cancel := upstreamConfig.Context(ctx) - defer cancel() - return dnsResolver.Resolve(resolveCtx, msg) + + ctrld.Log(ctx, p.Debug(), "No cache hit, trying upstreams") + if res := p.tryUpstreams(ctx, req, upstreams, upstreamConfigs); res != nil { + ctrld.Log(ctx, p.Debug(), "Upstream query successful") + return res } - resolve := func(upstream string, upstreamConfig *ctrld.UpstreamConfig, msg *dns.Msg) *dns.Msg { - if upstreamConfig.UpstreamSendClientInfo() && req.ci != nil { - ctrld.Log(ctx, mainLog.Load().Debug(), "including client info with the request") - ctx = context.WithValue(ctx, ctrld.ClientInfoCtxKey{}, req.ci) - } - answer, err := resolve1(upstream, upstreamConfig, msg) - // if we have an answer, we should reset the failure count - // we dont use reset here since we dont want to prevent failure counts from being incremented - if answer != nil { - p.um.mu.Lock() - p.um.failureReq[upstream] = 0 - p.um.down[upstream] = false - p.um.mu.Unlock() - return answer - } - ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to resolve query") + ctrld.Log(ctx, p.Debug(), "All upstreams failed, handling failure") + return p.handleAllUpstreamsFailure(ctx, req, upstreams) +} + +// initializeUpstreams determines which upstreams and configurations to use for a given proxyRequest. +// If no upstreams are configured, it defaults to the operating system's resolver configuration. +// Returns a slice of upstream names and their corresponding configurations. +func (p *prog) initializeUpstreams(req *proxyRequest) ([]string, []*ctrld.UpstreamConfig) { + upstreams := req.ufr.upstreams + upstreamConfigs := p.upstreamConfigsFromUpstreamNumbers(upstreams) + if len(upstreamConfigs) == 0 { + return []string{upstreamOS}, []*ctrld.UpstreamConfig{osUpstreamConfig} + } + return upstreams, upstreamConfigs +} - // increase failure count when there is no answer - // rehardless of what kind of error we get - p.um.increaseFailureCount(upstream) +// tryCache attempts to retrieve a cached response for the given DNS request from specified upstreams. +// Returns a proxyResponse if a cache hit occurs; otherwise, returns nil. +// Skips cache checking if caching is disabled or the request is a PTR query. +// Iterates through the provided upstreams to find a cached response using the checkCache method. +func (p *prog) tryCache(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse { + if p.cache == nil || req.msg.Question[0].Qtype == dns.TypePTR || req.ufr.zpaDNSEcho { // https://www.rfc-editor.org/rfc/rfc1035#section-7.4 + ctrld.Log(ctx, p.Debug(), "Cache disabled, PTR query, or ZPA DNS echo, skipping cache lookup") + return nil + } - if err != nil { - // For timeout error (i.e: context deadline exceed), force re-bootstrapping. - var e net.Error - if errors.As(err, &e) && e.Timeout() { - upstreamConfig.ReBootstrap() - } - // For network error, turn ipv6 off if enabled. - if ctrld.HasIPv6() && (errUrlNetworkError(err) || errNetworkError(err)) { - ctrld.DisableIPv6() - } + ctrld.Log(ctx, p.Debug(), "Checking cache for upstreams: %v", upstreams) + for _, upstream := range upstreams { + if res := p.checkCache(ctx, req, upstream); res != nil { + ctrld.Log(ctx, p.Debug(), "Cache hit found for upstream: %s", upstream) + return res } + } + ctrld.Log(ctx, p.Debug(), "No cache hit found") + return nil +} + +// checkCache checks if a cached DNS response exists for the given request and upstream. +// Returns a proxyResponse with the cached response if found and valid, or nil otherwise. +func (p *prog) checkCache(ctx context.Context, req *proxyRequest, upstream string) *proxyResponse { + dns64Prefix, dns64Active := netip.Prefix{}, false + if req.msg.Question[0].Qtype == dns.TypeAAAA { + dns64Prefix, dns64Active = p.activeDNS64Prefix() + } + + answer, stale, hit, dns64Hit, dns64Bypass := p.cachedResponse(req.msg, upstream, dns64Prefix, dns64Active, time.Now()) + if stale != nil { + req.staleAnswer = stale + } + if dns64Bypass { + ctrld.Log(ctx, p.Debug(), "DNS64: bypassing cached empty-AAAA answer for synthesis") + } + if !hit { + ctrld.Log(ctx, p.Debug(), "No usable cached value found for upstream: %s", upstream) return nil } - for n, upstreamConfig := range upstreamConfigs { - if upstreamConfig == nil { - continue - } - logger := mainLog.Load().Debug(). - Str("upstream", upstreamConfig.String()). - Str("query", req.msg.Question[0].Name). - Bool("is_ad_query", p.isAdDomainQuery(req.msg)). - Bool("is_lan_query", isLanOrPtrQuery) - if p.isLoop(upstreamConfig) { - ctrld.Log(ctx, logger, "DNS loop detected") - continue - } - answer := resolve(upstreams[n], upstreamConfig, req.msg) - if answer == nil { - if serveStaleCache && staleAnswer != nil { - ctrld.Log(ctx, mainLog.Load().Debug(), "serving stale cached response") - now := time.Now() - setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL)) - res.answer = staleAnswer - res.cached = true - return res - } - continue + if dns64Hit { + ctrld.Log(ctx, p.Debug(), "DNS64: hit cached response variant") + } else { + ctrld.Log(ctx, p.Debug(), "Hit cached response") + } + + // Firewall mode: refresh allowlist entries from cached responses. + // Even though these IPs were already added when the response was first + // resolved, the allowlist entries may have expired (TTL-based reaper) + // while the DNS cache entry is still valid. Refreshing here ensures + // the allowlist stays populated for as long as the cached DNS entry is served. + if p.firewallModeEnabled() { + domain := canonicalName(req.msg.Question[0].Name) + p.firewallRecordResolvedIPs(answer, domain) + } + + return &proxyResponse{answer: answer, cached: true} +} + +// updateCache updates the DNS response cache with the given request, response, TTL, and upstream information. +func (p *prog) updateCache(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string) { + if req.ufr.zpaDNSEcho { + return + } + p.addCachedResponse(dnscache.NewKey(req.msg, upstream), answer) + ctrld.Log(ctx, p.Debug(), "Added cached response") +} + +// serveStaleResponse serves a stale cached DNS response when an upstream query fails, updating TTL for cached records. +func (p *prog) serveStaleResponse(ctx context.Context, staleAnswer *dns.Msg) *proxyResponse { + ctrld.Log(ctx, p.Debug(), "Serving stale cached response") + p.health.countCacheHit() + now := time.Now() + setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL)) + return &proxyResponse{answer: staleAnswer, cached: true} +} + +// handleAllUpstreamsFailure handles the failure scenario when all upstream resolvers fail to respond or process the request. +func (p *prog) handleAllUpstreamsFailure(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse { + if req.internalFallback { + if res := p.internalDomainNetworkFallback(ctx, req, upstreams); res != nil { + return res } - // We are doing LAN/PTR lookup using private resolver, so always process next one. - // Except for the last, we want to send response instead of saying all upstream failed. - if answer.Rcode != dns.RcodeSuccess && isLanOrPtrQuery && n != len(upstreamConfigs)-1 { - ctrld.Log(ctx, mainLog.Load().Debug(), "no response from %s, process to next upstream", upstreams[n]) - continue + } + ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "All %v endpoints failed", journalUpstreamNames(upstreams)) + + // An Internal Domain with explicit resolvers never takes the OS-resolver + // catch-all below: it can reach public nameservers, ctrld's own fallback + // included, which no mode selects for a private name. Fallback mode has + // already tried the VPN resolvers and the network's LAN nameservers above. + // The recovery flow exists for the loss of general DNS, not for one + // unavailable internal server, so it is not started either. + if p.internalDomainExplicitUpstreams(upstreams) { + ctrld.Log(ctx, p.Debug(), "Internal domain resolvers unreachable; not using the OS resolver catch all") + } else if p.leakOnUpstreamFailure() { + ctrld.Log(ctx, p.Debug(), "Leak on upstream failure enabled") + if p.um.countHealthy(upstreams) == 0 { + ctrld.Log(ctx, p.Debug(), "Selected upstreams unavailable; requesting recovery") + p.triggerRecovery(upstreams[0] == upstreamOS) + } else { + ctrld.Log(ctx, p.Debug(), "One upstream is down but at least one is healthy; skipping recovery trigger") } - if answer.Rcode != dns.RcodeSuccess && len(upstreamConfigs) > 1 && containRcode(req.failoverRcodes, answer.Rcode) { - ctrld.Log(ctx, mainLog.Load().Debug(), "failover rcode matched, process to next upstream") - continue + + if upstreams[0] != upstreamOS { + ctrld.Log(ctx, p.Debug(), "Trying OS resolver as fallback") + if answer := p.tryOSResolver(ctx, req); answer != nil { + ctrld.Log(ctx, p.Debug(), "OS resolver fallback successful") + return answer + } } + } - // set compression, as it is not set by default when unpacking - answer.Compress = true + ctrld.Log(ctx, p.Debug(), "Returning server failure response") + p.countFailedClientQuery(upstreams) + answer := new(dns.Msg) + answer.SetRcode(req.msg, dns.RcodeServerFailure) + return &proxyResponse{answer: answer} +} +// internalDomainNetworkFallback runs the network fallback of an Internal Domain +// in explicit-with-network-fallback mode, once its configured resolvers have +// failed, or answered SERVFAIL or NXDOMAIN. It returns nil when nothing on the +// network resolved the name and no resolver gave a negative answer either. +func (p *prog) internalDomainNetworkFallback(ctx context.Context, req *proxyRequest, upstreams []string) *proxyResponse { + ctrld.Log(ctx, p.Debug(), "Internal domain resolvers did not resolve the query; trying network resolvers") + answer, negative := p.resolveInternalDomainOnNetwork(ctx, req.msg) + if answer != nil { + answer.Compress = true if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR { - ttl := ttlFromMsg(answer) - now := time.Now() - expired := now.Add(time.Duration(ttl) * time.Second) - if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 { - expired = now.Add(time.Duration(cachedTTL) * time.Second) - } - setCachedAnswerTTL(answer, now, expired) - p.cache.Add(dnscache.NewKey(req.msg, upstreams[n]), dnscache.NewValue(answer, expired)) - ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response") + p.updateCache(ctx, req, answer, upstreams[0]) } hostname := "" if req.ci != nil { hostname = req.ci.Hostname } - ctrld.Log(ctx, mainLog.Load().Info(), "REPLY: %s -> %s (%s): %s", upstreams[n], req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode]) - res.answer = answer - res.upstream = upstreamConfig.Endpoint - return res + ctrld.Log(ctx, p.Info(), "REPLY: internal domain network fallback -> %s (%s): %s", req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode]) + return &proxyResponse{answer: answer} + } + if negative = betterInternalDomainNegative(req.internalNegative, negative); negative != nil { + ctrld.Log(ctx, p.Debug(), "No resolver resolved the internal domain; returning %s", dns.RcodeToString[negative.Rcode]) + negative.Compress = true + return &proxyResponse{answer: negative} } - ctrld.Log(ctx, mainLog.Load().Error(), "all %v endpoints failed", upstreams) + return nil +} - // if we have no healthy upstreams, trigger recovery flow - if p.leakOnUpstreamFailure() { - if p.um.countHealthy(upstreams) == 0 { - p.recoveryCancelMu.Lock() - if p.recoveryCancel == nil { - var reason RecoveryReason - if upstreams[0] == upstreamOS { - reason = RecoveryReasonOSFailure - } else { - reason = RecoveryReasonRegularFailure +// countFailedClientQuery grades one client query that ended without an answer. +// A query that only Internal Domain resolvers serve stays out of the grade: an +// endpoint away from the organization network never reaches them, and that is +// not an outage of the query path. +func (p *prog) countFailedClientQuery(upstreams []string) { + if p.internalDomainExplicitUpstreams(upstreams) { + return + } + p.health.countFailedQuery() +} + +// shouldContinueWithNextUpstream determines whether processing should continue with the next upstream based on response conditions. +func (p *prog) shouldContinueWithNextUpstream(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string, lastUpstream bool) bool { + if answer.Rcode == dns.RcodeSuccess { + ctrld.Log(ctx, p.Debug(), "Successful response, not continuing to next upstream") + return false + } + + // We are doing LAN/PTR lookup using private resolver, so always process the next one. + // Except for the last, we want to send a response instead of saying all upstream failed. + if req.isLanOrPtrQuery && !lastUpstream { + ctrld.Log(ctx, p.Debug(), "No response for LAN/PTR query from %s, process to next upstream", upstream) + return true + } + + if len(req.upstreamConfigs) > 1 && slices.Contains(req.failoverRcodes, answer.Rcode) { + ctrld.Log(ctx, p.Debug(), "Failover rcode matched, process to next upstream") + return true + } + + // An Internal Domain in explicit-with-network-fallback mode moves on from a + // configured resolver that answers SERVFAIL or NXDOMAIN, as it does from + // one that does not answer. + if req.internalFallback && internalDomainFallbackRcode(answer.Rcode) { + ctrld.Log(ctx, p.Debug(), "Internal domain resolver %s answered %s, trying next resolver", + upstream, dns.RcodeToString[answer.Rcode]) + req.internalNegative = betterInternalDomainNegative(req.internalNegative, answer) + return true + } + + ctrld.Log(ctx, p.Debug(), "Not continuing to next upstream") + return false +} + +// prepareSuccessResponse prepares a successful DNS response for a given request, logs it, and updates the cache if applicable. +func (p *prog) prepareSuccessResponse(ctx context.Context, req *proxyRequest, answer *dns.Msg, upstream string, upstreamConfig *ctrld.UpstreamConfig) *proxyResponse { + ctrld.Log(ctx, p.Debug(), "Preparing success response") + + answer.Compress = true + + // The echo must reach Client Connector on every poll, including reconnects. + cacheable := p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR && !req.ufr.zpaDNSEcho + if cacheable { + ctrld.Log(ctx, p.Debug(), "Updating cache with successful response") + p.updateCache(ctx, req, answer, upstream) + } + + // Apply DNS64 only after policy processing, and resolve the companion A + // question through the same upstream that produced the approved answer. + var synthesizedPrefix netip.Prefix + answer, synthesizedPrefix = p.maybeDNS64(ctx, req.msg, answer, func(aReq *dns.Msg) *dns.Msg { + key := dnscache.NewKey(aReq, upstream) + if cacheable { + if cachedValue := p.cache.Get(key); cachedValue != nil { + now := time.Now() + if cachedValue.Expire.After(now) { + cached := cachedValue.Msg.Copy() + ctrld.SetCacheReply(cached, aReq, cached.Rcode) + setCachedAnswerTTL(cached, now, cachedValue.Expire) + return cached } - mainLog.Load().Debug().Msgf("No healthy upstreams, triggering recovery with reason: %v", reason) - go p.handleRecovery(reason) - } else { - mainLog.Load().Debug().Msg("Recovery already in progress; skipping duplicate trigger from down detection") } - p.recoveryCancelMu.Unlock() - } else { - mainLog.Load().Debug().Msg("One upstream is down but at least one is healthy; skipping recovery trigger") } - // attempt query to OS resolver while as a retry catch all - // we dont want this to happen if leakOnUpstreamFailure is false - if upstreams[0] != upstreamOS { - ctrld.Log(ctx, mainLog.Load().Debug(), "attempting query to OS resolver as a retry catch all") - answer := resolve(upstreamOS, osUpstreamConfig, req.msg) - if answer != nil { - ctrld.Log(ctx, mainLog.Load().Debug(), "OS resolver retry query successful") - res.answer = answer - res.upstream = osUpstreamConfig.Endpoint - return res - } - ctrld.Log(ctx, mainLog.Load().Debug(), "OS resolver retry query failed") + aProxyReq := *req + aProxyReq.msg = aReq + resolved := p.queryUpstream(ctx, &aProxyReq, upstream, upstreamConfig) + if cacheable && resolved != nil && sameQuestion(aReq, resolved) { + p.addCachedResponse(key, resolved) } + return resolved + }) + if cacheable && synthesizedPrefix.IsValid() { + p.addCachedResponse(dns64CacheKey(req.msg, upstream, synthesizedPrefix), answer) + ctrld.Log(ctx, p.Debug(), "DNS64: added cached response variant") } - answer := new(dns.Msg) - answer.SetRcode(req.msg, dns.RcodeServerFailure) - res.answer = answer - return res + hostname := "" + if req.ci != nil { + hostname = req.ci.Hostname + } + + ctrld.Log(ctx, p.Info(), "REPLY: %s -> %s (%s): %s", + upstream, req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode]) + + return &proxyResponse{ + answer: answer, + upstream: upstreamConfig.Endpoint, + } } +// tryUpstreams attempts to proxy a DNS request through the provided upstreams and their configurations sequentially. +// It returns a successful proxyResponse if any upstream processes the request successfully, or nil otherwise. +// The function supports "serve stale" for cache by utilizing cached responses when upstreams fail. +func (p *prog) tryUpstreams(ctx context.Context, req *proxyRequest, upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) *proxyResponse { + serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale && !req.ufr.zpaDNSEcho + req.upstreamConfigs = upstreamConfigs + req.internalFallback = p.internalDomainFallbackUpstreams(upstreams) + + ctrld.Log(ctx, p.Debug(), "Trying %d upstreams", len(upstreamConfigs)) + + for n, upstreamConfig := range upstreamConfigs { + last := n == len(upstreamConfigs)-1 + // A fallback-mode Internal Domain resolver that the monitor reports as + // down is not waited on: off the organization network every query + // would otherwise pay its timeout before the network fallback runs. + // Nothing else marks a generated resolver up again, so skipping it + // starts a background re-check that does. + if req.internalFallback && upstreamConfig != nil && p.um.isDown(upstreams[n]) { + ctrld.Log(ctx, p.Debug(), "Internal domain resolver %s is down, skipping it", upstreams[n]) + p.internalDomainProbes.probe(p.um, upstreams[n], upstreamConfig, req.msg) + continue + } + ctrld.Log(ctx, p.Debug(), "Processing upstream %d/%d: %s", n+1, len(upstreamConfigs), upstreams[n]) + + if res := p.processUpstream(ctx, req, upstreams[n], upstreamConfig, serveStaleCache, last); res != nil { + ctrld.Log(ctx, p.Debug(), "Upstream %s succeeded", upstreams[n]) + return res + } + + ctrld.Log(ctx, p.Debug(), "Upstream %s failed", upstreams[n]) + } + + ctrld.Log(ctx, p.Debug(), "All upstreams failed") + return nil +} + +// processUpstream proxies a DNS query to a given upstream server and processes the response based on the provided configuration. +// It supports serving stale cache when upstream queries fail, and checks if processing should continue to another upstream. +// Returns a proxyResponse on success or nil if the upstream query fails or processing conditions are not met. +func (p *prog) processUpstream(ctx context.Context, req *proxyRequest, upstream string, upstreamConfig *ctrld.UpstreamConfig, serveStaleCache, lastUpstream bool) *proxyResponse { + if upstreamConfig == nil { + ctrld.Log(ctx, p.Debug(), "Upstream config is nil, skipping") + return nil + } + if p.isLoop(upstreamConfig) { + logger := p.Debug(). + Str("upstream", upstreamConfig.String()). + Str("query", req.msg.Question[0].Name). + Bool("is_lan_query", req.isLanOrPtrQuery) + ctrld.Log(ctx, logger, "DNS loop detected") + return nil + } + + ctrld.Log(ctx, p.Debug(), "Querying upstream: %s", upstream) + answer := p.queryUpstream(ctx, req, upstream, upstreamConfig) + if answer == nil { + ctrld.Log(ctx, p.Debug(), "Upstream query failed") + if serveStaleCache && req.staleAnswer != nil { + ctrld.Log(ctx, p.Debug(), "Serving stale response due to upstream failure") + return p.serveStaleResponse(ctx, req.staleAnswer) + } + return nil + } + + ctrld.Log(ctx, p.Debug(), "Upstream query successful") + // Reject an answer whose question does not match the request before it + // can be served or cached. A mismatched question means the upstream + // answered a different name/type than asked; caching it would poison + // the shared cache with wrong-domain records for the requested name. + // See github.com/Control-D-Inc/ctrld/issues/322. + if !sameQuestion(req.msg, answer) { + ctrld.Log(ctx, p.Debug(), + "Discarding answer from %s: question mismatch (asked %q, got %q)", + upstream, questionString(req.msg), questionString(answer)) + return nil + } + if p.shouldContinueWithNextUpstream(ctx, req, answer, upstream, lastUpstream) { + return nil + } + return p.prepareSuccessResponse(ctx, req, answer, upstream, upstreamConfig) +} + +// queryUpstream sends a DNS query to a specified upstream using its configuration and handles errors and retries. +func (p *prog) queryUpstream(ctx context.Context, req *proxyRequest, upstream string, upstreamConfig *ctrld.UpstreamConfig) *dns.Msg { + if isGeneratedInternalDomainUpstream(upstreamConfig) { + ctx = ctrld.PrivateResolverCtx(ctx) + } + if upstreamConfig.UpstreamSendClientInfo() && req.ci != nil { + ctrld.Log(ctx, p.Debug(), "Adding client info to upstream query") + ctx = context.WithValue(ctx, ctrld.ClientInfoCtxKey{}, req.ci) + } + + ctrld.Log(ctx, p.Debug(), "Sending query to %s: %s", upstream, upstreamConfig.Name) + dnsResolver, err := newResolverFn(ctx, upstreamConfig) + if err != nil { + ctrld.Log(ctx, p.Error().Err(err), "Failed to create resolver") + return nil + } + + resolveCtx, cancel := upstreamConfig.Context(ctx) + defer cancel() + + ctrld.Log(ctx, p.Debug(), "Resolving query with upstream") + answer, err := dnsResolver.Resolve(resolveCtx, req.msg) + if answer != nil { + ctrld.Log(ctx, p.Debug(), "Upstream resolution successful") + // reset is not used here, because it also stops the failure count for + // one second, and the failures already in flight must still count. + p.um.noteSuccess(upstream) + return answer + } + + // A failing minute must not push the state events out of the retained log, + // so the sampler sets the level of these lines. + // Transport errors contain the organization's private resolver address. + // Keep that detail at debug while retaining a visible failure classification. + if isGeneratedInternalDomainUpstream(upstreamConfig) { + ctrld.Log(ctx, p.Debug().Err(err), "Failed to resolve query") + ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)), + "Failed to resolve query using an Internal Domain resolver") + } else { + ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Err(err), "Failed to resolve query") + } + // Increasing the failure count when there is no answer regardless of what kind of error we get + p.um.increaseFailureCount(upstream) + if err != nil { + // For timeout error (i.e: context deadline exceed), force re-bootstrapping. + var e net.Error + if errors.As(err, &e) && e.Timeout() { + ctrld.Log(ctx, p.Debug(), "Timeout error, forcing re-bootstrapping") + upstreamConfig.ReBootstrap(ctx) + } + // For network error, turn ipv6 off if enabled. + if ctrld.HasIPv6(ctx) && (errUrlNetworkError(err) || errNetworkError(err)) { + ctrld.Log(ctx, p.Debug(), "Network error, disabling IPv6") + ctrld.DisableIPv6(ctx) + } + } + return nil +} + +// triggerRecovery attempts to initiate a recovery process if no healthy upstreams are detected. +// If "isOSFailure" is true, the recovery will account for an operating system failure. +// Logs are generated to indicate whether recovery is triggered or already in progress. +func (p *prog) triggerRecovery(isOSFailure bool) { + p.recoveryCancelMu.Lock() + defer p.recoveryCancelMu.Unlock() + + if p.recoveryCancel == nil { + var reason RecoveryReason + if isOSFailure { + reason = RecoveryReasonOSFailure + } else { + reason = RecoveryReasonRegularFailure + } + p.Debug().Msgf("Selected upstreams unavailable; requesting recovery with reason: %v", reason) + go queryRecoveryFn(p, reason) + } else { + p.Debug().Msg("Recovery already in progress; skipping duplicate trigger from down detection") + } +} + +// tryOSResolver attempts to query the OS resolver as a fallback mechanism when other upstreams fail. +// Logs success or failure of the query attempt and returns a proxyResponse or nil based on query result. +func (p *prog) tryOSResolver(ctx context.Context, req *proxyRequest) *proxyResponse { + ctrld.Log(ctx, p.Debug(), "Attempting query to OS resolver as a retry catch all") + // An Active Directory name that falls back from its explicit + // upstream still must not reach Control D's public resolver. + if len(req.msg.Question) > 0 && inActiveDirectoryDomain(req.msg.Question[0].Name) { + ctx = ctrld.LanQueryCtx(ctx) + ctrld.Log(ctx, p.Debug(), "Active Directory domain lookup, skipping public DNS") + } + answer := p.queryUpstream(ctx, req, upstreamOS, osUpstreamConfig) + if answer != nil { + ctrld.Log(ctx, p.Debug(), "OS resolver retry query successful") + return &proxyResponse{answer: answer, upstream: osUpstreamConfig.Endpoint} + } + ctrld.Log(ctx, p.Debug(), "OS resolver retry query failed") + return nil +} + +// upstreamsAndUpstreamConfigForPtr returns the updated upstreams and upstreamConfigs for a private PTR lookup scenario. func (p *prog) upstreamsAndUpstreamConfigForPtr(upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) ([]string, []*ctrld.UpstreamConfig) { if len(p.localUpstreams) > 0 { tmp := make([]string, 0, len(p.localUpstreams)+len(upstreams)) @@ -667,6 +1381,7 @@ func (p *prog) upstreamsAndUpstreamConfigForPtr(upstreams []string, upstreamConf return append([]string{upstreamOS}, upstreams...), append([]*ctrld.UpstreamConfig{privateUpstreamConfig}, upstreamConfigs...) } +// upstreamConfigsFromUpstreamNumbers converts a list of upstream names into their corresponding UpstreamConfig objects. func (p *prog) upstreamConfigsFromUpstreamNumbers(upstreams []string) []*ctrld.UpstreamConfig { upstreamConfigs := make([]*ctrld.UpstreamConfig, 0, len(upstreams)) for _, upstream := range upstreams { @@ -676,14 +1391,6 @@ func (p *prog) upstreamConfigsFromUpstreamNumbers(upstreams []string) []*ctrld.U return upstreamConfigs } -func (p *prog) isAdDomainQuery(msg *dns.Msg) bool { - if p.adDomain == "" { - return false - } - cDomainName := canonicalName(msg.Question[0].Name) - return dns.IsSubDomain(p.adDomain, cDomainName) -} - // canonicalName returns canonical name from FQDN with "." trimmed. func canonicalName(fqdn string) string { q := strings.TrimSpace(fqdn) @@ -720,10 +1427,12 @@ func wildcardMatches(wildcard, str string) bool { return false } +// fmtRemoteToLocal formats a remote address to indicate its mapping to a local listener using listener number and hostname. func fmtRemoteToLocal(listenerNum, hostname, remote string) string { return fmt.Sprintf("%s (%s) -> listener.%s", remote, hostname, listenerNum) } +// requestID generates a random 6-character hexadecimal string to uniquely identify a request. It panics on error. func requestID() string { b := make([]byte, 3) // 6 chars if _, err := rand.Read(b); err != nil { @@ -732,15 +1441,34 @@ func requestID() string { return hex.EncodeToString(b) } -func containRcode(rcodes []int, rcode int) bool { - for i := range rcodes { - if rcodes[i] == rcode { - return true - } +// sameQuestion reports whether the upstream answer echoes the request's +// question. A well-behaved resolver always copies the question section from +// the query (RFC 1035 section 4.1.2); names are compared case-insensitively +// because DNS names are case-insensitive. A mismatch means the upstream +// answered a different name/type than asked - malformed or malicious - and the +// answer must not be served or cached, or it would poison the shared cache with +// wrong-domain records. See github.com/Control-D-Inc/ctrld/issues/322. +func sameQuestion(req, answer *dns.Msg) bool { + if req == nil || answer == nil { + return false } - return false + if len(req.Question) == 0 || len(answer.Question) == 0 { + return false + } + rq, aq := req.Question[0], answer.Question[0] + return rq.Qtype == aq.Qtype && rq.Qclass == aq.Qclass && strings.EqualFold(rq.Name, aq.Name) +} + +// questionString renders a message's first question as "name/type" for logging. +func questionString(msg *dns.Msg) string { + if msg == nil || len(msg.Question) == 0 { + return "" + } + q := msg.Question[0] + return q.Name + "/" + dns.TypeToString[q.Qtype] } +// setCachedAnswerTTL updates the TTL of each DNS record in the provided message based on the current and expiration times. func setCachedAnswerTTL(answer *dns.Msg, now, expiredTime time.Time) { ttlSecs := expiredTime.Sub(now).Seconds() if ttlSecs < 0 { @@ -761,6 +1489,8 @@ func setCachedAnswerTTL(answer *dns.Msg, now, expiredTime time.Time) { } } +// ttlFromMsg extracts and returns the TTL value from the first record in the Answer or Ns sections of a DNS message. +// If no records exist in either section, the function returns 0. func ttlFromMsg(msg *dns.Msg) uint32 { for _, rr := range msg.Answer { return rr.Header().Ttl @@ -771,6 +1501,7 @@ func ttlFromMsg(msg *dns.Msg) uint32 { return 0 } +// needLocalIPv6Listener checks if a local IPv6 listener is required on Windows by verifying IPv6 support and the OS type. func needLocalIPv6Listener() bool { // On Windows, there's no easy way for disabling/removing IPv6 DNS resolver, so we check whether we can // listen on ::1, then spawn a listener for receiving DNS requests. @@ -842,6 +1573,8 @@ func spoofRemoteAddr(addr net.Addr, ci *ctrld.ClientInfo) net.Addr { // // It's the caller responsibility to call Shutdown to close the server. func runDNSServer(addr, network string, handler dns.Handler) (*dns.Server, <-chan error) { + mainLog.Load().Debug().Str("address", addr).Str("network", network).Msg("Starting DNS server") + s := &dns.Server{ Addr: addr, Net: network, @@ -851,16 +1584,17 @@ func runDNSServer(addr, network string, handler dns.Handler) (*dns.Server, <-cha startedCh := make(chan struct{}) s.NotifyStartedFunc = func() { sync.OnceFunc(func() { close(startedCh) })() } - errCh := make(chan error) + errCh := make(chan error, 1) go func() { defer close(errCh) if err := s.ListenAndServe(); err != nil { s.NotifyStartedFunc() - mainLog.Load().Error().Err(err).Msgf("could not listen and serve on: %s", s.Addr) + mainLog.Load().Error().Err(err).Msgf("Could not listen and serve on: %s", s.Addr) errCh <- err } }() <-startedCh + mainLog.Load().Debug().Str("address", addr).Str("network", network).Msg("DNS server started successfully") return s, errCh } @@ -908,15 +1642,38 @@ func (p *prog) getClientInfo(remoteIP string, msg *dns.Msg) *ctrld.ClientInfo { } } } else { - ci.Hostname = p.ciTable.LookupHostname(ci.IP, ci.Mac) + ci.Hostname = p.ciTable.LookupHostname(ci.IP, ci.Mac) + } + + if ci.IP == "" { + p.Debug().Msgf("client info entry with empty IP address: %v", ci) + } else { + ci.Self = p.queryFromSelf(ci.IP) + } + + // In DNS intercept mode, ALL queries are from the local machine — pf/WFP + // intercepts outbound DNS and redirects to ctrld. The source IP may be a + // virtual interface (Tailscale, VPN) that has no ARP/MAC entry, causing + // missing x-cd-mac, x-cd-host, and x-cd-os headers. Force Self=true and + // populate from the primary physical interface info. + if dnsIntercept && !ci.Self { + ci.Self = true } - ci.Self = p.queryFromSelf(ci.IP) + // If this is a query from self, but ci.IP is not loopback IP, // try using hostname mapping for lookback IP if presents. if ci.Self { if name := p.ciTable.LocalHostname(); name != "" { ci.Hostname = name } + // If MAC is still empty (e.g., query arrived via virtual interface IP + // like Tailscale), fall back to the loopback MAC mapping which addSelf() + // populates from the primary physical interface. + if ci.Mac == "" { + if mac := p.ciTable.LookupMac("127.0.0.1"); mac != "" { + ci.Mac = mac + } + } } p.spoofLoopbackIpInClientInfo(ci) return ci @@ -940,8 +1697,9 @@ func (p *prog) spoofLoopbackIpInClientInfo(ci *ctrld.ClientInfo) { // - There is only 1 ControlD upstream in-use. // - Number of refused queries seen so far equals to selfUninstallMaxQueries. // - The cdUID is deleted. -func (p *prog) doSelfUninstall(answer *dns.Msg) { - if !p.canSelfUninstall.Load() || answer == nil || answer.Rcode != dns.RcodeRefused { +func (p *prog) doSelfUninstall(pr *proxyResponse) { + answer := pr.answer + if pr.refused || !p.canSelfUninstall.Load() || answer == nil || answer.Rcode != dns.RcodeRefused { return } @@ -951,15 +1709,21 @@ func (p *prog) doSelfUninstall(answer *dns.Msg) { return } - logger := mainLog.Load().With().Str("mode", "self-uninstall").Logger() + logger := p.logger.Load().With().Str("mode", "self-uninstall") if p.refusedQueryCount > selfUninstallMaxQueries { p.checkingSelfUninstall = true - _, err := controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev) - logger.Debug().Msg("maximum number of refused queries reached, checking device status") + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + req := &controld.ResolverConfigRequest{ + RawUID: cdUID, + Version: appVersion, + Metadata: ctrld.SystemMetadataRuntime(loggerCtx), + } + _, err := fetchResolverConfig(loggerCtx, req, cdDev) + logger.Debug().Msg("Maximum number of refused queries reached, checking device status") selfUninstallCheck(err, p, logger) if err != nil { - logger.Warn().Err(err).Msg("could not fetch resolver config") + logResolverConfigFetchFailure(logger, err) } // Cool-of period to prevent abusing the API. go p.selfUninstallCoolOfPeriod() @@ -983,6 +1747,10 @@ func (p *prog) selfUninstallCoolOfPeriod() { // forceFetchingAPI sends signal to force syncing API config if run in cd mode, // and the domain == "cdUID.verify.controld.com" func (p *prog) forceFetchingAPI(domain string) { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() if cdUID == "" { return } @@ -999,9 +1767,32 @@ func (p *prog) forceFetchingAPI(domain string) { return } _ = p.apiForceReloadGroup.DoChan("force_sync_api", func() (interface{}, error) { - p.apiForceReloadCh <- struct{}{} + if !p.beginNetworkActivity() { + return nil, nil + } + defer p.netMonitorWG.Done() + shutdown := p.networkActivityDone() + select { + case <-shutdown: + return nil, nil + case p.apiForceReloadCh <- struct{}{}: + case <-p.stopCh: + return nil, nil + case <-p.runAbortCh: + return nil, nil + } // Wait here to prevent abusing API if we are flooded. - time.Sleep(timeDurationOrDefault(p.cfg.Service.ForceRefetchWaitTime, 30) * time.Second) + p.mu.Lock() + wait := timeDurationOrDefault(p.cfg.Service.ForceRefetchWaitTime, 30) * time.Second + p.mu.Unlock() + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-timer.C: + case <-shutdown: + case <-p.stopCh: + case <-p.runAbortCh: + } return nil, nil }) } @@ -1020,10 +1811,15 @@ func (p *prog) queryFromSelf(ip string) bool { if val, ok := p.queryFromSelfMap.Load(ip); ok { return val.(bool) } - netIP := netip.MustParseAddr(ip) + netIP, err := netip.ParseAddr(ip) + if err != nil { + p.Debug().Err(err).Msgf("could not parse IP: %q", ip) + return false + } + regularIPs, loopbackIPs, err := netmon.LocalAddresses() if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not get local addresses") + p.Warn().Err(err).Msg("Could not get local addresses") return false } for _, localIP := range slices.Concat(regularIPs, loopbackIPs) { @@ -1090,7 +1886,8 @@ func isPrivatePtrLookup(m *dns.Msg) bool { return addr.IsPrivate() || addr.IsLoopback() || addr.IsLinkLocalUnicast() || - tsaddr.CGNATRange().Contains(addr) + tsaddr.CGNATRange().Contains(addr) || + isServiceContinuityAddr(addr) } } return false @@ -1128,6 +1925,20 @@ func isLanHostname(name string) bool { strings.HasSuffix(name, ".local") } +// ipv4ServiceContinuityPrefix is the RFC 7335 IPv4 Service Continuity Prefix +// (192.0.0.0/29), used by the CLAT in 464XLAT/DS-Lite transition setups. On such +// networks (common on IPv6-only cellular carriers and iPhone hotspots) the local +// machine's DNS queries reach ctrld with a source in this range (e.g. 192.0.0.2), +// so they must be treated as local, not WAN. Go's netip.IsPrivate does not cover +// this range — the same reason the CGNAT range is special-cased below. See #552. +var ipv4ServiceContinuityPrefix = netip.MustParsePrefix("192.0.0.0/29") + +// isServiceContinuityAddr reports whether ip is in the RFC 7335 IPv4 Service +// Continuity Prefix (464XLAT/DS-Lite CLAT). +func isServiceContinuityAddr(ip netip.Addr) bool { + return ipv4ServiceContinuityPrefix.Contains(ip) +} + // isWanClient reports whether the input is a WAN address. func isWanClient(na net.Addr) bool { var ip netip.Addr @@ -1138,12 +1949,14 @@ func isWanClient(na net.Addr) bool { !ip.IsPrivate() && !ip.IsLinkLocalUnicast() && !ip.IsLinkLocalMulticast() && - !tsaddr.CGNATRange().Contains(ip) + !tsaddr.CGNATRange().Contains(ip) && + !isServiceContinuityAddr(ip) } // resolveInternalDomainTestQuery resolves internal test domain query, returning the answer to the caller. func resolveInternalDomainTestQuery(ctx context.Context, domain string, m *dns.Msg) *dns.Msg { - ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain test query") + logger := ctrld.LoggerFromCtx(ctx) + ctrld.Log(ctx, logger.Debug(), "Internal domain test query") q := m.Question[0] answer := new(dns.Msg) @@ -1180,165 +1993,433 @@ func FlushDNSCache() error { return nil } +// networkChangeCallback fences even callbacks already dispatched by netmon +// when Close begins. The monitor does not track those callback goroutines. +func (p *prog) networkChangeCallback(fn netmon.ChangeFunc) netmon.ChangeFunc { + return func(delta *netmon.ChangeDelta) { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + fn(delta) + } +} + // monitorNetworkChanges starts monitoring for network interface changes -func (p *prog) monitorNetworkChanges() error { - mon, err := netmon.New(func(format string, args ...any) { +func (p *prog) monitorNetworkChanges(ctx context.Context) error { + mon, err := newNetworkChangeMonitorFn(func(format string, args ...any) { // Always fetch the latest logger (and inject the prefix) - mainLog.Load().Printf("netmon: "+format, args...) + p.logger.Load().Printf("netmon: "+format, args...) }) if err != nil { return fmt.Errorf("creating network monitor: %w", err) } - mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { - // Get map of valid interfaces - validIfaces := validInterfacesMap() - - isMajorChange := mon.IsMajorChangeFrom(delta.Old, delta.New) + mon.RegisterChangeCallback(p.networkChangeCallback(func(delta *netmon.ChangeDelta) { + p.handleNetworkChange(ctx, delta, mon.IsMajorChangeFrom(delta.Old, delta.New)) + })) + if !p.setNetMonitor(mon) { + _ = mon.Close() + p.Debug().Msg("Network monitor discarded, ctrld is stopping") + return nil + } - mainLog.Load().Debug(). - Interface("old_state", delta.Old). - Interface("new_state", delta.New). - Bool("is_major_change", isMajorChange). - Msg("Network change detected") + mon.Start() + p.logger.Load().Debug().Msg("Network monitor started") + return nil +} - changed := false - activeInterfaceExists := false - var changeIPs []netip.Prefix - // Check each valid interface for changes - for ifaceName := range validIfaces { - oldIface, oldExists := delta.Old.Interface[ifaceName] - newIface, newExists := delta.New.Interface[ifaceName] - if !newExists { - continue +// handleNetworkChange is the network monitor callback, kept separate for synthetic deltas. +func (p *prog) handleNetworkChange(ctx context.Context, delta *netmon.ChangeDelta, isMajorChange bool) { + // netmon v1.74.0 dispatches callbacks concurrently and caches only major + // snapshots. Minor events refer to that cache through Old. The synthetic + // delta.Major flag after a time jump does not change the cache. + p.networkSourceMu.Lock() + currentState := networkChangeCurrentStateFn(delta) + current := networkSnapshotCurrent(delta, isMajorChange, currentState) + if delta.TimeJumped { + // A late callback still tells that the host woke, but the network of + // the wake is the one that netmon holds now. + wakeState := delta.New + if !current { + wakeState = currentState + } + p.noteHostWoke("netmon", 0, wakeState) + } + // Only a current callback becomes the baseline of the next diff. A late + // callback that a newer snapshot replaced would hide the changes between + // the callback before it and the callback after it. + before := p.deltaBeforeState(delta, currentState, current, isMajorChange) + changes := diffNetworkDelta(before, delta.New) + if current && hasAddOrRemove(changes) { + refreshInterfaceMeta() + } + // The noise test runs before any other work, because the point of the class + // is that a storm starts no pfctl and no scutil process. + if current && noiseDelta(before, delta.New, delta.TimeJumped, changes) { + p.networkSourceMu.Unlock() + p.noteNoiseDelta(changes) + return + } + // A superseded snapshot never becomes a transition, so it takes no ID and + // reports zero. + var transitionID uint64 + changed := false + changedInterface := "" + sourceSnapshotAvailable := false + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + after4, after6 := before4, before6 + // The newer callback of this epoch carries the same interface changes, so + // this outcome reports at debug and stays out of the journal. + outcome := transitionOutcomeSnapshotSuperseded + defer func() { + stateBefore, stateAfter := networkStateOrEmpty(before), networkStateOrEmpty(delta.New) + networkTransitionEvent(outcome).Uint64("transition_id", transitionID). + Bool("is_major_change", isMajorChange).Bool("changed", changed). + Str("interface", changedInterface). + Strs("changed_interfaces", changedInterfaceNames(changes)). + Bool("time_jumped", delta.TimeJumped). + Str("default_route_before", stateBefore.DefaultRouteInterface). + Str("default_route_after", stateAfter.DefaultRouteInterface). + Bool("have_v4_before", stateBefore.HaveV4).Bool("have_v4_after", stateAfter.HaveV4). + Bool("have_v6_before", stateBefore.HaveV6).Bool("have_v6_after", stateAfter.HaveV6). + Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). + Str("source_ipv4_after", after4.String()). + Str("source_ipv6_after", after6.String()). + Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome). + Msg(networkTransitionMessage) + }() + if !current { + p.networkSourceMu.Unlock() + return + } + p.flushNoiseSummary() + outcome = transitionOutcomeIgnored + transitionID = p.networkTransitionGen.Add(1) + sourceState := delta.New + // A late major callback and reordered minor callbacks share one cache + // epoch. For these cases, source validity comes from the OS, not ordering. + if delta.Monitor != nil && (!isMajorChange || (p.networkSourceEpoch == currentState && p.networkSourceState != currentState)) { + fresh, err := readNetworkSourceStateFn() + if err != nil || fresh == nil { + if !p.networkSourceReadFailed { + p.logger.Load().Warn().Uint64("transition_id", transitionID). + Msg("Network source snapshot unavailable; source writes deferred") } - - oldIPs := delta.Old.InterfaceIPs[ifaceName] - newIPs := delta.New.InterfaceIPs[ifaceName] - - // if a valid interface did not exist in old - // check that its up and has usable IPs - if !oldExists { - // The interface is new (was not present in the old state). - usableNewIPs := filterUsableIPs(newIPs) - if newIface.IsUp() && len(usableNewIPs) > 0 { - changed = true - changeIPs = usableNewIPs - mainLog.Load().Debug(). - Str("interface", ifaceName). - Interface("new_ips", usableNewIPs). - Msg("Interface newly appeared (was not present in old state)") - break - } - continue + p.networkSourceReadFailed = true + sourceState = nil + } else { + snapshot := *fresh + snapshot.DefaultRouteInterface = currentState.DefaultRouteInterface + sourceState = &snapshot + if p.networkSourceReadFailed { + p.logger.Load().Warn().Uint64("transition_id", transitionID).Msg("Network source snapshot available again") } + p.networkSourceReadFailed = false + } + } + p.networkSourceState, p.networkSourceEpoch = sourceState, currentState + sourceSnapshotAvailable = sourceState != nil + validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(ctx, p.logger.Load()), sourceState, transitionID) + after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + // The network of this callback belongs to the epoch that the lines above + // store. A store after the unlock can overwrite the network of a newer + // callback. The header render reads the hardware ports, so it runs after + // the unlock and starts no process under the lock. + p.lastNetworkState.Store(currentState) + // HasIPv6 reads the flag that this callback stores, so one monitor serves + // the whole process. + ctrld.SetIPv6Available(currentState.HaveV6) + p.networkSourceMu.Unlock() + p.refreshLogHeader() + describeInterfaceChanges(changes, interfaceMetaFor) + logInterfaceChanges(transitionID, changes) + + p.handleDNS64NetworkChange(delta, isMajorChange) + validIfaces := networkChangeValidInterfacesFn(ctrld.LoggerCtx(ctx, p.logger.Load())) + + activeInterfaceExists := false + var changeIPs []netip.Prefix + // Check each valid interface for changes + for ifaceName := range validIfaces { + oldIface, oldExists := delta.Old.Interface[ifaceName] + newIface, newExists := delta.New.Interface[ifaceName] + if !newExists { + continue + } - // Filter new IPs to only those that are usable. - usableNewIPs := filterUsableIPs(newIPs) + oldIPs := delta.Old.InterfaceIPs[ifaceName] + newIPs := delta.New.InterfaceIPs[ifaceName] - // Check if interface is up and has usable IPs. + // if a valid interface did not exist in old + // check that its up and has usable IPs + if !oldExists { + // The interface is new (was not present in the old state). + usableNewIPs := filterUsableIPs(newIPs) if newIface.IsUp() && len(usableNewIPs) > 0 { + // The new interface is the active one, or the change ends as + // no_active_interface and nothing reconciles. activeInterfaceExists = true + changed = true + changeIPs = usableNewIPs + changedInterface = ifaceName + p.logger.Load().Debug(). + Str("interface", ifaceName). + Interface("new_ips", usableNewIPs). + Msg("Interface newly appeared (was not present in old state)") + break } - - // Compare interface states and IPs (interfaceIPsEqual will itself filter the IPs). - if !interfaceStatesEqual(&oldIface, &newIface) || !interfaceIPsEqual(oldIPs, newIPs) { - if newIface.IsUp() && len(usableNewIPs) > 0 { - changed = true - changeIPs = usableNewIPs - mainLog.Load().Debug(). - Str("interface", ifaceName). - Interface("old_ips", oldIPs). - Interface("new_ips", usableNewIPs). - Msg("Interface state or IPs changed") - break - } - } + continue } - // if the default route changed, set changed to true - if delta.New.DefaultRouteInterface != delta.Old.DefaultRouteInterface { - changed = true - mainLog.Load().Debug().Msgf("Default route changed from %s to %s", delta.Old.DefaultRouteInterface, delta.New.DefaultRouteInterface) - } + // Filter new IPs to only those that are usable. + usableNewIPs := filterUsableIPs(newIPs) - if !changed { - mainLog.Load().Debug().Msg("Ignoring interface change - no valid interfaces affected") - // check if the default IPs are still on an interface that is up - ValidateDefaultLocalIPsFromDelta(delta.New) - return + // Check if interface is up and has usable IPs. + if newIface.IsUp() && len(usableNewIPs) > 0 { + activeInterfaceExists = true } - if !activeInterfaceExists { - mainLog.Load().Debug().Msg("No active interfaces found, skipping reinitialization") - return + // Compare interface states and IPs (interfaceIPsEqual will itself filter the IPs). + if !interfaceStatesEqual(&oldIface, &newIface) || !interfaceIPsEqual(oldIPs, newIPs) { + if newIface.IsUp() && len(usableNewIPs) > 0 { + changed = true + changeIPs = usableNewIPs + changedInterface = ifaceName + p.logger.Load().Debug(). + Str("interface", ifaceName). + Interface("old_ips", oldIPs). + Interface("new_ips", usableNewIPs). + Msg("Interface state or IPs changed") + break + } } + } - // Get IPs from default route interface in new state - selfIP := defaultRouteIP() + // if the default route changed, set changed to true + if delta.New.DefaultRouteInterface != delta.Old.DefaultRouteInterface { + changed = true + p.logger.Load().Debug().Msgf("Default route changed from %s to %s", delta.Old.DefaultRouteInterface, delta.New.DefaultRouteInterface) + } - // Ensure that selfIP is an IPv4 address. - // If defaultRouteIP mistakenly returns an IPv6 (such as a ULA), clear it - if ip := net.ParseIP(selfIP); ip != nil && ip.To4() == nil { - mainLog.Load().Debug().Msgf("defaultRouteIP returned a non-IPv4 address: %s, ignoring it", selfIP) - selfIP = "" + if !changed { + p.logger.Load().Debug().Msg("Ignoring interface change - no valid interfaces affected") + // Minor interface changes can still accompany pf/WFP or VPN DNS changes. + // On macOS, bound the immediate full reconciliation so link-local-only + // notification storms do not run pfctl/scutil work for every event. + // Windows keeps the existing immediate behavior. Tunnel changes always + // bypass the macOS limit, and delayed checks provide a trailing refresh. + if dnsIntercept && p.dnsInterceptState != nil { + networkChangeIgnoredInterceptFn(p, delta, time.Now()) } - var ipv6 string + return + } - if delta.New.DefaultRouteInterface != "" { - mainLog.Load().Debug().Msgf("default route interface: %s, IPs: %v", delta.New.DefaultRouteInterface, delta.New.InterfaceIPs[delta.New.DefaultRouteInterface]) - for _, ip := range delta.New.InterfaceIPs[delta.New.DefaultRouteInterface] { - ipAddr, _ := netip.ParsePrefix(ip.String()) - addr := ipAddr.Addr() - if selfIP == "" && addr.Is4() { - mainLog.Load().Debug().Msgf("checking IP: %s", addr.String()) - if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - selfIP = addr.String() - } - } - if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - ipv6 = addr.String() - } + if !activeInterfaceExists { + outcome = "no_active_interface" + p.logger.Load().Debug().Msg("No active interfaces found, skipping reinitialization") + return + } + + // An ignored event must not cancel this accepted transition. Only a newer + // accepted transition can replace its source commit or debounced recovery. + p.networkSourceMu.Lock() + if p.networkAcceptedGen.Load() > transitionID { + p.networkSourceMu.Unlock() + outcome = "superseded" + return + } + p.networkAcceptedGen.Store(transitionID) + p.networkSourceMu.Unlock() + + p.logger.Load().Debug().Msg("Link state changed, re-bootstrapping") + for _, uc := range p.cfg.Upstream { + uc.ReBootstrap(ctrld.LoggerCtx(ctx, p.logger.Load())) + } + + // Get IPs from default route interface in new state + selfIP := networkChangeDefaultRouteIPFn(p) + + // Ensure that selfIP is an IPv4 address. + // If defaultRouteIP mistakenly returns an IPv6 (such as a ULA), clear it + if ip := net.ParseIP(selfIP); ip != nil && ip.To4() == nil { + p.logger.Load().Debug().Msgf("defaultRouteIP returned a non-IPv4 address: %s, ignoring it", selfIP) + selfIP = "" + } + // Route discovery can lag the delta; do not reintroduce an invalid source. + if ip := net.ParseIP(selfIP); ip != nil && sourceInvalidReason(delta.New, ip) != "" { + selfIP = "" + } + var ipv6 string + + if delta.New.DefaultRouteInterface != "" { + p.logger.Load().Debug().Msgf("default route interface: %s, IPs: %v", delta.New.DefaultRouteInterface, delta.New.InterfaceIPs[delta.New.DefaultRouteInterface]) + for _, ip := range delta.New.InterfaceIPs[delta.New.DefaultRouteInterface] { + if sourceInvalidReason(delta.New, net.ParseIP(ip.Addr().String())) != "" { + continue } - } else { - // If no default route interface is set yet, use the changed IPs - mainLog.Load().Debug().Msgf("no default route interface found, using changed IPs: %v", changeIPs) - for _, ip := range changeIPs { - ipAddr, _ := netip.ParsePrefix(ip.String()) - addr := ipAddr.Addr() - if selfIP == "" && addr.Is4() { - mainLog.Load().Debug().Msgf("checking IP: %s", addr.String()) - if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - selfIP = addr.String() - } + ipAddr, _ := netip.ParsePrefix(ip.String()) + addr := ipAddr.Addr() + if selfIP == "" && addr.Is4() { + p.logger.Load().Debug().Msgf("checking IP: %s", addr.String()) + if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + selfIP = addr.String() } - if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - ipv6 = addr.String() + } + if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + ipv6 = addr.String() + } + } + } else { + // If no default route interface is set yet, use the changed IPs + p.logger.Load().Debug().Msgf("no default route interface found, using changed IPs: %v", changeIPs) + for _, ip := range changeIPs { + ipAddr, _ := netip.ParsePrefix(ip.String()) + addr := ipAddr.Addr() + if selfIP == "" && addr.Is4() { + p.logger.Load().Debug().Msgf("checking IP: %s", addr.String()) + if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + selfIP = addr.String() } } + if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + ipv6 = addr.String() + } } + } - // Only set the IPv4 default if selfIP is a valid IPv4 address. - if ip := net.ParseIP(selfIP); ip != nil && ip.To4() != nil { - ctrld.SetDefaultLocalIPv4(ip) - if !isMobile() && p.ciTable != nil { - p.ciTable.SetSelfIP(selfIP) - } + // An ignored event can change source validity without replacing recovery. + // Recheck addresses against the newest state after blocking route lookup. + p.networkSourceMu.Lock() + if p.networkAcceptedGen.Load() != transitionID { + p.networkSourceMu.Unlock() + outcome = "superseded" + return + } + latestState := p.sourceCommitState(delta) + sourceSnapshotAvailable = latestState != nil + validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(ctx, p.logger.Load()), latestState, transitionID) + // Only keep candidate addresses that still exist on an up interface. + if ip := net.ParseIP(selfIP); ip != nil && ip.To4() != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" { + ctrld.SetDefaultLocalIPv4(ctrld.LoggerCtx(ctx, p.logger.Load()), ip) + if !isMobile() && p.ciTable != nil { + p.ciTable.SetSelfIP(selfIP) + } + } + if ip := net.ParseIP(ipv6); ip != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" { + ctrld.SetDefaultLocalIPv6(ctrld.LoggerCtx(ctx, p.logger.Load()), ip) + } + after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + p.networkSourceMu.Unlock() + p.logger.Load().Debug().Msgf("Set default local IPv4: %s, IPv6: %s", selfIP, ipv6) + + outcome = "accepted" + networkChangeReconcileFn(p, ctx, transitionID) + p.logNetworkSnapshot("transition") + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } +} + +// reconcileNetworkChange is the first DNS/PF mutation boundary after source updates. +func (p *prog) reconcileNetworkChange(ctx context.Context, transitionID uint64) { + p.debounceRecovery(transitionID) + + // Drop stale firewall allowances from the previous network. + p.firewallOnNetworkChange() + + // After network changes, verify our pf anchor is still active and + // refresh VPN DNS state. Order matters: tunnel checks first (may rebuild + // anchor), then VPN DNS refresh (updates exemptions in anchor), then + // delayed re-checks for async VPN teardown. + if dnsIntercept && p.dnsInterceptState != nil { + if !p.pfStabilizing.Load() { + p.ensurePFAnchorActive() } - if ip := net.ParseIP(ipv6); ip != nil { - ctrld.SetDefaultLocalIPv6(ip) + // Check tunnel interfaces unconditionally — it decides internally + // whether to enter stabilization or rebuild immediately. + p.checkTunnelInterfaceChanges() + // Refresh VPN DNS routes — runs after tunnel checks so the anchor + // rebuild includes current VPN DNS exemptions. + if p.vpnDNS != nil { + p.vpnDNS.Refresh(ctrld.LoggerCtx(ctx, p.logger.Load()), true) } - mainLog.Load().Debug().Msgf("Set default local IPv4: %s, IPv6: %s", selfIP, ipv6) + // Preserve master VM/container forwarded-source reconciliation. + p.reconcileForwardedSources() + // Schedule delayed re-checks to catch async VPN teardown changes. + p.scheduleDelayedRechecks() + } +} - // we only trigger recovery flow for network changes on non router devices - if router.Name() == "" { - p.handleRecovery(RecoveryReasonNetworkChange) +// handleDNSInterceptIgnoredNetworkChange runs the DNS-intercept work for a +// network delta that did not affect a usable interface. Keeping this path in a +// method lets tests exercise the callback wiring with synthetic deltas. +func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, now time.Time) { + reconcileNow := false + // Stabilization owns PF repair. Do not consume the next leading-edge slot + // until an ignored delta can actually perform the corresponding PF check. + if !p.pfStabilizing.Load() { + reconcileNow = p.dnsInterceptIgnoredChangeReconcileDue(now) + if reconcileNow { + p.ensurePFAnchorActive() } - }) + } - mon.Start() - mainLog.Load().Debug().Msg("Network monitor started") - return nil + // Check tunnel interfaces unconditionally — it decides internally whether + // to enter stabilization or rebuild immediately. + tunnelChanged := p.checkTunnelInterfaceChanges() + // Schedule delayed re-checks to catch async VPN teardown changes. These also + // refresh the OS resolver and VPN DNS routes. + p.scheduleDelayedRechecks() + + // Detect interface appearance/disappearance — hypervisors (Parallels, + // VMware, VirtualBox) reload pf when creating/destroying virtual network + // interfaces, which can corrupt pf's internal translation state. The rdr + // rules survive in text form (watchdog says "intact") but stop evaluating. + // Spawn an async monitor that probes pf interception with backoff and forces + // a full pf reload if broken. + if delta.Old != nil { + changedAction := "removed" + changedIface := interfaceOnlyIn(delta.Old.Interface, delta.New.Interface) + if changedIface == "" { + changedAction = "added" + changedIface = interfaceOnlyIn(delta.New.Interface, delta.Old.Interface) + } + if changedIface != "" { + journal(mainLog.Load().Info()).Str("interface", changedIface). + Str("class", interfaceMetaFor(changedIface).Class). + Str("action", changedAction). + Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor") + go p.pfInterceptMonitor() + // A VM/container bridge appearing/disappearing changes the effective + // forwarded-source set; an intact anchor/probe does not detect that. + p.reconcileForwardedSources() + } + } + + // Refresh VPN DNS immediately for real tunnel changes even when the periodic + // ignored-change reconciliation is currently rate-limited - but not while + // stabilization owns pf. A refresh rebuilds the anchor, and these deltas arrive + // exactly when a VPN is bringing its own ruleset up, which is the collision + // stabilization is there to prevent. checkTunnelInterfaceChanges keeps the + // observation pending, so the transition is retried rather than dropped. + if p.vpnDNS != nil && (reconcileNow || tunnelChanged) && !p.pfStabilizing.Load() { + p.vpnDNS.Refresh(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true) + } +} + +// interfaceOnlyIn names one interface that have holds and lack does not. The +// loopback stays out, because it never appears and never goes away. +func interfaceOnlyIn(have, lack map[string]netmon.Interface) string { + for name := range have { + if name == "lo0" { + continue + } + if _, exists := lack[name]; !exists { + return name + } + } + return "" } // interfaceStatesEqual compares two interface states @@ -1389,14 +2470,32 @@ func interfaceIPsEqual(a, b []netip.Prefix) bool { return true } +var errOsHealthcheckSuppressed = errors.New("upstream os health check suppressed") + +// upstreamFailureLog keeps one error line for each upstream of one recovery +// pass. The pass retries every two seconds until the upstream answers, and the +// first line already names the fault, so the failures that follow it go to +// debug. +type upstreamFailureLog struct{ reported bool } + +// report logs one failed check of upstream. +func (l *upstreamFailureLog) report(p *prog, upstream string, uc *ctrld.UpstreamConfig, err error, duration time.Duration) { + level := p.Error + if l.reported { + level = p.Debug + } + l.reported = true + p.logUpstreamProbeFailure(upstream, uc, err, level, "Upstream check failed after %v", duration) +} + // checkUpstreamOnce sends a test query to the specified upstream. // Returns nil if the upstream responds successfully. -func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) error { - mainLog.Load().Debug().Msgf("Starting check for upstream: %s", upstream) +func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, failures *upstreamFailureLog) error { + p.Debug().Msgf("Starting check for upstream: %s", upstream) - resolver, err := ctrld.NewResolver(uc) + resolver, err := ctrld.NewResolver(ctrld.LoggerCtx(context.Background(), p.logger.Load()), uc) if err != nil { - mainLog.Load().Error().Err(err).Msgf("Failed to create resolver for upstream %s", upstream) + p.logUpstreamProbeFailure(upstream, uc, err, p.Error, "Failed to create resolver") return err } @@ -1404,13 +2503,13 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro if uc.Timeout > 0 { timeout = time.Millisecond * time.Duration(uc.Timeout) } - mainLog.Load().Debug().Msgf("Timeout for upstream %s: %s", upstream, timeout) + p.Debug().Msgf("Timeout for upstream %s: %s", upstream, timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - uc.ReBootstrap() - mainLog.Load().Debug().Msgf("Rebootstrapping resolver for upstream: %s", upstream) + uc.ReBootstrap(ctrld.LoggerCtx(ctx, p.logger.Load())) + p.Debug().Msgf("Rebootstrapping resolver for upstream: %s", upstream) start := time.Now() msg := uc.VerifyMsg() @@ -1418,149 +2517,466 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro duration := time.Since(start) if err != nil { - mainLog.Load().Error().Err(err).Msgf("Upstream %s check failed after %v", upstream, duration) + // Demote upstream.os check failures to debug while WFP loopback + // protect is active: an external WFP block filter is interfering + // with plain DNS so repeated failures here are expected. Other + // upstreams keep error level so real outages stay visible. + if upstream == upstreamOS && p.osHealthcheckSuppressed() { + p.Debug().Err(err).Msgf("Upstream %s check failed after %v (WFP loopback protect active)", upstream, duration) + return errOsHealthcheckSuppressed + } + // A no-route/network-unreachable failure means the endpoint's address + // family is available locally but unroutable (e.g. an IPv6 DoH endpoint + // while IPv6 is up but has no route). These repeat until the route + // returns and are handled by bounded backoff in the recovery loop, so + // keep them at debug to avoid sustained error-log spam. + if ctrldnet.IsUnreachable(err) { + p.Debug().Err(err).Msgf("Upstream %s check failed after %v (network unreachable)", upstream, duration) + return err + } + failures.report(p, upstream, uc, err, duration) + return err + } + p.Debug().Msgf("Upstream %s responded successfully in %v", upstream, duration) + return nil +} + +// handleRecovery orchestrates the recovery process by coordinating multiple smaller methods. +// It handles recovery cancellation logic, creates recovery context, prepares the system, +// waits for upstream recovery with timeout, and completes the recovery process. +// The method is designed to be called from a goroutine and handles different recovery reasons +// (network changes, regular failures, OS failures) with appropriate logic for each. +// recoveryDebounceWindow is the time to wait after the last network change +// before triggering handleRecovery. This coalesces rapid consecutive network +// changes (e.g., hotspot→LAN causing en1 drop + en0 pickup + en1 re-pickup) +// into a single recovery pass, avoiding the cancel-and-restart race that +// leaves DoH transports in a stale state. +const recoveryDebounceWindow = 500 * time.Millisecond + +// recoveryResolverReason names this flow in the OS resolver events, so a +// reader tells a resolver read of the recovery from every other read. +const recoveryResolverReason = "recovery" + +// debounceRecovery schedules a handleRecovery(NetworkChange) call after a debounce +// window. If called again before the window expires, the timer is reset so that +// recovery runs once with the final network state. All other state updates (IP, +// pf anchor, VPN DNS, tunnel checks) run immediately — only the recovery flow +// with its upstream probing and DHCP bypass logic is debounced. +func (p *prog) debounceRecovery(transitionID uint64) { + p.networkSourceMu.Lock() + defer p.networkSourceMu.Unlock() + if p.networkAcceptedGen.Load() != transitionID { + return + } + p.recoveryDebounceMu.Lock() + defer p.recoveryDebounceMu.Unlock() + if p.networkActivityClosed() { + return + } + + if p.recoveryDebounceTimer != nil { + p.recoveryDebounceTimer.Stop() + p.Debug().Msg("Recovery debounce: resetting timer (rapid network change)") + } + p.recoveryDebounceTimer = time.AfterFunc(recoveryDebounceWindow, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + p.networkSourceMu.Lock() + if p.networkAcceptedGen.Load() != transitionID { + p.networkSourceMu.Unlock() + return + } + p.recoveryDebounceMu.Lock() + p.recoveryDebounceTimer = nil + p.recoveryDebounceMu.Unlock() + p.networkSourceMu.Unlock() + handleRecoveryForTransitionFn(p, RecoveryReasonNetworkChange, transitionID) + }) + p.Debug().Msg("Recovery debounce: scheduled (500ms window)") +} + +// endRecovery closes one recovery pass. Support reads the end event and the +// snapshot that follows it to learn what the recovery left behind. +func (p *prog) endRecovery(diagnostic *recoveryDiagnostic, outcome, targetBefore string) { + // A superseded pass changed nothing that it owns any more. Its successor + // owns the loopback target and the bypass flag, so reading them here would + // report the state of the successor. + if outcome == recoveryOutcomeSuperseded { + diagnostic.interceptTargetAction = interceptTargetActionUnchanged + diagnostic.bypassActive = false } else { - mainLog.Load().Debug().Msgf("Upstream %s responded successfully in %v", upstream, duration) + diagnostic.interceptTargetAction = interceptTargetAction(targetBefore, p.interceptTargetSnapshot()) + diagnostic.bypassActive = p.recoveryBypass.Load() + } + diagnostic.end(outcome) + p.logNetworkSnapshot("recovery_end") + // The resolver table of the host settles after the recovery, so the poll + // stays fast until it does. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } +} + +// The actions that one recovery takes on the loopback DNS target. +const ( + interceptTargetActionUnchanged = "unchanged" + interceptTargetActionRemoved = "removed" + interceptTargetActionSet = "set" +) + +// interceptTargetAction names what one recovery did to the loopback DNS +// target. The target value alone does not say whether this pass wrote it. +func interceptTargetAction(before, after string) string { + switch { + case before == after: + return interceptTargetActionUnchanged + case after == "": + return interceptTargetActionRemoved + default: + return interceptTargetActionSet } - return err } -// handleRecovery performs a unified recovery by removing DNS settings, -// canceling existing recovery checks for network changes, but coalescing duplicate -// upstream failure recoveries, waiting for recovery to complete (using a cancellable context without timeout), -// and then re-applying the DNS settings. func (p *prog) handleRecovery(reason RecoveryReason) { - mainLog.Load().Debug().Msg("Starting recovery process: removing DNS settings") + p.handleRecoveryForTransition(reason, 0) +} + +func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID uint64) { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + p.networkSourceMu.Lock() + if transitionID != 0 && p.networkAcceptedGen.Load() != transitionID { + p.networkSourceMu.Unlock() + p.logger.Load().Debug().Uint64("transition_id", transitionID).Msg("Recovery skipped: transition superseded") + return + } + // Admission and completion use the same pool. A failed OS-only policy + // is not evidence that configured general DNS is unavailable. Check here, + // rather than only at the query, because a queued trigger can arrive late. + upstreams := p.buildRecoveryUpstreams(reason) + upstreamNames := make([]string, 0, len(upstreams)) + for name := range upstreams { + upstreamNames = append(upstreamNames, name) + } + slices.Sort(upstreamNames) + if reason == RecoveryReasonOSFailure { + if _, osOnly := upstreams[upstreamOS]; !osOnly { + for _, name := range upstreamNames { + if !p.um.isDown(name) { + p.networkSourceMu.Unlock() + p.refreshOSResolverAfterRecoverySkip(name) + return + } + } + } + } + recoveryCtx, gen, interceptRecovery, ok := p.beginRecovery(reason) + p.networkSourceMu.Unlock() + if !ok { + p.logger.Load().Debug().Uint64("transition_id", transitionID). + Uint64("recovery_generation", p.recoveryGen.Load()). + Msg("Upstream recovery already in progress; skipping duplicate trigger") + return + } + diagnostic := recoveryDiagnostic{logger: p.logger.Load(), transitionID: transitionID, generation: gen, reason: reason, started: time.Now()} + journal(diagnostic.event(p.logger.Load().Info())).Bool("intercept", interceptRecovery). + Strs("probe_upstreams", journalUpstreamNames(upstreamNames)).Msg("Recovery begin") + p.logNetworkSnapshot("recovery_begin") + targetBefore := p.interceptTargetSnapshot() + outcome := recoveryOutcomeSuperseded + defer func() { p.endRecovery(&diagnostic, outcome, targetBefore) }() + // Clean every exit before the final snapshot, including cancellation before mutation. + defer p.recoveryCanceledCleanup(gen) + if recoveryCtx.Err() != nil { + if p.recoveryGen.Load() == gen { + outcome = recoveryOutcomeCanceled + } + return + } + if reason == RecoveryReasonNetworkChange { + p.logger.Load().Debug().Msg("Network change recovery now owns shared recovery state") + } - // For network changes, cancel any existing recovery check because the network state has changed. + // For network changes, force-reset all upstream transports synchronously. + // The lazy ReBootstrap() called earlier in the network change callback only + // sets a flag — the old transport's dead connections can still be used by + // recovery probes, causing context deadline timeouts. ForceReBootstrap() + // closes old connections and creates fresh transports so probes succeed on + // first attempt. This runs outside recoveryCancelMu to avoid holding the + // mutex during transport work. if reason == RecoveryReasonNetworkChange { - p.recoveryCancelMu.Lock() - if p.recoveryCancel != nil { - mainLog.Load().Debug().Msg("Cancelling existing recovery check (network change)") - p.recoveryCancel() - p.recoveryCancel = nil + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + for _, uc := range p.cfg.Upstream { + if uc != nil { + uc.ForceReBootstrap(loggerCtx) + } } - p.recoveryCancelMu.Unlock() - } else { - // For upstream failures, if a recovery is already in progress, do nothing new. - p.recoveryCancelMu.Lock() - if p.recoveryCancel != nil { - mainLog.Load().Debug().Msg("Upstream recovery already in progress; skipping duplicate trigger") - p.recoveryCancelMu.Unlock() - return + p.Info().Msg("Force-reset upstream transports for network change recovery") + } + + if err := p.prepareForRecovery(reason, interceptRecovery, &diagnostic); err != nil { + p.Error().Err(err).Msg("Failed to prepare for recovery") + outcome = recoveryOutcomeFailed + p.recoveryCanceledCleanup(gen) + return + } + + // Wait for the same upstream pool used to admit this recovery. + recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic) + if err != nil { + if p.recoveryGen.Load() == gen { + outcome = recoveryOutcomeCanceled } - p.recoveryCancelMu.Unlock() + return + } + diagnostic.recoveredUpstream = recovered + if !p.recoveryOwnsState(gen) { + p.Debug().Msgf("Recovery generation %d was superseded after upstream success; skipping stale completion", gen) + return } - // Create a new recovery context without a fixed timeout. - p.recoveryCancelMu.Lock() - recoveryCtx, cancel := context.WithCancel(context.Background()) - p.recoveryCancel = cancel - p.recoveryCancelMu.Unlock() + // The reset ends the outage, so the down time is read before it. The name + // of the line is bounded, because an upstream key is operator text. + journalName := journalUpstreamName(recovered) + journal(p.Info()).Str("upstream", journalName). + Int64("down_for_ms", p.um.downFor(recovered).Milliseconds()). + Msgf("Upstream %q recovered; re-applying DNS settings", journalName) + if err := p.completeRecoveryWork(reason, recovered, interceptRecovery); err != nil { + p.Error().Err(err).Msg("Failed to complete recovery") + p.recoveryCanceledCleanup(gen) + return + } + if !p.completeRecoveryState(gen) { + p.Debug().Msgf("Recovery generation %d was superseded during completion; preserving successor state", gen) + return + } + + outcome = recoveryOutcomeCompleted + if interceptRecovery { + journal(p.Info()).Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow") + } + p.Info().Msgf("Recovery completed successfully for upstream %q", journalName) +} + +// prepareForRecovery removes DNS settings and initializes OS resolver if needed. +func (p *prog) prepareForRecovery(reason RecoveryReason, interceptRecovery bool, diagnostic *recoveryDiagnostic) error { + // In DNS intercept mode, don't tear down WFP/pf filters. + // Instead, enable recovery bypass so proxy() forwards queries to + // the OS/DHCP resolver. This handles captive portal authentication + // without the overhead of filter teardown/rebuild. + if interceptRecovery { + journal(p.Info()).Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)") + + // Reinitialize OS resolver to discover DHCP servers on the new network. + // This is critical for captive portals — we need the network's DNS servers + // to resolve the auth page. + p.Debug().Msg("DNS intercept recovery: discovering DHCP nameservers") + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + // The effective list adds a synthetic public resolver when the network + // gave none, so the journal reports the discovered list instead. + resolverNameservers, systemNameservers := initializeOsResolverWithSystemNameserversFn(loggerCtx, true, recoveryResolverReason) + diagnostic.dhcpServers = systemNameservers + if len(systemNameservers) == 0 { + journal(p.Warn()).Msg("DNS intercept recovery: no DHCP nameservers found") + } else { + journal(p.Info()).Strs("dhcp_servers", systemNameservers). + Msgf("DNS intercept recovery: found DHCP nameservers: %v", systemNameservers) + } - // Immediately remove our DNS settings from the interface. - // set recoveryRunning to true to prevent watchdogs from putting the listener back on the interface - p.recoveryRunning.Store(true) + ensureInterceptDNSTargetFn(p, systemNameservers) + + // Exempt DHCP nameservers from intercept filters so the OS resolver + // can actually reach them on port 53. Without this, the WFP block + // or pf redirect would intercept ctrld's own recovery queries. + // The OS resolver queries the effective list, so the exemptions follow it. + if len(resolverNameservers) > 0 { + // Strip :53 port suffix if present (exemptVPNDNSServers expects bare IPs). + var dhcpExemptions []vpnDNSExemption + for _, s := range resolverNameservers { + host := s + if h, _, err := net.SplitHostPort(s); err == nil { + host = h + } + dhcpExemptions = append(dhcpExemptions, vpnDNSExemption{Server: host}) + } + p.Info().Msgf("DNS intercept recovery: exempting DHCP nameservers from filters: %v", resolverNameservers) + if err := p.exemptVPNDNSServers(dhcpExemptions); err != nil { + p.Warn().Err(err).Msg("DNS intercept recovery: failed to exempt DHCP nameservers — recovery queries may fail") + } + } + + return nil + } + + // Traditional flow: remove DNS settings to expose DHCP nameservers // we do not want to restore any static DNS settings // we must try to get the DHCP values, any static DNS settings // will be appended to nameservers from the saved interface values - p.resetDNS(false, false) + recoveryResetDNSFn(p, false, false) // For an OS failure, reinitialize OS resolver nameservers immediately. if reason == RecoveryReasonOSFailure { - mainLog.Load().Debug().Msg("OS resolver failure detected; reinitializing OS resolver nameservers") - ns := ctrld.InitializeOsResolver(true) - if len(ns) == 0 { - mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values") - } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + if err := p.reinitializeOSResolver("OS resolver failure detected"); err != nil { + return fmt.Errorf("failed to reinitialize OS resolver: %w", err) } } - // Build upstream map based on the recovery reason. - upstreams := p.buildRecoveryUpstreams(reason) + return nil +} - // Wait indefinitely until one of the upstreams recovers. - recovered, err := p.waitForUpstreamRecovery(recoveryCtx, upstreams) - if err != nil { - mainLog.Load().Error().Err(err).Msg("Recovery canceled; DNS settings remain removed") - p.recoveryCancelMu.Lock() - p.recoveryCancel = nil - p.recoveryCancelMu.Unlock() - return +// reinitializeOSResolver reinitializes the OS resolver and logs the results. +func (p *prog) reinitializeOSResolver(message string) error { + p.Debug().Msg(message) + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + ns := ctrld.InitializeOsResolverWithReason(loggerCtx, true, recoveryResolverReason) + if len(ns) == 0 { + p.Warn().Msg("No nameservers found for OS resolver; using existing values") + } else { + journal(p.Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } - mainLog.Load().Info().Msgf("Upstream %q recovered; re-applying DNS settings", recovered) + return nil +} - // reset the upstream failure count and down state +// completeRecoveryWork performs owner-specific recovery work. Shared recovery +// flags are released separately by completeRecoveryState under the ownership lock. +func (p *prog) completeRecoveryWork(reason RecoveryReason, recovered string, interceptRecovery bool) error { p.um.reset(recovered) - // For network changes we also reinitialize the OS resolver. - if reason == RecoveryReasonNetworkChange { - ns := ctrld.InitializeOsResolver(true) - if len(ns) == 0 { - mainLog.Load().Warn().Msg("No nameservers found for OS resolver during network-change recovery; using existing values") - } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + if interceptRecovery { + // Refresh VPN DNS routes in case VPN state changed during recovery. + if p.vpnDNS != nil { + p.vpnDNS.Refresh(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true) + } + if reason == RecoveryReasonNetworkChange { + if err := p.reinitializeOSResolver("Network change detected during recovery"); err != nil { + return fmt.Errorf("failed to reinitialize OS resolver during network change: %w", err) + } + } + return nil + } + + var systemNameservers []string + if dnsIntercept { + // Intercept was requested but was not active when recovery began. A + // retry must use a completed raw discovery result, never nil. + systemNameservers = p.systemNameserversForInterceptRetry() + } else if reason == RecoveryReasonNetworkChange { + if err := p.reinitializeOSResolver("Network change detected during recovery"); err != nil { + return fmt.Errorf("failed to reinitialize OS resolver during network change: %w", err) } } - // Apply our DNS settings back and log the interface state. - p.setDNS() - p.logInterfacesState() + // Apply our DNS settings back. The snapshot that follows the end event + // reports the interfaces. + p.setDNS(systemNameservers) + return nil +} - // allow watchdogs to put the listener back on the interface if its changed for any reason - p.recoveryRunning.Store(false) +// refreshOSResolverAfterRecoverySkip repairs stale OS resolver discovery without +// enabling bypass, changing host DNS, or clearing the monitor's failure state. +// Only a successful query establishes that the OS resolver recovered. +func (p *prog) refreshOSResolverAfterRecoverySkip(healthy string) { + if !p.osRecoveryRefreshMu.TryLock() { + return + } + defer p.osRecoveryRefreshMu.Unlock() - // Clear the recovery cancellation for a clean slate. - p.recoveryCancelMu.Lock() - p.recoveryCancel = nil - p.recoveryCancelMu.Unlock() + now := networkEventsNowFn() + if p.osRecoverySkipLogAt.IsZero() || healthy != p.osRecoverySkipUpstream || now.Sub(p.osRecoverySkipLogAt) >= 5*time.Minute { + journal(p.Info()).Str("recovery_reason", recoveryReasonName(RecoveryReasonOSFailure)). + Str("healthy_upstream", journalUpstreamName(healthy)). + Msg("Recovery skipped: configured upstream is not marked down") + p.osRecoverySkipLogAt = now + p.osRecoverySkipUpstream = healthy + } + // Failed policy queries can arrive concurrently and at high volume. Retry + // discovery on later traffic, but never run overlapping or per-query reads. + if !p.osRecoveryRefreshAt.IsZero() && now.Sub(p.osRecoveryRefreshAt) < upstreamDownDelay { + return + } + p.osRecoveryRefreshAt = now + initializeOsResolverWithSystemNameserversFn(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true, recoveryResolverReason) } +// waitForUpstreamRecoveryFn is the seam of the upstream probe. A test drives +// the whole recovery flow without a query to a real upstream. +var waitForUpstreamRecoveryFn = (*prog).waitForUpstreamRecovery + +var queryRecoveryFn = (*prog).handleRecovery + // waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers. // It returns the name of the recovered upstream or an error if the check times out. -func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig) (string, error) { +func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { + recoveryCtx, cancel := context.WithCancel(ctx) + defer cancel() + recoveredCh := make(chan string, 1) var wg sync.WaitGroup - mainLog.Load().Debug().Msgf("Starting upstream recovery check for %d upstreams", len(upstreams)) + p.Debug().Msgf("Starting upstream recovery check for %d upstreams", len(upstreams)) + defer func() { cancel(); wg.Wait() }() for name, uc := range upstreams { wg.Add(1) go func(name string, uc *ctrld.UpstreamConfig) { defer wg.Done() - mainLog.Load().Debug().Msgf("Starting recovery check loop for upstream: %s", name) + p.Debug().Msgf("Starting recovery check loop for upstream: %s", name) attempts := 0 + unreachableStreak := 0 + var failures upstreamFailureLog for { select { - case <-ctx.Done(): - mainLog.Load().Debug().Msgf("Context canceled for upstream %s", name) + case <-recoveryCtx.Done(): + p.Debug().Msgf("Context canceled for upstream %s", name) return default: attempts++ - // checkUpstreamOnce will reset any failure counters on success. - if err := p.checkUpstreamOnce(name, uc); err == nil { - mainLog.Load().Debug().Msgf("Upstream %s recovered successfully", name) + // The owning recovery resets the monitor after this probe succeeds. + err := p.checkUpstreamOnce(name, uc, &failures) + if err == nil || errors.Is(err, errOsHealthcheckSuppressed) { + p.Debug().Msgf("Upstream %s recovered successfully", name) select { case recoveredCh <- name: - mainLog.Load().Debug().Msgf("Sent recovery notification for upstream %s", name) + p.Debug().Msgf("Sent recovery notification for upstream %s", name) + cancel() default: - mainLog.Load().Debug().Msg("Recovery channel full, another upstream already recovered") + p.Debug().Msg("Recovery channel full, another upstream already recovered") } return } - mainLog.Load().Debug().Msgf("Upstream %s check failed, sleeping before retry", name) - time.Sleep(checkUpstreamBackoffSleep) + diagnostic.failure(err) + // Back off the retry cadence for an unroutable endpoint so a + // host with IPv6 up but no route to the IPv6 DoH endpoint does + // not re-bootstrap/re-check every checkUpstreamBackoffSleep and + // spam the log. The backoff is bounded (checkUpstreamUnreachableBackoffMax) + // so the endpoint is still re-probed and recovers when the route + // returns; any other failure resets to the base cadence. + sleep := checkUpstreamBackoffSleep + if ctrldnet.IsUnreachable(err) { + unreachableStreak++ + sleep = unreachableRecoveryBackoff(unreachableStreak) + p.Debug().Msgf("Upstream %s unreachable (streak %d), backing off %s before retry", name, unreachableStreak, sleep) + } else { + unreachableStreak = 0 + p.Debug().Msgf("Upstream %s check failed, sleeping before retry", name) + } + if !sleepWithContext(recoveryCtx, sleep) { + return + } // if this is the upstreamOS and it's the 3rd attempt (or multiple of 3), // we should try to reinit the OS resolver to ensure we can recover if name == upstreamOS && attempts%3 == 0 { - mainLog.Load().Debug().Msgf("UpstreamOS check failed on attempt %d, reinitializing OS resolver", attempts) - ns := ctrld.InitializeOsResolver(true) + p.Debug().Msgf("UpstreamOS check failed on attempt %d, reinitializing OS resolver", attempts) + ns := ctrld.InitializeOsResolverWithReason(ctrld.LoggerCtx(ctx, p.logger.Load()), true, recoveryResolverReason) if len(ns) == 0 { - mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values") + p.Warn().Msg("No nameservers found for OS resolver; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(p.Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } @@ -1570,29 +2986,61 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string var recovered string select { + case <-ctx.Done(): + return "", ctx.Err() + default: + } + select { case recovered = <-recoveredCh: + if err := ctx.Err(); err != nil { + return "", err + } case <-ctx.Done(): return "", ctx.Err() } - wg.Wait() return recovered, nil } +func sleepWithContext(ctx context.Context, d time.Duration) bool { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-timer.C: + return true + case <-ctx.Done(): + return false + } +} + // buildRecoveryUpstreams constructs the map of upstream configurations to test. -// For OS failures we supply the manual OS resolver upstream configuration. -// For network change or regular failure we use the upstreams defined in p.cfg (ignoring OS). +// OS failures use configured non-OS upstreams for both admission and recovery, +// excluding generated Internal Domain resolvers. OS-only configurations retain +// the OS recovery path. Other recovery reasons retain their configured pool. func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.UpstreamConfig { upstreams := make(map[string]*ctrld.UpstreamConfig) switch reason { case RecoveryReasonOSFailure: - upstreams[upstreamOS] = osUpstreamConfig + for k, uc := range p.cfg.Upstream { + name := upstreamPrefix + k + if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isGeneratedInternalDomainUpstream(uc) { + upstreams[name] = uc + } + } + if len(upstreams) == 0 { + upstreams[upstreamOS] = osUpstreamConfig + } case RecoveryReasonNetworkChange, RecoveryReasonRegularFailure: // Use all configured upstreams except any OS type. for k, uc := range p.cfg.Upstream { - if uc.Type != ctrld.ResolverTypeOS { + if uc != nil && uc.Type != ctrld.ResolverTypeOS { upstreams[upstreamPrefix+k] = uc } } + // An OS-only configuration still needs a recovery worker. Do not add + // OS to mixed pools: their existing recovery boundaries stay intact. + if len(upstreams) == 0 { + upstreams[upstreamOS] = osUpstreamConfig + } } return upstreams } @@ -1601,27 +3049,8 @@ func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.U // are still present in the new network state (provided by delta.New). // If a stored default IP is no longer active, it resets that default (sets it to nil) // so that it won't be used in subsequent custom dialer contexts. +// Deprecated: retained for callers of the existing exported cli API. +// The network monitor uses transition-aware validation instead. func ValidateDefaultLocalIPsFromDelta(newState *netmon.State) { - currentIPv4 := ctrld.GetDefaultLocalIPv4() - currentIPv6 := ctrld.GetDefaultLocalIPv6() - - // Build a map of active IP addresses from the new state. - activeIPs := make(map[string]bool) - for _, prefixes := range newState.InterfaceIPs { - for _, prefix := range prefixes { - activeIPs[prefix.Addr().String()] = true - } - } - - // Check if the default IPv4 is still active. - if currentIPv4 != nil && !activeIPs[currentIPv4.String()] { - mainLog.Load().Debug().Msgf("DefaultLocalIPv4 %s is no longer active in the new state. Resetting.", currentIPv4) - ctrld.SetDefaultLocalIPv4(nil) - } - - // Check if the default IPv6 is still active. - if currentIPv6 != nil && !activeIPs[currentIPv6.String()] { - mainLog.Load().Debug().Msgf("DefaultLocalIPv6 %s is no longer active in the new state. Resetting.", currentIPv6) - ctrld.SetDefaultLocalIPv6(nil) - } + validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(context.Background(), mainLog.Load()), newState, 0) } diff --git a/cmd/cli/dns_proxy_sampling_test.go b/cmd/cli/dns_proxy_sampling_test.go new file mode 100644 index 00000000..e6a77d00 --- /dev/null +++ b/cmd/cli/dns_proxy_sampling_test.go @@ -0,0 +1,232 @@ +package cli + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" +) + +// deadUpstreamEndpoint is a port that nothing listens on, so every query to it +// fails at once. +const deadUpstreamEndpoint = "127.0.0.1:1" + +// countSamplingLines counts the captured lines of one level whose message +// starts with the given text. A request id rides in front of the message. +func countSamplingLines(t *testing.T, logs *syncBuffer, level, message string) int { + t.Helper() + count := 0 + for _, event := range jsonLogEvents(t, logs, "") { + text, _ := event["message"].(string) + if !strings.HasPrefix(text, message) || event["level"] != level { + continue + } + count++ + } + return count +} + +func Test_querySamplingBoundsPerQueryErrors(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + for i := 0; i < querySampleLimit+1; i++ { + ctrld.Log(context.Background(), p.querySampler.event(sampleClassResolveFailed, "upstream.0"), samplerTestMessage) + } + + if got := countSamplingLines(t, logs, "error", samplerTestMessage); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSamplingLines(t, logs, "debug", samplerTestMessage); got != 1 { + t.Fatalf("debug lines: got %d, want 1", got) + } +} + +// askThroughProxy sends one query through the real proxy path and returns the +// answer. +func askThroughProxy(t *testing.T, p *prog, ctx context.Context) *dns.Msg { + t.Helper() + msg := new(dns.Msg) + msg.SetQuestion("sampling.test.", dns.TypeA) + res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: &upstreamForResult{srcAddr: "192.168.0.1:1234"}}) + if res == nil || res.answer == nil { + t.Fatal("the proxy returned no answer") + } + return res.answer +} + +// Test_querySamplingOnTheQueryPath drives the real query path with an upstream +// that answers nothing: the flood keeps its request id, the sampler bounds it, +// and one state event brackets each end of the outage. +func Test_querySamplingOnTheQueryPath(t *testing.T) { + logs := captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + // A new monitor holds back the failure counts for one second. The test + // clears the flag instead of waiting for that timer. + p.um.clearRecovered(upstreamOS) + var armed []func() + p.um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + } + + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, "r1") + for i := 0; i < querySampleLimit+2; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + + const failedMessage = "[r1] " + samplerTestMessage + if got := countSamplingLines(t, logs, "error", failedMessage); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSamplingLines(t, logs, "debug", failedMessage); got != 2 { + t.Fatalf("debug lines: got %d, want 2", got) + } + + if len(armed) != 1 { + t.Fatalf("armed timers: got %d, want 1", len(armed)) + } + armed[0]() + startOSResolverStub(t) + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeSuccess { + t.Fatalf("the answered query returned %s, want NOERROR", dns.RcodeToString[rcode]) + } + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + wantField(t, events[0], "upstream", upstreamOS) + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "timer") + wantField(t, events[1], "upstream", upstreamOS) + wantField(t, events[1], "state", "up") +} + +// Test_allEndpointsFailedBoundsTheUpstreamName drives the query path with an +// operator-chosen upstream key. The line that reports the loss of every +// endpoint reaches the journal, so it must name the upstream by its bounded +// name. +func Test_allEndpointsFailedBoundsTheUpstreamName(t *testing.T) { + logs := captureDebugMainLog(t) + uc := &ctrld.UpstreamConfig{ + Name: probeSecret, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 200, + } + uc.Init(context.Background()) + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{probeSecret: uc}} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + msg := new(dns.Msg) + msg.SetQuestion("private.test.", dns.TypeA) + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{ + srcAddr: "192.168.0.1:1234", + upstreams: []string{upstreamPrefix + probeSecret}, + }, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Fatal("the proxy answered although every endpoint failed") + } + + var reported string + for _, event := range jsonLogEvents(t, logs, "") { + text, _ := event["message"].(string) + if strings.Contains(text, "endpoints failed") { + reported = text + } + } + if !strings.Contains(reported, upstreamPrefix+"custom") { + t.Fatalf("the all-endpoints line does not name the bounded upstream: %q", reported) + } + if strings.Contains(reported, probeSecret) { + t.Fatalf("the all-endpoints line holds the operator key: %q", reported) + } +} + +// Test_upstreamOutageOnTheQueryPathLogsOneStatePair drives 633 failed queries +// through the real query path, the count of the customer upload that sized the +// journal. The outage must give one down event, and the recovery one up event +// with the length of the outage. +func Test_upstreamOutageOnTheQueryPathLogsOneStatePair(t *testing.T) { + const failedQueries = 633 + logs := captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + // A new monitor holds back the failure counts for one second. The test + // clears the flag instead of waiting for that timer. + p.um.clearRecovered(upstreamOS) + p.um.after = func(time.Duration, func()) {} + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + // Windows reports no port-unreachable error on a UDP socket, so each query + // waits for the full timeout there. A short timeout keeps 633 queries fast. + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 20, + } + + ctx := context.Background() + for i := 0; i < failedQueries; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events after %d failed queries: got %d, want 1", failedQueries, len(events)) + } + wantField(t, events[0], "upstream", upstreamOS) + wantField(t, events[0], "state", "down") + + startOSResolverStub(t) + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeSuccess { + t.Fatalf("the answered query returned %s, want NOERROR", dns.RcodeToString[rcode]) + } + + events = jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events after the recovery: got %d, want 2", len(events)) + } + wantField(t, events[1], "upstream", upstreamOS) + wantField(t, events[1], "state", "up") + if _, carried := events[1]["down_for_ms"].(float64); !carried { + t.Fatalf("the up event carries no down_for_ms: %v", events[1]) + } +} diff --git a/cmd/cli/dns_proxy_test.go b/cmd/cli/dns_proxy_test.go index 4a4e5b4e..4867150c 100644 --- a/cmd/cli/dns_proxy_test.go +++ b/cmd/cli/dns_proxy_test.go @@ -77,7 +77,8 @@ func Test_prog_upstreamFor(t *testing.T) { cfg := testhelper.SampleConfig(t) cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) p := &prog{cfg: cfg} - p.um = newUpstreamMonitor(p.cfg) + p.logger.Store(mainLog.Load()) + p.um = newUpstreamMonitor(p.cfg, mainLog.Load()) p.lanLoopGuard = newLoopGuard() p.ptrLoopGuard = newLoopGuard() for _, nc := range p.cfg.Network { @@ -142,9 +143,94 @@ func Test_prog_upstreamFor(t *testing.T) { } } +func Test_prog_upstreamForWithCustomMatching(t *testing.T) { + cfg := testhelper.SampleConfig(t) + prog := &prog{cfg: cfg} + prog.logger.Store(mainLog.Load()) + for _, nc := range prog.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatal(err) + } + nc.IPNets = append(nc.IPNets, ipNet) + } + } + + // Create a custom policy with domain-first matching order + customPolicy := &ctrld.ListenerPolicyConfig{ + Name: "Custom Policy", + Networks: []ctrld.Rule{ + {"network.0": []string{"upstream.1", "upstream.0"}}, + }, + Macs: []ctrld.Rule{ + {"14:45:A0:67:83:0A": []string{"upstream.2"}}, + }, + Rules: []ctrld.Rule{ + {"*.ru": []string{"upstream.1"}}, + }, + Matching: &ctrld.MatchingConfig{ + Order: []string{"domain", "mac", "network"}, + }, + } + + customListener := &ctrld.ListenerConfig{ + Policy: customPolicy, + } + + tests := []struct { + name string + ip string + mac string + domain string + upstreams []string + matched bool + }{ + { + name: "Domain rule should match first with custom order", + ip: "192.168.0.1:0", + mac: "14:45:A0:67:83:0A", + domain: "example.ru", + upstreams: []string{"upstream.1"}, + matched: true, + }, + { + name: "MAC rule should match when no domain rule", + ip: "192.168.0.1:0", + mac: "14:45:A0:67:83:0A", + domain: "example.com", + upstreams: []string{"upstream.2"}, + matched: true, + }, + { + name: "Network rule should match when no domain or MAC rule", + ip: "192.168.0.1:0", + mac: "00:11:22:33:44:55", + domain: "example.com", + upstreams: []string{"upstream.1", "upstream.0"}, + matched: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + addr, err := net.ResolveUDPAddr("udp", tc.ip) + require.NoError(t, err) + require.NotNil(t, addr) + + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := prog.upstreamFor(ctx, "0", customListener, addr, tc.mac, tc.domain) + + assert.Equal(t, tc.matched, ufr.matched) + assert.Equal(t, tc.upstreams, ufr.upstreams) + }) + } +} + func TestCache(t *testing.T) { cfg := testhelper.SampleConfig(t) prog := &prog{cfg: cfg} + prog.logger.Store(mainLog.Load()) for _, nc := range prog.cfg.Network { for _, cidr := range nc.Cidrs { _, ipNet, err := net.ParseCIDR(cidr) @@ -200,6 +286,52 @@ func TestCache(t *testing.T) { assert.Equal(t, answer2.Rcode, got2.answer.Rcode) } +func TestDNS64CacheLookup(t *testing.T) { + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + cache, err := dnscache.NewLRUCache(16) + require.NoError(t, err) + p.cache = cache + + now := time.Now() + prefix := dns64WellKnownPrefix + req := mkAAAAReq("legacy.example") + upstream := "upstream.0" + empty := new(dns.Msg) + empty.SetReply(req) + synthesized := new(dns.Msg) + synthesized.SetReply(req) + synthesized.Answer = []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{Name: req.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}, AAAA: net.ParseIP("64:ff9b::c000:201")}} + + t.Run("fresh variant hit", func(t *testing.T) { + p.cache.Purge() + p.cache.Add(dns64CacheKey(req, upstream, prefix), dnscache.NewValue(synthesized, now.Add(time.Minute))) + answer, stale, hit, dns64Hit, bypass := p.cachedResponse(req, upstream, prefix, true, now) + if answer == nil || !answerHasAAAA(answer) || stale != nil || !hit || !dns64Hit || bypass { + t.Fatalf("unexpected lookup result: answer=%v stale=%v hit=%v dns64Hit=%v bypass=%v", answer, stale, hit, dns64Hit, bypass) + } + }) + + t.Run("fresh empty normal answer is retained as stale while bypassed", func(t *testing.T) { + p.cache.Purge() + p.cache.Add(dnscache.NewKey(req, upstream), dnscache.NewValue(empty, now.Add(time.Minute))) + answer, stale, hit, dns64Hit, bypass := p.cachedResponse(req, upstream, prefix, true, now) + if answer != nil || stale == nil || hit || dns64Hit || !bypass { + t.Fatalf("unexpected lookup result: answer=%v stale=%v hit=%v dns64Hit=%v bypass=%v", answer, stale, hit, dns64Hit, bypass) + } + }) + + t.Run("expired variant is preferred as stale", func(t *testing.T) { + p.cache.Purge() + p.cache.Add(dns64CacheKey(req, upstream, prefix), dnscache.NewValue(synthesized, now.Add(-time.Minute))) + p.cache.Add(dnscache.NewKey(req, upstream), dnscache.NewValue(empty, now.Add(-time.Minute))) + answer, stale, hit, dns64Hit, bypass := p.cachedResponse(req, upstream, prefix, true, now) + if answer != nil || stale == nil || !answerHasAAAA(stale) || hit || dns64Hit || bypass { + t.Fatalf("unexpected lookup result: answer=%v stale=%v hit=%v dns64Hit=%v bypass=%v", answer, stale, hit, dns64Hit, bypass) + } + }) +} + func Test_ipAndMacFromMsg(t *testing.T) { tests := []struct { name string @@ -405,6 +537,8 @@ func Test_isPrivatePtrLookup(t *testing.T) { {"CGNAT", newDnsMsgPtr("100.66.27.28", t), true}, {"Loopback", newDnsMsgPtr("127.0.0.1", t), true}, {"Link Local Unicast", newDnsMsgPtr("fe80::69f6:e16e:8bdb:433f", t), true}, + // RFC 7335 IPv4 Service Continuity Prefix (464XLAT/DS-Lite CLAT), see #552. + {"464XLAT CLAT host", newDnsMsgPtr("192.0.0.2", t), true}, {"Public IP", newDnsMsgPtr("8.8.8.8", t), false}, } for _, tc := range tests { @@ -452,6 +586,11 @@ func Test_isWanClient(t *testing.T) { {"CGNAT", &net.UDPAddr{IP: net.ParseIP("100.66.27.28")}, false}, {"Loopback", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")}, false}, {"Link Local Unicast", &net.UDPAddr{IP: net.ParseIP("fe80::69f6:e16e:8bdb:433f")}, false}, + // RFC 7335 IPv4 Service Continuity Prefix (464XLAT/DS-Lite CLAT), see #552. + {"464XLAT PLAT side", &net.UDPAddr{IP: net.ParseIP("192.0.0.1")}, false}, + {"464XLAT CLAT host", &net.UDPAddr{IP: net.ParseIP("192.0.0.2")}, false}, + // Outside the /29 but inside 192.0.0.0/24: still WAN (fix is scoped to /29). + {"192.0.0.0/24 outside /29", &net.UDPAddr{IP: net.ParseIP("192.0.0.100")}, true}, {"Public", &net.UDPAddr{IP: net.ParseIP("8.8.8.8")}, true}, } for _, tc := range tests { @@ -464,3 +603,61 @@ func Test_isWanClient(t *testing.T) { }) } } + +func Test_reinitializeOSResolver(t *testing.T) { + p := newTestProg(t) + + err := p.reinitializeOSResolver("Test message") + + // This function should not return an error under normal circumstances + // The actual behavior depends on the OS resolver implementation + assert.NoError(t, err) +} + +func Test_prog_queryFromSelf(t *testing.T) { + p := newTestProg(t) + require.NotPanics(t, func() { + p.queryFromSelf("") + }) + require.NotPanics(t, func() { + p.queryFromSelf("foo") + }) +} + +func Test_sameQuestion(t *testing.T) { + mk := func(name string, qtype uint16) *dns.Msg { + m := new(dns.Msg) + m.SetQuestion(name, qtype) + return m + } + tests := []struct { + name string + req *dns.Msg + answer *dns.Msg + want bool + }{ + {"identical", mk("example.com.", dns.TypeA), mk("example.com.", dns.TypeA), true}, + {"case insensitive", mk("Example.COM.", dns.TypeA), mk("example.com.", dns.TypeA), true}, + {"different name", mk("victim.example.", dns.TypeA), mk("attacker.example.", dns.TypeA), false}, + {"different type", mk("example.com.", dns.TypeA), mk("example.com.", dns.TypeAAAA), false}, + {"nil req", nil, mk("example.com.", dns.TypeA), false}, + {"nil answer", mk("example.com.", dns.TypeA), nil, false}, + {"empty answer question", mk("example.com.", dns.TypeA), new(dns.Msg), false}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + if got := sameQuestion(tc.req, tc.answer); got != tc.want { + t.Errorf("sameQuestion() = %v, want %v", got, tc.want) + } + }) + } +} + +// newTestProg creates a properly initialized *prog for testing. +func newTestProg(t *testing.T) *prog { + p := &prog{cfg: testhelper.SampleConfig(t)} + p.logger.Store(mainLog.Load()) + p.um = newUpstreamMonitor(p.cfg, mainLog.Load()) + return p +} diff --git a/cmd/cli/dns_proxy_zpa_test.go b/cmd/cli/dns_proxy_zpa_test.go new file mode 100644 index 00000000..08989781 --- /dev/null +++ b/cmd/cli/dns_proxy_zpa_test.go @@ -0,0 +1,386 @@ +package cli + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// withZPADNSEchoBypass sets the three gates that guard the built-in Zscaler +// DNS echo bypass: the macOS-only platform gate, Intercept Mode, and the +// hard-mode opt-out. +func withZPADNSEchoBypass(t *testing.T, onDarwin, intercept, hard bool) { + t.Helper() + oldBypass, oldIntercept, oldHard := zpaDNSEchoBypassEnabled, dnsIntercept, hardIntercept + zpaDNSEchoBypassEnabled = onDarwin + dnsIntercept = intercept + hardIntercept = hard + t.Cleanup(func() { + zpaDNSEchoBypassEnabled = oldBypass + dnsIntercept = oldIntercept + hardIntercept = oldHard + }) +} + +// listenerWithRules builds a listener whose policy carries only domain rules. +func listenerWithRules(rules ...ctrld.Rule) *ctrld.ListenerConfig { + return &ctrld.ListenerConfig{ + IP: "127.0.0.1", + Port: 53, + Policy: &ctrld.ListenerPolicyConfig{ + Name: "Test Policy", + Rules: rules, + }, + } +} + +// Test_prog_upstreamFor_ZPADNSEchoBypass pins the routing decision for issue +// 601: under macOS Intercept Mode the exact Zscaler health-check name — and +// nothing else — is routed like a Control D bypass rule, i.e. with an empty +// upstream list that proxy() then resolves through upstream.os. +func Test_prog_upstreamFor_ZPADNSEchoBypass(t *testing.T) { + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + p.logger.Store(mainLog.Load()) + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + + // A listener whose policy routes the echo name to a specific non-OS + // upstream. That is a deliberate route for this exact name, so it wins. + lcWithEchoRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{"upstream.2"}}) + lcWithZscalerWildcardRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{"upstream.2"}}) + + // Empty targets are how a Control D profile's bypass list reaches ctrld + // (cfg.Listener rules built from resolverConfig.Exclude). This is the + // documented pre-fix workaround, so it must keep routing to the OS path AND + // stay classified as the health check, or upgrading a machine that still + // carries the workaround would lose the no-cache behavior. + lcWithEchoBypassRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{}}) + lcWithZscalerWildcardBypassRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{}}) + + // network.0 targets, i.e. what an un-bypassed query from 192.168.0.1 gets. + notBypassed := []string{"upstream.1", "upstream.0"} + const ( + inNetwork0 = "192.168.0.1:0" // matches network.0, so listener.0's policy matches + outOfPolicy = "10.0.0.1:0" // matches no configured network + ) + + tests := []struct { + name string + onDarwin bool + intercept bool + hard bool + lc *ctrld.ListenerConfig + defaultNum string + srcAddr string + domain string + wantUpstreams []string + wantMatched bool + wantZPAEcho bool + }{ + {"exact echo name routes to the OS resolver path", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, nil, true, true}, + {"echo name is matched case-insensitively and with a trailing dot", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "DNSEchoTest.ZScaler.COM.", nil, true, true}, + {"echo name is bypassed on a listener with no policy", true, true, false, p.cfg.Listener["1"], "1", inNetwork0, zpaDNSEchoDomain, nil, false, true}, + + // The bypass must not double as source authorization. serveDNS refuses a + // query on a Restricted listener when matched is false, so an + // unauthorized source must stay unmatched even for this name — while + // still being routed to the OS resolver path if the listener does serve + // it. See TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener. + {"unauthorized source is still unmatched", true, true, false, p.cfg.Listener["0"], "0", outOfPolicy, zpaDNSEchoDomain, nil, false, true}, + + // An explicit domain rule routing the name to a specific upstream + // outranks the built-in route; network/MAC policy targets, which say + // nothing about this name, do not. + {"explicit domain rule for the name wins", true, true, false, lcWithEchoRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false}, + {"explicit wildcard rule covering the name wins", true, true, false, lcWithZscalerWildcardRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false}, + + // An explicit rule that itself selects the OS path (empty targets, i.e. + // the Control D bypass-list shape) keeps its own labels but is still the + // health check, so proxy() keeps it out of the cache. + {"existing exact bypass rule stays classified as the health check", true, true, false, lcWithEchoBypassRule, "0", inNetwork0, zpaDNSEchoDomain, []string{}, true, true}, + {"existing wildcard bypass rule stays classified as the health check", true, true, false, lcWithZscalerWildcardBypassRule, "0", inNetwork0, zpaDNSEchoDomain, []string{}, true, true}, + + // Gating: the bypass exists only where the problem does. + {"not bypassed off macOS", false, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + {"not bypassed outside Intercept Mode", true, false, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + {"not bypassed in hard intercept mode", true, true, true, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + + // Scope: only the one exact name, so no neighbouring name loses filtering. + {"subdomain of the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "foo." + zpaDNSEchoDomain, notBypassed, true, false}, + {"echo name as a prefix of another domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain + ".evil.example", notBypassed, true, false}, + {"name that merely ends with the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "evil" + zpaDNSEchoDomain, notBypassed, true, false}, + {"parent Zscaler domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "zscaler.com", notBypassed, true, false}, + {"ordinary domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "example.com", notBypassed, true, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + withZPADNSEchoBypass(t, tc.onDarwin, tc.intercept, tc.hard) + addr, err := net.ResolveUDPAddr("udp", tc.srcAddr) + require.NoError(t, err) + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + + ufr := p.upstreamFor(ctx, tc.defaultNum, tc.lc, addr, "", tc.domain) + + assert.Equal(t, tc.wantUpstreams, ufr.upstreams) + assert.Equal(t, tc.wantMatched, ufr.matched, "matched is the source-authorization bit") + assert.Equal(t, tc.wantZPAEcho, ufr.zpaDNSEcho) + if tc.wantZPAEcho { + // An empty upstream list is the whole mechanism: it is what + // proxy() turns into upstream.os. Guard the translation step + // that makes that substitution fire. + assert.Empty(t, p.upstreamConfigsFromUpstreamNumbers(ufr.upstreams)) + if tc.lc.Policy == nil || len(tc.lc.Policy.Rules) == 0 { + // Only relabelled when the built-in route is the one that + // made the decision; an operator's own matching rule keeps + // its own labels. + assert.Equal(t, zpaDNSEchoPolicyName, ufr.matchedPolicy) + assert.Equal(t, zpaDNSEchoDomain, ufr.matchedRule) + } + } + }) + } +} + +// TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener pins the serveDNS +// consequence of keeping `matched` out of the bypass: a Restricted listener +// answers only sources that match its policy, and the built-in echo route must +// not become a hole in that for every macOS DNS-intercept instance. +func TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener(t *testing.T) { + withZPADNSEchoBypass(t, true, true, false) + + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + p.logger.Store(mainLog.Load()) + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + lc := p.cfg.Listener["0"] + lc.Restricted = true + + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + for _, tc := range []struct { + name string + srcAddr string + wantRefused bool + }{ + {"source outside every network policy is refused", "10.0.0.1:0", true}, + {"source inside network.0 is served", "192.168.0.1:0", false}, + } { + t.Run(tc.name, func(t *testing.T) { + addr, err := net.ResolveUDPAddr("udp", tc.srcAddr) + require.NoError(t, err) + ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain) + + // The route itself is unconditional; only the authorization differs. + require.True(t, ufr.zpaDNSEcho) + // This is the exact condition serveDNS uses to answer REFUSED. + assert.Equal(t, tc.wantRefused, !ufr.matched && lc.Restricted) + }) + } +} + +// osResolverStub stands in for the OS resolver during a proxy() test. It +// answers NOERROR for anything, and records what it was actually asked, so a +// test can tell an answer that came off the wire from one served locally. +type osResolverStub struct { + addr string + + mu sync.Mutex + queries []string +} + +// startOSResolverStub points osUpstreamConfig at a local DNS server for the +// duration of the test. It uses the legacy (plain UDP) resolver type because +// ResolverTypeOS resolves through process-wide state in the ctrld package that +// a test in this package cannot address. +func startOSResolverStub(t *testing.T) *osResolverStub { + t.Helper() + + pc, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + require.NoError(t, err) + + stub := &osResolverStub{addr: pc.LocalAddr().String()} + srv := &dns.Server{PacketConn: pc, Net: "udp"} + srv.Handler = dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { + stub.mu.Lock() + stub.queries = append(stub.queries, dns.TypeToString[m.Question[0].Qtype]+" "+m.Question[0].Name) + stub.mu.Unlock() + answer := new(dns.Msg) + answer.SetReply(m) + _ = w.WriteMsg(answer) + }) + + started := make(chan struct{}) + srv.NotifyStartedFunc = func() { close(started) } + go func() { _ = srv.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the OS resolver stub to start") + } + + old := osUpstreamConfig + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "OS resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: stub.addr, + Timeout: 2000, + } + t.Cleanup(func() { + osUpstreamConfig = old + _ = srv.Shutdown() + }) + return stub +} + +func (s *osResolverStub) received() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.queries...) +} + +// TestZPADNSEchoBypassReachesOSResolverUncached is the regression test for the +// two ways this route could look correct while still leaving ZPA broken. +// +// 1. It must not be served from ctrld's response cache. Client Connector +// enables Private Access only after it observes the echo answer arrive on +// its own DNS path; an answer we serve locally is silence to it. The nasty +// case is an entry cached while ZPA was disconnected, which would otherwise +// outlive InitializeOsResolver() on reconnect and keep Private Access +// disabled for the entry's whole TTL. +// 2. It must apply to every record type, not just A, matching the Control D +// bypass folder the customer confirmed as a workaround. +// +// It runs against three listener shapes, because the profile bypass list the +// customer is using today arrives as a listener domain rule with empty targets +// (see cfg.Listener rules built from resolverConfig.Exclude). Such a rule +// routes to the OS path on its own, so it would silently keep the cache — a +// machine upgraded while still carrying the workaround must get the fix too. +// +// A pre-existing upstream.os entry with a distinguishable rcode is seeded +// before every query, so a locally served answer is detectable. +func TestZPADNSEchoBypassReachesOSResolverUncached(t *testing.T) { + oldDNS64 := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { return true, false, nil } + t.Cleanup(func() { dns64NetworkClassFn = oldDNS64 }) + + listeners := []struct { + name string + rules []ctrld.Rule + }{ + {"no-rule", nil}, + {"exact-bypass-rule", []ctrld.Rule{{zpaDNSEchoDomain: []string{}}}}, + {"wildcard-bypass-rule", []ctrld.Rule{{"*.zscaler.com": []string{}}}}, + } + qtypes := []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeTXT, dns.TypeHTTPS, dns.TypeMX} + for _, lcCase := range listeners { + for _, qtype := range qtypes { + for _, bypass := range []bool{true, false} { + name := lcCase.name + "/" + dns.TypeToString[qtype] + if bypass { + name += "/bypass" + } else { + name += "/no-bypass" + } + t.Run(name, func(t *testing.T) { + withZPADNSEchoBypass(t, bypass, true, false) + stub := startOSResolverStub(t) + + cfg := testhelper.SampleConfig(t) + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load())} + p.logger.Store(mainLog.Load()) + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + cache, err := dnscache.NewLRUCache(64) + require.NoError(t, err) + p.cache = cache + + lc := p.cfg.Listener["0"] + if lcCase.rules != nil { + lc = listenerWithRules(lcCase.rules...) + } + + msg := newDnsMsgWithHostname(zpaDNSEchoDomain+".", qtype) + // Every candidate upstream has a fresh REFUSED answer + // waiting. The stub answers NOERROR, so the rcode says + // whether we went to the wire or were served from cache. + seedCache := func() { + for _, upstream := range []string{upstreamOS, "upstream.0", "upstream.1", "upstream.2"} { + cached := new(dns.Msg) + cached.SetRcode(msg, dns.RcodeRefused) + p.cache.Add(dnscache.NewKey(msg, upstream), dnscache.NewValue(cached, time.Now().Add(time.Hour))) + } + } + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + require.NoError(t, err) + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + + // Query twice. The second pass is what a Client Connector + // poll after a ZPA reconnect looks like: if the first answer + // had been cached, this one would never reach the resolver. + for pass := 1; pass <= 2; pass++ { + seedCache() + ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain) + assert.Equal(t, bypass, ufr.zpaDNSEcho) + res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: ufr}) + + require.NotNil(t, res) + require.NotNil(t, res.answer) + if bypass { + assert.False(t, res.cached, "pass %d: the echo query must not be answered from cache", pass) + assert.Equal(t, dns.RcodeSuccess, res.answer.Rcode, + "pass %d: %s echo query must be answered by the OS resolver", pass, dns.TypeToString[qtype]) + } else { + assert.True(t, res.cached, "pass %d: an ordinary query still uses the cache", pass) + assert.Equal(t, dns.RcodeRefused, res.answer.Rcode, + "pass %d: %s echo query must not reach the OS resolver when the bypass is off", pass, dns.TypeToString[qtype]) + } + } + + got := stub.received() + want := dns.TypeToString[qtype] + " " + zpaDNSEchoDomain + "." + if bypass { + assert.Equal(t, []string{want, want}, got, + "the OS resolver must see the echo query on every poll") + // Write side: nothing may be stored under the OS + // upstream either, or the next poll would be local. + cached := p.cache.Get(dnscache.NewKey(msg, upstreamOS)) + require.NotNil(t, cached, "seeded entry disappeared") + assert.Equal(t, dns.RcodeRefused, cached.Msg.Rcode, + "the echo answer must not be written to the cache") + } else { + assert.Empty(t, got, "the OS resolver must not be queried when the bypass is off") + } + }) + } + } + } +} diff --git a/cmd/cli/dns_reset_missing_iface_test.go b/cmd/cli/dns_reset_missing_iface_test.go new file mode 100644 index 00000000..316097a6 --- /dev/null +++ b/cmd/cli/dns_reset_missing_iface_test.go @@ -0,0 +1,317 @@ +package cli + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "net/netip" + "os" + "strings" + "testing" + + "tailscale.com/net/netmon" +) + +// Test_logIfaceLookupFailure is a regression test for issue-608: a macOS +// upgrade that succeeded still printed +// +// ERR could not get interface error="interface not found" iface=en5 +// +// because the DNS cleanup after the upgrade logged every interface-lookup +// failure at error level, including the interface it had been bound to simply +// being gone. +// +// This covers the helper's own branching. The conditions it must tell apart +// are exercised through the real code in Test_netInterfaceLookupOutcomes and +// Test_resetDNSForRunningIfaceFailures. +func Test_logIfaceLookupFailure(t *testing.T) { + for _, tc := range []struct { + name string + skipping string + err error + wantLevel string + wantMsg string + }{ + { + name: "the previous interface is gone", + skipping: "DNS restoration", + err: errInterfaceNotFound, + wantLevel: "debug", + wantMsg: "Skipping DNS restoration: previous interface is no longer present", + }, + { + name: "the missing interface is reported through a wrapped error", + skipping: "DNS restoration", + err: fmt.Errorf("looking up %q: %w", "en5", errInterfaceNotFound), + wantLevel: "debug", + wantMsg: "Skipping DNS restoration: previous interface is no longer present", + }, + { + // The caller names the work it skipped, so the helper stays usable + // outside the DNS reset path. + name: "another caller names its own work", + skipping: "interface snapshot", + err: errInterfaceNotFound, + wantLevel: "debug", + wantMsg: "Skipping interface snapshot: previous interface is no longer present", + }, + { + // What patchNetIfaceName reports for an interface that does exist. + name: "the lookup failed for another reason", + skipping: "DNS restoration", + err: fmt.Errorf("patching interface name: %w", os.ErrPermission), + wantLevel: "error", + wantMsg: "Could not get interface", + }, + } { + t.Run(tc.name, func(t *testing.T) { + var buf syncBuffer + logger := newTestJSONLogger(&buf).With().Str("iface", "en5") + + logIfaceLookupFailure(logger, tc.skipping, tc.err) + + entry := soleLogEntry(t, buf.String()) + if got := entry["level"]; got != tc.wantLevel { + t.Errorf("level = %v, want %q (line: %s)", got, tc.wantLevel, strings.TrimSpace(buf.String())) + } + if got := entry["message"]; got != tc.wantMsg { + t.Errorf("message = %v, want %q", got, tc.wantMsg) + } + if got := entry["iface"]; got != "en5" { + t.Errorf("iface = %v, want %q: the diagnostic must name the interface", got, "en5") + } + if tc.wantLevel == "error" { + if got := entry["error"]; got == nil { + t.Error("error field missing: a genuine failure must keep its cause") + } + } else if got, ok := entry["error"]; ok { + t.Errorf("error = %v, want no error field on an expected skip", got) + } + }) + } +} + +// Test_netInterfaceLookupOutcomes pins which lookup failures netInterface may +// report as a missing interface. Only that error is quiet downstream, so an +// enumeration that never ran must not be reported as one: the host may well +// still have the interface. +func Test_netInterfaceLookupOutcomes(t *testing.T) { + errEnumerate := errors.New("route socket closed") + + t.Run("enumeration failed", func(t *testing.T) { + stubForeachInterface(t, func(fn func(netmon.Interface, []netip.Prefix)) error { + return errEnumerate + }) + + iface, err := netInterface("en5") + + if iface != nil { + t.Errorf("iface = %v, want nil", iface) + } + if !errors.Is(err, errEnumerate) { + t.Errorf("err = %v, want it to keep the enumeration cause %v", err, errEnumerate) + } + if errors.Is(err, errInterfaceNotFound) { + t.Error("a failed enumeration must not report the interface as missing: " + + "the caller would then skip DNS restoration on an interface that still exists") + } + }) + + t.Run("enumeration ran and the interface is absent", func(t *testing.T) { + stubForeachInterface(t, func(fn func(netmon.Interface, []netip.Prefix)) error { + fn(netmon.Interface{Interface: &net.Interface{Index: 1, Name: "en0"}}, nil) + return nil + }) + + iface, err := netInterface("en5") + + if iface != nil { + t.Errorf("iface = %v, want nil", iface) + } + if !errors.Is(err, errInterfaceNotFound) { + t.Errorf("err = %v, want errInterfaceNotFound", err) + } + }) +} + +// Test_resetDNSForRunningIfaceFailures drives the real cleanup path and asserts +// how each outcome is reported: the interface being gone is a debug skip, while +// a failure to restore DNS on an interface that does exist stays an error. The +// OS boundaries are stubbed, so no host DNS or NetworkManager state is touched. +func Test_resetDNSForRunningIfaceFailures(t *testing.T) { + errRestore := errors.New("permission denied") + + t.Run("the previous interface is gone", func(t *testing.T) { + buf := captureJSONMainLog(t) + // The real lookup over an enumeration that does not list the + // interface, which is what an upgrade sees after the adapter ctrld + // was bound to went away. + stubForeachInterface(t, func(fn func(netmon.Interface, []netip.Prefix)) error { + fn(netmon.Interface{Interface: &net.Interface{Index: 1, Name: "en0"}}, nil) + return nil + }) + stubDNSResetBoundaries(t, func(iface *net.Interface, nameservers []string) error { + t.Fatalf("setDNS called for a missing interface: %v", nameservers) + return nil + }, func(iface *net.Interface) error { + t.Fatal("resetDNS called for a missing interface") + return nil + }) + + p := &prog{runningIface: "en5"} + p.logger.Store(mainLog.Load()) + if got := p.resetDNSForRunningIface(false, true); got != nil { + t.Errorf("resetDNSForRunningIface returned %v, want nil for a missing interface", got) + } + + entry := findLogEntry(t, buf.String(), "previous interface is no longer present") + if got := entry["level"]; got != "debug" { + t.Errorf("level = %v, want %q: a missing interface must not make a successful upgrade look broken", + got, "debug") + } + if got := entry["iface"]; got != "en5" { + t.Errorf("iface = %v, want %q", got, "en5") + } + }) + + t.Run("restoring the saved static config failed", func(t *testing.T) { + buf := captureJSONMainLog(t) + iface := stubExistingIface(t) + savedStaticNameserversFn = func(*net.Interface) []string { return []string{"192.0.2.1"} } + called := false + stubDNSResetBoundaries(t, func(i *net.Interface, nameservers []string) error { + called = true + return errRestore + }, func(i *net.Interface) error { + t.Fatal("resetDNS called although a saved static config exists") + return nil + }) + + p := &prog{runningIface: iface.Name} + p.logger.Store(mainLog.Load()) + p.resetDNSForRunningIface(false, true) + + if !called { + t.Fatal("setDNS was never called: the saved static config was not restored") + } + entry := findLogEntry(t, buf.String(), "Failed to restore static DNS config") + if got := entry["level"]; got != "error" { + t.Errorf("level = %v, want %q: the interface exists and its DNS was left wrong", + got, "error") + } + if got := entry["error"]; got != errRestore.Error() { + t.Errorf("error = %v, want %q", got, errRestore) + } + }) + + t.Run("resetting to DHCP failed", func(t *testing.T) { + buf := captureJSONMainLog(t) + iface := stubExistingIface(t) + // No saved static config, so the cleanup falls back to DHCP. + called := false + stubDNSResetBoundaries(t, func(i *net.Interface, nameservers []string) error { + t.Fatalf("setDNS called without a saved static config: %v", nameservers) + return nil + }, func(i *net.Interface) error { + called = true + return errRestore + }) + + p := &prog{runningIface: iface.Name} + p.logger.Store(mainLog.Load()) + p.resetDNSForRunningIface(false, true) + + if !called { + t.Fatal("resetDNS was never called: the interface was not reset to DHCP") + } + entry := findLogEntry(t, buf.String(), "Failed to reset DNS to DHCP") + if got := entry["level"]; got != "error" { + t.Errorf("level = %v, want %q: the interface exists and its DNS was left wrong", + got, "error") + } + if got := entry["error"]; got != errRestore.Error() { + t.Errorf("error = %v, want %q", got, errRestore) + } + }) +} + +// stubForeachInterface replaces the host interface enumeration for one test. +func stubForeachInterface(t *testing.T, fn func(func(netmon.Interface, []netip.Prefix)) error) { + t.Helper() + old := foreachInterface + foreachInterface = fn + t.Cleanup(func() { foreachInterface = old }) +} + +// stubDNSResetBoundaries replaces the parts of the DNS reset path that change +// host state: the DNS setters and the NetworkManager restore. +func stubDNSResetBoundaries(t *testing.T, set func(*net.Interface, []string) error, reset func(*net.Interface) error) { + t.Helper() + oldSet, oldReset, oldNM := setIfaceDNSFn, resetIfaceDNSFn, restoreNetworkManagerFn + setIfaceDNSFn, resetIfaceDNSFn, restoreNetworkManagerFn = set, reset, func(*prog) error { return nil } + t.Cleanup(func() { + setIfaceDNSFn, resetIfaceDNSFn, restoreNetworkManagerFn = oldSet, oldReset, oldNM + }) +} + +// stubExistingIface makes the lookup report an interface that is present. +func stubExistingIface(t *testing.T) *net.Interface { + t.Helper() + iface := &net.Interface{Index: 1, Name: "ctrld-test0"} + oldSaved := savedStaticNameserversFn + savedStaticNameserversFn = func(*net.Interface) []string { return nil } + t.Cleanup(func() { savedStaticNameserversFn = oldSaved }) + old := netInterfaceFn + netInterfaceFn = func(name string) (*net.Interface, error) { + if name != iface.Name { + return nil, errInterfaceNotFound + } + return iface, nil + } + t.Cleanup(func() { netInterfaceFn = old }) + return iface +} + +// soleLogEntry decodes the single JSON log line in out. +func soleLogEntry(t *testing.T, out string) map[string]any { + t.Helper() + entries := logEntries(t, out) + if len(entries) != 1 { + t.Fatalf("want exactly one log line, got %d:\n%s", len(entries), out) + } + return entries[0] +} + +// findLogEntry returns the one log line whose message contains want. +func findLogEntry(t *testing.T, out, want string) map[string]any { + t.Helper() + var found []map[string]any + for _, entry := range logEntries(t, out) { + if msg, _ := entry["message"].(string); strings.Contains(msg, want) { + found = append(found, entry) + } + } + if len(found) != 1 { + t.Fatalf("want exactly one log line containing %q, got %d:\n%s", want, len(found), out) + } + return found[0] +} + +// logEntries decodes the JSON log lines in out. +func logEntries(t *testing.T, out string) []map[string]any { + t.Helper() + out = strings.TrimSpace(out) + if out == "" { + t.Fatal("no log output") + } + var entries []map[string]any + for _, line := range strings.Split(out, "\n") { + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("decoding log line %q: %v", line, err) + } + entries = append(entries, entry) + } + return entries +} diff --git a/cmd/cli/dns_target.go b/cmd/cli/dns_target.go new file mode 100644 index 00000000..e1451a0b --- /dev/null +++ b/cmd/cli/dns_target.go @@ -0,0 +1,116 @@ +package cli + +import "net" + +// interceptDNSRdrTarget is the loopback address used as the macOS service +// DNS value when ctrld's listener is NOT reachable at :53 +// directly (non-53 port, e.g. 127.0.0.1:5354 when mDNSResponder holds *:53). +// +// macOS resolvers always send DNS to port 53, so a direct-hit value is +// impossible in that case; delivery must go through the pf rdr rule +// ("rdr on lo0 ... to ! port 53 -> port "). +// The value therefore must be a loopback address DIFFERENT from the listener +// IP so the rdr's "! " matches. Any 127/8 address routes via lo0 +// on macOS. +const interceptDNSRdrTarget = "127.0.0.53" + +// interceptDNSTargetValue returns the nameserver value to set on a DNS-less +// macOS service so the OS emits DNS queries that reach ctrld, respecting the +// configured listener. The listener IP/port derivation mirrors +// buildPFAnchorRulesForTunnels so the value and the pf rules always agree. +// +// - listener on port 53: return the effective listener IP — queries hit the +// listener directly, no pf dependency for this leg. +// - listener on another port: return interceptDNSRdrTarget so the lo0 rdr +// rule fires and rewrites to the real listener address. +func (p *prog) interceptDNSTargetValue() string { + listenerIP := "127.0.0.1" + listenerPort := 53 + // FirstListener panics when no listener is configured; guard like the + // startup paths do. + if p.cfg != nil && len(p.cfg.Listener) > 0 { + if lc := p.cfg.FirstListener(); lc != nil { + if lc.IP != "" && lc.IP != "0.0.0.0" && lc.IP != "::" { + listenerIP = lc.IP + } + if lc.Port != 0 { + listenerPort = lc.Port + } + } + } + if listenerPort == 53 { + return listenerIP + } + if listenerIP == interceptDNSRdrTarget { + // Pathological config: the listener itself sits on the rdr target + // address (with a non-53 port). Pick a different loopback so the + // rdr's "! " still matches. + return "127.0.0.54" + } + return interceptDNSRdrTarget +} + +// hasIPv4DNS reports whether any of the given nameserver strings (bare IPs or +// host:port) is an IPv4 address. Loopback counts: an existing local resolver +// is treated conservatively as an intentional emittable DNS target; ctrld does +// not probe or replace another resolver's ownership. +func hasIPv4DNS(nameservers []string) bool { + for _, s := range nameservers { + host := s + if h, _, err := net.SplitHostPort(s); err == nil { + host = h + } + ip := net.ParseIP(host) + if ip == nil { + continue + } + if ip.To4() != nil { + return true + } + } + return false +} + +// needsInterceptDNSTarget reports whether the OS is left without any usable +// IPv4 DNS target: neither the default-route service's static DNS nor the +// discovered (DHCP/scutil) nameservers contain an IPv4 address. +// +// IPv6-only DNS is not usable under DNS intercept mode on macOS: the pf +// ruleset blocks all outbound IPv6 port-53 traffic (IPv6 interception is not +// supported, see issues #507/#533), and with no IPv4 DNS configured +// mDNSResponder emits no DNS packets at all — leaving pf nothing to +// intercept despite a healthy upstream. Observed in production on IPv6-only +// iPhone tethering with 464XLAT (issue #533). +func needsInterceptDNSTarget(staticDNS, discovered []string) bool { + return !hasIPv4DNS(staticDNS) && !hasIPv4DNS(discovered) +} + +// isInterceptDNSTargetOnly reports whether the given static DNS list is +// exactly the entry ctrld set via ensureInterceptDNSTarget (recorded in +// target), meaning it is safe for ctrld to remove. +func isInterceptDNSTargetOnly(nameservers []string, target string) bool { + return target != "" && len(nameservers) == 1 && nameservers[0] == target +} + +// filterOwnTarget returns nameservers with ctrld's own recorded target +// removed. A previously-set target must never be mistaken for user/network +// IPv4 DNS when judging whether the network still needs one — otherwise the +// second recovery on the same DNS-less network would see "IPv4 DNS present" +// and remove the entry, and the third would re-add it, oscillating on every +// recovery. +func filterOwnTarget(nameservers []string, target string) []string { + if target == "" { + return nameservers + } + out := nameservers[:0:0] + for _, s := range nameservers { + host := s + if h, _, err := net.SplitHostPort(s); err == nil { + host = h + } + if host != target { + out = append(out, s) + } + } + return out +} diff --git a/cmd/cli/dns_target_darwin.go b/cmd/cli/dns_target_darwin.go new file mode 100644 index 00000000..08f706c5 --- /dev/null +++ b/cmd/cli/dns_target_darwin.go @@ -0,0 +1,523 @@ +//go:build darwin + +package cli + +import ( + "context" + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "slices" + "time" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +// interceptDNSTargetStateFile persists which service/value ctrld set, so a +// daemon restart (crash, upgrade, plain restart) does not orphan the entry: +// without it a restarted daemon would not know the entry is ctrld's own and +// could neither remove it on shutdown nor keep its bookkeeping consistent. +const interceptDNSTargetStateFile = ".intercept_dns_target" + +// ipv6OnlyConfirmDelay is how long "no IPv4" must persist before it counts +// as an IPv6-only network rather than DHCPv4 still in progress. A DHCPv4 +// client normally completes in a few seconds; issue #629 allows 45s from +// joining to working DNS, which leaves room for recovery and RFC 7050. +const ipv6OnlyConfirmDelay = 15 * time.Second + +var ( + interceptDNSTargetStatePathFn = interceptDNSTargetStatePath + interceptDefaultRouteInterfaceFn = netmon.DefaultRouteInterface + interceptInterfaceByNameFn = net.InterfaceByName + interceptPatchNetIfaceNameFn = patchNetIfaceName + interceptCurrentStaticDNSFn = readTargetStaticDNS + interceptNativeCLATDefaultServiceFn = nativeCLATDefaultService + interceptNativeIPv6OnlyServiceFn = nativeIPv6OnlyDefaultService + interceptNativeStaticDNSFn = readNativeTargetStaticDNS + interceptSaveStaticDNSSnapshotFn = func(iface *net.Interface, dns []string, owned string) error { + return saveTargetStaticDNSSnapshot(ctrld.SavedStaticDnsSettingsFilePath(iface), dns, owned) + } + interceptSaveCurrentStaticDNSFn = saveCurrentStaticDNS + interceptRemoveSavedDNSFn = func(iface *net.Interface) error { + err := os.Remove(savedStaticDNSPathFn(iface)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + interceptSetDNSFn = setDNS + interceptSavedStaticNameserversFn = ctrld.SavedStaticNameservers + interceptResetDNSIgnoreUnusableIfaceFn = resetDnsIgnoreUnusableInterface + interceptDHCPNameserversForInterfaceFn = readTargetDHCPNameservers + interceptNowFn = time.Now + // interceptIPv6OnlyAfterFn schedules the reconcile that can complete an + // IPv6-only confirmation. The pf watchdog would also get there, but only + // after up to pfWatchdogInterval more. + interceptIPv6OnlyAfterFn = time.AfterFunc +) + +func interceptDNSTargetStatePath() string { + dir, err := userHomeDir() + if err != nil { + return interceptDNSTargetStateFile + } + return filepath.Join(dir, interceptDNSTargetStateFile) +} + +// A retained target record means cleanup was not completed. The broad saved +// static sweep must not bypass the target's ownership check or retry failures +// through an unguarded restore. Unknown state is conservatively left alone. +func interceptTargetAllowsStaticRestore(service string) bool { + data, err := os.ReadFile(interceptDNSTargetStatePathFn()) + if errors.Is(err, os.ErrNotExist) { + return true + } + if err != nil { + return false + } + var st interceptDNSTargetState + if json.Unmarshal(data, &st) != nil || st.Service == "" { + return false + } + return st.Service != service +} + +type interceptDNSTargetState struct { + Service string `json:"service"` + Value string `json:"value"` +} + +// loadInterceptDNSTargetStateLocked hydrates in-memory tracking from the +// state file once (only when memory is empty). Callers must hold +// interceptDNSTargetMu. +func (p *prog) loadInterceptDNSTargetStateLocked() { + if p.interceptDNSTargetService != "" || p.interceptDNSTargetLoaded { + return + } + p.interceptDNSTargetLoaded = true + data, err := os.ReadFile(interceptDNSTargetStatePathFn()) + if err != nil { + return + } + var st interceptDNSTargetState + if err := json.Unmarshal(data, &st); err != nil || st.Service == "" || st.Value == "" { + return + } + p.setInterceptDNSTargetLocked(st.Service, st.Value) + mainLog.Load().Debug().Msgf("intercept DNS target: restored tracking of %s on %q from previous run", st.Value, st.Service) +} + +// persistInterceptDNSTargetStateLocked writes (or clears) the state file to +// match in-memory tracking. Callers must hold interceptDNSTargetMu. +func (p *prog) persistInterceptDNSTargetStateLocked() error { + file := interceptDNSTargetStatePathFn() + if p.interceptDNSTargetService == "" { + if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + data, err := json.Marshal(interceptDNSTargetState{Service: p.interceptDNSTargetService, Value: p.interceptDNSTargetSetValue}) + if err != nil { + return err + } + // Replace atomically; a failed write must not truncate prior ownership. + tmp, err := os.CreateTemp(filepath.Dir(file), ".intercept-target-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + _, writeErr := tmp.Write(data) + err = errors.Join(writeErr, tmp.Close()) + if err != nil { + return err + } + return os.Rename(tmp.Name(), file) +} + +// ensureInterceptDNSTarget provides an emittable DNS target when ownership +// and discovery permit it; externally changed DNS is not permission to reinstall. +// +// Intercept mode deliberately never manages interface DNS: pf redirects DNS +// packets in flight. But pf can only redirect packets macOS actually sends, +// and mDNSResponder emits none when the active network service has no DNS +// configured. IPv6-only networks (e.g. iPhone tethering with 464XLAT) supply +// no IPv4 DNS, and the pf ruleset blocks all outbound IPv6 port 53, so such +// networks otherwise end in a total DNS outage with a healthy upstream +// (issue #533). +// +// Only when the default-route service has no usable IPv4 DNS at all does +// ctrld set a loopback DNS value on it — chosen by interceptDNSTargetValue to +// respect the configured listener: the listener IP directly when it serves +// port 53, else a distinct loopback address so the pf lo0 rdr rule rewrites +// to the listener's real port. The entry is removed when the network regains +// IPv4 DNS and on intercept shutdown. Networks that provide IPv4 DNS are +// never modified. +// +// A failed DHCP option-6 read is not absence on its own. It admits a target +// only with positive native evidence that the network has no IPv4: either +// macOS's own CLAT on the primary service, or (before macOS has started a +// CLAT) no usable IPv4 with the device as the IPv6 primary. "No usable IPv4" +// includes the 169.254.0.0/16 address macOS self-assigns once DHCPv4 gives +// up, about 25 s after joining such a network. Without a +// target there, macOS cannot emit the ipv4only.arpa lookup it needs to start +// the CLAT, so the CLAT evidence alone can never arrive (issue #629). +// +// "No IPv4 yet" also describes a dual-stack network whose router +// advertisement arrived before the DHCPv4 ACK. The IPv6-only evidence must +// therefore hold for ipv6OnlyConfirmDelay on the same service and device +// before it admits a target. A lease or usable IPv4 state arriving in that window +// fails the evidence on the next pass (at the latest the scheduled one) and +// ends the confirmation without a write. +// +// Callers pass a non-nil raw system discovery result to prove discovery ran; +// an empty slice is a valid DNS-less result. The decision itself uses static +// DNS plus DHCP option 6 from the default-route interface, so resolvers on a +// second physical interface cannot suppress the target. Invoked during +// startup, debounced network recovery, and periodic pf watchdog reconciliation. +func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { + if !dnsIntercept || p.dnsInterceptState == nil { + return + } + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.loadInterceptDNSTargetStateLocked() + // A pass that reads the native evidence and does not end waiting on + // IPv6-only confirmation restarts it: continuity means every observation + // in between agreed. Passes that fail before reading evidence know + // nothing new and leave a pending confirmation alone. + keepConfirm := true + defer func() { + if !keepConfirm { + p.resetIPv6OnlyConfirmLocked() + } + }() + var diagnostic *dnsTargetDecisionDiagnostic + if state, ok := p.dnsInterceptState.(*pfState); ok { + diagnostic = &state.targetDiagnostic + } + decision := dnsTargetDecisionContext{ + ownership: p.interceptDNSTargetOwnershipLocked(), + generation: p.recoveryGen.Load(), + transitionID: p.networkAcceptedGen.Load(), + } + // Only completed discovery resolves an outstanding diagnostic. A resolution + // describes the decision and observed ownership, not successful DNS repair. + resolvedReason := "" + defer func() { + if resolvedReason != "" { + diagnostic.resolved(decision, p.interceptDNSTargetOwnershipLocked(), resolvedReason) + } + }() + if systemDiscovery == nil { + diagnostic.failed(decision, "system_discovery", nil) + mainLog.Load().Debug().Msg("intercept DNS target: system DNS discovery was not performed; not changing DNS") + return + } + + drIfaceName, err := interceptDefaultRouteInterfaceFn() + decision.iface = drIfaceName + if err != nil || drIfaceName == "" { + // Mid-transition with no default route; the next recovery decides. + diagnostic.failed(decision, "default_route", err) + return + } + iface, err := interceptInterfaceByNameFn(drIfaceName) + if err != nil || iface == nil { + diagnostic.failed(decision, "interface_lookup", err) + return + } + routeDHCPDNS, dhcpErr := interceptDHCPNameserversForInterfaceFn(drIfaceName) + keepConfirm = false + nativeFallback := false + ctx, cancel := context.WithTimeout(context.Background(), nativeTargetReadBudget) + defer cancel() + var nativeService nativeTargetService + nativeEvidence := "" + nativeServiceFn := interceptNativeCLATDefaultServiceFn + readStatic := interceptCurrentStaticDNSFn + if dhcpErr != nil { + // Resolve the exact primary UUID before any expanded mutation. A + // device can have multiple services; the first listed is not proof. + var nativeErr error + nativeService, nativeErr = interceptNativeCLATDefaultServiceFn(ctx, drIfaceName) + nativeFallback = nativeErr == nil && nativeService.ID != "" && nativeService.Name != "" && nativeService.Device == drIfaceName && !hasIPv4DNS(routeDHCPDNS) + mainLog.Load().Debug().Err(nativeErr).Bool("native_clat_verified", nativeFallback). + Str("interface", drIfaceName).Msg("intercept DNS target: native fallback evidence") + if nativeFallback { + nativeEvidence = "clat" + } else { + // Before macOS starts its CLAT, an IPv6-only network without + // DHCPv4 has neither a lease to read nor a CLAT to prove. Its + // absence of IPv4 is the evidence instead (issue #629). + var v6Err error + nativeService, v6Err = interceptNativeIPv6OnlyServiceFn(ctx, drIfaceName) + nativeFallback = v6Err == nil && nativeService.ID != "" && nativeService.Name != "" && nativeService.Device == drIfaceName && !hasIPv4DNS(routeDHCPDNS) + mainLog.Load().Debug().Err(v6Err).Bool("native_ipv6_only_verified", nativeFallback). + Str("interface", drIfaceName).Msg("intercept DNS target: native fallback evidence") + if nativeFallback { + nativeEvidence = "ipv6_only" + nativeServiceFn = interceptNativeIPv6OnlyServiceFn + } + } + } + if nativeFallback { + iface.Name = nativeService.Name + readStatic = func(iface *net.Interface) ([]string, error) { return interceptNativeStaticDNSFn(ctx, iface) } + } else if _, err := interceptPatchNetIfaceNameFn(iface); err != nil { + diagnostic.failed(decision, "service_lookup", err) + mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not resolve network service for %s", drIfaceName) + return + } + decision.service = iface.Name + + staticDNS, err := readStatic(iface) + if err != nil { + diagnostic.failed(decision, "static_dns", err) + mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not read static DNS for %q", iface.Name) + return + } + // Retain the exact validated snapshot for backup and equality checks, + // including an owned non-loopback value that must never be snapshotted. + snapshot := staticDNS + if nativeFallback { + device, routeErr := interceptDefaultRouteInterfaceFn() + check, serviceErr := nativeServiceFn(ctx, drIfaceName) + if routeErr != nil || serviceErr != nil || device != drIfaceName || check != nativeService { + diagnostic.failed(decision, "native_recheck", errors.Join(routeErr, serviceErr)) + return + } + current, readErr := readStatic(iface) + if readErr != nil || ctx.Err() != nil || !slices.Equal(current, snapshot) { + diagnostic.failed(decision, "static_recheck", errors.Join(readErr, ctx.Err())) + return + } + } + // Tracking is not proof that the target is still installed. An external + // edit (including clearing DNS) is not permission to reinstall it. Retain + // the cleanup record until shutdown, preventing periodic reconciliation + // from reinterpreting the same edit as a new DNS-less admission. + if p.interceptDNSTargetService == iface.Name && !isInterceptDNSTargetOnly(snapshot, p.interceptDNSTargetSetValue) && !hasIPv4DNS(filterOwnTarget(snapshot, p.interceptDNSTargetSetValue)) { + diagnostic.failed(decision, "owned_target_changed", nil) + return + } + // Never count ctrld's own previously-set entry as network-provided DNS, + // or the next recovery on the same DNS-less network would remove it and + // the one after re-add it. + if p.interceptDNSTargetService == iface.Name { + staticDNS = filterOwnTarget(staticDNS, p.interceptDNSTargetSetValue) + } + if hasIPv4DNS(staticDNS) { + resolvedReason = "static_ipv4_dns" + p.removeInterceptDNSTargetLocked("network has usable static IPv4 DNS") + return + } + if nativeFallback && slices.ContainsFunc(staticDNS, func(s string) bool { + ip := net.ParseIP(s) + return ip != nil && ip.To4() == nil && ip.IsLoopback() + }) { + // lo0 can serve another local resolver. The static backup reader + // discards loopback, so expanded admission cannot safely replace it. + resolvedReason = "static_loopback_dns" + return + } + + if dhcpErr != nil && !nativeFallback { + diagnostic.failed(decision, "dhcp_dns", dhcpErr) + mainLog.Load().Debug().Err(dhcpErr).Msgf("intercept DNS target: could not read DHCP DNS for default-route service %q", iface.Name) + return + } + if hasIPv4DNS(routeDHCPDNS) { + resolvedReason = "dhcp_ipv4_dns" + // The default-route service regained DHCP option 6. Remove a target + // previously set on this or another service. + p.removeInterceptDNSTargetLocked("network has usable DHCP IPv4 DNS") + return + } + + resolvedReason = "dns_less_network" + target := p.interceptDNSTargetValue() + if p.interceptDNSTargetService == iface.Name && p.interceptDNSTargetSetValue == target { + return // already set on this service + } + if nativeEvidence == "ipv6_only" && !p.ipv6OnlyConfirmedLocked(nativeService) { + keepConfirm = true + resolvedReason = "ipv6_only_confirming" + return + } + if nativeFallback { + owned := "" + if p.interceptDNSTargetService == iface.Name { + owned = p.interceptDNSTargetSetValue + } + if err := interceptSaveStaticDNSSnapshotFn(iface, snapshot, owned); err != nil { + return // no expanded mutation without a restorable backup + } + } + // Default route moved to a different DNS-less service (or the listener + // config changed): clear the stale entry first. + p.removeInterceptDNSTargetLocked("default route service changed") + if p.interceptDNSTargetService != "" { + return // cleanup failed; never overwrite ownership of the old service + } + + // Preserve any existing (IPv6-only) static entries for later restore. + // saveCurrentStaticDNS filters loopback on write, and + // savedStaticNameservers filters loopback on read, so ctrld's own + // loopback target can never be recorded or restored as user DNS. + if !nativeFallback { + if err := interceptSaveCurrentStaticDNSFn(iface); err != nil { + mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not save static DNS for %q", iface.Name) + } + } + p.setInterceptDNSTargetLocked(iface.Name, target) + if err := p.persistInterceptDNSTargetStateLocked(); err != nil { + p.setInterceptDNSTargetLocked("", "") + resolvedReason = "" + diagnostic.failed(decision, "ownership_persistence", err) + mainLog.Load().Warn().Err(err).Msg("intercept DNS target: not changing DNS without persisted ownership") + return + } + if err := interceptSetDNSFn(iface, []string{target}); err != nil { + // A command error can follow a partial OS mutation. Keep the durable + // cleanup record; the next observation decides what still belongs to us. + resolvedReason = "" + mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not set %s on %q", target, iface.Name) + return + } + journal(mainLog.Load().Warn()).Str("service", iface.Name).Str("target", target). + Bool("native_clat_fallback", nativeFallback). + Str("native_evidence", nativeEvidence). + Str("reason", "dns_less_network"). + Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) +} + +// ipv6OnlyConfirmedLocked reports whether the same IPv6-only observation has +// held for ipv6OnlyConfirmDelay. The first observation starts the clock and +// schedules one follow-up reconcile; a different service or device starts it +// again. Callers must hold interceptDNSTargetMu. +func (p *prog) ipv6OnlyConfirmedLocked(svc nativeTargetService) bool { + now := interceptNowFn() + seen := &p.interceptIPv6OnlySeen + if seen.since.IsZero() || seen.service != svc { + p.resetIPv6OnlyConfirmLocked() + *seen = ipv6OnlyObservation{service: svc, since: now} + p.interceptIPv6OnlyTimer = interceptIPv6OnlyAfterFn(ipv6OnlyConfirmDelay, p.confirmIPv6OnlyTarget) + journal(mainLog.Load().Info()).Str("service", svc.Name).Str("interface", svc.Device). + Int64("confirm_after_ms", ipv6OnlyConfirmDelay.Milliseconds()). + Msgf("intercept DNS target: service %q has no IPv4 yet; confirming no DHCPv4 arrives before setting a target", svc.Name) + return false + } + return now.Sub(seen.since) >= ipv6OnlyConfirmDelay +} + +// confirmIPv6OnlyTarget is the scheduled confirming reconcile. It runs as +// network activity so shutdown waits for or skips it. +func (p *prog) confirmIPv6OnlyTarget() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + if p.dnsInterceptState == nil { + return + } + p.ensureInterceptDNSTarget([]string{}) +} + +// resetIPv6OnlyConfirmLocked drops a pending IPv6-only confirmation and its +// follow-up. Callers must hold interceptDNSTargetMu. +func (p *prog) resetIPv6OnlyConfirmLocked() { + if p.interceptIPv6OnlyTimer != nil { + p.interceptIPv6OnlyTimer.Stop() + p.interceptIPv6OnlyTimer = nil + } + p.interceptIPv6OnlySeen = ipv6OnlyObservation{} +} + +// removeInterceptDNSTarget removes a previously set intercept DNS target, +// restoring the service's saved static DNS (or empty). Safe no-op when no +// target was set. +func (p *prog) removeInterceptDNSTarget(reason string) { + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.loadInterceptDNSTargetStateLocked() + p.resetIPv6OnlyConfirmLocked() + p.removeInterceptDNSTargetLocked(reason) +} + +// removeInterceptDNSTargetLocked is removeInterceptDNSTarget without locking; +// callers must hold interceptDNSTargetMu. +func (p *prog) removeInterceptDNSTargetLocked(reason string) { + svc := p.interceptDNSTargetService + val := p.interceptDNSTargetSetValue + if svc == "" { + return + } + iface := &net.Interface{Name: svc} + // Only remove what ctrld set. If the service's DNS changed externally, + // leave that value alone and discard our stale ownership record. + cur, err := interceptCurrentStaticDNSFn(iface) + if err != nil { + mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not read %q DNS; retaining cleanup state (%s)", svc, reason) + return + } + if !isInterceptDNSTargetOnly(cur, val) { + // The generic stop/uninstall sweep must not replay a stale snapshot + // over the external edit. On failure keep ownership so its guard skips. + if err := interceptRemoveSavedDNSFn(iface); err != nil { + mainLog.Load().Warn().Err(err).Msg("intercept DNS target: could not discard stale backup; retaining cleanup state") + return + } + if !p.clearInterceptDNSTargetStateLocked() { + return + } + // ctrld owns the DNS of the service no longer, and a later outage + // report needs the moment that ownership ended. + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val). + Str("reason", "external_change"). + Msgf("intercept DNS target: %q DNS changed externally; not removing (%s)", svc, reason) + return + } + if saved := interceptSavedStaticNameserversFn(iface); len(saved) > 0 { + if err := interceptSetDNSFn(iface, saved); err != nil { + mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not restore saved DNS on %q; retaining cleanup state", svc) + return + } + } else if err := interceptResetDNSIgnoreUnusableIfaceFn(iface); err != nil { + mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not reset DNS on %q; retaining cleanup state", svc) + return + } + if !p.clearInterceptDNSTargetStateLocked() { + return + } + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val).Str("reason", reason). + Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) +} + +func (p *prog) interceptDNSTargetOwnershipLocked() dnsTargetOwnership { + return dnsTargetOwnership{p.interceptDNSTargetService, p.interceptDNSTargetSetValue} +} + +func (p *prog) clearInterceptDNSTargetStateLocked() bool { + service, value := p.interceptDNSTargetService, p.interceptDNSTargetSetValue + p.setInterceptDNSTargetLocked("", "") + if err := p.persistInterceptDNSTargetStateLocked(); err != nil { + p.setInterceptDNSTargetLocked(service, value) + mainLog.Load().Warn().Err(err).Msg("intercept DNS target: could not clear ownership; retaining cleanup state") + return false + } + return true +} + +// setInterceptDNSTargetLocked stores the service that ctrld owns and the value +// it wrote, and publishes the value for the readers that take no lock. Callers +// must hold interceptDNSTargetMu. +func (p *prog) setInterceptDNSTargetLocked(service, value string) { + p.interceptDNSTargetService = service + p.interceptDNSTargetSetValue = value + p.publishInterceptTarget(value) +} diff --git a/cmd/cli/dns_target_darwin_test.go b/cmd/cli/dns_target_darwin_test.go new file mode 100644 index 00000000..7d4b5765 --- /dev/null +++ b/cmd/cli/dns_target_darwin_test.go @@ -0,0 +1,420 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +type interceptTargetHarness struct { + dns map[string][]string + saved map[string][]string + serviceByDev map[string]string + dhcp []string + dhcpErr error + nativeCLAT bool + nativeErr error + ipv6Only bool + ipv6OnlyErr error + ipv6Calls int + now time.Time + confirms []time.Duration + confirmFns []func() + saveErr error + readErr error + setErr error + resetErr error + setCalls []string + resetCalls []string + statePath string +} + +func newInterceptTargetHarness(t *testing.T) *interceptTargetHarness { + t.Helper() + h := &interceptTargetHarness{ + dns: make(map[string][]string), + saved: make(map[string][]string), + serviceByDev: map[string]string{"en1": "Wi-Fi"}, + statePath: filepath.Join(t.TempDir(), interceptDNSTargetStateFile), + } + + // Master's saved DNS path belongs to ctrld, not the CLI homedir. + // Isolate the production remove/sweep paths without touching host backups. + savedDir := t.TempDir() + origSavedPath := savedStaticDNSPathFn + savedStaticDNSPathFn = func(iface *net.Interface) string { + return filepath.Join(savedDir, ".dns_"+iface.Name) + } + t.Cleanup(func() { savedStaticDNSPathFn = origSavedPath }) + origPath := interceptDNSTargetStatePathFn + origRoute := interceptDefaultRouteInterfaceFn + origIface := interceptInterfaceByNameFn + origPatch := interceptPatchNetIfaceNameFn + origCurrent := interceptCurrentStaticDNSFn + origSave := interceptSaveCurrentStaticDNSFn + origSet := interceptSetDNSFn + origSaved := interceptSavedStaticNameserversFn + origReset := interceptResetDNSIgnoreUnusableIfaceFn + origDHCP := interceptDHCPNameserversForInterfaceFn + origNative := interceptNativeCLATDefaultServiceFn + origIPv6Only := interceptNativeIPv6OnlyServiceFn + origNativeStatic := interceptNativeStaticDNSFn + origSnapshot := interceptSaveStaticDNSSnapshotFn + origIntercept := dnsIntercept + origNow := interceptNowFn + origAfter := interceptIPv6OnlyAfterFn + t.Cleanup(func() { + interceptDNSTargetStatePathFn = origPath + interceptDefaultRouteInterfaceFn = origRoute + interceptInterfaceByNameFn = origIface + interceptPatchNetIfaceNameFn = origPatch + interceptCurrentStaticDNSFn = origCurrent + interceptSaveCurrentStaticDNSFn = origSave + interceptSetDNSFn = origSet + interceptSavedStaticNameserversFn = origSaved + interceptResetDNSIgnoreUnusableIfaceFn = origReset + interceptDHCPNameserversForInterfaceFn = origDHCP + interceptNativeCLATDefaultServiceFn = origNative + interceptNativeIPv6OnlyServiceFn = origIPv6Only + interceptNativeStaticDNSFn = origNativeStatic + interceptSaveStaticDNSSnapshotFn = origSnapshot + dnsIntercept = origIntercept + interceptNowFn = origNow + interceptIPv6OnlyAfterFn = origAfter + }) + h.now = time.Unix(1_700_000_000, 0) + interceptNowFn = func() time.Time { return h.now } + // Keep the scheduled follow-up so a test can run it as the timer would. + interceptIPv6OnlyAfterFn = func(d time.Duration, f func()) *time.Timer { + h.confirms = append(h.confirms, d) + h.confirmFns = append(h.confirmFns, f) + return nil + } + + interceptNativeCLATDefaultServiceFn = func(_ context.Context, device string) (nativeTargetService, error) { + if !h.nativeCLAT || h.nativeErr != nil { + return nativeTargetService{}, h.nativeErr + } + return nativeTargetService{ID: nativeTargetTestServiceID, Name: h.serviceByDev[device], Device: device}, nil + } + interceptNativeIPv6OnlyServiceFn = func(_ context.Context, device string) (nativeTargetService, error) { + h.ipv6Calls++ + if !h.ipv6Only || h.ipv6OnlyErr != nil { + return nativeTargetService{}, h.ipv6OnlyErr + } + return nativeTargetService{ID: nativeTargetTestServiceID, Name: h.serviceByDev[device], Device: device}, nil + } + interceptNativeStaticDNSFn = func(_ context.Context, iface *net.Interface) ([]string, error) { + return interceptCurrentStaticDNSFn(iface) + } + interceptSaveStaticDNSSnapshotFn = func(iface *net.Interface, snapshot []string, owned string) error { + if h.saveErr != nil { + return h.saveErr + } + if !isInterceptDNSTargetOnly(snapshot, owned) { + h.saved[iface.Name] = slices.Clone(filterOwnTarget(snapshot, owned)) + } + return nil + } + dnsIntercept = true + interceptDNSTargetStatePathFn = func() string { return h.statePath } + interceptDefaultRouteInterfaceFn = func() (string, error) { return "en1", nil } + interceptInterfaceByNameFn = func(name string) (*net.Interface, error) { return &net.Interface{Name: name}, nil } + interceptPatchNetIfaceNameFn = func(iface *net.Interface) (bool, error) { + service, ok := h.serviceByDev[iface.Name] + if !ok { + return false, errors.New("unknown network service") + } + iface.Name = service + return true, nil + } + interceptCurrentStaticDNSFn = func(iface *net.Interface) ([]string, error) { + if h.readErr != nil { + return nil, h.readErr + } + return slices.Clone(h.dns[iface.Name]), nil + } + interceptSaveCurrentStaticDNSFn = func(iface *net.Interface) error { + if h.saveErr != nil { + return h.saveErr + } + h.saved[iface.Name] = slices.Clone(h.dns[iface.Name]) + return nil + } + interceptSetDNSFn = func(iface *net.Interface, nameservers []string) error { + h.setCalls = append(h.setCalls, iface.Name) + if h.setErr != nil { + return h.setErr + } + h.dns[iface.Name] = slices.Clone(nameservers) + return nil + } + interceptSavedStaticNameserversFn = func(iface *net.Interface) []string { + return slices.Clone(h.saved[iface.Name]) + } + interceptResetDNSIgnoreUnusableIfaceFn = func(iface *net.Interface) error { + h.resetCalls = append(h.resetCalls, iface.Name) + if h.resetErr != nil { + return h.resetErr + } + h.dns[iface.Name] = nil + return nil + } + interceptDHCPNameserversForInterfaceFn = func(iface string) ([]string, error) { + if iface != "en1" { + return nil, errors.New("DHCP lookup used a non-default interface") + } + return slices.Clone(h.dhcp), h.dhcpErr + } + return h +} + +func newInterceptTargetProg() *prog { + p := &prog{ + cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: "127.0.0.1", Port: 5354}, + }}, + dnsInterceptState: &pfState{}, + } + p.logger.Store(mainLog.Load()) + return p +} + +func persistInterceptTargetForTest(t *testing.T, p *prog, service, value string) { + t.Helper() + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.interceptDNSTargetLoaded = true + p.setInterceptDNSTargetLocked(service, value) + if err := p.persistInterceptDNSTargetStateLocked(); err != nil { + t.Fatal(err) + } +} + +func TestEnsureInterceptDNSTargetRequiresCompletedDiscovery(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget(nil) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("nil system discovery changed service DNS") + } +} + +func TestEnsureInterceptDNSTargetMigratesService(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "iPhone USB", "127.0.0.53") + h.dns["iPhone USB"] = []string{"127.0.0.53"} + h.dns["Wi-Fi"] = nil + + p.ensureInterceptDNSTarget([]string{}) + + if len(h.dns["iPhone USB"]) != 0 { + t.Fatalf("old service DNS = %v, want empty", h.dns["iPhone USB"]) + } + if got := h.dns["Wi-Fi"]; !slices.Equal(got, []string{"127.0.0.53"}) { + t.Fatalf("new service DNS = %v, want [127.0.0.53]", got) + } + if p.interceptDNSTargetService != "Wi-Fi" || p.interceptDNSTargetSetValue != "127.0.0.53" { + t.Fatalf("tracking = %q/%q, want Wi-Fi/127.0.0.53", p.interceptDNSTargetService, p.interceptDNSTargetSetValue) + } +} + +func TestEnsureInterceptDNSTargetUsesDefaultRouteDHCPOnly(t *testing.T) { + t.Run("other interface IPv4 does not suppress target", func(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{"10.10.10.1"}) + if got := h.dns["Wi-Fi"]; !slices.Equal(got, []string{"127.0.0.53"}) { + t.Fatalf("other interface DNS suppressed target: %v", got) + } + }) + + t.Run("returned default route DHCP removes target", func(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + h.dhcp = []string{"192.168.10.1"} + + p.ensureInterceptDNSTarget([]string{"10.10.10.1"}) + + if len(h.dns["Wi-Fi"]) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("returned default-route DHCP DNS did not remove target: dns=%v service=%q", h.dns["Wi-Fi"], p.interceptDNSTargetService) + } + }) +} + +func TestRemoveInterceptDNSTargetRestoresStateFileAfterRestart(t *testing.T) { + h := newInterceptTargetHarness(t) + h.dns["iPhone USB"] = []string{"127.0.0.53"} + if err := os.WriteFile(h.statePath, []byte(`{"service":"iPhone USB","value":"127.0.0.53"}`), 0600); err != nil { + t.Fatal(err) + } + p := newInterceptTargetProg() + + p.removeInterceptDNSTarget("intercept mode inactive") + + if len(h.dns["iPhone USB"]) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("restart cleanup failed: dns=%v service=%q", h.dns["iPhone USB"], p.interceptDNSTargetService) + } + if _, err := os.Stat(h.statePath); !os.IsNotExist(err) { + t.Fatalf("state file still exists after cleanup: %v", err) + } +} + +func TestRemoveInterceptDNSTargetKeepsExternalDNS(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"8.8.8.8"} + + p.removeInterceptDNSTarget("test") + + if !slices.Equal(h.dns["Wi-Fi"], []string{"8.8.8.8"}) || len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatalf("external DNS was changed: dns=%v set=%v reset=%v", h.dns["Wi-Fi"], h.setCalls, h.resetCalls) + } + if p.interceptDNSTargetService != "" { + t.Fatal("external change left stale ownership tracking") + } +} + +func TestRemoveInterceptDNSTargetRetainsStateOnFailure(t *testing.T) { + for _, tc := range []struct { + name string + readErr error + resetErr error + }{ + {"read failure", errors.New("networksetup read failed"), nil}, + {"restore failure", nil, errors.New("networksetup reset failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + h.readErr = tc.readErr + h.resetErr = tc.resetErr + + p.removeInterceptDNSTarget("test") + + if p.interceptDNSTargetService != "Wi-Fi" || p.interceptDNSTargetSetValue != "127.0.0.53" { + t.Fatal("failed cleanup discarded retry state") + } + if _, err := os.Stat(h.statePath); err != nil { + t.Fatalf("failed cleanup removed persisted retry state: %v", err) + } + }) + } +} + +// interceptTargetEvent returns the single event whose message starts with the +// given text. The two target events name the service and the value in their +// message, so no test can match a whole message. +func interceptTargetEvent(t *testing.T, logs *syncBuffer, prefix string) map[string]any { + t.Helper() + var found []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + found = append(found, event) + } + } + if len(found) != 1 { + t.Fatalf("events with prefix %q: got %d, want 1", prefix, len(found)) + } + return found[0] +} + +func TestEnsureInterceptDNSTargetJournalsTheSetTarget(t *testing.T) { + logs := captureDebugMainLog(t) + newInterceptTargetHarness(t) + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + + event := interceptTargetEvent(t, logs, "intercept DNS target: service ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "dns_less_network") +} + +func TestRemoveInterceptDNSTargetJournalsTheRemovedTarget(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + + p.removeInterceptDNSTarget("network has usable static IPv4 DNS") + + event := interceptTargetEvent(t, logs, "intercept DNS target: removed ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "network has usable static IPv4 DNS") +} + +// TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock covers the read +// that a network report uses. The target mutex is held across networksetup +// calls that take seconds, so a report that takes it waits for them. +func TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dns["Wi-Fi"] = nil + + p.ensureInterceptDNSTarget([]string{}) + + p.interceptDNSTargetMu.Lock() + written := p.interceptDNSTargetSetValue + got := p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if written == "" { + t.Fatal("the pass wrote no target") + } + if got != written { + t.Fatalf("target snapshot = %q, want %q", got, written) + } + + p.removeInterceptDNSTarget("test") + + p.interceptDNSTargetMu.Lock() + got = p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if got != "" { + t.Fatalf("target snapshot after the removal = %q, want an empty value", got) + } +} + +// TestRemoveInterceptDNSTargetJournalsAnExternalChange covers the branch that +// drops the ownership record. The user or another program owns the DNS of the +// service from that moment, and the journal held nothing about it. +func TestRemoveInterceptDNSTargetJournalsAnExternalChange(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"9.9.9.9"} + + p.removeInterceptDNSTarget("intercept shutdown") + + event := oneRecoveryEvent(t, logs, `intercept DNS target: "Wi-Fi" DNS changed externally; not removing (intercept shutdown)`) + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "external_change") + if p.interceptTargetSnapshot() != "" { + t.Fatal("the ownership record survived an external change") + } +} diff --git a/cmd/cli/dns_target_dhcp_darwin_test.go b/cmd/cli/dns_target_dhcp_darwin_test.go new file mode 100644 index 00000000..747409a8 --- /dev/null +++ b/cmd/cli/dns_target_dhcp_darwin_test.go @@ -0,0 +1,54 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "reflect" + "slices" + "testing" + "time" +) + +func TestInterceptDNSTargetDHCPDeadline(t *testing.T) { + if reflect.ValueOf(interceptDHCPNameserversForInterfaceFn).Pointer() != reflect.ValueOf(readTargetDHCPNameservers).Pointer() { + t.Fatal("target discovery bypasses bounded DHCP reader") + } + h := newInterceptTargetHarness(t) + h.dns["Wi-Fi"] = []string{"192.0.2.53"} + original := targetDHCPReadFn + t.Cleanup(func() { targetDHCPReadFn = original }) + entered := make(chan struct{}) + targetDHCPReadFn = func(ctx context.Context, device string) ([]string, error) { + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > nativeTargetReadBudget || device != "en1" { + t.Error("missing bounded context or wrong interface") + close(entered) + return nil, errors.New("invalid context") + } + close(entered) + <-ctx.Done() + return nil, ctx.Err() + } + interceptDHCPNameserversForInterfaceFn = readTargetDHCPNameservers + p := newInterceptTargetProg() + finished := make(chan struct{}) + go func() { p.ensureInterceptDNSTarget([]string{}); close(finished) }() + <-entered + cleaned := make(chan struct{}) + go func() { p.removeInterceptDNSTarget("test shutdown"); close(cleaned) }() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Fatal("DHCP lookup did not release reconciliation") + } + select { + case <-cleaned: + case <-time.After(time.Second): + t.Fatal("cleanup remained blocked after DHCP deadline") + } + if !slices.Equal(h.dns["Wi-Fi"], []string{"192.0.2.53"}) || len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("deadline changed existing static DNS") + } +} diff --git a/cmd/cli/dns_target_diagnostic.go b/cmd/cli/dns_target_diagnostic.go new file mode 100644 index 00000000..8841171e --- /dev/null +++ b/cmd/cli/dns_target_diagnostic.go @@ -0,0 +1,126 @@ +package cli + +import ( + "context" + "errors" + "net" + "os" + "os/exec" + + "github.com/Control-D-Inc/ctrld" +) + +// dnsTargetDecisionDiagnostic keeps one outstanding discovery failure, not a +// history keyed by interface or recovery generation. The target mutex serializes +// access. Correlation IDs are sampled before discovery, never after a slow read. +// A successful read (including empty DNS) closes the failure; it does not claim +// that a subsequent DNS write succeeded. repeat_count counts suppressed reads +// across the whole failure episode, including condition changes; first retains +// its original correlation IDs until discovery succeeds. ID changes alone do +// not reopen the condition or turn periodic reconciliation into a log flood. +type dnsTargetDecisionDiagnostic struct { + pending bool + condition dnsTargetFailureCondition + first dnsTargetDecisionContext + repeats uint64 +} + +type dnsTargetOwnership struct { + service string + value string +} + +type dnsTargetDecisionContext struct { + iface string + service string + ownership dnsTargetOwnership + generation uint64 + transitionID uint64 +} + +type dnsTargetFailureCondition struct { + iface string + service string + ownership dnsTargetOwnership + stage string + class string +} + +// dnsTargetReadErrorClass deliberately never serializes command output or error +// text. Classify wrapped causes by type; unknown errors remain read_failed and +// must never be guessed to mean "no DNS". +func dnsTargetReadErrorClass(err error) string { + var networkError net.Error + var exitError *exec.ExitError + switch { + case err == nil: + return "unavailable" + case errors.Is(err, context.DeadlineExceeded), errors.As(err, &networkError) && networkError.Timeout(): + return "timeout" + case errors.Is(err, context.Canceled): + return "canceled" + case errors.Is(err, os.ErrPermission): + return "permission_denied" + case errors.As(err, &exitError): + return "command_failed" + default: + return "read_failed" + } +} + +func dnsTargetDecisionEvent(e *ctrld.LogEvent, c dnsTargetDecisionContext, after dnsTargetOwnership) *ctrld.LogEvent { + action := "unchanged" + if c.ownership != after { + switch { + case after.service == "": + action = "removed" + case c.ownership.service == "": + action = "set" + default: + action = "changed" + } + } + return journal(e).Str("interface", c.iface).Str("service", c.service). + Uint64("recovery_generation", c.generation).Uint64("transition_id", c.transitionID). + Str("action", action). + Bool("ownership_before", c.ownership.service != "").Bool("ownership_after", after.service != ""). + Str("owned_service_before", c.ownership.service).Str("owned_service_after", after.service). + Str("target_before", c.ownership.value).Str("target_after", after.value) +} + +func (d *dnsTargetDecisionDiagnostic) failed(c dnsTargetDecisionContext, stage string, err error) { + if d == nil { + return + } + condition := dnsTargetFailureCondition{c.iface, c.service, c.ownership, stage, dnsTargetReadErrorClass(err)} + if d.pending && d.condition == condition { + // Saturate rather than wrap during an arbitrarily long failure. + if d.repeats != ^uint64(0) { + d.repeats++ + } + return + } + if !d.pending { + d.first = c + } + d.pending = true + d.condition = condition + dnsTargetDecisionEvent(mainLog.Load().Warn(), c, c.ownership). + Str("outcome", "discovery_failed").Str("stage", stage).Str("error_class", condition.class). + Uint64("repeat_count", d.repeats). + Uint64("failure_recovery_generation", d.first.generation).Uint64("failure_transition_id", d.first.transitionID). + Msg("intercept DNS target: decision unavailable; DNS unchanged") +} + +func (d *dnsTargetDecisionDiagnostic) resolved(c dnsTargetDecisionContext, after dnsTargetOwnership, reason string) { + if d == nil || !d.pending { + return + } + dnsTargetDecisionEvent(mainLog.Load().Info(), c, after). + Str("outcome", "decision_available").Str("reason", reason). + Str("stage", d.condition.stage).Str("error_class", d.condition.class). + Uint64("repeat_count", d.repeats). + Uint64("failure_recovery_generation", d.first.generation).Uint64("failure_transition_id", d.first.transitionID). + Msg("intercept DNS target: decision available again") + *d = dnsTargetDecisionDiagnostic{} +} diff --git a/cmd/cli/dns_target_diagnostic_darwin_test.go b/cmd/cli/dns_target_diagnostic_darwin_test.go new file mode 100644 index 00000000..2dd48924 --- /dev/null +++ b/cmd/cli/dns_target_diagnostic_darwin_test.go @@ -0,0 +1,269 @@ +//go:build darwin + +package cli + +import ( + "encoding/json" + "errors" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestEnsureInterceptDNSTargetFailedDecision(t *testing.T) { + for _, owned := range []bool{false, true} { + name := "without target" + if owned { + name = "existing target" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.recoveryGen.Store(3) + p.networkAcceptedGen.Store(7) + if owned { + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + } + before := p.interceptDNSTargetOwnershipLocked() + fileBefore, _ := os.ReadFile(h.statePath) + // This is the first mutating call in the no-target path. Replacing it + // with a fatal stub makes any failure-to-absence regression safe/red. + save := interceptSaveCurrentStaticDNSFn + interceptSaveCurrentStaticDNSFn = func(*net.Interface) error { t.Fatal("discovery failure attempted a DNS backup"); return nil } + // Exercise the production getoption/getpacket wrapper, not a synthetic + // error. PATH contains only our fixture so no host ipconfig can run. + binDir := t.TempDir() + const script = `#!/bin/sh +case "$*" in + 'getoption en1 domain_name_server') code=17 ;; + 'getpacket en1') code=19 ;; + *) exit 99 ;; +esac +printf 'private.example command output\n' +printf 'private.example command stderr\n' >&2 +exit "$code" +` + if err := os.WriteFile(filepath.Join(binDir, "ipconfig"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir) + dhcpStub := interceptDHCPNameserversForInterfaceFn + interceptDHCPNameserversForInterfaceFn = func(iface string) ([]string, error) { + servers, err := ctrld.DHCPNameserversForInterface(iface) + h.dhcpErr = err + return servers, err + } + for range 4 { + p.ensureInterceptDNSTarget([]string{}) + } + wrapped, ok := h.dhcpErr.(interface{ Unwrap() []error }) + if !ok || len(wrapped.Unwrap()) != 2 { + t.Fatalf("DHCP error did not preserve both command causes: %v", h.dhcpErr) + } + for i, cause := range wrapped.Unwrap() { + var exitErr *exec.ExitError + if !errors.As(cause, &exitErr) || exitErr.ExitCode() != []int{17, 19}[i] { + t.Fatalf("command cause %d = %v, want original exit error", i, cause) + } + if !strings.Contains(string(exitErr.Stderr), "private.example") { + t.Fatal("command fixture did not exercise private stderr") + } + } + if got := dnsTargetReadErrorClass(h.dhcpErr); got != "command_failed" { + t.Fatalf("production DHCP error class = %q", got) + } + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed discovery mutated DNS") + } + if p.interceptDNSTargetOwnershipLocked() != before { + t.Fatal("failed discovery changed ownership") + } + fileAfter, _ := os.ReadFile(h.statePath) + if !slices.Equal(fileBefore, fileAfter) { + t.Fatal("failed discovery changed persisted ownership") + } + failed := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, failed, "stage", "dhcp_dns") + wantField(t, failed, "error_class", "command_failed") + wantField(t, failed, "ownership_before", owned) + wantField(t, failed, "ownership_after", owned) + wantField(t, failed, "action", "unchanged") + wantField(t, failed, "service", "Wi-Fi") + wantField(t, failed, "interface", "en1") + wantField(t, failed, "journal", true) + detail := jsonLogEvents(t, logs, `intercept DNS target: could not read DHCP DNS for default-route service "Wi-Fi"`) + if len(detail) != 4 { + t.Fatal("detailed discovery errors disappeared from the debug stream") + } + wantField(t, detail[0], "level", "debug") + wantField(t, detail[0], "error", h.dhcpErr.Error()) + + // Successful empty discovery is different from failure: only now may + // ctrld install the loopback target (or keep an existing one). + interceptSaveCurrentStaticDNSFn = save + interceptDHCPNameserversForInterfaceFn = dhcpStub + h.dhcpErr = nil + p.recoveryGen.Store(4) + p.networkAcceptedGen.Store(8) + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + resolved := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, resolved, "error_class", "command_failed") + for _, event := range []map[string]any{failed, resolved} { + encoded, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{"error reading DHCP", "getoption", "getpacket", "exit status", "private.example"} { + if strings.Contains(string(encoded), raw) { + t.Fatalf("raw command error/output leaked into decision diagnostic: %s", encoded) + } + } + if _, ok := event["error"]; ok { + t.Fatal("decision diagnostic included a raw error field") + } + } + wantField(t, resolved, "reason", "dns_less_network") + wantField(t, resolved, "repeat_count", float64(3)) + wantField(t, resolved, "failure_recovery_generation", float64(3)) + wantField(t, resolved, "failure_transition_id", float64(7)) + wantField(t, resolved, "recovery_generation", float64(4)) + wantField(t, resolved, "transition_id", float64(8)) + wantField(t, resolved, "ownership_before", owned) + wantField(t, resolved, "ownership_after", true) + action := "set" + if owned { + action = "unchanged" + } + wantField(t, resolved, "action", action) + wantSets := 1 + if owned { + wantSets = 0 + } + if len(h.setCalls) != wantSets || len(h.resetCalls) != 0 { + t.Fatalf("set/reset = %v/%v", h.setCalls, h.resetCalls) + } + if !slices.Equal(h.dns["Wi-Fi"], []string{"127.0.0.53"}) { + t.Fatal("empty discovery did not preserve a target") + } + + // DHCP IPv4 arriving later must still remove it exactly once. + h.dhcp = []string{"192.0.2.1"} + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + if len(h.resetCalls) != 1 || p.interceptDNSTargetService != "" { + t.Fatal("later IPv4 DNS did not remove target once") + } + }) + } +} + +func TestEnsureInterceptDNSTargetFailureThenIPv4(t *testing.T) { + for _, static := range []bool{false, true} { + name := "dhcp" + if static { + name = "static" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + h.dhcpErr = errors.New("failed read") + p.ensureInterceptDNSTarget([]string{}) + h.dhcpErr = nil + h.dhcp = []string{"192.0.2.1"} + reason := "dhcp_ipv4_dns" + if static { + h.dns["Wi-Fi"] = []string{"192.0.2.1"} + reason = "static_ipv4_dns" + } + p.ensureInterceptDNSTarget([]string{}) + resolved := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, resolved, "reason", reason) + wantField(t, resolved, "action", "removed") + wantField(t, resolved, "ownership_before", true) + wantField(t, resolved, "ownership_after", false) + if static && !slices.Equal(h.dns["Wi-Fi"], []string{"192.0.2.1"}) { + t.Fatal("external static DNS was changed") + } + }) + } +} + +func TestEnsureInterceptDNSTargetDiscoveryFailureStages(t *testing.T) { + for _, stage := range []string{"system_discovery", "default_route", "interface_lookup", "service_lookup", "static_dns"} { + t.Run(stage, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + input := []string{} + err := errors.New("synthetic read error") + switch stage { + case "system_discovery": + input = nil + case "default_route": + interceptDefaultRouteInterfaceFn = func() (string, error) { return "", err } + case "interface_lookup": + interceptInterfaceByNameFn = func(string) (*net.Interface, error) { return nil, err } + case "service_lookup": + interceptPatchNetIfaceNameFn = func(*net.Interface) (bool, error) { return false, err } + case "static_dns": + h.readErr = err + } + interceptSaveCurrentStaticDNSFn = func(*net.Interface) error { t.Fatal("failed discovery attempted backup"); return nil } + p.ensureInterceptDNSTarget(input) + p.ensureInterceptDNSTarget(input) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed discovery mutated DNS") + } + wantField(t, oneRecoveryEvent(t, logs, dnsTargetFailedMessage), "stage", stage) + }) + } +} + +func TestEnsureInterceptDNSTargetResolutionDoesNotClaimWriteSuccess(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dhcpErr = errors.New("read failed") + p.ensureInterceptDNSTarget([]string{}) + h.dhcpErr = nil + h.setErr = errors.New("synthetic set failure") + p.ensureInterceptDNSTarget([]string{}) + if events := jsonLogEvents(t, logs, dnsTargetResolvedMessage); len(events) != 0 { + t.Fatal("failed target write reported a resolved decision") + } + if len(h.setCalls) != 1 || p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("failed target write did not retain cleanup state for possible partial mutation") + } +} + +func TestEnsureInterceptDNSTargetSamplesCorrelationBeforeRead(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.recoveryGen.Store(2) + p.networkAcceptedGen.Store(5) + interceptDHCPNameserversForInterfaceFn = func(string) ([]string, error) { + p.recoveryGen.Store(3) + p.networkAcceptedGen.Store(6) + return nil, errors.New("read failed") + } + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, event, "recovery_generation", float64(2)) + wantField(t, event, "transition_id", float64(5)) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed discovery mutated DNS") + } +} diff --git a/cmd/cli/dns_target_diagnostic_test.go b/cmd/cli/dns_target_diagnostic_test.go new file mode 100644 index 00000000..9a323dcd --- /dev/null +++ b/cmd/cli/dns_target_diagnostic_test.go @@ -0,0 +1,200 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +const ( + dnsTargetFailedMessage = "intercept DNS target: decision unavailable; DNS unchanged" + dnsTargetResolvedMessage = "intercept DNS target: decision available again" +) + +func TestDNSTargetReadErrorClass(t *testing.T) { + for _, tc := range []struct { + name string + err error + want string + }{ + {"missing", nil, "unavailable"}, + {"deadline", fmt.Errorf("wrapped: %w", context.DeadlineExceeded), "timeout"}, + {"canceled", context.Canceled, "canceled"}, + {"permission", &os.PathError{Op: "read", Path: "private", Err: os.ErrPermission}, "permission_denied"}, + {"command", &exec.ExitError{}, "command_failed"}, + {"wrapped command", fmt.Errorf("wrapped: %w", &exec.ExitError{}), "command_failed"}, + {"untyped error", errors.New("exit status 1"), "read_failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := dnsTargetReadErrorClass(tc.err); got != tc.want { + t.Fatalf("class = %q, want %q", got, tc.want) + } + }) + } +} + +func TestDNSTargetDiagnosticFailureAndResolution(t *testing.T) { + for _, owned := range []bool{false, true} { + t.Run(fmt.Sprintf("owned=%t", owned), func(t *testing.T) { + logs := captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{iface: "en1", service: "Wi-Fi", generation: 3, transitionID: 7} + if owned { + c.ownership = dnsTargetOwnership{"Wi-Fi", "127.0.0.53"} + } + // A healthy pass must not turn the watchdog into a journal heartbeat. + d.resolved(c, c.ownership, "dns_less_network") + failure := errors.New("private.example must not be retained") + d.failed(c, "dhcp_dns", failure) + for range 100 { + c.generation++ + c.transitionID++ + d.failed(c, "dhcp_dns", failure) + } + failed := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, failed, "journal", true) + wantField(t, failed, "level", "warn") + wantField(t, failed, "interface", "en1") + wantField(t, failed, "service", "Wi-Fi") + wantField(t, failed, "stage", "dhcp_dns") + wantField(t, failed, "error_class", "read_failed") + wantField(t, failed, "action", "unchanged") + wantField(t, failed, "ownership_before", owned) + wantField(t, failed, "ownership_after", owned) + wantField(t, failed, "recovery_generation", float64(3)) + wantField(t, failed, "transition_id", float64(7)) + d.resolved(c, c.ownership, "dns_less_network") + d.resolved(c, c.ownership, "dns_less_network") + resolved := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, resolved, "journal", true) + wantField(t, resolved, "level", "info") + wantField(t, resolved, "repeat_count", float64(100)) + wantField(t, resolved, "failure_recovery_generation", float64(3)) + wantField(t, resolved, "failure_transition_id", float64(7)) + wantField(t, resolved, "recovery_generation", float64(103)) + wantField(t, resolved, "transition_id", float64(107)) + if strings.Contains(logs.String(), "private.example") { + t.Fatal("raw read error leaked into diagnostics") + } + if d != (dnsTargetDecisionDiagnostic{}) { + t.Fatal("resolved diagnostic retained failure state") + } + if len(jsonLogEvents(t, logs, "")) != 2 { + t.Fatal("repeated failures or healthy decisions were logged") + } + }) + } +} + +func TestDNSTargetDiagnosticConditionChanges(t *testing.T) { + for _, change := range []string{"interface", "service", "ownership", "stage", "class"} { + t.Run(change, func(t *testing.T) { + logs := captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{iface: "en1", service: "Wi-Fi"} + err := errors.New("read failed") + stage := "dhcp_dns" + d.failed(c, stage, err) + d.failed(c, stage, err) + switch change { + case "interface": + c.iface = "en2" + case "service": + c.service = "USB" + case "ownership": + c.ownership = dnsTargetOwnership{"USB", "127.0.0.53"} + case "stage": + stage = "static_dns" + case "class": + err = context.DeadlineExceeded + } + d.failed(c, stage, err) + d.failed(c, stage, err) + if got := len(jsonLogEvents(t, logs, dnsTargetFailedMessage)); got != 2 { + t.Fatalf("condition-change failures = %d, want 2", got) + } + d.resolved(c, dnsTargetOwnership{}, "dhcp_ipv4_dns") + wantField(t, oneRecoveryEvent(t, logs, dnsTargetResolvedMessage), "repeat_count", float64(2)) + // A new episode must log again and reset the suppressed count. + d.failed(c, stage, err) + if d.repeats != 0 { + t.Fatal("new episode retained old repeat count") + } + }) + } +} + +func TestDNSTargetDiagnosticRepeatCountSaturates(t *testing.T) { + captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{} + d.failed(c, "dhcp_dns", nil) + d.repeats = ^uint64(0) + d.failed(c, "dhcp_dns", nil) + if d.repeats != ^uint64(0) { + t.Fatal("repeat count wrapped") + } +} + +func TestDNSTargetDiagnosticOwnershipActions(t *testing.T) { + owned := dnsTargetOwnership{"Wi-Fi", "127.0.0.53"} + for _, tc := range []struct { + action string + before, after dnsTargetOwnership + }{ + {"set", dnsTargetOwnership{}, owned}, + {"removed", owned, dnsTargetOwnership{}}, + {"changed", owned, dnsTargetOwnership{"USB", "127.0.0.53"}}, + {"unchanged", owned, owned}, + } { + t.Run(tc.action, func(t *testing.T) { + logs := captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{ownership: tc.before} + d.failed(c, "dhcp_dns", os.ErrPermission) + d.resolved(c, tc.after, "dns_less_network") + event := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, event, "action", tc.action) + wantField(t, event, "owned_service_before", tc.before.service) + wantField(t, event, "owned_service_after", tc.after.service) + wantField(t, event, "target_before", tc.before.value) + wantField(t, event, "target_after", tc.after.value) + }) + } +} + +func TestDNSTargetDiagnosticSurvivesDebugTruncation(t *testing.T) { + p, _ := startInternalLogging(t) + useTestSendBudget(t, 1024) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{iface: "en1", service: "Wi-Fi"} + d.failed(c, "dhcp_dns", errors.New("private.example")) + for range 5 { + d.failed(c, "dhcp_dns", errors.New("private.example")) + } + d.resolved(c, dnsTargetOwnership{"Wi-Fi", "127.0.0.53"}, "dns_less_network") + for range 40 { + mainLog.Load().Debug().Msg(strings.Repeat("debug filler", 20)) + } + start := time.Now() + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start) + for _, message := range []string{dnsTargetFailedMessage, dnsTargetResolvedMessage} { + if strings.Contains(string(debugPart), message) { + t.Fatalf("fixture did not truncate %q from debug", message) + } + if strings.Count(string(journalPart), message) != 1 { + t.Fatalf("journal did not retain exactly one %q", message) + } + } + if !strings.Contains(string(journalPart), `"repeat_count":5`) { + t.Fatal("journal lost repeat count") + } + if strings.Contains(string(journalPart), "private.example") { + t.Fatal("raw error leaked into journal") + } +} diff --git a/cmd/cli/dns_target_filter_test.go b/cmd/cli/dns_target_filter_test.go new file mode 100644 index 00000000..88b0a9d9 --- /dev/null +++ b/cmd/cli/dns_target_filter_test.go @@ -0,0 +1,57 @@ +package cli + +import "testing" + +func TestFilterOwnTarget(t *testing.T) { + tests := []struct { + name string + in []string + target string + wantLen int + }{ + // The oscillation guard (MR !997 review): the second recovery on the + // same DNS-less network must not count ctrld's own entry as + // network-provided IPv4 DNS. + {"removes own entry", []string{"127.0.0.1"}, "127.0.0.1", 0}, + {"removes own entry with resolver port", []string{"127.0.0.53:53"}, "127.0.0.53", 0}, + {"keeps user entries", []string{"127.0.0.1", "1.1.1.1"}, "127.0.0.1", 1}, + {"empty target keeps all", []string{"127.0.0.1"}, "", 1}, + {"no match keeps all", []string{"1.1.1.1"}, "127.0.0.53", 1}, + {"nil input", nil, "127.0.0.1", 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := filterOwnTarget(tc.in, tc.target) + if len(got) != tc.wantLen { + t.Errorf("filterOwnTarget(%v, %q) = %v, want len %d", tc.in, tc.target, got, tc.wantLen) + } + for _, s := range got { + if tc.target != "" && s == tc.target { + t.Errorf("filterOwnTarget(%v, %q) retained the target entry", tc.in, tc.target) + } + } + }) + } +} + +// TestFilterOwnTargetStability pins the recovery-cycle contract: on a +// DNS-less network where ctrld already set its target, needsInterceptDNSTarget +// over the filtered list must still report true (entry kept, no oscillation), +// while a genuine user-added IPv4 server must report false (entry removed). +func TestFilterOwnTargetStability(t *testing.T) { + target := "127.0.0.1" + + // Second recovery, same tether: only our own entry present. The OS resolver + // reports it with :53, while networksetup reports the bare address. + static := filterOwnTarget([]string{target}, target) + discovered := filterOwnTarget([]string{target + ":53"}, target) + if !needsInterceptDNSTarget(static, discovered) { + t.Error("second recovery on the same DNS-less network would remove the target (oscillation)") + } + + // User manually added a public server meanwhile: target no longer needed. + static = filterOwnTarget([]string{target, "1.1.1.1"}, target) + if needsInterceptDNSTarget(static, nil) { + t.Error("user-added IPv4 DNS not recognized; target would be kept unnecessarily") + } +} diff --git a/cmd/cli/dns_target_integration_darwin_test.go b/cmd/cli/dns_target_integration_darwin_test.go new file mode 100644 index 00000000..31838500 --- /dev/null +++ b/cmd/cli/dns_target_integration_darwin_test.go @@ -0,0 +1,97 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "net" + "testing" +) + +func TestNativeCLATDecisionJournalIntegration(t *testing.T) { + for _, local := range []bool{false, true} { + name := "target installed" + if local { + name = "local resolver preserved" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dhcpErr = errors.New("no DHCP packet") + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + wantField(t, oneRecoveryEvent(t, logs, dnsTargetFailedMessage), "stage", "dhcp_dns") + h.nativeCLAT = true + reason := "dns_less_network" + action := "set" + if local { + h.dns["Wi-Fi"] = []string{"::1"} + reason = "static_loopback_dns" + action = "unchanged" + } + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, event, "reason", reason) + wantField(t, event, "action", action) + wantField(t, event, "repeat_count", float64(1)) + if p.dnsInterceptState.(*pfState).targetDiagnostic.pending { + t.Fatal("native decision left stale failure") + } + if local && (len(h.setCalls) != 0 || len(h.resetCalls) != 0 || h.dns["Wi-Fi"][0] != "::1") { + t.Fatal("local resolver changed") + } + if !local && len(h.setCalls) != 1 { + t.Fatal("native target not installed exactly once") + } + }) + } +} + +func TestNativeCLATRecheckFailureJournal(t *testing.T) { + for _, static := range []bool{false, true} { + name := "native recheck" + if static { + name = "static recheck" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("no DHCP packet") + h.nativeCLAT = true + p := newInterceptTargetProg() + stage := "native_recheck" + calls := 0 + if static { + stage = "static_recheck" + orig := interceptNativeStaticDNSFn + interceptNativeStaticDNSFn = func(ctx context.Context, i *net.Interface) ([]string, error) { + calls++ + if calls%2 == 0 { + return nil, context.DeadlineExceeded + } + return orig(ctx, i) + } + } else { + orig := interceptNativeCLATDefaultServiceFn + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + calls++ + if calls%2 == 0 { + return nativeTargetService{}, context.DeadlineExceeded + } + return orig(ctx, device) + } + } + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, event, "stage", stage) + wantField(t, event, "error_class", "timeout") + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed recheck changed DNS") + } + }) + } +} diff --git a/cmd/cli/dns_target_ipv6only_darwin_test.go b/cmd/cli/dns_target_ipv6only_darwin_test.go new file mode 100644 index 00000000..24c41cd7 --- /dev/null +++ b/cmd/cli/dns_target_ipv6only_darwin_test.go @@ -0,0 +1,298 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "slices" + "testing" + "time" +) + +// confirmIPv6Only runs the first observation and the confirming pass one +// ipv6OnlyConfirmDelay later, as the scheduled follow-up would. +func confirmIPv6Only(h *interceptTargetHarness, p *prog) { + p.ensureInterceptDNSTarget([]string{}) + h.now = h.now.Add(ipv6OnlyConfirmDelay) + p.ensureInterceptDNSTarget([]string{}) +} + +// Issue #629: an IPv6-only network without DHCPv4 and before macOS starts +// its CLAT has neither a lease to read nor a CLAT to prove. +func TestIPv6OnlyTargetAdmission(t *testing.T) { + for _, tc := range []struct { + name string + ipv6Only, clat bool + ipv6Err error + static, dhcp []string + dhcpErr error + want bool + wantIPv6Checked bool + }{ + {name: "no IPv4 at all", ipv6Only: true, dhcpErr: errors.New("getoption: exit status 1; getpacket: exit status 1"), want: true, wantIPv6Checked: true}, + {name: "IPv6 static DNS kept for restore", ipv6Only: true, static: []string{"2001:db8::53"}, dhcpErr: errors.New("no lease"), want: true, wantIPv6Checked: true}, + {name: "unknown native state", ipv6Err: errors.New("scutil failed"), dhcpErr: errors.New("no lease"), wantIPv6Checked: true}, + {name: "IPv4 present", dhcpErr: errors.New("no lease"), wantIPv6Checked: true}, + {name: "static IPv4 wins", ipv6Only: true, static: []string{"9.9.9.9"}, dhcpErr: errors.New("no lease"), wantIPv6Checked: true}, + {name: "readable DHCP never consults IPv6-only evidence", ipv6Only: true, dhcp: []string{"192.168.1.1"}}, + {name: "CLAT evidence takes precedence", clat: true, ipv6Only: true, dhcpErr: errors.New("no lease"), want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only, h.ipv6OnlyErr, h.nativeCLAT = tc.ipv6Only, tc.ipv6Err, tc.clat + h.dns["Wi-Fi"], h.dhcp, h.dhcpErr = slices.Clone(tc.static), tc.dhcp, tc.dhcpErr + p := newInterceptTargetProg() + confirmIPv6Only(h, p) + if got := len(h.setCalls) > 0; got != tc.want || len(h.resetCalls) != 0 { + t.Fatalf("set=%v reset=%v, want set=%v", h.setCalls, h.resetCalls, tc.want) + } + if (h.ipv6Calls > 0) != tc.wantIPv6Checked { + t.Fatalf("IPv6-only evidence consulted %d times, want consulted=%v", h.ipv6Calls, tc.wantIPv6Checked) + } + if tc.want && !slices.Equal(h.dns["Wi-Fi"], []string{"127.0.0.53"}) { + t.Fatalf("target = %v, want [127.0.0.53]", h.dns["Wi-Fi"]) + } + }) + } +} + +// One IPv6-only observation is not enough: it also matches a dual-stack +// network between its router advertisement and DHCPv4 ACK. The first pass +// only starts the clock and schedules the confirming reconcile. +func TestIPv6OnlyTargetWaitsForConfirmation(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || !slices.Equal(h.confirms, []time.Duration{ipv6OnlyConfirmDelay}) { + t.Fatalf("first observation: set=%v confirms=%v, want no set and one %s follow-up", h.setCalls, h.confirms, ipv6OnlyConfirmDelay) + } + h.now = h.now.Add(ipv6OnlyConfirmDelay - 1) + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.confirms) != 1 { + t.Fatalf("before the delay: set=%v confirms=%v", h.setCalls, h.confirms) + } + h.now = h.now.Add(1) + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.dns["Wi-Fi"], []string{"127.0.0.53"}) { + t.Fatalf("confirmed IPv6-only network not admitted: %v", h.dns["Wi-Fi"]) + } + if p.interceptIPv6OnlySeen != (ipv6OnlyObservation{}) { + t.Fatal("confirmation state kept after admission") + } +} + +// The scheduled follow-up itself admits the target after the delay. It runs +// as network activity, so the WaitGroup must be balanced when it returns. +func TestIPv6OnlyTargetScheduledConfirmationAdmits(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + if len(h.confirmFns) != 1 || len(h.setCalls) != 0 { + t.Fatalf("first observation: confirms=%d set=%v", len(h.confirmFns), h.setCalls) + } + h.now = h.now.Add(ipv6OnlyConfirmDelay) + h.confirmFns[0]() + if !slices.Equal(h.dns["Wi-Fi"], []string{"127.0.0.53"}) { + t.Fatalf("scheduled confirmation did not admit: %v", h.dns["Wi-Fi"]) + } + waitNetworkActivity(t, p) +} + +// After shutdown has closed network activity, the scheduled follow-up does +// nothing and leaves the WaitGroup balanced. With intercept already torn +// down it also does nothing. +func TestIPv6OnlyTargetScheduledConfirmationSkips(t *testing.T) { + for _, tc := range []struct { + name string + stop func(*prog) + }{ + {name: "network activity closed", stop: func(p *prog) { p.closeNetMonitor() }}, + {name: "intercept stopped", stop: func(p *prog) { p.dnsInterceptState = nil }}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + calls := h.ipv6Calls + tc.stop(p) + h.now = h.now.Add(ipv6OnlyConfirmDelay) + h.confirmFns[0]() + if len(h.setCalls) != 0 || h.ipv6Calls != calls { + t.Fatalf("follow-up reconciled after stop: set=%v evidence reads=%d->%d", h.setCalls, calls, h.ipv6Calls) + } + waitNetworkActivity(t, p) + }) + } +} + +// waitNetworkActivity fails if a follow-up left netMonitorWG unbalanced. +func waitNetworkActivity(t *testing.T, p *prog) { + t.Helper() + done := make(chan struct{}) + go func() { p.netMonitorWG.Wait(); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("netMonitorWG not balanced after the follow-up") + } +} + +// Dual-stack network, DHCPv4 late: the RA arrives first, so the first pass +// sees the IPv6-only shape. The lease then arrives before the confirming +// pass, and the service is never modified. +func TestIPv6OnlyTargetDualStackDHCPv4Late(t *testing.T) { + for _, tc := range []struct { + name string + late func(*interceptTargetHarness) + }{ + {name: "lease readable", late: func(h *interceptTargetHarness) { + h.dhcpErr, h.dhcp = nil, []string{"192.168.1.1"} + }}, + {name: "IPv4 state published before the lease is readable", late: func(h *interceptTargetHarness) { + h.ipv6Only = false + }}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + tc.late(h) + h.now = h.now.Add(ipv6OnlyConfirmDelay) + p.ensureInterceptDNSTarget([]string{}) + // Even a later return to the IPv6-only shape restarts the clock. + h.ipv6Only, h.dhcpErr, h.dhcp = true, errors.New("no lease"), nil + p.ensureInterceptDNSTarget([]string{}) + + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 || !slices.Equal(h.dns["Wi-Fi"], []string{"2001:db8::53"}) { + t.Fatalf("dual-stack service modified: set=%v reset=%v dns=%v", h.setCalls, h.resetCalls, h.dns["Wi-Fi"]) + } + if len(h.confirms) != 2 { + t.Fatalf("confirmation not restarted after IPv4 appeared: %v", h.confirms) + } + }) + } +} + +// Failing before the evidence is read (no default route mid-transition) +// learns nothing new, so it neither admits nor restarts a confirmation. +func TestIPv6OnlyTargetConfirmationSurvivesNoDecision(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + route := interceptDefaultRouteInterfaceFn + interceptDefaultRouteInterfaceFn = func() (string, error) { return "", errors.New("no default route") } + p.ensureInterceptDNSTarget([]string{}) + interceptDefaultRouteInterfaceFn = route + h.now = h.now.Add(ipv6OnlyConfirmDelay) + p.ensureInterceptDNSTarget([]string{}) + + if len(h.setCalls) != 1 || len(h.confirms) != 1 { + t.Fatalf("set=%v confirms=%v, want one admission from one confirmation", h.setCalls, h.confirms) + } +} + +// A different service on the same evidence path starts a new confirmation. +func TestIPv6OnlyTargetServiceChangeRestartsConfirmation(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + read := interceptNativeIPv6OnlyServiceFn + interceptNativeIPv6OnlyServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + svc, err := read(ctx, device) + svc.ID = nativeTargetOtherServiceID + return svc, err + } + h.now = h.now.Add(ipv6OnlyConfirmDelay) + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.confirms) != 2 { + t.Fatalf("service change admitted on the old clock: set=%v confirms=%v", h.setCalls, h.confirms) + } +} + +// Shutdown drops a pending confirmation and its follow-up. +func TestIPv6OnlyTargetShutdownDropsConfirmation(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + p.removeInterceptDNSTarget("intercept shutdown") + if p.interceptIPv6OnlySeen != (ipv6OnlyObservation{}) || p.interceptIPv6OnlyTimer != nil { + t.Fatal("shutdown kept a pending IPv6-only confirmation") + } +} + +// The admitted target is removed when the network gains IPv4, and the +// original IPv6 static DNS comes back. +func TestIPv6OnlyTargetLifecycle(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + p := newInterceptTargetProg() + confirmIPv6Only(h, p) + if p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("no target on IPv6-only network") + } + // Repeated reconciliation is idempotent. + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 1 { + t.Fatalf("target churn: %v", h.setCalls) + } + // Unknown native state later never removes or rewrites ownership. + h.ipv6Only, h.ipv6OnlyErr = false, errors.New("store unavailable") + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "Wi-Fi" || len(h.setCalls) != 1 || len(h.resetCalls) != 0 { + t.Fatal("unknown state changed ownership") + } + // Back on a DHCPv4 network: the target goes, the user's IPv6 DNS returns. + h.ipv6OnlyErr, h.dhcpErr, h.dhcp = nil, nil, []string{"192.168.1.1"} + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "" || !slices.Equal(h.dns["Wi-Fi"], []string{"2001:db8::53"}) { + t.Fatalf("target not removed on DHCPv4 return: owner=%q dns=%v", p.interceptDNSTargetService, h.dns["Wi-Fi"]) + } +} + +// The IPv6-only path rechecks with its own reader, not the CLAT reader, and a +// changed tuple during the transaction blocks the write. +func TestIPv6OnlyTargetRecheck(t *testing.T) { + h := newInterceptTargetHarness(t) + h.ipv6Only = true + h.dhcpErr = errors.New("no lease") + read := interceptNativeIPv6OnlyServiceFn + calls := 0 + interceptNativeIPv6OnlyServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + calls++ + svc, err := read(ctx, device) + if calls == 4 { + svc.ID = nativeTargetOtherServiceID + } + return svc, err + } + p := newInterceptTargetProg() + confirmIPv6Only(h, p) + if calls != 4 || len(h.setCalls) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("changed IPv6-only tuple admitted a target: calls=%d set=%v", calls, h.setCalls) + } +} diff --git a/cmd/cli/dns_target_ipv6only_reader_test.go b/cmd/cli/dns_target_ipv6only_reader_test.go new file mode 100644 index 00000000..6621afcb --- /dev/null +++ b/cmd/cli/dns_target_ipv6only_reader_test.go @@ -0,0 +1,173 @@ +package cli + +import ( + "context" + "errors" + "net" + "strings" + "testing" +) + +// Captured on the ctrld-qa macOS 26 guest on an IPv6-only network without +// DHCPv4 or PREF64, before the CLAT started (ctrld#629). +const ( + ipv6OnlyTestGlobal6 = " {\n PrimaryInterface : en1\n PrimaryService : " + nativeTargetTestServiceID + "\n Router : fe80::5054:ff:fe51:3\n}" + ipv6OnlyTestNoKey = " No such key" + // The same guest about 25 s after joining, once DHCPv4 gave up and macOS + // self-assigned an IPv4 link-local address (Global/IPv4 still absent). + ipv6OnlyTestLinkLocal4 = " {\n AdditionalRoutes : {\n 0 : {\n DestinationAddress : 169.254.131.113\n SubnetMask : 255.255.255.255\n }\n }\n Addresses : {\n 0 : 169.254.131.113\n }\n ConfirmedInterfaceName : en1\n InterfaceName : en1\n SubnetMasks : {\n 0 : 255.255.0.0\n }\n}" +) + +func ipv6OnlyTestOutputs() []string { + return []string{ + ipv6OnlyTestNoKey, // State:/Network/Global/IPv4 + ipv6OnlyTestGlobal6, // State:/Network/Global/IPv6 + ipv6OnlyTestNoKey, // State:/Network/Service//IPv4 + " {\n UserDefinedName : Primary Wi-Fi\n}", + nativeTargetTestServiceOrder("Primary Wi-Fi", "en1"), + ipv6OnlyTestNoKey, // recheck Global/IPv4 + ipv6OnlyTestGlobal6, // recheck Global/IPv6 + } +} + +func ipv6OnlyTestReader(t *testing.T, outputs []string, calls *[]string, addrs ...string) nativeTargetReader { + t.Helper() + r := nativeTargetTestReader(t, outputs, calls) + if len(addrs) == 0 { + addrs = []string{"fd66:6:6:0:1876:fb78:b312:107b/64", "fe80::440:5b5a:3e67:23fe/64"} + } + r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { return nativeTargetTestAddrs(addrs...), nil } + return r +} + +func TestNativeIPv6OnlyReader(t *testing.T) { + for _, tc := range []struct { + name string + change func(*nativeTargetReader, []string) + addrs []string + want bool + }{ + {name: "captured IPv6-only state", want: true}, + {name: "global IPv4 present", change: func(_ *nativeTargetReader, out []string) { + out[0] = nativeTargetTestGlobal(nativeTargetTestServiceID, "en1") + }}, + {name: "service IPv4 present", change: func(_ *nativeTargetReader, out []string) { + out[2] = nativeTargetTestState() + }}, + {name: "IPv6 primary is another device", change: func(_ *nativeTargetReader, out []string) { + out[1] = strings.Replace(ipv6OnlyTestGlobal6, "en1", "en9", 1) + }}, + {name: "CLAT address on the interface", addrs: []string{"192.0.0.2/32", "2001:db8::1/64"}}, + {name: "routable IPv4 address on the interface", addrs: []string{"10.0.0.2/24", "2001:db8::1/64"}}, + // After DHCPv4 gives up, macOS self-assigns 169.254.x.x. That is still + // no usable IPv4, so the evidence holds (issue #629 review). + {name: "DHCPv4 gave up: IPv4 link-local on service and interface", want: true, + addrs: []string{"169.254.131.113/16", "fd66:6:6:0:1876:fb78:b312:107b/64", "fe80::440:5b5a:3e67:23fe/64"}, + change: func(_ *nativeTargetReader, out []string) { out[2] = ipv6OnlyTestLinkLocal4 }}, + {name: "link-local service state with a router", change: func(_ *nativeTargetReader, out []string) { + out[2] = strings.Replace(ipv6OnlyTestLinkLocal4, " InterfaceName : en1\n", " InterfaceName : en1\n Router : 169.254.0.1\n", 1) + }}, + {name: "link-local service state plus a routable address", change: func(_ *nativeTargetReader, out []string) { + out[2] = strings.Replace(ipv6OnlyTestLinkLocal4, " 0 : 169.254.131.113\n }\n Confirmed", " 0 : 169.254.131.113\n 1 : 10.0.0.2\n }\n Confirmed", 1) + }}, + {name: "link-local service state on another device", change: func(_ *nativeTargetReader, out []string) { + out[2] = strings.ReplaceAll(ipv6OnlyTestLinkLocal4, ": en1", ": en9") + }}, + {name: "link-local service state with an unknown field", change: func(_ *nativeTargetReader, out []string) { + out[2] = strings.Replace(ipv6OnlyTestLinkLocal4, " InterfaceName : en1\n", " InterfaceName : en1\n Surprise : 1\n", 1) + }}, + {name: "link-local IPv6 only", addrs: []string{"fe80::1/64"}}, + {name: "unknown store response", change: func(_ *nativeTargetReader, out []string) { + out[0] = "scutil: permission denied" + }}, + {name: "unknown IPv6 field", change: func(_ *nativeTargetReader, out []string) { + out[1] = strings.Replace(ipv6OnlyTestGlobal6, "}", " Surprise : 1\n}", 1) + }}, + {name: "IPv4 appears before recheck", change: func(_ *nativeTargetReader, out []string) { + out[5] = nativeTargetTestGlobal(nativeTargetTestServiceID, "en1") + }}, + {name: "primary changes before recheck", change: func(_ *nativeTargetReader, out []string) { + out[6] = strings.Replace(ipv6OnlyTestGlobal6, nativeTargetTestServiceID, nativeTargetOtherServiceID, 1) + }}, + {name: "ambiguous service mapping", change: func(_ *nativeTargetReader, out []string) { + out[4] += "(2) Secondary Wi-Fi\n(Hardware Port: Wi-Fi, Device: en1)\n" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + var calls []string + out := ipv6OnlyTestOutputs() + r := ipv6OnlyTestReader(t, out, &calls, tc.addrs...) + if tc.change != nil { + tc.change(&r, out) + } + svc, err := r.ipv6OnlyService(context.Background(), "en1") + got := err == nil && svc.ID == nativeTargetTestServiceID && svc.Name == "Primary Wi-Fi" && svc.Device == "en1" + if got != tc.want { + t.Fatalf("service=%+v err=%v, want admitted=%v", svc, err, tc.want) + } + if !tc.want && svc != (nativeTargetService{}) { + t.Fatalf("rejected state returned a service: %+v", svc) + } + }) + } +} + +func TestNativeIPv6OnlyReaderHonorsBudget(t *testing.T) { + var calls []string + r := ipv6OnlyTestReader(t, ipv6OnlyTestOutputs(), &calls) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if svc, err := r.ipv6OnlyService(ctx, "en1"); !errors.Is(err, context.Canceled) || svc != (nativeTargetService{}) || len(calls) != 0 { + t.Fatalf("expired budget read state: svc=%+v err=%v calls=%v", svc, err, calls) + } +} + +func TestNativeStoreAbsent(t *testing.T) { + for out, want := range map[string]bool{" No such key\n": true, " {\n}": false} { + if got, err := nativeStoreAbsent(out); err != nil || got != want { + t.Fatalf("%q: got %v %v", out, got, err) + } + } + for _, out := range []string{"", "No such key here", "error"} { + if _, err := nativeStoreAbsent(out); err == nil { + t.Fatalf("%q accepted as known", out) + } + } +} + +func TestParseNativeIPv6GlobalRejectsUnsafe(t *testing.T) { + for _, out := range []string{ + strings.Replace(ipv6OnlyTestGlobal6, "fe80::5054:ff:fe51:3", "192.168.1.1", 1), + strings.Replace(ipv6OnlyTestGlobal6, nativeTargetTestServiceID, "not-a-uuid", 1), + strings.Replace(ipv6OnlyTestGlobal6, "PrimaryInterface : en1", "PrimaryInterface : en1 en2", 1), + strings.Replace(ipv6OnlyTestGlobal6, "}", " PrimaryInterface : en1\n}", 1), + " {\n Router : fe80::1\n}", + } { + if _, _, err := parseNativeIPv6Global(out); err == nil { + t.Fatalf("accepted %q", out) + } + } +} + +func TestNativeIPv6OnlyAddresses(t *testing.T) { + for _, tc := range []struct { + name string + addrs []string + want bool + }{ + {name: "global IPv6", addrs: []string{"2001:db8::1/64"}, want: true}, + {name: "ULA with link-local", addrs: []string{"fd66:6:6::10/64", "fe80::1/64"}, want: true}, + {name: "link-local only", addrs: []string{"fe80::1/64"}}, + {name: "no addresses"}, + {name: "IPv4 address beside global IPv6", addrs: []string{"2001:db8::1/64", "10.0.0.2/24"}}, + {name: "IPv4 link-local beside global IPv6", addrs: []string{"2001:db8::1/64", "169.254.131.113/16"}, want: true}, + {name: "IPv4 link-local only", addrs: []string{"169.254.131.113/16"}}, + {name: "CLAT address beside global IPv6", addrs: []string{"2001:db8::1/64", "192.0.0.2/32"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := nativeIPv6OnlyAddresses(nativeTargetTestAddrs(tc.addrs...)); got != tc.want { + t.Fatalf("got %v, want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/dns_target_native.go b/cmd/cli/dns_target_native.go new file mode 100644 index 00000000..f8d5caa8 --- /dev/null +++ b/cmd/cli/dns_target_native.go @@ -0,0 +1,532 @@ +package cli + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "net" + "os" + "os/exec" + "regexp" + "strings" + "time" +) + +const ( + // One budget covers the entire native evidence transaction, including + // both identity/static rechecks; it is not renewed for each subprocess. + nativeTargetReadBudget = 2 * time.Second + nativeTargetMaxOutput = 64 * 1024 +) + +// ipv6OnlyObservation is the first sighting of IPv6-only evidence that an +// admission must still match after ipv6OnlyConfirmDelay. +type ipv6OnlyObservation struct { + //lint:ignore U1000 used in Darwin code. + service nativeTargetService + //lint:ignore U1000 used in Darwin code. + since time.Time +} + +var errNativeTargetOutputLimit = errors.New("native DNS evidence output exceeds limit") + +type nativeTargetOutput struct{ buf bytes.Buffer } + +func (w *nativeTargetOutput) Write(p []byte) (int, error) { + if len(p) > nativeTargetMaxOutput-w.buf.Len() { + return 0, errNativeTargetOutputLimit + } + return w.buf.Write(p) +} + +func runNativeTargetCommand(ctx context.Context, input, path string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, path, args...) + cmd.Stdin = strings.NewReader(input) + var out nativeTargetOutput + cmd.Stdout = &out + cmd.Stderr = io.Discard + // Also bound waiting for inherited pipes if a child outlives the command. + cmd.WaitDelay = 100 * time.Millisecond + if err := cmd.Run(); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + return out.buf.Bytes(), nil +} + +type nativeTargetService struct { + ID, Name, Device string +} + +type nativeTargetReader struct { + run func(context.Context, string, string, ...string) ([]byte, error) + interfaceByName func(string) (*net.Interface, error) + interfaceAddrs func(*net.Interface) ([]net.Addr, error) +} + +func (r nativeTargetReader) store(ctx context.Context, key string) (map[string]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + out, err := r.run(ctx, "show "+key+"\nquit\n", "/usr/sbin/scutil") + if err != nil { + return nil, err + } + return parseNativeIPv4Store(string(out)) +} + +// The service root contains only the configured name. Reject multiline or +// unfamiliar layouts rather than searching arbitrary DHCP/SSID/packet text. +// The UUID in the key comes exclusively from the validated Global dictionary. +func parseNativeServiceName(out string) (string, error) { + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 3 || lines[0] != " {" || lines[2] != "}" { + return "", errors.New("invalid native service dictionary") + } + name, ok := strings.CutPrefix(strings.TrimLeft(lines[1], " \t"), "UserDefinedName : ") + if !ok || name == "" || strings.ContainsAny(name, "\r\n\x00/{}") { + return "", errors.New("invalid native service name") + } + return name, nil +} + +// serviceName validates the configured service root for id and returns the +// unique enabled networksetup service mapped to device. Both native readers +// share it, so a fix to either check applies to both. +func (r nativeTargetReader) serviceName(ctx context.Context, id, device string) (string, error) { + out, err := r.run(ctx, "show Setup:/Network/Service/"+id+"\nquit\n", "/usr/sbin/scutil") + if err != nil { + return "", err + } + if _, err := parseNativeServiceName(string(out)); err != nil { + return "", err + } + out, err = r.run(ctx, "", "/usr/sbin/networksetup", "-listnetworkserviceorder") + if err != nil { + return "", err + } + return targetNetworkServiceName(string(out), device) +} + +// defaultService returns the actual primary service, not the first service +// networksetup happens to list for the device. Unknown or changing state is +// never absence evidence. The caller must carry and recheck the whole tuple. +func (r nativeTargetReader) defaultService(ctx context.Context, device string) (nativeTargetService, error) { + unknown := nativeTargetService{} + before, err := r.store(ctx, "State:/Network/Global/IPv4") + if err != nil { + return unknown, err + } + id := before["PrimaryService"] + if !nativeServiceID.MatchString(id) || before["PrimaryInterface"] != device { + return unknown, nil + } + state, err := r.store(ctx, "State:/Network/Service/"+id+"/IPv4") + if err != nil { + return unknown, err + } + if state["CLAT46"] != "TRUE" || state["InterfaceName"] != device || state["Router"] != "192.0.0.1" { + return unknown, nil + } + iface, err := r.interfaceByName(device) + if err != nil { + return unknown, err + } + if iface == nil || iface.Name != device || iface.Flags&net.FlagUp == 0 { + return unknown, nil + } + addrs, err := r.interfaceAddrs(iface) + if err != nil || !nativeCLATAddresses(addrs) { + return unknown, err + } + published := strings.Fields(state["AddressList"]) + if len(published) != 1 { + return unknown, nil + } + matched := false + for _, addr := range addrs { + ip, _, parseErr := net.ParseCIDR(addr.String()) + if parseErr == nil && ip.String() == published[0] { + matched = true + } + } + if !matched { + return unknown, nil + } + name, err := r.serviceName(ctx, id, device) + if err != nil { + return unknown, err + } + after, err := r.store(ctx, "State:/Network/Global/IPv4") + if err != nil { + return unknown, err + } + if after["PrimaryService"] != id || after["PrimaryInterface"] != device { + return unknown, errors.New("default service changed during CLAT discovery") + } + return nativeTargetService{ID: id, Name: name, Device: device}, nil +} + +// saveTargetStaticDNSSnapshot saves only the already validated/rechecked +// snapshot. Never re-read via the legacy permissive static DNS reader, and +// never replace the original backup with ctrld's own (possibly non-loopback) +// target when a listener changes after restart. +func saveTargetStaticDNSSnapshot(path string, snapshot []string, owned string) error { + if isInterceptDNSTargetOnly(snapshot, owned) { + // Missing means the original service used automatic DNS. A present + // but unreadable/corrupt backup must not authorize a listener change. + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if len(data) == 0 { + return nil + } + for _, s := range strings.Split(string(data), ",") { + if net.ParseIP(s) == nil || s == owned { + return errors.New("invalid retained static DNS backup") + } + } + return nil + } + var dns []string + for _, s := range snapshot { + ip := net.ParseIP(s) + if ip == nil { + return errors.New("invalid static DNS snapshot") + } + if s == owned { + continue + } + // The restore reader filters loopbacks. Refuse an unrestorable + // user snapshot rather than silently discarding a local resolver. + if ip.IsLoopback() { + return errors.New("non-owned loopback in static DNS snapshot") + } + dns = append(dns, ip.String()) + } + if len(dns) == 0 { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + return os.WriteFile(path, []byte(strings.Join(dns, ",")), 0600) +} + +var nativeServiceID = regexp.MustCompile(`^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$`) + +// parseNativeIPv4Store reads only the two native IPv4 dictionaries, never +// ipconfig getsummary (which embeds unescaped, network-controlled DHCP text). +// Unknown layouts fail closed. These fields are generated by configd from +// interface names, service UUIDs, booleans and binary IP addresses. +func parseNativeIPv4Store(out string) (map[string]string, error) { + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) < 2 || lines[0] != " {" || lines[len(lines)-1] != "}" { + return nil, errors.New("invalid native IPv4 dictionary") + } + values := make(map[string]string) + depth := 1 + arrayKey := "" + for _, line := range lines[1 : len(lines)-1] { + s := strings.TrimSpace(line) + if s == "}" { + depth-- + if depth < 1 { + return nil, errors.New("unbalanced native IPv4 dictionary") + } + continue + } + key, value, ok := strings.Cut(s, " : ") + if !ok { + return nil, errors.New("invalid native IPv4 field") + } + if depth == 1 { + if _, exists := values[key]; exists { + return nil, errors.New("duplicate native IPv4 field") + } + switch key { + case "PrimaryService": + if !nativeServiceID.MatchString(value) { + return nil, errors.New("invalid service ID") + } + case "PrimaryInterface", "InterfaceName", "ConfirmedInterfaceName": + if value == "" || strings.ContainsAny(value, " \t\r\n{}") { + return nil, errors.New("invalid interface") + } + case "CLAT46": + if value != "TRUE" && value != "FALSE" { + return nil, errors.New("invalid CLAT flag") + } + case "Router": + if net.ParseIP(value).To4() == nil { + return nil, errors.New("invalid router") + } + case "Addresses", "SubnetMasks", "DestinationAddresses", "AdditionalRoutes", "ExcludedRoutes": + if value != " {" { + return nil, errors.New("invalid IPv4 array") + } + default: + return nil, errors.New("unknown native IPv4 field") + } + values[key] = value + arrayKey = key + } else if depth == 2 && arrayKey == "Addresses" { + ip := net.ParseIP(value) + if ip.To4() == nil { + return nil, errors.New("invalid native IPv4 address") + } + values["AddressList"] += ip.String() + " " + } + if strings.HasSuffix(value, "{") { + depth++ + } + } + if depth != 1 { + return nil, errors.New("unbalanced native IPv4 dictionary") + } + return values, nil +} + +// A successful networksetup exit alone is not absence evidence: reject all +// text except IP literals and its exact no-static-DNS response. +func parseTargetStaticDNS(out, service string) ([]string, error) { + out = strings.TrimSpace(out) + if out == "There aren't any DNS Servers set on "+service+"." { + return nil, nil + } + if out == "" { + return nil, errors.New("empty static DNS response") + } + var dns []string + scanner := bufio.NewScanner(strings.NewReader(out)) + for scanner.Scan() { + ip := net.ParseIP(strings.TrimSpace(scanner.Text())) + if ip == nil { + return nil, errors.New("invalid static DNS response") + } + dns = append(dns, ip.String()) + } + return dns, scanner.Err() +} + +func nativeCLATAddresses(addrs []net.Addr) bool { + clat, ipv6 := false, false + for _, addr := range addrs { + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return false + } + if v4 := ip.To4(); v4 != nil { + // Apple allocates one of these four CLAT addresses, not only 192.0.0.2. + if v4[0] != 192 || v4[1] != 0 || v4[2] != 0 || v4[3] < 2 || v4[3] > 5 { + return false + } + clat = true + } else if ip.IsGlobalUnicast() { + // ULA is legitimate IPv6 connectivity. Addresses alone never + // authorize a target: configd must independently publish CLAT. + ipv6 = true + } + } + return clat && ipv6 +} + +// nativeStoreAbsent reports whether scutil printed its exact missing-key +// response. Absence is evidence only in that exact form; a dictionary means +// present, and anything else is unknown. +func nativeStoreAbsent(out string) (bool, error) { + s := strings.TrimSpace(out) + if s == "No such key" { + return true, nil + } + if strings.HasPrefix(s, " {") { + return false, nil + } + return false, errors.New("unknown native store response") +} + +func (r nativeTargetReader) absent(ctx context.Context, key string) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + out, err := r.run(ctx, "show "+key+"\nquit\n", "/usr/sbin/scutil") + if err != nil { + return false, err + } + return nativeStoreAbsent(string(out)) +} + +// parseNativeIPv6Global reads only State:/Network/Global/IPv6. Unknown, +// nested or duplicate fields fail closed, like the IPv4 reader. +func parseNativeIPv6Global(out string) (id, device string, err error) { + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) < 2 || lines[0] != " {" || lines[len(lines)-1] != "}" { + return "", "", errors.New("invalid native IPv6 dictionary") + } + seen := make(map[string]bool) + for _, line := range lines[1 : len(lines)-1] { + key, value, ok := strings.Cut(strings.TrimSpace(line), " : ") + if !ok || strings.HasSuffix(value, "{") { + return "", "", errors.New("invalid native IPv6 field") + } + if seen[key] { + return "", "", errors.New("duplicate native IPv6 field") + } + seen[key] = true + switch key { + case "PrimaryService": + if !nativeServiceID.MatchString(value) { + return "", "", errors.New("invalid service ID") + } + id = value + case "PrimaryInterface": + if value == "" || strings.ContainsAny(value, " \t\r\n{}") { + return "", "", errors.New("invalid interface") + } + device = value + case "Router": + if ip := net.ParseIP(value); ip == nil || ip.To4() != nil { + return "", "", errors.New("invalid IPv6 router") + } + default: + return "", "", errors.New("unknown native IPv6 field") + } + } + if id == "" || device == "" { + return "", "", errors.New("incomplete native IPv6 dictionary") + } + return id, device, nil +} + +// nativeIPv6OnlyAddresses requires routable IPv6 and no usable IPv4. An +// IPv4 link-local address (169.254.0.0/16) is allowed: macOS self-assigns one +// when DHCPv4 gives up, and it reaches no DNS server. +func nativeIPv6OnlyAddresses(addrs []net.Addr) bool { + ipv6 := false + for _, addr := range addrs { + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return false + } + if ip.To4() != nil { + if !ip.IsLinkLocalUnicast() { + return false + } + continue + } + if ip.IsGlobalUnicast() { + ipv6 = true + } + } + return ipv6 +} + +// ipv6OnlyService returns the primary service of a network that gives the +// device no usable IPv4: configd publishes no global IPv4 state and at most a +// link-local per-service IPv4 state, the IPv6 primary is this device, and the +// interface holds only IPv6 plus at most an IPv4 link-local address. Before +// macOS starts its CLAT, such a network has no DHCPv4 lease to read and no +// CLAT to prove, yet it is just as DNS-less (issue #629). Unknown or changing +// state is never absence evidence; the caller rechecks the tuple. +func (r nativeTargetReader) ipv6OnlyService(ctx context.Context, device string) (nativeTargetService, error) { + unknown := nativeTargetService{} + if gone, err := r.absent(ctx, "State:/Network/Global/IPv4"); err != nil || !gone { + return unknown, err + } + out, err := r.run(ctx, "show State:/Network/Global/IPv6\nquit\n", "/usr/sbin/scutil") + if err != nil { + return unknown, err + } + id, primary, err := parseNativeIPv6Global(string(out)) + if err != nil || primary != device { + return unknown, err + } + if none, err := r.serviceNoUsableIPv4(ctx, id, device); err != nil || !none { + return unknown, err + } + iface, err := r.interfaceByName(device) + if err != nil { + return unknown, err + } + if iface == nil || iface.Name != device || iface.Flags&net.FlagUp == 0 { + return unknown, nil + } + addrs, err := r.interfaceAddrs(iface) + if err != nil || !nativeIPv6OnlyAddresses(addrs) { + return unknown, err + } + name, err := r.serviceName(ctx, id, device) + if err != nil { + return unknown, err + } + if gone, err := r.absent(ctx, "State:/Network/Global/IPv4"); err != nil || !gone { + return unknown, errors.Join(err, errors.New("IPv4 appeared during IPv6-only discovery")) + } + out, err = r.run(ctx, "show State:/Network/Global/IPv6\nquit\n", "/usr/sbin/scutil") + if err != nil { + return unknown, err + } + if afterID, afterDevice, err := parseNativeIPv6Global(string(out)); err != nil || afterID != id || afterDevice != device { + return unknown, errors.Join(err, errors.New("default service changed during IPv6-only discovery")) + } + return nativeTargetService{ID: id, Name: name, Device: device}, nil +} + +// serviceNoUsableIPv4 reports whether the per-service IPv4 state is absent or +// only the link-local address macOS self-assigns after DHCPv4 gives up. On +// the ctrld-qa macOS 26 guest (IPv6-only, no DHCPv4, CLAT blocked) that state +// appears about 25 s after joining: one 169.254.0.0/16 address, no Router, no +// CLAT46, and Global/IPv4 still absent. A DHCP, static or CLAT configuration +// always fails this check, so it never hides a network that provides IPv4. +func (r nativeTargetReader) serviceNoUsableIPv4(ctx context.Context, id, device string) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + out, err := r.run(ctx, "show State:/Network/Service/"+id+"/IPv4\nquit\n", "/usr/sbin/scutil") + if err != nil { + return false, err + } + if gone, err := nativeStoreAbsent(string(out)); err != nil || gone { + return gone, err + } + values, err := parseNativeIPv4Store(string(out)) + if err != nil { + return false, err + } + return nativeIPv4LinkLocalOnly(values, device), nil +} + +// nativeIPv4LinkLocalOnly accepts only an IPv4 link-local configuration of +// device: no router, no CLAT, not a primary, and every address in +// 169.254.0.0/16. +func nativeIPv4LinkLocalOnly(values map[string]string, device string) bool { + for _, key := range []string{"Router", "CLAT46", "PrimaryService", "PrimaryInterface", "DestinationAddresses", "ExcludedRoutes"} { + if _, ok := values[key]; ok { + return false + } + } + if values["InterfaceName"] != device { + return false + } + if c, ok := values["ConfirmedInterfaceName"]; ok && c != device { + return false + } + addrs := strings.Fields(values["AddressList"]) + if len(addrs) == 0 { + return false + } + for _, a := range addrs { + if ip := net.ParseIP(a); ip == nil || ip.To4() == nil || !ip.IsLinkLocalUnicast() { + return false + } + } + return true +} diff --git a/cmd/cli/dns_target_native_capture_test.go b/cmd/cli/dns_target_native_capture_test.go new file mode 100644 index 00000000..76051062 --- /dev/null +++ b/cmd/cli/dns_target_native_capture_test.go @@ -0,0 +1,87 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" +) + +// Captured macOS 15.7.7 CLAT dictionaries; UUID and interface IPv6 addresses +// are sanitized. Keep the native field shapes, including nested routes. +func TestCapturedCLATNativeReader(t *testing.T) { + const id = "11111111-2222-3333-4444-555555555555" + const global = ` { + PrimaryInterface : en0 + PrimaryService : 11111111-2222-3333-4444-555555555555 + Router : 192.0.0.1 +}` + const state = ` { + AdditionalRoutes : { + 0 : { + DestinationAddress : 192.0.0.2 + SubnetMask : 255.255.255.255 + } + } + Addresses : { + 0 : 192.0.0.2 + } + CLAT46 : TRUE + InterfaceName : en0 + Router : 192.0.0.1 +}` + const setup = ` { + UserDefinedName : Wi-Fi +}` + iface := &net.Interface{Name: "en0", Flags: net.FlagUp | net.FlagRunning} + var addrs []net.Addr + for _, s := range []string{"192.0.0.2/32", "fe80::1/64", "2001:db8::1/64", "2001:db8::2/64", "2001:db8::3/64"} { + ip, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + n.IP = ip + addrs = append(addrs, n) + } + calls := 0 + reader := nativeTargetReader{ + run: func(_ context.Context, input, path string, args ...string) ([]byte, error) { + calls++ + if path == "/usr/sbin/networksetup" && len(args) == 1 && args[0] == "-listnetworkserviceorder" && input == "" { + return []byte(nativeTargetTestServiceOrder("Wi-Fi", "en0")), nil + } + if path != "/usr/sbin/scutil" || len(args) != 0 { + t.Fatal("unexpected command") + } + switch strings.TrimSpace(input) { + case "show State:/Network/Global/IPv4\nquit": + return []byte(global), nil + case "show State:/Network/Service/" + id + "/IPv4\nquit": + return []byte(state), nil + case "show Setup:/Network/Service/" + id + "\nquit": + return []byte(setup), nil + default: + t.Fatal("unexpected key", input) + return nil, nil + } + }, + interfaceByName: func(name string) (*net.Interface, error) { + if name != "en0" { + t.Fatal(name) + } + return iface, nil + }, + interfaceAddrs: func(got *net.Interface) ([]net.Addr, error) { + if got != iface { + t.Fatal("wrong interface") + } + return addrs, nil + }, + } + got, err := reader.defaultService(context.Background(), "en0") + want := nativeTargetService{ID: id, Name: "Wi-Fi", Device: "en0"} + if err != nil || got != want || calls != 5 { + t.Fatalf("got=%+v err=%v reads=%d", got, err, calls) + } + t.Log("Captured native dictionaries accepted; primary Wi-Fi service on en0 selected") +} diff --git a/cmd/cli/dns_target_native_darwin.go b/cmd/cli/dns_target_native_darwin.go new file mode 100644 index 00000000..12869cb5 --- /dev/null +++ b/cmd/cli/dns_target_native_darwin.go @@ -0,0 +1,49 @@ +//go:build darwin + +package cli + +import ( + "context" + "net" + + ctrld "github.com/Control-D-Inc/ctrld" +) + +var targetDHCPReadFn = ctrld.DHCPNameserversForInterfaceContext + +// Bound the DHCP prerequisite separately from the native evidence transaction. +func readTargetDHCPNameservers(device string) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), nativeTargetReadBudget) + defer cancel() + return targetDHCPReadFn(ctx, device) +} + +func readTargetStaticDNS(iface *net.Interface) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), nativeTargetReadBudget) + defer cancel() + return readNativeTargetStaticDNS(ctx, iface) +} + +func readNativeTargetStaticDNS(ctx context.Context, iface *net.Interface) ([]string, error) { + out, err := runNativeTargetCommand(ctx, "", "/usr/sbin/networksetup", "-getdnsservers", iface.Name) + if err != nil { + return nil, err + } + return parseTargetStaticDNS(string(out), iface.Name) +} + +func nativeCLATDefaultService(ctx context.Context, device string) (nativeTargetService, error) { + return newNativeTargetReader().defaultService(ctx, device) +} + +func nativeIPv6OnlyDefaultService(ctx context.Context, device string) (nativeTargetService, error) { + return newNativeTargetReader().ipv6OnlyService(ctx, device) +} + +func newNativeTargetReader() nativeTargetReader { + return nativeTargetReader{ + run: runNativeTargetCommand, + interfaceByName: net.InterfaceByName, + interfaceAddrs: (*net.Interface).Addrs, + } +} diff --git a/cmd/cli/dns_target_native_darwin_test.go b/cmd/cli/dns_target_native_darwin_test.go new file mode 100644 index 00000000..dca408ca --- /dev/null +++ b/cmd/cli/dns_target_native_darwin_test.go @@ -0,0 +1,382 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestNativeCLATTargetPreservesNonOwnedLoopback(t *testing.T) { + for _, static := range [][]string{{"::1"}, {"::1", "2001:db8::53"}, {"127.0.0.1"}, {"127.0.0.1", "::1"}} { + t.Run(strings.Join(static, ","), func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = slices.Clone(static) + backup := filepath.Join(t.TempDir(), ".dns_Wi-Fi") + interceptSaveStaticDNSSnapshotFn = func(_ *net.Interface, snapshot []string, owned string) error { + return saveTargetStaticDNSSnapshot(backup, snapshot, owned) + } + p := newInterceptTargetProg() + p.cfg.Listener["0"].Port = 53 + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("local resolver overwritten: set=%v reset=%v owner=%q", h.setCalls, h.resetCalls, p.interceptDNSTargetService) + } + p.removeInterceptDNSTarget("shutdown") + if !slices.Equal(h.dns["Wi-Fi"], static) { + t.Fatalf("lost user DNS: got %v, want %v", h.dns["Wi-Fi"], static) + } + if _, err := os.Stat(backup); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("backup unexpectedly changed: %v", err) + } + }) + } +} + +func TestNativeCLATTargetRejectsLoopbackBeforeSnapshot(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = []string{"::1", "2001:db8::53"} + h.dns["Old service"] = []string{"127.0.0.53"} + interceptSaveStaticDNSSnapshotFn = func(*net.Interface, []string, string) error { + t.Fatal("non-owned IPv6 loopback admitted to snapshot") + return nil + } + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "Old service" { + t.Fatal("rejected admission changed old ownership") + } +} + +func TestNativeCLATTargetLoopbackKeepsIPv4CleanupFirst(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = []string{"::1", "127.0.0.1"} + h.dns["Old service"] = []string{"127.0.0.53"} + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.resetCalls, []string{"Old service"}) || len(h.setCalls) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("IPv4 cleanup skipped: reset=%v set=%v owner=%q", h.resetCalls, h.setCalls, p.interceptDNSTargetService) + } + if !slices.Equal(h.dns["Wi-Fi"], []string{"::1", "127.0.0.1"}) { + t.Fatal("external local resolvers changed") + } +} + +func TestNativeCLATTargetSnapshotRejectsNonOwnedLoopback(t *testing.T) { + for _, snapshot := range [][]string{{"::1"}, {"2001:db8::53", "::1"}, {"127.0.0.1"}, {"127.0.0.53", "::1"}} { + t.Run(strings.Join(snapshot, ","), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + const original = "2001:db8::54" + if err := os.WriteFile(path, []byte(original), 0600); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, snapshot, "127.0.0.53"); err == nil { + t.Fatal("accepted unrestorable loopback snapshot") + } + if got, err := os.ReadFile(path); err != nil || string(got) != original { + t.Fatalf("failed snapshot changed backup: %q %v", got, err) + } + }) + } + for _, owned := range []string{"127.0.0.53", "::1", "10.0.0.53"} { + t.Run("owned "+owned, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + const original = "2001:db8::54" + if err := os.WriteFile(path, []byte(original), 0600); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, []string{owned}, owned); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != original { + t.Fatalf("owned target replaced backup: %q %v", got, err) + } + }) + } +} + +func TestNativeCLATTargetFallback(t *testing.T) { + for _, tc := range []struct { + name string + clat bool + nativeErr, readErr, saveErr error + static, dhcp []string + want bool + }{ + {name: "CLAT", clat: true, want: true}, + {name: "unknown"}, + {name: "native read error", clat: true, nativeErr: errors.New("read failed")}, + {name: "static read error", clat: true, readErr: errors.New("read failed")}, + {name: "backup error", clat: true, saveErr: errors.New("disk full")}, + {name: "static IPv4", clat: true, static: []string{"9.9.9.9"}}, + {name: "partial DHCP IPv4", clat: true, dhcp: []string{"1.1.1.1"}}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("getoption and getpacket exit 1") + h.nativeCLAT, h.nativeErr, h.readErr, h.saveErr = tc.clat, tc.nativeErr, tc.readErr, tc.saveErr + h.dns["Wi-Fi"], h.dhcp = tc.static, tc.dhcp + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if (len(h.setCalls) > 0) != tc.want || len(h.resetCalls) != 0 { + t.Fatalf("set=%v reset=%v", h.setCalls, h.resetCalls) + } + }) + } +} + +func TestNativeCLATTargetUnknownPreservesStaticIPv4Cleanup(t *testing.T) { + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("DHCP read failed") + h.nativeErr = errors.New("native state unavailable") + h.dns["Wi-Fi"] = []string{"192.0.2.53"} + h.dns["Old service"] = []string{"127.0.0.53"} + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "" || !slices.Equal(h.resetCalls, []string{"Old service"}) || len(h.setCalls) != 0 || !slices.Equal(h.dns["Wi-Fi"], []string{"192.0.2.53"}) { + t.Fatalf("existing static IPv4 cleanup changed: owner=%q reset=%v set=%v", p.interceptDNSTargetService, h.resetCalls, h.setCalls) + } +} + +func TestNativeCLATTargetTransitionAndReadFailure(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4 packet") + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("no target") + } + h.nativeErr = errors.New("store unavailable") + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "Wi-Fi" || len(h.setCalls) != 1 || len(h.resetCalls) != 0 { + t.Fatal("unknown changed ownership") + } + h.nativeErr = nil + h.dhcpErr = nil + h.dhcp = []string{"192.168.1.1"} + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "" || !slices.Equal(h.dns["Wi-Fi"], []string{"2001:db8::53"}) { + t.Fatal("dual stack did not restore IPv6 static DNS") + } +} + +// Use the real evidence reader: the native primary UUID is still checked, +// and the command-facing service is unambiguously mapped to its device. +func TestNativeCLATTargetExactPrimaryService(t *testing.T) { + for _, static := range [][]string{{"9.9.9.9"}, {"2001:db8::53"}} { + t.Run(strings.Join(static, ","), func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("no packet") + h.serviceByDev["en1"] = "First Wi-Fi" + h.dns["First Wi-Fi"] = nil + h.dns["Primary Wi-Fi"] = slices.Clone(static) + var calls []string + out := nativeTargetTestOutputs() + out[3] = nativeTargetTestServiceOrder("First Wi-Fi", "en9") + "(2) Primary Wi-Fi\n(Hardware Port: Wi-Fi, Device: en1)\n" + outputs := append(out, out...) + r := nativeTargetTestReader(t, outputs, &calls) + interceptNativeCLATDefaultServiceFn = r.defaultService + interceptPatchNetIfaceNameFn = func(*net.Interface) (bool, error) { + t.Fatal("used first-match mapping for native fallback") + return false, nil + } + var reads []string + read := interceptNativeStaticDNSFn + interceptNativeStaticDNSFn = func(ctx context.Context, iface *net.Interface) ([]string, error) { + reads = append(reads, iface.Name) + return read(ctx, iface) + } + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(reads, []string{"Primary Wi-Fi", "Primary Wi-Fi"}) || len(calls) != 10 { + t.Fatalf("reads=%v calls=%v", reads, calls) + } + if len(h.dns["First Wi-Fi"]) != 0 { + t.Fatal("inactive service mutated") + } + if hasIPv4DNS(static) { + if !slices.Equal(h.dns["Primary Wi-Fi"], static) || len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("primary static IPv4 overwritten") + } + } else { + if !slices.Equal(h.setCalls, []string{"Primary Wi-Fi"}) || p.interceptDNSTargetService != "Primary Wi-Fi" { + t.Fatalf("set=%v ownership=%q", h.setCalls, p.interceptDNSTargetService) + } + p.removeInterceptDNSTarget("test shutdown") + if !slices.Equal(h.dns["Primary Wi-Fi"], static) { + t.Fatal("original static IPv6 not restored") + } + } + }) + } +} + +func TestNativeCLATTargetIdentityRecheck(t *testing.T) { + for _, field := range []string{"UUID", "name", "device", "read error", "static DNS"} { + t.Run(field, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + h.dns["Old service"] = []string{"127.0.0.53"} + read := interceptNativeCLATDefaultServiceFn + calls := 0 + var shared context.Context + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + calls++ + if calls == 1 { + shared = ctx + } else if shared != ctx { + t.Fatal("budget renewed on recheck") + } + svc, err := read(ctx, device) + if calls == 2 { + switch field { + case "UUID": + svc.ID = nativeTargetOtherServiceID + case "name": + svc.Name = "Other Wi-Fi" + case "device": + svc.Device = "en9" + case "read error": + err = errors.New("unknown") + case "static DNS": + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + } + } + return svc, err + } + p.ensureInterceptDNSTarget([]string{}) + if calls != 2 || len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "Old service" { + t.Fatalf("uncertain recheck mutated state: calls=%d set=%v reset=%v owner=%q", calls, h.setCalls, h.resetCalls, p.interceptDNSTargetService) + } + }) + } +} + +func TestNativeCLATTargetRestartListenerChangePreservesBackup(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + backup := filepath.Join(t.TempDir(), ".dns_Wi-Fi") + original := "2001:db8::53" + if err := os.WriteFile(backup, []byte(original), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(h.statePath, []byte(`{"service":"Wi-Fi","value":"10.0.0.53"}`), 0600); err != nil { + t.Fatal(err) + } + h.dns["Wi-Fi"] = []string{"10.0.0.53"} + var snapshots [][]string + interceptSaveStaticDNSSnapshotFn = func(_ *net.Interface, snapshot []string, owned string) error { + snapshots = append(snapshots, slices.Clone(snapshot)) + return saveTargetStaticDNSSnapshot(backup, snapshot, owned) + } + interceptSaveCurrentStaticDNSFn = func(*net.Interface) error { t.Fatal("legacy unvalidated snapshot read"); return nil } + interceptSavedStaticNameserversFn = func(*net.Interface) []string { + data, err := os.ReadFile(backup) + if err != nil { + t.Fatal(err) + } + return strings.Split(string(data), ",") + } + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP, p.cfg.Listener["0"].Port = "10.0.0.54", 53 + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.dns["Wi-Fi"], []string{"10.0.0.54"}) || p.interceptDNSTargetSetValue != "10.0.0.54" || len(snapshots) != 1 { + t.Fatalf("listener migration failed: dns=%v target=%q snapshots=%v", h.dns["Wi-Fi"], p.interceptDNSTargetSetValue, snapshots) + } + if got, err := os.ReadFile(backup); err != nil || string(got) != original { + t.Fatalf("original backup lost: %q %v", got, err) + } + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 2 { + t.Fatalf("target churn after reconciliation: %v", h.setCalls) + } + // A second restart must recover new ownership and still restore the original. + restarted := newInterceptTargetProg() + restarted.removeInterceptDNSTarget("shutdown") + if !slices.Equal(h.dns["Wi-Fi"], []string{original}) || restarted.interceptDNSTargetService != "" { + t.Fatalf("shutdown restored owned target: %v", h.dns["Wi-Fi"]) + } + if _, err := os.Stat(h.statePath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("ownership not removed", err) + } +} + +func TestNativeCLATTargetSnapshotFailureRetainsOwnership(t *testing.T) { + for _, mode := range []string{"write", "remove", "cleanup"} { + t.Run(mode, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "10.0.0.53") + h.dns["Old service"] = []string{"10.0.0.53"} + path := t.TempDir() + if err := os.WriteFile(filepath.Join(path, "block"), nil, 0600); err != nil { + t.Fatal(err) + } + if mode == "write" { + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + } + if mode == "cleanup" { + path = filepath.Join(t.TempDir(), "backup") + h.resetErr = errors.New("cleanup failed") + } + interceptSaveStaticDNSSnapshotFn = func(_ *net.Interface, snapshot []string, owned string) error { + return saveTargetStaticDNSSnapshot(path, snapshot, owned) + } + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "Old service" || len(h.setCalls) != 0 { + t.Fatal("failed backup/cleanup lost old ownership") + } + if mode != "cleanup" && len(h.resetCalls) != 0 { + t.Fatal("backup failure attempted cleanup") + } + if got := h.dns["Old service"]; !slices.Equal(got, []string{"10.0.0.53"}) { + t.Fatal("old service changed", got) + } + if _, err := os.Stat(h.statePath); err != nil { + t.Fatal("ownership file lost", err) + } + }) + } +} + +func TestNativeCLATTargetRouteChange(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + calls := 0 + interceptDefaultRouteInterfaceFn = func() (string, error) { + calls++ + if calls > 1 { + return "en9", nil + } + return "en1", nil + } + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("route change mutated DNS") + } +} diff --git a/cmd/cli/dns_target_native_reader_test.go b/cmd/cli/dns_target_native_reader_test.go new file mode 100644 index 00000000..1acf7a17 --- /dev/null +++ b/cmd/cli/dns_target_native_reader_test.go @@ -0,0 +1,317 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +const nativeTargetTestServiceID = "22222222-2222-2222-2222-222222222222" +const nativeTargetOtherServiceID = "11111111-1111-1111-1111-111111111111" + +func nativeTargetTestGlobal(id, device string) string { + return fmt.Sprintf(" {\n PrimaryInterface : %s\n PrimaryService : %s\n Router : 192.0.0.1\n}", device, id) +} + +func nativeTargetTestState() string { + return " {\n CLAT46 : TRUE\n InterfaceName : en1\n Router : 192.0.0.1\n Addresses : {\n 0 : 192.0.0.2\n }\n}" +} + +func nativeTargetTestAddrs(ips ...string) []net.Addr { + var out []net.Addr + for _, s := range ips { + ip, n, err := net.ParseCIDR(s) + if err != nil { + panic(err) + } + n.IP = ip + out = append(out, n) + } + return out +} + +func nativeTargetTestReader(t *testing.T, outputs []string, calls *[]string) nativeTargetReader { + t.Helper() + return nativeTargetReader{ + run: func(ctx context.Context, input, path string, args ...string) ([]byte, error) { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if path == "/usr/sbin/networksetup" && slices.Equal(args, []string{"-listnetworkserviceorder"}) && input == "" { + input = "networksetup -listnetworkserviceorder" + } else if path != "/usr/sbin/scutil" || len(args) != 0 { + t.Fatalf("unexpected command %q %v", path, args) + } + *calls = append(*calls, input) + if len(*calls) > len(outputs) { + t.Fatalf("unexpected read: %q", input) + } + return []byte(outputs[len(*calls)-1]), nil + }, + interfaceByName: func(name string) (*net.Interface, error) { return &net.Interface{Name: name, Flags: net.FlagUp}, nil }, + interfaceAddrs: func(*net.Interface) ([]net.Addr, error) { + return nativeTargetTestAddrs("192.0.0.2/32", "2001:db8::1/64"), nil + }, + } +} + +func nativeTargetTestOutputs() []string { + global := nativeTargetTestGlobal(nativeTargetTestServiceID, "en1") + return []string{global, nativeTargetTestState(), " {\n UserDefinedName : Primary Wi-Fi\n}", nativeTargetTestServiceOrder("Primary Wi-Fi", "en1"), global} +} + +func TestNativeTargetReaderIdentityAndEvidence(t *testing.T) { + for _, tc := range []struct { + name string + change func(*nativeTargetReader, []string) + want bool + }{ + {name: "primary UUID exact service", want: true}, + {name: "matching native name does not disambiguate another service on same device", want: false, change: func(_ *nativeTargetReader, out []string) { + out[3] += "(2) Secondary Wi-Fi\n(Hardware Port: Wi-Fi, Device: en1)\n" + }}, + {name: "ULA with native proof", want: true, change: func(r *nativeTargetReader, _ []string) { + r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { + return nativeTargetTestAddrs("192.0.0.2/32", "fd12:3456::1/64"), nil + } + }}, + {name: "ULA without native proof", change: func(_ *nativeTargetReader, out []string) { out[1] = strings.Replace(out[1], "TRUE", "FALSE", 1) }}, + {name: "down interface", change: func(r *nativeTargetReader, _ []string) { + r.interfaceByName = func(name string) (*net.Interface, error) { return &net.Interface{Name: name}, nil } + }}, + {name: "missing interface", change: func(r *nativeTargetReader, _ []string) { + r.interfaceByName = func(string) (*net.Interface, error) { return nil, errors.New("gone") } + }}, + {name: "address error", change: func(r *nativeTargetReader, _ []string) { + r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { return nil, errors.New("gone") } + }}, + {name: "ordinary IPv4", change: func(r *nativeTargetReader, _ []string) { + r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { + return nativeTargetTestAddrs("192.0.0.2/32", "2001:db8::1/64", "10.0.0.2/24"), nil + } + }}, + {name: "published address mismatch", change: func(_ *nativeTargetReader, out []string) { + out[1] = strings.Replace(out[1], "192.0.0.2", "192.0.0.5", 1) + }}, + {name: "wrong device", change: func(_ *nativeTargetReader, out []string) { + out[0] = nativeTargetTestGlobal(nativeTargetTestServiceID, "en9") + }}, + {name: "wrong service device", change: func(_ *nativeTargetReader, out []string) { out[1] = strings.Replace(out[1], "en1", "en9", 1) }}, + {name: "invalid UUID", change: func(_ *nativeTargetReader, out []string) { out[0] = nativeTargetTestGlobal("untrusted", "en1") }}, + {name: "primary changed", change: func(_ *nativeTargetReader, out []string) { + out[4] = nativeTargetTestGlobal(nativeTargetOtherServiceID, "en1") + }}, + {name: "missing name", change: func(_ *nativeTargetReader, out []string) { out[2] = "No such key" }}, + {name: "DHCP embedded dictionary", change: func(_ *nativeTargetReader, out []string) { out[1] = "DHCPPacket : " + out[1] }}, + {name: "SSID embedded name", change: func(_ *nativeTargetReader, out []string) { + out[2] = " {\n SSID : network\n UserDefinedName : Primary Wi-Fi\n}" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + out := nativeTargetTestOutputs() + var calls []string + r := nativeTargetTestReader(t, out, &calls) + if tc.change != nil { + tc.change(&r, out) + } + got, err := r.defaultService(context.Background(), "en1") + if (got.ID != "" && err == nil) != tc.want { + t.Fatalf("service=%+v err=%v calls=%v", got, err, calls) + } + if tc.want { + want := nativeTargetService{ID: nativeTargetTestServiceID, Name: "Primary Wi-Fi", Device: "en1"} + if got != want { + t.Fatalf("got %+v want %+v", got, want) + } + wantCalls := []string{"show State:/Network/Global/IPv4\nquit\n", "show State:/Network/Service/" + nativeTargetTestServiceID + "/IPv4\nquit\n", "show Setup:/Network/Service/" + nativeTargetTestServiceID + "\nquit\n", "networksetup -listnetworkserviceorder", "show State:/Network/Global/IPv4\nquit\n"} + if !slices.Equal(calls, wantCalls) { + t.Fatalf("calls=%v", calls) + } + } + }) + } +} + +func TestNativeTargetReaderFailuresAndSharedBudget(t *testing.T) { + for failAt := 1; failAt <= 5; failAt++ { + t.Run(fmt.Sprint(failAt), func(t *testing.T) { + var calls []string + r := nativeTargetTestReader(t, nativeTargetTestOutputs(), &calls) + run := r.run + count := 0 + r.run = func(ctx context.Context, input, path string, args ...string) ([]byte, error) { + count++ + if count == failAt { + return nil, errors.New("read failed") + } + return run(ctx, input, path, args...) + } + if got, err := r.defaultService(context.Background(), "en1"); got.ID != "" || err == nil || count != failAt { + t.Fatalf("got=%v err=%v count=%d", got, err, count) + } + }) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + var calls []string + r := nativeTargetTestReader(t, nativeTargetTestOutputs(), &calls) + run := r.run + r.run = func(actual context.Context, input, path string, args ...string) ([]byte, error) { + if actual != ctx { + t.Fatal("reader renewed the shared context") + } + out, err := run(actual, input, path, args...) + if len(calls) == 2 { + cancel() + } + return out, err + } + if got, err := r.defaultService(ctx, "en1"); got.ID != "" || !errors.Is(err, context.Canceled) { + t.Fatalf("got=%v err=%v", got, err) + } + if len(calls) != 2 { + t.Fatalf("reads after cancellation: %v", calls) + } +} + +func TestNativeServiceNameStrict(t *testing.T) { + for _, name := range []string{"Wi-Fi", "iPhone USB", "Office : Wi-Fi", "Téléphone", "Wi-Fi ", " Wi-Fi"} { + got, err := parseNativeServiceName(" {\n UserDefinedName : " + name + "\n}") + if err != nil || got != name { + t.Fatalf("%q: %q %v", name, got, err) + } + } + for _, out := range []string{"", "No such key", " {\n UserDefinedName : \n}", " {\n UserDefinedName : Wi-Fi\n UserDefinedName : Evil\n}", " {\n DHCPPacket : {\n UserDefinedName : Evil\n}\n}", " {\n UserDefinedName : ../bad\n}"} { + if _, err := parseNativeServiceName(out); err == nil { + t.Fatalf("accepted %q", out) + } + } +} + +// Exercise the real exec runner without macOS tools or host network effects. +func TestNativeTargetCommandHelper(t *testing.T) { + i := slices.Index(os.Args, "--native-target-helper") + if i < 0 { + return + } + switch os.Args[i+1] { + case "large": + _, _ = os.Stdout.Write([]byte(strings.Repeat("x", nativeTargetMaxOutput+1))) + case "exact": + _, _ = os.Stdout.Write([]byte(strings.Repeat("x", nativeTargetMaxOutput))) + case "slow": + time.Sleep(10 * time.Second) + case "fail": + fmt.Print(" {}") + os.Exit(1) + default: + fmt.Print("ok") + } + os.Exit(0) +} + +func TestNativeTargetCommandBounds(t *testing.T) { + for _, mode := range []string{"ok", "exact", "large", "slow", "fail"} { + t.Run(mode, func(t *testing.T) { + budget := 3 * time.Second + if mode == "slow" { + budget = 100 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), budget) + defer cancel() + start := time.Now() + out, err := runNativeTargetCommand(ctx, "", os.Args[0], "-test.run=^TestNativeTargetCommandHelper$", "--", "--native-target-helper", mode) + if mode == "ok" || mode == "exact" { + want := 2 + if mode == "exact" { + want = nativeTargetMaxOutput + } + if err != nil || len(out) != want { + t.Fatalf("len=%d err=%v", len(out), err) + } + } else if err == nil || out != nil { + t.Fatalf("accepted incomplete evidence len=%d err=%v", len(out), err) + } + if time.Since(start) > budget+2*time.Second { + t.Fatal("command exceeded deadline slack") + } + }) + } + var out nativeTargetOutput + if _, err := out.Write([]byte(strings.Repeat("x", nativeTargetMaxOutput))); err != nil { + t.Fatal(err) + } + if _, err := out.Write([]byte("x")); !errors.Is(err, errNativeTargetOutputLimit) || out.buf.Len() != nativeTargetMaxOutput { + t.Fatalf("unbounded writer: %d %v", out.buf.Len(), err) + } +} + +func TestSaveTargetStaticDNSSnapshotRetainedBackup(t *testing.T) { + for _, content := range []string{"", "2001:db8::53", "corrupt", "10.0.0.53"} { + t.Run(content, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + err := saveTargetStaticDNSSnapshot(path, []string{"10.0.0.53"}, "10.0.0.53") + wantErr := content == "corrupt" || content == "10.0.0.53" + if (err != nil) != wantErr { + t.Fatalf("retained %q: %v", content, err) + } + if got, readErr := os.ReadFile(path); readErr != nil || string(got) != content { + t.Fatalf("retained backup changed: %q %v", got, readErr) + } + }) + } + if err := saveTargetStaticDNSSnapshot(t.TempDir(), []string{"10.0.0.53"}, "10.0.0.53"); err == nil { + t.Fatal("unreadable retained backup accepted") + } + if err := saveTargetStaticDNSSnapshot(filepath.Join(t.TempDir(), "missing"), []string{"10.0.0.53"}, "10.0.0.53"); err != nil { + t.Fatal("original automatic DNS rejected", err) + } +} + +func TestSaveTargetStaticDNSSnapshot(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + original := "2001:db8::53" + if err := saveTargetStaticDNSSnapshot(path, []string{original}, ""); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, []string{"10.0.0.53"}, "10.0.0.53"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != original { + t.Fatalf("owned value replaced original: %q %v", got, err) + } + if err := saveTargetStaticDNSSnapshot(path, []string{"bad"}, ""); err == nil { + t.Fatal("invalid snapshot accepted") + } + if err := saveTargetStaticDNSSnapshot(path, nil, ""); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("empty snapshot left stale backup") + } + if err := saveTargetStaticDNSSnapshot(path, nil, ""); err != nil { + t.Fatal("missing backup removal should succeed", err) + } + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "block"), nil, 0600); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, nil, ""); err == nil { + t.Fatal("remove failure swallowed") + } + if err := saveTargetStaticDNSSnapshot(path, []string{original}, ""); err == nil { + t.Fatal("write failure swallowed") + } +} diff --git a/cmd/cli/dns_target_native_test.go b/cmd/cli/dns_target_native_test.go new file mode 100644 index 00000000..9d207029 --- /dev/null +++ b/cmd/cli/dns_target_native_test.go @@ -0,0 +1,55 @@ +package cli + +import ( + "net" + "strings" + "testing" +) + +func TestNativeIPv4Store(t *testing.T) { + valid := " {\n CLAT46 : TRUE\n InterfaceName : en7\n Router : 192.0.0.1\n Addresses : {\n 0 : 192.0.0.2\n }\n}" + if got, err := parseNativeIPv4Store(valid); err != nil || got["CLAT46"] != "TRUE" { + t.Fatalf("%v %v", got, err) + } + for _, bad := range []string{"", "No such key", strings.Replace(valid, "TRUE", "1", 1), strings.Replace(valid, "InterfaceName", "DHCPPacket", 1), strings.Replace(valid, " Router", " CLAT46 : TRUE\n Router", 1), valid + "\n {}", " {\nPrimaryService : bad\n}"} { + if _, err := parseNativeIPv4Store(bad); err == nil { + t.Fatalf("accepted %q", bad) + } + } +} + +func TestTargetStaticDNSStrict(t *testing.T) { + for _, good := range []string{"There aren't any DNS Servers set on Wi-Fi.", "2001:db8::53\n", "1.1.1.1\n2001:db8::53"} { + if _, err := parseTargetStaticDNS(good, "Wi-Fi"); err != nil { + t.Fatal(err) + } + } + for _, bad := range []string{"", "read failed", "There aren't any DNS Servers set on Other.", "1.1.1.1\nunexpected"} { + if _, err := parseTargetStaticDNS(bad, "Wi-Fi"); err == nil { + t.Fatalf("accepted %q", bad) + } + } +} + +func TestNativeCLATAddresses(t *testing.T) { + for _, tc := range []struct { + ips []string + want bool + }{ + {[]string{"192.0.0.2/32", "2001:db8::1/64"}, true}, + {[]string{"192.0.0.5/32", "2001:db8::1/64", "fe80::1/64"}, true}, + {[]string{"192.0.0.2/32", "2001:db8::1/64", "192.168.1.2/24"}, false}, + {[]string{"192.0.0.2/32", "fe80::1/64"}, false}, + {[]string{"2001:db8::1/64"}, false}, + } { + var addrs []net.Addr + for _, s := range tc.ips { + ip, n, _ := net.ParseCIDR(s) + n.IP = ip + addrs = append(addrs, n) + } + if got := nativeCLATAddresses(addrs); got != tc.want { + t.Fatalf("%v: %v", tc.ips, got) + } + } +} diff --git a/cmd/cli/dns_target_ownership_darwin_test.go b/cmd/cli/dns_target_ownership_darwin_test.go new file mode 100644 index 00000000..d2a78dc2 --- /dev/null +++ b/cmd/cli/dns_target_ownership_darwin_test.go @@ -0,0 +1,168 @@ +//go:build darwin + +package cli + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "testing" +) + +func TestInterceptDNSTargetOwnershipPersistenceBeforeMutation(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + h.statePath = filepath.Join(t.TempDir(), "absent", "ownership") + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.9" + p.cfg.Listener["0"].Port = 5353 + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || p.interceptDNSTargetService != "" || !slices.Equal(h.dns["Wi-Fi"], []string{"2001:db8::53"}) { + t.Fatalf("persistence failure mutated DNS: calls=%v dns=%v owner=%s", h.setCalls, h.dns["Wi-Fi"], p.interceptDNSTargetService) + } +} + +func TestInterceptDNSTargetExternalEditInvalidatesBackup(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.9" + p.cfg.Listener["0"].Port = 5353 + p.ensureInterceptDNSTarget([]string{}) + target := p.interceptDNSTargetValue() + if !slices.Equal(h.dns["Wi-Fi"], []string{target}) { + t.Fatal("listener-aware target not installed") + } + backup := savedStaticDNSPathFn(&net.Interface{Name: "Wi-Fi"}) + if err := os.WriteFile(backup, []byte("2001:db8::old"), 0600); err != nil { + t.Fatal(err) + } + external := []string{"2001:db8::54"} + h.dns["Wi-Fi"] = slices.Clone(external) + for range 2 { + p.ensureInterceptDNSTarget([]string{}) + } + if !slices.Equal(h.dns["Wi-Fi"], external) || len(h.setCalls) != 1 { + t.Fatal("reconciliation overwrote external DNS") + } + p.removeInterceptDNSTarget("stop") + oldEach, oldRestore := restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn + t.Cleanup(func() { restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn = oldEach, oldRestore }) + restoreSavedStaticDNSInterfacesFn = func(_, _ string, f func(*net.Interface) error) { _ = f(&net.Interface{Name: "Wi-Fi"}) } + restoreSavedStaticDNSRestoreFn = func(*net.Interface) error { t.Fatal("full stop sweep replayed stale DNS"); return nil } + restoreSavedStaticDNS("", false) + if _, err := os.Stat(backup); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("stale backup remains available to stop sweep: %v", err) + } + if !slices.Equal(h.dns["Wi-Fi"], external) { + t.Fatal("stop overwrote external DNS") + } +} + +func TestInterceptDNSTargetFailedOwnershipClearRetainsState(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if err := os.Remove(h.statePath); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(h.statePath, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(h.statePath, "busy"), nil, 0600); err != nil { + t.Fatal(err) + } + p.removeInterceptDNSTarget("stop") + if p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("failed ownership clear reported completed cleanup") + } +} + +func TestInterceptDNSTargetTrackedValueIsNotActualDNS(t *testing.T) { + for _, port := range []int{53, 5353} { + t.Run(fmt.Sprint(port), func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.9" + p.cfg.Listener["0"].Port = port + p.ensureInterceptDNSTarget([]string{}) + h.dns["Wi-Fi"] = nil + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, event, "stage", "owned_target_changed") + if len(h.setCalls) != 1 || len(h.dns["Wi-Fi"]) != 0 { + t.Fatal("reinstalled after external clear") + } + }) + } +} + +func TestInterceptDNSTargetFailedCleanupGuardsFullSweep(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + backup := savedStaticDNSPathFn(&net.Interface{Name: "Wi-Fi"}) + if err := os.WriteFile(backup, []byte("2001:db8::53"), 0600); err != nil { + t.Fatal(err) + } + h.resetErr = errors.New("reset failed") + p.removeInterceptDNSTarget("stop") + oldEach, oldRestore := restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn + t.Cleanup(func() { restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn = oldEach, oldRestore }) + restoreSavedStaticDNSInterfacesFn = func(_, _ string, f func(*net.Interface) error) { _ = f(&net.Interface{Name: "Wi-Fi"}) } + restoreSavedStaticDNSRestoreFn = func(*net.Interface) error { t.Fatal("sweep bypassed failed guarded cleanup"); return nil } + restoreSavedStaticDNS("", true) + if _, err := os.Stat(backup); err != nil { + t.Fatal("failed cleanup lost backup:", err) + } + if p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("failed cleanup lost state") + } +} + +func TestInterceptDNSTargetStaticRestoreSkipIsNotFailure(t *testing.T) { + for _, state := range []string{`{"service":"Wi-Fi","value":"127.0.0.53"}`, `invalid ownership`} { + t.Run(state, func(t *testing.T) { + h := newInterceptTargetHarness(t) + logs := captureDebugMainLog(t) + if err := os.WriteFile(h.statePath, []byte(state), 0600); err != nil { + t.Fatal(err) + } + backup := savedStaticDNSPathFn(&net.Interface{Name: "Wi-Fi"}) + const savedDNS = "2001:db8::53" + if err := os.WriteFile(backup, []byte(savedDNS), 0600); err != nil { + t.Fatal(err) + } + oldEach, oldRestore := restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn + t.Cleanup(func() { restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn = oldEach, oldRestore }) + calls := 0 + restoreSavedStaticDNSInterfacesFn = func(_, _ string, f func(*net.Interface) error) { + calls++ + if err := f(&net.Interface{Name: "Wi-Fi"}); err != nil { + t.Fatalf("ownership skip reported as restore failure: %v", err) + } + } + restoreSavedStaticDNSRestoreFn = func(*net.Interface) error { + t.Fatal("ownership skip restored DNS") + return nil + } + restoreSavedStaticDNS("", true) + if calls != 1 { + t.Fatalf("interface sweeps=%d, want 1", calls) + } + event := oneRecoveryEvent(t, logs, "Saved static DNS restore skipped on interface Wi-Fi: intercept target cleanup is pending or ownership is unreadable") + wantField(t, event, "level", "debug") + if data, err := os.ReadFile(backup); err != nil || string(data) != savedDNS { + t.Fatalf("ownership skip changed backup: %q, %v", data, err) + } + if data, err := os.ReadFile(h.statePath); err != nil || string(data) != state { + t.Fatalf("ownership skip changed state: %q, %v", data, err) + } + }) + } +} diff --git a/cmd/cli/dns_target_service.go b/cmd/cli/dns_target_service.go new file mode 100644 index 00000000..441984b9 --- /dev/null +++ b/cmd/cli/dns_target_service.go @@ -0,0 +1,45 @@ +package cli + +import ( + "errors" + "strings" +) + +// networksetup uses its own service name, which can differ from the Dynamic +// Store's UserDefinedName (for example, iPhone USB versus iPhone). +// Its listing has no UUID, so require exactly one service on the proven +// device, and require that service to be enabled. Do not use patchNetIfaceName: +// its first-enabled-match policy cannot prove a unique target for a DNS write. +func targetNetworkServiceName(output, device string) (string, error) { + if !serviceOrderDeviceName.MatchString(device) { + return "", errors.New("invalid target device") + } + entries, err := parseNetworkServiceOrder(strings.NewReader(output)) + if err != nil { + return "", err + } + seen := make(map[string]bool) + name := "" + matches := 0 + for _, entry := range entries { + key := strings.ToLower(entry.Name) + if seen[key] { + return "", errors.New("ambiguous network service name") + } + seen[key] = true + if entry.Device != device { + continue + } + if strings.ContainsAny(entry.Name, "/{}") { + return "", errors.New("unsafe target service name") + } + matches++ + if !entry.Disabled { + name = entry.Name + } + } + if matches != 1 || name == "" { + return "", errors.New("network service mapping is missing or ambiguous") + } + return name, nil +} diff --git a/cmd/cli/dns_target_service_test.go b/cmd/cli/dns_target_service_test.go new file mode 100644 index 00000000..83a84d95 --- /dev/null +++ b/cmd/cli/dns_target_service_test.go @@ -0,0 +1,94 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "testing" +) + +func nativeTargetTestServiceOrder(name, device string) string { + return fmt.Sprintf("An asterisk (*) denotes that a network service is disabled.\n(1) %s\n(Hardware Port: Test, Device: %s)\n", name, device) +} + +// networksetup output captured on the failing USB fixture. The Dynamic Store +// name below models the name observed in ctrld's native-reader diagnostic. +const capturedUSBServiceOrder = `An asterisk (*) denotes that a network service is disabled. +(1) USB 10/100/1000 LAN +(Hardware Port: USB 10/100/1000 LAN, Device: en7) + +(2) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) + +(3) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) + +(4) iPhone USB +(Hardware Port: iPhone USB, Device: en8) + +(5) Tailscale +(Hardware Port: io.tailscale.ipn.macos, Device: ) +` + +func TestNativeUSBServiceName(t *testing.T) { + global := nativeTargetTestGlobal(nativeTargetTestServiceID, "en8") + outputs := []string{global, strings.ReplaceAll(nativeTargetTestState(), "en1", "en8"), " {\n UserDefinedName : iPhone\n}", capturedUSBServiceOrder, global} + var calls []string + reader := nativeTargetTestReader(t, outputs, &calls) + got, err := reader.defaultService(context.Background(), "en8") + if err != nil || got.ID != nativeTargetTestServiceID || got.Device != "en8" || got.Name != "iPhone USB" { + t.Fatalf("service=%+v err=%v", got, err) + } + dns, err := parseTargetStaticDNS("There aren't any DNS Servers set on iPhone USB.\n", got.Name) + if err != nil || len(dns) != 0 { + t.Fatalf("recognized USB DNS response rejected: %v %v", dns, err) + } +} + +func TestTargetNetworkServiceName(t *testing.T) { + base := nativeTargetTestServiceOrder("iPhone USB", "en8") + for _, tc := range []struct{ name, listing, want string }{ + {"captured USB", capturedUSBServiceOrder, "iPhone USB"}, + {"renamed", nativeTargetTestServiceOrder("Téléphone personnel", "en8"), "Téléphone personnel"}, + {"spaces preserved", nativeTargetTestServiceOrder(" iPhone USB ", "en8"), " iPhone USB "}, + {"enabled and disabled same device", base + "(*) Old USB\n(Hardware Port: Test, Device: en8)\n", ""}, + {"disabled duplicate on another device", base + "(*) iPhone USB\n(Hardware Port: Test, Device: en9)\n", ""}, + {"device boundary", nativeTargetTestServiceOrder("Other", "en80"), ""}, + {"disabled", strings.Replace(base, "(1)", "(*)", 1), ""}, + {"two enabled same device", base + "(2) Backup USB\n(Hardware Port: Test, Device: en8)\n", ""}, + {"duplicate command name", base + "(2) iPhone USB\n(Hardware Port: Test, Device: en9)\n", ""}, + {"truncated", base + "(2) Incomplete\n", ""}, + {"malformed device", strings.Replace(base, "Device: en8", "Device: en8 extra", 1), ""}, + {"unrecognized output", "Error: invalid parameter", ""}, + {"unsafe name", nativeTargetTestServiceOrder("../other", "en8"), ""}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := targetNetworkServiceName(tc.listing, "en8") + if tc.want == "" { + if err == nil { + t.Fatalf("accepted %q", got) + } + return + } + if err != nil || got != tc.want { + t.Fatalf("got=%q err=%v", got, err) + } + }) + } +} + +func TestNativeUSBServiceMappingRechecked(t *testing.T) { + outputs := nativeTargetTestOutputs() + var calls []string + reader := nativeTargetTestReader(t, outputs, &calls) + first, err := reader.defaultService(context.Background(), "en1") + if err != nil { + t.Fatal(err) + } + outputs[3] = nativeTargetTestServiceOrder("Renamed Wi-Fi", "en1") + calls = nil + second, err := reader.defaultService(context.Background(), "en1") + if err != nil || first == second || second.Name != "Renamed Wi-Fi" { + t.Fatalf("mapping change invisible: %+v %+v %v", first, second, err) + } +} diff --git a/cmd/cli/dns_target_stub.go b/cmd/cli/dns_target_stub.go new file mode 100644 index 00000000..459e7b2f --- /dev/null +++ b/cmd/cli/dns_target_stub.go @@ -0,0 +1,16 @@ +//go:build !darwin + +package cli + +func interceptTargetAllowsStaticRestore(_ string) bool { return true } + +// ensureInterceptDNSTarget is a no-op on non-Darwin platforms: the DNS-less +// network problem it solves is specific to macOS pf interception blocking +// IPv6 port 53 with no IPv4 fallback (issue #533). Windows intercept mode +// uses NRPT, which routes queries regardless of adapter DNS configuration. +func (p *prog) ensureInterceptDNSTarget(_ []string) {} + +// removeInterceptDNSTarget is a no-op on non-Darwin platforms. +// +//lint:ignore U1000 called from Darwin-only intercept shutdown; kept for API symmetry. +func (p *prog) removeInterceptDNSTarget(_ string) {} diff --git a/cmd/cli/dns_target_test.go b/cmd/cli/dns_target_test.go new file mode 100644 index 00000000..6a5ab63b --- /dev/null +++ b/cmd/cli/dns_target_test.go @@ -0,0 +1,113 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestHasIPv4DNS(t *testing.T) { + tests := []struct { + name string + in []string + want bool + }{ + {"empty", nil, false}, + {"ipv4", []string{"8.8.8.8"}, true}, + {"ipv4 with port", []string{"192.168.1.1:53"}, true}, + {"loopback counts", []string{"127.0.0.1"}, true}, + {"ipv6 only", []string{"2001:4860:4860::8888"}, false}, + {"ipv6 with port", []string{"[2001:4860:4860::8888]:53"}, false}, + {"mixed", []string{"2001:4860:4860::8888", "9.9.9.9"}, true}, + {"garbage ignored", []string{"not-an-ip", ""}, false}, + {"garbage plus v4", []string{"not-an-ip", "1.1.1.1"}, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hasIPv4DNS(tc.in); got != tc.want { + t.Errorf("hasIPv4DNS(%v) = %v, want %v", tc.in, got, tc.want) + } + }) + } +} + +func TestNeedsInterceptDNSTarget(t *testing.T) { + tests := []struct { + name string + static, discovered []string + want bool + }{ + {"no dns at all", nil, nil, true}, + {"ipv6-only tether (464XLAT, issue #533)", nil, []string{"2605:8d80::1"}, true}, + {"static v4 present", []string{"1.1.1.1"}, nil, false}, + {"discovered v4 present", nil, []string{"192.168.1.1:53"}, false}, + {"existing ctrld target satisfies", []string{"127.0.0.1"}, nil, false}, + {"ipv6 static, v4 discovered", []string{"2001:db8::1"}, []string{"10.0.0.1"}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := needsInterceptDNSTarget(tc.static, tc.discovered); got != tc.want { + t.Errorf("needsInterceptDNSTarget(%v, %v) = %v, want %v", tc.static, tc.discovered, got, tc.want) + } + }) + } +} + +func TestIsInterceptDNSTargetOnly(t *testing.T) { + tests := []struct { + name string + in []string + target string + want bool + }{ + {"exactly ours (direct listener)", []string{"127.0.0.1"}, "127.0.0.1", true}, + {"exactly ours (rdr target)", []string{"127.0.0.53"}, "127.0.0.53", true}, + {"empty list", nil, "127.0.0.1", false}, + {"empty target never matches", []string{"127.0.0.1"}, "", false}, + {"ours plus user entry", []string{"127.0.0.1", "1.1.1.1"}, "127.0.0.1", false}, + {"user entry only", []string{"1.1.1.1"}, "127.0.0.1", false}, + {"different loopback than ours", []string{"127.0.0.53"}, "127.0.0.1", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isInterceptDNSTargetOnly(tc.in, tc.target); got != tc.want { + t.Errorf("isInterceptDNSTargetOnly(%v, %q) = %v, want %v", tc.in, tc.target, got, tc.want) + } + }) + } +} + +func TestInterceptDNSTargetValue(t *testing.T) { + tests := []struct { + name string + ip string + port int + want string + }{ + {"default direct listener :53", "127.0.0.1", 53, "127.0.0.1"}, + {"custom loopback listener :53", "127.0.0.2", 53, "127.0.0.2"}, + {"non-53 port uses rdr target", "127.0.0.1", 5354, "127.0.0.53"}, + {"listener on rdr target with non-53 port", "127.0.0.53", 5354, "127.0.0.54"}, + {"wildcard ip :53 falls back to loopback", "0.0.0.0", 53, "127.0.0.1"}, + {"wildcard ip non-53 uses rdr target", "0.0.0.0", 5354, "127.0.0.53"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: tc.ip, Port: tc.port}, + }, + }} + if got := p.interceptDNSTargetValue(); got != tc.want { + t.Errorf("interceptDNSTargetValue() with listener %s:%d = %q, want %q", tc.ip, tc.port, got, tc.want) + } + }) + } +} + +func TestInterceptDNSTargetValue_NoListener(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}} + if got := p.interceptDNSTargetValue(); got != "127.0.0.1" { + t.Errorf("interceptDNSTargetValue() with no listener = %q, want 127.0.0.1", got) + } +} diff --git a/cmd/cli/dns_target_usb_darwin_test.go b/cmd/cli/dns_target_usb_darwin_test.go new file mode 100644 index 00000000..b86a914a --- /dev/null +++ b/cmd/cli/dns_target_usb_darwin_test.go @@ -0,0 +1,89 @@ +//go:build darwin + +package cli + +import ( + "context" + "fmt" + "net" + "slices" + "strings" + "testing" +) + +func usbTargetTestReader(t *testing.T, listing string) nativeTargetReader { + t.Helper() + global := nativeTargetTestGlobal(nativeTargetTestServiceID, "en8") + outputs := []string{global, strings.ReplaceAll(nativeTargetTestState(), "en1", "en8"), " {\n UserDefinedName : iPhone\n}", listing, global} + var calls []string + return nativeTargetTestReader(t, outputs, &calls) +} + +func TestUSBServiceReaderDrivesTargetLifecycle(t *testing.T) { + for _, initial := range [][]string{nil, {"2001:db8::53"}} { + for _, port := range []int{53, 5353, 5354} { + t.Run(fmt.Sprintf("%d/%v", port, initial), func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.serviceByDev["en8"] = "iPhone USB" + h.dns["iPhone USB"] = slices.Clone(initial) + interceptDefaultRouteInterfaceFn = func() (string, error) { return "en8", nil } + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + return usbTargetTestReader(t, capturedUSBServiceOrder).defaultService(ctx, device) + } + interceptNativeStaticDNSFn = func(_ context.Context, iface *net.Interface) ([]string, error) { + if iface.Name != "iPhone USB" { + t.Fatalf("invalid command service %q", iface.Name) + } + return slices.Clone(h.dns[iface.Name]), nil + } + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.2" + p.cfg.Listener["0"].Port = port + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.dns["iPhone USB"], []string{p.interceptDNSTargetValue()}) || len(h.setCalls) != 1 { + t.Fatalf("USB target=%v writes=%v", h.dns, h.setCalls) + } + if _, ok := h.dns["iPhone"]; ok { + t.Fatal("wrote display-name service") + } + p.removeInterceptDNSTarget("USB test cleanup") + if !slices.Equal(h.dns["iPhone USB"], initial) || p.interceptDNSTargetService != "" { + t.Fatal("USB DNS not restored") + } + }) + } + } +} + +func TestUSBServiceMappingChangeAndAmbiguityDoNotWrite(t *testing.T) { + for _, scenario := range []string{"renamed on recheck", "ambiguous device"} { + t.Run(scenario, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.serviceByDev["en8"] = "iPhone USB" + original := []string{"2001:db8::53"} + h.dns["iPhone USB"] = slices.Clone(original) + interceptDefaultRouteInterfaceFn = func() (string, error) { return "en8", nil } + reads := 0 + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + reads++ + listing := capturedUSBServiceOrder + if scenario == "renamed on recheck" && reads == 2 { + listing = strings.Replace(listing, "(4) iPhone USB", "(4) Renamed USB", 1) + } + if scenario == "ambiguous device" { + listing += "(6) iPhone\n(Hardware Port: iPhone USB, Device: en8)\n" + } + return usbTargetTestReader(t, listing).defaultService(ctx, device) + } + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "" || !slices.Equal(h.dns["iPhone USB"], original) { + t.Fatalf("uncertain mapping changed DNS: %+v", h) + } + if scenario == "renamed on recheck" && reads != 2 { + t.Fatal("did not exercise final mapping recheck") + } + }) + } +} diff --git a/cmd/cli/firewall.go b/cmd/cli/firewall.go new file mode 100644 index 00000000..e250992a --- /dev/null +++ b/cmd/cli/firewall.go @@ -0,0 +1,811 @@ +package cli + +import ( + "context" + "net" + "net/netip" + "net/url" + "strings" + "time" + + "github.com/kardianos/service" + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld/internal/controld" + "github.com/Control-D-Inc/ctrld/internal/firewall" +) + +// firewallModeEnabled reports whether firewall mode is active for this prog instance. +func (p *prog) firewallModeEnabled() bool { + return p.firewallAllowList() != nil +} + +// firewallAllowList returns the allowlist this run enforces, or nil when Firewall +// Mode is off. +// +// syncFirewallMode replaces the field from the reload goroutine while refreshes +// read it, so a read that is followed by a dereference has to work from a +// snapshot rather than from the field: otherwise a reload landing in between +// turns the pointer to nil under the caller. Acting on a superseded allowlist is +// harmless - reconcileDestinations re-checks the firewall generation under +// destinationsMu and does nothing for a generation that has ended. +func (p *prog) firewallAllowList() *firewall.AllowList { + p.mu.Lock() + defer p.mu.Unlock() + return p.allowList +} + +// setFirewallAllowList publishes this run's allowlist. Only syncFirewallMode +// calls it; every other goroutine reads through firewallAllowList. +func (p *prog) setFirewallAllowList(al *firewall.AllowList) { + p.mu.Lock() + defer p.mu.Unlock() + p.allowList = al +} + +// initFirewallAllowList populates the permanent allowlist entries and starts the +// background reaper. Called once during prog.run() when firewall_mode is "on". +// +// Permanent entries include: +// - Loopback (127.0.0.0/8, ::1) +// - RFC1918 private ranges (configurable — enabled by default) +// - Link-local (169.254.0.0/16, fe80::/10) +// - CGNAT range (100.64.0.0/10) — used by Tailscale, carrier NAT +// - ctrld listener IPs +// - DoH/DoT/DoQ upstream resolver IPs +// - ControlD API and upgrade download server IPs +func (p *prog) initFirewallAllowList(ctx context.Context, al *firewall.AllowList) { + // Loopback. + al.AddPermanentPrefix(netip.MustParsePrefix("127.0.0.0/8")) + al.AddPermanent(netip.MustParseAddr("::1")) + + // RFC1918 private ranges — needed for LAN access, printers, NAS, etc. + al.AddPermanentPrefix(netip.MustParsePrefix("10.0.0.0/8")) + al.AddPermanentPrefix(netip.MustParsePrefix("172.16.0.0/12")) + al.AddPermanentPrefix(netip.MustParsePrefix("192.168.0.0/16")) + + // Link-local. + al.AddPermanentPrefix(netip.MustParsePrefix("169.254.0.0/16")) + al.AddPermanentPrefix(netip.MustParsePrefix("fe80::/10")) + + // CGNAT range — used by Tailscale (100.x.x.x), carrier-grade NAT, etc. + al.AddPermanentPrefix(netip.MustParsePrefix("100.64.0.0/10")) + + // Multicast. + al.AddPermanentPrefix(netip.MustParsePrefix("224.0.0.0/4")) + al.AddPermanentPrefix(netip.MustParsePrefix("ff00::/8")) + + // ctrld listener IPs — traffic to ourselves must always be allowed. + for _, lc := range p.cfg.Listener { + if ip, err := netip.ParseAddr(lc.IP); err == nil { + al.AddPermanent(ip) + } + } + + // Upstream resolver IPs — ctrld needs to reach its upstreams. + p.addUpstreamIPsToPermanent(al) + + // ControlD API and download IPs — ctrld needs to reach its own control plane. + p.addControlDEndpointIPsToPermanent(al) + + // Platform-specific enforcement (pf on macOS, WFP on Windows) is initialized + // from postRun() after startDNSIntercept() has prepared dnsInterceptState. + + p.Info().Msgf("Firewall allowlist initialized with %d permanent entries", + al.Stats().PermanentIPs) +} + +// syncFirewallMode applies the current firewall_mode setting for this run. +// Reloads create a new run-scoped context, so background firewall workers must +// be restarted each run even when the allowlist object is reused. +func (p *prog) syncFirewallMode(ctx context.Context) { + // This is the only writer of p.allowList, so its own reads need no lock; every + // other goroutine reads the published pointer through firewallAllowList. + al := p.allowList + + if p.cfg.Service.FirewallMode != "on" { + if al != nil || p.platformFirewallState != nil { + p.Info().Msg("Firewall mode disabled: removing platform enforcement and clearing allowlist") + } + if al != nil { + al.SetOnChange(nil) + al.SetOnBatchChange(nil) + p.setFirewallAllowList(nil) + } + // Platform enforcement is torn down below, so nothing may be mirrored any + // more: end this generation, which retires the maintenance worker, and + // forget what was applied so a later re-enable reinstalls it. + p.retireFirewallDestinations() + if p.platformFirewallState != nil { + p.shutdownPlatformFirewall() + p.platformFirewallState = nil + } + return + } + + if al == nil { + al = firewall.New() + p.initFirewallAllowList(ctx, al) + p.setFirewallAllowList(al) + if service.Interactive() { + p.Warn().Msg("Firewall mode has no effect in interactive mode; run ctrld as a service for enforcement") + } else { + p.Info().Msg("Firewall mode enabled: only DNS-resolved IPs will be allowed") + } + } else { + p.addUpstreamIPsToPermanent(al) + p.addControlDEndpointIPsToPermanent(al) + } + + // Open this run's firewall generation before any work is scheduled against it, + // so the previous run's maintenance worker stops acting on enforcement this + // run is now responsible for. + gen := p.startFirewallGeneration() + + // Apply the organization's allowed destinations from the resolver config this + // run started with. A reload that turns Firewall Mode on builds a fresh + // allowlist, so the set has to be re-applied rather than assumed present. + p.syncAllowedDestinations() + + // The run context is canceled on each reload. Restart the reaper/maintenance + // workers for this run so reused allowlists keep expiring entries. + al.StartReaper(ctx) + go p.firewallMaintenance(ctx, al, gen) + + // On reload, postRun() is not called, so initialize platform enforcement here + // if intercept state already exists. Initial startup still defers to postRun() + // because DNS intercept state is prepared there. + // + // Called whether or not enforcement is already up, because the permanent adds + // above reach memory only: AddPermanent fires no change callback, so a reload + // that resolves a new API address would log it as permitted while the platform + // never hears about it. Each platform's re-entry is a refresh - Windows + // reinstalls the permanent filters it is missing, macOS returns early - so + // calling it when enforcement is already up costs nothing and closes that gap. + if p.dnsInterceptState != nil { + p.initPlatformFirewall() + } +} + +// addUpstreamIPsToPermanent resolves upstream endpoint hostnames and adds their +// IPs to the permanent allowlist. Called at startup and on config reload. +func (p *prog) addUpstreamIPsToPermanent(al *firewall.AllowList) { + for _, uc := range p.cfg.Upstream { + if uc == nil || uc.Endpoint == "" { + continue + } + // Extract host from the endpoint URL. + host := extractHostFromEndpoint(uc.Endpoint) + if host == "" { + continue + } + + // If it's already an IP, add directly. + if ip, err := netip.ParseAddr(host); err == nil { + al.AddPermanent(ip) + p.Debug().Msgf("Firewall: added upstream IP %s to permanent allowlist", ip) + continue + } + + // Resolve hostname to IPs. + ips, err := net.LookupHost(host) + if err != nil { + p.Warn().Err(err).Msgf("Firewall: could not resolve upstream host %s", host) + continue + } + for _, ipStr := range ips { + if ip, err := netip.ParseAddr(ipStr); err == nil { + al.AddPermanent(ip) + p.Debug().Msgf("Firewall: added upstream IP %s (%s) to permanent allowlist", ip, host) + } + } + } +} + +// syncAllowedDestinations applies the organization's Allowed Destination IP list +// from the resolver config currently held by prog. +// +// Called whenever that config could have changed: on every start and reload (via +// syncFirewallMode) and after every API refresh, forced or scheduled (via +// apiConfigReload). Reading the list from p.rc rather than taking it as an +// argument keeps those callers from having to know whether Firewall Mode is on. +func (p *prog) syncAllowedDestinations() { + p.mu.Lock() + rc := p.rc + al := p.allowList + p.mu.Unlock() + + var entries []string + if rc != nil { + entries = rc.DestinationIPs + } else if al != nil && p.startedInAPIMaintenance.Load() { + // A maintenance fallback start holds no resolver config, and the + // organization's list is not persisted with the configuration on disk, + // so the exception set below is empty: raw-IP destinations the + // organization allowed stay blocked until the API answers and the + // recovery refresh re-applies the list. Failing closed is deliberate - + // the list is the organization's network topology, which is kept out of + // persisted state - so say so in the retained journal rather than leave + // the blocked destinations unexplained. + journal(p.Warn()).Str("reason", "api_maintenance"). + Msg("Firewall: organization allowed destinations are not enforced until the Control D API answers; started on the configuration on disk during maintenance") + } + p.applyAllowedDestinations(al, entries) +} + +// applyAllowedDestinations records entries as the desired Firewall Mode exception +// set and mirrors it into platform enforcement. Entries the API sent that are not +// a valid address or CIDR are dropped individually, so one bad entry never voids +// the rest of an organization's list. +// +// A no-op when Firewall Mode is off: with no allowlist there is nothing to +// except from, and the set is re-applied from p.rc if the mode is turned on. +// +// The allowlist is an argument rather than a field read because a reload can +// replace p.allowList - including with nil - between the nil check and the +// SetExceptions call below, which would dereference nil. Callers snapshot it +// once under mu (see firewallAllowList) and pass what they snapshotted. +func (p *prog) applyAllowedDestinations(al *firewall.AllowList, entries []string) { + if al == nil { + return + } + + prefixes, rejected, wide := parseAllowedDestinations(entries) + p.warnRejectedAllowedDestinations(rejected) + p.warnWideAllowedDestinations(wide) + + al.SetExceptions(prefixes) + p.reconcileDestinations(al, p.firewallGen.Load()) +} + +// reconcileAllowedDestinations brings platform enforcement in line with the +// desired allowed-destination set, installing what is missing and removing what +// the organization has withdrawn. +// +// The applied snapshot advances ONLY after the platform accepted the change. A +// failed pfctl call or WFP filter operation therefore leaves the previous +// snapshot recorded, so the same delta is recomputed - and retried - by the next +// refresh and by the periodic reconcile, instead of being silently dropped while +// the logs claim the new set is in force. Retrying the whole delta is safe +// because both mirrors are idempotent: installing an entry that is already there +// and removing one that is already gone are no-ops. +// +// Callers must not hold destinationsMu; the mirror can block on pfctl. +func (p *prog) reconcileAllowedDestinations() { + p.reconcileDestinations(p.firewallAllowList(), p.firewallGen.Load()) +} + +// reconcileDestinations is reconcileAllowedDestinations for one firewall +// generation. Background workers pass the allowlist and generation they were +// started with, and the generation is re-checked under destinationsMu: teardown +// bumps it while holding the same lock, so a worker from a previous run can never +// mirror anything into enforcement that is being (or has been) removed. +func (p *prog) reconcileDestinations(al *firewall.AllowList, gen uint64) { + if al == nil { + return + } + desired := al.Exceptions() + + p.destinationsMu.Lock() + defer p.destinationsMu.Unlock() + + if p.firewallGen.Load() != gen { + return + } + + // A resync owes the platform the whole set, not a delta: it means enforcement + // started with state ctrld does not know (a persist pf table from a previous + // run) or with none at all. Until the replace succeeds nothing about the + // applied set can be assumed, so the flag stays set and it is retried. + if p.destinationsNeedResync { + if err := firewallReplaceExceptionsFn(p, desired); err != nil { + p.Warn().Err(err).Int("total", len(desired)). + Msg("Firewall: could not install organization allowed destinations, will retry") + return + } + p.destinationsNeedResync = false + p.appliedDestinations = desired + p.logDestinationChange(len(desired), 0, desired, nil) + return + } + + added := prefixesNotIn(desired, p.appliedDestinations) + removed := prefixesNotIn(p.appliedDestinations, desired) + if len(added) == 0 && len(removed) == 0 { + return + } + + if err := firewallMirrorExceptionsFn(p, added, removed); err != nil { + p.Warn().Err(err). + Int("pending_add", len(added)). + Int("pending_remove", len(removed)). + Msg("Firewall: could not apply all organization allowed destinations, will retry") + return + } + + p.appliedDestinations = desired + p.logDestinationChange(len(added), len(removed), added, removed) +} + +// logDestinationChange reports an applied change: counts at Info, addresses at +// Debug. The list is an organization's network topology, and Info-level logs are +// persisted and uploaded with support bundles, so the counts are all that goes +// into the routine record. +func (p *prog) logDestinationChange(nAdded, nRemoved int, added, removed []netip.Prefix) { + p.Info(). + Int("added", nAdded). + Int("removed", nRemoved). + Int("total", len(p.appliedDestinations)). + Msg("Firewall: applied organization allowed destination IPs") + p.Debug(). + Strs("added", prefixStrings(added)). + Strs("removed", prefixStrings(removed)). + Msg("Firewall: organization allowed destination changes") +} + +// pendingDestinations reports how many allowed-destination changes platform +// enforcement has not accepted yet. Non-zero means a mirror attempt failed and +// the reconcile is still retrying, which is the difference between "the set is in +// force" and "the set is what we want" - the stats line must not conflate them. +func (p *prog) pendingDestinations(al *firewall.AllowList) int { + if al == nil { + return 0 + } + desired := al.Exceptions() + + p.destinationsMu.Lock() + defer p.destinationsMu.Unlock() + + if p.destinationsNeedResync { + // Nothing about the applied set is known, so everything is outstanding - + // and an empty desired set still owes the platform a flush of whatever it + // is holding, which is one pending operation, not zero. + return max(len(desired), 1) + } + return len(prefixesNotIn(desired, p.appliedDestinations)) + len(prefixesNotIn(p.appliedDestinations, desired)) +} + +// startFirewallGeneration opens a new firewall generation and returns it, +// retiring the workers of the previous one. Called for every run (start or +// reload) that has Firewall Mode on; the applied snapshot is left alone because +// platform enforcement survives a reload. +func (p *prog) startFirewallGeneration() uint64 { + p.destinationsMu.Lock() + defer p.destinationsMu.Unlock() + return p.firewallGen.Add(1) +} + +// markDestinationsForResync records that platform enforcement holds unknown +// state, so the next reconcile replaces its whole allowed-destination set rather +// than applying a delta against a snapshot that no longer describes anything. +// Called when enforcement starts: a fresh WFP session holds nothing, and a pf +// persist table may still hold what a previous run put there. +func (p *prog) markDestinationsForResync() { + p.destinationsMu.Lock() + defer p.destinationsMu.Unlock() + p.appliedDestinations = nil + p.destinationsNeedResync = true +} + +// retireFirewallDestinations ends the current firewall generation and forgets the +// applied set, for teardown: enforcement is about to be removed, so there is +// nothing left to reconcile against and no resync to owe. +// +// Bumping the generation under destinationsMu is what makes teardown safe against +// the maintenance worker: either the worker is mid-reconcile and this blocks +// until it finishes, or it reaches its own reconcile afterwards, sees a +// generation it does not own, and does nothing. +func (p *prog) retireFirewallDestinations() { + p.destinationsMu.Lock() + defer p.destinationsMu.Unlock() + p.firewallGen.Add(1) + p.appliedDestinations = nil + p.destinationsNeedResync = false +} + +// firewallMirrorExceptionsFn mirrors an allowed-destination delta into platform +// enforcement, and firewallReplaceExceptionsFn makes enforcement hold exactly the +// given set regardless of what it held before. Indirected so the +// failure-and-retry paths are testable without pf or WFP. +var ( + firewallMirrorExceptionsFn = (*prog).firewallApplyExceptionsPlatform + firewallReplaceExceptionsFn = (*prog).firewallReplaceExceptionsPlatform +) + +// prefixesNotIn returns the members of a that are absent from b. +func prefixesNotIn(a, b []netip.Prefix) []netip.Prefix { + if len(a) == 0 { + return nil + } + inB := make(map[netip.Prefix]struct{}, len(b)) + for _, prefix := range b { + inB[prefix] = struct{}{} + } + var out []netip.Prefix + for _, prefix := range a { + if _, ok := inB[prefix]; ok { + continue + } + out = append(out, prefix) + } + return out +} + +// parseAllowedDestinations converts the API's Allowed Destination IP entries into +// prefixes, returning the usable ones and the raw entries that were rejected. +// +// The API reports a single host as a bare address ("1.2.3.4", "2606:1a40::1") and +// anything wider in CIDR form, so both spellings are accepted; a bare address +// becomes a single-host prefix. IPv4-in-IPv6 forms are unmapped to match how the +// allowlist stores addresses, otherwise a "::ffff:1.2.3.4" entry would never +// match the IPv4 address it denotes. +// The third result is the accepted prefixes that are wide enough to be worth +// reporting; see wideAllowedDestination. +func parseAllowedDestinations(entries []string) (accepted []netip.Prefix, rejected []string, wide []netip.Prefix) { + if len(entries) == 0 { + return nil, nil, nil + } + accepted = make([]netip.Prefix, 0, len(entries)) + for _, entry := range entries { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + if prefix, err := netip.ParsePrefix(entry); err == nil { + prefix = unmapPrefix(prefix) + accepted = append(accepted, prefix) + if wideAllowedDestination(prefix) { + wide = append(wide, prefix) + } + continue + } + if addr, err := netip.ParseAddr(entry); err == nil { + addr = addr.Unmap() + accepted = append(accepted, netip.PrefixFrom(addr, addr.BitLen())) + continue + } + rejected = append(rejected, entry) + } + return accepted, rejected, wide +} + +// An accepted prefix with fewer mask bits than these is reported. The floors are +// set below anything an organization plausibly means: /8 is the widest classical +// IPv4 network and the size of RFC1918's 10.0.0.0/8, and /32 is a whole IPv6 RIR +// allocation. A bare address always parses to a single-host prefix, so only a +// CIDR entry can reach either floor. +const ( + minSaneAllowedDestinationV4Bits = 8 + minSaneAllowedDestinationV6Bits = 32 +) + +// wideAllowedDestination reports whether an accepted prefix covers enough of the +// address space to deserve a line in the log. +// +// netip.ParsePrefix takes "1.2.3.4/0", and normalizeExceptions masks it to +// 0.0.0.0/0; "::/0" and - through unmapPrefix - "::ffff:0:0/96" do the same for +// IPv6. One such entry lets every destination of that family bypass Firewall +// Mode while the mode still reports on, and logDestinationChange records only +// counts at Info, so without this the bypass is invisible outside Debug logs. +// +// This is not a defence against a hostile API, which can already turn the mode +// off through custom_config. It is a defence against a wide prefix arriving by +// accident - a dashboard bug, or an admin who typed the wrong mask - and nobody +// noticing. +func wideAllowedDestination(prefix netip.Prefix) bool { + if prefix.Addr().Is4() { + return prefix.Bits() < minSaneAllowedDestinationV4Bits + } + return prefix.Bits() < minSaneAllowedDestinationV6Bits +} + +// unmapPrefix rewrites an IPv4-in-IPv6 prefix to its IPv4 form, adjusting the +// mask by the 96-bit IPv4-mapped prefix length. Prefixes of other families are +// returned unchanged. +func unmapPrefix(prefix netip.Prefix) netip.Prefix { + addr := prefix.Addr() + if !addr.Is4In6() { + return prefix + } + bits := prefix.Bits() - 96 + if bits < 0 { + // A mask wider than the mapped range does not denote an IPv4 network; + // leave it as the IPv6 prefix it literally is. + return prefix + } + return netip.PrefixFrom(addr.Unmap(), bits) +} + +// warnRejectedAllowedDestinations reports unusable entries, but only when the set +// of rejections changes. The list is re-parsed on every refresh (hourly by +// default), so warning unconditionally would repeat the same lines for the life +// of the process while still saying nothing new. +// The rejected values themselves go to Debug, never to Warn. A rejected entry is +// still an organization's topology - "10.0.0.0/33" names a real network - and +// Warn logs are persisted and travel in support bundles exactly like Info ones, +// so they follow the same rule as logDestinationChange: counts in the routine +// record, addresses only when someone turned Debug on to look. +func (p *prog) warnRejectedAllowedDestinations(rejected []string) { + key := strings.Join(rejected, ",") + + p.mu.Lock() + unchanged := p.rejectedDestinationsKey == key + p.rejectedDestinationsKey = key + p.mu.Unlock() + + if unchanged || len(rejected) == 0 { + return + } + p.Warn().Int("rejected", len(rejected)). + Msg("Firewall: ignoring organization allowed destinations that are not a valid IP address or CIDR") + p.Debug().Strs("values", rejected). + Msg("Firewall: rejected organization allowed destinations") +} + +// warnWideAllowedDestinations reports accepted entries wide enough to blanket an +// address family, with the mask width but not the address, and only when the set +// of them changes - the list is re-parsed on every refresh, so an unconditional +// warning would repeat the same lines for the life of the process. +func (p *prog) warnWideAllowedDestinations(wide []netip.Prefix) { + key := strings.Join(prefixStrings(wide), ",") + + p.mu.Lock() + unchanged := p.wideDestinationsKey == key + p.wideDestinationsKey = key + p.mu.Unlock() + + if unchanged || len(wide) == 0 { + return + } + for _, prefix := range wide { + family := "ipv6" + if prefix.Addr().Is4() { + family = "ipv4" + } + p.Warn().Str("family", family).Int("bits", prefix.Bits()). + Msg("Firewall: organization allowed destination covers a very wide range; traffic to it bypasses Firewall Mode") + } + p.Debug().Strs("values", prefixStrings(wide)). + Msg("Firewall: wide organization allowed destinations") +} + +// prefixStrings renders prefixes for logging. +func prefixStrings(prefixes []netip.Prefix) []string { + out := make([]string, 0, len(prefixes)) + for _, prefix := range prefixes { + out = append(out, prefix.String()) + } + return out +} + +// addControlDEndpointIPsToPermanent permits the ControlD endpoints ctrld dials on +// its own behalf. Called at startup and on config reload, like the upstream IPs. +// +// Firewall Mode permits what ctrld's listener resolved, and each of these has a +// hardcoded address it dials when DNS is unusable - which is exactly the state a +// ctrld blocked by its own filters is in. Nothing teaches the allowlist about +// those addresses, so the block-all filters deny ctrld's own sockets. See +// controld.APIEndpointIPs for the incident this comes from. +func (p *prog) addControlDEndpointIPsToPermanent(al *firewall.AllowList) { + // The API. Its transport resolves with ctrld.LookupIP, which queries the OS + // nameservers directly rather than through the listener, so neither what it + // resolves nor what it falls back to is ever learned - both are permitted here. + p.addPermanentIPs(al, "ControlD API", controld.APIEndpointIPs(cdDev)) + p.addPermanentResolvedIPs(al, "ControlD API", controld.APIDomain(cdDev)) + + // The upgrade download server. performUpgrade spawns a detached child process, + // which WFP's block-all filters deny exactly like this one: they carry no + // process condition. Its hostname lookup does go through the listener and is + // learned, so only the direct IP it falls back to needs permitting - and that + // fallback is the one an upgrade on a blocked host depends on. + p.addPermanentIPs(al, "ControlD download server", []string{downloadServerIp}) +} + +// addPermanentIPs permits literal addresses, ignoring any that do not parse. +func (p *prog) addPermanentIPs(al *firewall.AllowList, what string, ips []string) { + for _, ipStr := range ips { + if ip, err := netip.ParseAddr(ipStr); err == nil { + al.AddPermanent(ip) + p.Debug().Msgf("Firewall: added %s IP %s to permanent allowlist", what, ip) + } + } +} + +// addPermanentResolvedIPs permits whatever domain resolves to right now. +func (p *prog) addPermanentResolvedIPs(al *firewall.AllowList, what, domain string) { + ips, err := net.LookupHost(domain) + if err != nil { + // Neither fatal nor surprising during early startup, and not a Warn: the + // direct addresses are permitted regardless, and they are what the + // transport itself falls back to in this same situation. + p.Debug().Err(err).Msgf("Firewall: could not resolve %s for the permanent allowlist; its direct IPs are permitted", domain) + return + } + for _, ipStr := range ips { + if ip, err := netip.ParseAddr(ipStr); err == nil { + al.AddPermanent(ip) + p.Debug().Msgf("Firewall: added %s IP %s (%s) to permanent allowlist", what, ip, domain) + } + } +} + +// extractHostFromEndpoint extracts the hostname or IP from a DoH/DoT/DoQ endpoint URL. +// Handles formats like: +// - "https://dns.controld.com/abcdef" +// - "tls://dns.controld.com" +// - "quic://dns.controld.com:784" +// - "1.2.3.4:53" +// - "sdns://..." (DNS stamps — host is encoded inside, skip) +func extractHostFromEndpoint(endpoint string) string { + // DNS stamps encode the server info in base64 — we can't extract the host + // without decoding. The upstream IPs will be resolved by the sdns upstream + // initialization path at runtime. + if strings.HasPrefix(endpoint, "sdns://") { + return "" + } + + // Try parsing as URL first (covers https://, tls://, quic://). + if host := extractHostFromURL(endpoint); host != "" { + return host + } + + // Try as host:port. + host, _, err := net.SplitHostPort(endpoint) + if err == nil { + return host + } + + // Try as bare IP. + if _, err := netip.ParseAddr(endpoint); err == nil { + return endpoint + } + + return "" +} + +// extractHostFromURL extracts the host from a URL string. +func extractHostFromURL(s string) string { + u, err := url.Parse(s) + if err != nil || u.Scheme == "" || u.Host == "" { + return "" + } + return u.Hostname() +} + +// firewallRecordResolvedIPs extracts A and AAAA records from a DNS response +// and adds them to the firewall allowlist. Called from postProcessStandardQuery() +// after a successful DNS resolution. +// +// This is the primary feed for the allowlist — every IP that ctrld resolves +// gets added here, making it allowed for outbound connections. +func (p *prog) firewallRecordResolvedIPs(answer *dns.Msg, domain string) { + if p.allowList == nil || answer == nil { + return + } + + // Only record IPs from successful responses. + if answer.Rcode != dns.RcodeSuccess { + return + } + + for _, rr := range answer.Answer { + switch r := rr.(type) { + case *dns.A: + if ip, ok := netip.AddrFromSlice(r.A); ok { + ttl := time.Duration(r.Hdr.Ttl) * time.Second + if ttl < 30*time.Second { + // Enforce minimum TTL to prevent constant churn for very short TTLs. + ttl = 30 * time.Second + } + p.allowList.Add(ip, domain, ttl) + } + case *dns.AAAA: + if ip, ok := netip.AddrFromSlice(r.AAAA); ok { + ttl := time.Duration(r.Hdr.Ttl) * time.Second + if ttl < 30*time.Second { + ttl = 30 * time.Second + } + p.allowList.Add(ip, domain, ttl) + } + case *dns.CNAME: + // For CNAME chains: the final A/AAAA records will be caught above. + // We don't need to do anything special for the CNAME itself, but we + // log it for debugging CNAME chain issues. + p.Debug().Msgf("Firewall: CNAME %s → %s (IPs from target will be allowlisted)", domain, r.Target) + } + } +} + +// firewallOnConfigReload is called when apiConfigReload() detects a config change. +// It flushes the entire allowlist so that DNS queries against the new policy +// repopulate it with the correct set of allowed IPs. +// +// This is the simple approach (vs. selective re-resolution per domain). +// The tradeoff is a brief window where connections may fail until DNS cache +// repopulates. Apps that reconnect directly to a previously resolved IP without +// making a fresh DNS query can remain blocked longer; this is an explicit v1 +// limitation to call out in release notes and app-compatibility testing. +func (p *prog) firewallOnConfigReload() { + if p.allowList == nil { + return + } + stats := p.allowList.Stats() + p.Info().Msgf("Firewall: config reload detected, flushing allowlist (%d IPs, %d domains)", + stats.AllowedIPs, stats.TrackedDomains) + // Flush platform-specific state first (pf table / WFP filters), + // then flush the allowlist. The AllowList's batch callbacks will + // also fire, but the platform flush handles the bulk operation more + // efficiently than removing IPs one-by-one. + p.firewallFlushPlatform() + p.allowList.Flush() +} + +// firewallOnNetworkChange is called when monitorNetworkChanges() detects a major +// network transition (WiFi↔cellular, interface IP changes). Stale IPs from the +// old network may no longer be valid, so we flush and let DNS repopulate. +func (p *prog) firewallOnNetworkChange() { + if p.allowList == nil { + return + } + stats := p.allowList.Stats() + p.Info().Msgf("Firewall: network change detected, flushing allowlist (%d IPs, %d domains)", + stats.AllowedIPs, stats.TrackedDomains) + p.firewallFlushPlatform() + p.allowList.Flush() +} + +// firewallMaintenance logs allowlist metrics immediately, then every 5 minutes +// while firewall mode is active, and retries any allowed-destination change that +// platform enforcement rejected. +// +// The retry has to be time-based, not only refresh-driven: configuration +// refreshes are hourly by default, so a transient pfctl or WFP failure would +// otherwise leave an approved destination blocked - or worse, a withdrawn one +// permitted - for up to an hour. +// It works on the allowlist and generation it was started with, not on +// p.allowList: a reload replaces that field from another goroutine, and this +// worker outlives the run whose context it was given by however long it takes to +// observe cancellation. Once its generation is over - a reload, or Firewall Mode +// being turned off - the worker retires rather than reconciling enforcement it no +// longer owns; reconcileDestinations re-checks the generation under +// destinationsMu, so even a worker that is already inside it cannot act late. +func (p *prog) firewallMaintenance(ctx context.Context, al *firewall.AllowList, gen uint64) { + if al == nil { + return + } + p.logFirewallStatsOnce(al) + + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + // A tick can win the select against an already-canceled context, and + // a generation can end without the context being canceled at all. + if ctx.Err() != nil || p.firewallGen.Load() != gen { + return + } + p.reconcileDestinations(al, gen) + p.logFirewallStatsOnce(al) + } + } +} + +func (p *prog) logFirewallStatsOnce(al *firewall.AllowList) { + if al == nil { + return + } + stats := al.Stats() + p.Info(). + Int("allowed_ips", stats.AllowedIPs). + Int("permanent_ips", stats.PermanentIPs). + Int("allowed_destinations", stats.ExceptionPrefixes). + Int("allowed_destinations_pending", p.pendingDestinations(al)). + Int("tracked_domains", stats.TrackedDomains). + Int64("total_hits", stats.TotalHits). + Int64("total_misses", stats.TotalMisses). + Msg("Firewall allowlist stats") +} diff --git a/cmd/cli/firewall_darwin.go b/cmd/cli/firewall_darwin.go new file mode 100644 index 00000000..80bdca91 --- /dev/null +++ b/cmd/cli/firewall_darwin.go @@ -0,0 +1,1186 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "os/exec" + "sort" + "strings" + "sync" + "time" +) + +// pfFirewallState holds the state for pf-based firewall mode enforcement on macOS. +// When firewall mode is active, we maintain a pf table of allowed IPs and add +// block/pass rules to the ctrld anchor that enforce the allowlist. +type pfFirewallState struct { + // mu protects batch accumulation. + mu sync.Mutex + + // pendingAdds and pendingRemoves accumulate changes for batched pf updates. + pendingAdds []netip.Addr + pendingRemoves []netip.Addr + + // batchTimer fires after the accumulation window to flush pending changes. + batchTimer *time.Timer + + // lastForwardedSources is the forwarded-source set (auto-detected VM networks + + // configured) that pf has actually accepted, and lastForwardedKey its + // order-independent signature. reconcileForwardedSources compares against these + // to rebuild the anchor when guests appear/disappear, and to know which subnets + // gained or lost trust so their stale pf states can be dropped. Both advance only + // after a successful anchor load, so a failed reload is retried, never latched. + lastForwardedSources []forwardedSource + lastForwardedKey string + + // applyForwardedMu serializes the compare-reload-record sequence in + // applyForwardedSourceChange, so a watchdog tick and a network change cannot both + // rebuild the anchor for the same transition or interleave their snapshot updates. + // Separate from mu, which must not be held across pfctl execution. + applyForwardedMu sync.Mutex +} + +// forwardedSourceWarnTracker dedupes the "unusable configured entry" warning by +// signature. Config is re-parsed on every anchor build and every watchdog tick, so +// warning unconditionally would repeat the same line for the life of the process; +// comparing signatures still reports an entry a config reload has just introduced. +// +// Package-level rather than a pfFirewallState field because it has to work before +// that state exists: initPlatformFirewall defers pf enforcement until intercept mode +// starts, so Firewall Mode can be on with no pfFirewallState yet. +var forwardedSourceWarnTracker struct { + mu sync.Mutex + key string +} + +const ( + // pfFirewallTable is the pf table name for dynamically-allowed IPs. + pfFirewallTable = "ctrld_allowed" + + // pfFirewallExceptionTable is the pf table name for the organization's + // Allowed Destination IP list. Kept separate from pfFirewallTable so the + // flushes that discard DNS-resolved IPs (config reload, network change) + // leave administratively allowed destinations in place, and so a destination + // removed upstream can be deleted without touching resolved entries. + pfFirewallExceptionTable = "ctrld_allowed_dst" + + // pfFirewallBatchInterval is the accumulation window for batching pf table updates. + // Short enough for responsiveness, long enough to avoid per-DNS-response pfctl calls. + pfFirewallBatchInterval = 200 * time.Millisecond +) + +// hypervisorVMNetPrefixes are interface-name prefixes that are specific to a +// virtualization vendor's private VM/NAT network, and therefore reliable proof of +// VM ownership by name alone. +// +// Two vendors still put the subnet directly on such an interface: Parallels (vnic0) +// and VirtualBox host-only (vboxnet0). vmnet* covers legacy kext-based VMware Fusion +// on Intel. They are also what proves ownership of a bridge they are a member of - +// see bridgeHasVMMember. +// +// On current macOS this is not sufficient on its own. vmnet.framework - which backs +// Virtualization.framework guests (UTM, Docker Desktop, Multipass, Fusion 12.1+ NAT) +// - puts the RFC1918 gateway address on a bridge10x interface and attaches the +// vendor-named vmenet* interface as an address-less member of it. Matching on name +// alone therefore never fires for those stacks, which is why bridge membership is +// checked too. +var hypervisorVMNetPrefixes = []string{ + "vmnet", // VMware Fusion (legacy kext-based, Intel) + "vmenet", // vmnet.framework member interface (UTM, Docker, Multipass, Fusion 12.1+) + "vnic", // Parallels Desktop + "vboxnet", // VirtualBox host-only +} + +// isHypervisorVMNetIface reports whether an interface name is a known +// vendor-specific VM/NAT network (see hypervisorVMNetPrefixes). +func isHypervisorVMNetIface(name string) bool { + for _, prefix := range hypervisorVMNetPrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// isBridgeIface reports whether name is a macOS bridge interface. +// +// A bridge name is deliberately NOT treated as proof of anything: macOS uses this +// namespace for Thunderbolt/aggregated links too (this repo's own tunnel-change code +// lists bridge0 among physical interfaces), and auto-trusting those could force-route +// unrelated same-subnet traffic. Ownership comes from the member list instead. +func isBridgeIface(name string) bool { + return strings.HasPrefix(name, "bridge") +} + +// bridgeMembersFn returns a bridge's member interfaces. Indirected so detection can +// be tested without a hypervisor or ifconfig. +var bridgeMembersFn = bridgeMembers + +// bridgeMembers returns the member interfaces of a macOS bridge, via ifconfig. +// There is no address-family-independent syscall for this that does not mean +// hand-rolling SIOCGDRVSPEC/BRDGGIFS structs, and this runs at most once per +// candidate bridge per anchor build. +func bridgeMembers(name string) []string { + out, err := exec.Command("ifconfig", name).CombinedOutput() + if err != nil { + return nil + } + return parseBridgeMembers(string(out)) +} + +// parseBridgeMembers extracts member interface names from ifconfig output, which +// lists each as a line of the form "\tmember: vmenet0 flags=3". +func parseBridgeMembers(ifconfigOut string) []string { + var members []string + for _, line := range strings.Split(ifconfigOut, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != "member:" { + continue + } + members = append(members, fields[1]) + } + return members +} + +// bridgeHasVMMember reports whether a bridge's member list proves VM/NAT ownership, +// i.e. whether a vendor-specific VM interface is bridged into it. +// +// This is the ownership evidence a bridge name lacks. A vmnet.framework bridge has a +// vmenet* member; a Thunderbolt bridge has en* members and so is never trusted. +func bridgeHasVMMember(members []string) bool { + for _, m := range members { + if isHypervisorVMNetIface(m) { + return true + } + } + return false +} + +// firewallFlushPlatform flushes the pf table on macOS. +func (p *prog) firewallFlushPlatform() { + p.pfFirewallFlushTable() +} + +// shutdownPlatformFirewall removes macOS firewall-mode dynamic state. The pf +// anchor itself is rebuilt by DNS intercept without firewall rules once +// p.allowList is nil. +func (p *prog) shutdownPlatformFirewall() { + p.pfFirewallFlushTable() + // The organization's allowed destinations live in their own persist table, + // which the dynamic flush does not touch. Clear it too so turning Firewall + // Mode off leaves no table content behind for a later run to inherit. + if out, err := pfExceptionTableCommand("flush", nil); err != nil { + p.Debug().Err(err).Str("output", strings.TrimSpace(string(out))). + Msgf("Firewall: failed to flush pf table %s during shutdown (may not exist)", pfFirewallExceptionTable) + } + + if p.dnsInterceptState == nil { + return + } + + var vpnExemptions []vpnDNSExemption + if p.vpnDNS != nil { + vpnExemptions = p.vpnDNS.CurrentExemptions() + } + rulesStr := p.buildPFAnchorRules(vpnExemptions) + if err := writePFAnchorFile(rulesStr); err != nil { + p.Warn().Err(err).Msg("Firewall: failed to write pf anchor during shutdown") + return + } + if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput(); err != nil { + p.Warn().Err(err).Str("output", strings.TrimSpace(string(out))).Msg("Firewall: failed to reload pf anchor during shutdown") + } +} + +// initPlatformFirewall initializes macOS-specific firewall enforcement (pf tables). +func (p *prog) initPlatformFirewall() { + if _, ok := p.platformFirewallState.(*pfFirewallState); ok { + return + } + + // pf enforcement is only meaningful when intercept mode is active - + // without it, we have no pf anchor to add rules to. + if dnsIntercept && p.dnsInterceptState != nil { + p.initPFFirewall() + } else { + p.Info().Msg("Firewall: pf enforcement deferred until intercept mode starts") + } +} + +// initPFFirewall sets up pf-based firewall mode enforcement. Called from +// initFirewallAllowList() when running on macOS with intercept mode active. +// +// Architecture: +// - Creates a pf table for dynamic IP allowlisting +// - Registers batch change callbacks on the AllowList +// - The actual pf anchor rules are injected via buildPFFirewallRules() which +// is called from buildPFAnchorRules() when firewall mode is active +// +// The table approach (vs. per-IP pass rules) is critical for performance: +// pfctl table operations are O(log n) and don't require a full anchor reload. +func (p *prog) initPFFirewall() { + state := &pfFirewallState{} + p.platformFirewallState = state + + // Register batch callback - AllowList reaper and FlushDomain use this. + p.allowList.SetOnBatchChange(func(added []netip.Addr, removed []netip.Addr) { + state.mu.Lock() + defer state.mu.Unlock() + + if len(added) > 0 { + state.pendingAdds = append(state.pendingAdds, added...) + } + if len(removed) > 0 { + state.pendingRemoves = append(state.pendingRemoves, removed...) + } + state.scheduleBatchFlush(p) + }) + + // Register individual change callback - Add() and Remove() use this. + p.allowList.SetOnChange(func(ip netip.Addr, added bool) { + state.mu.Lock() + defer state.mu.Unlock() + + if added { + state.pendingAdds = append(state.pendingAdds, ip) + } else { + state.pendingRemoves = append(state.pendingRemoves, ip) + } + state.scheduleBatchFlush(p) + }) + + // DNS responses may have populated the allowlist before platform callbacks + // were registered. Bulk-load that snapshot so pf starts with the same view + // as the in-memory allowlist. + p.pfFirewallPopulateTable() + + // Likewise for the organization's allowed destinations, which are applied as + // soon as the allowlist exists - before pf enforcement comes up. The table is + // a persist table that may still hold what a previous run put in it, so mark + // the set for a full replace rather than a delta; the reconcile retries until + // pf has exactly the current set. + p.markDestinationsForResync() + p.reconcileAllowedDestinations() + + // Seed the forwarded-source snapshot with what the anchor was just built with, + // so the first reconcile only fires on a real subsequent change. + sources := p.forwardedSources() + state.lastForwardedSources = sources + state.lastForwardedKey = forwardedSourceSetKey(sources) + + // Report the effective trust set, including the configured entries. Without this + // an admin who sets firewall_forwarded_sources has no way to confirm it took + // effect short of reading pf rules. + p.logForwardedSources(sources) + + p.Info().Msg("Firewall: pf table enforcement initialized") +} + +// scheduleBatchFlush starts or resets the batch timer. Must be called with state.mu held. +func (s *pfFirewallState) scheduleBatchFlush(p *prog) { + if s.batchTimer != nil { + return + } + s.batchTimer = time.AfterFunc(pfFirewallBatchInterval, func() { + s.flushBatch(p) + }) +} + +// flushBatch applies accumulated pf table changes in a single pfctl call per direction. +func (s *pfFirewallState) flushBatch(p *prog) { + s.mu.Lock() + adds := s.pendingAdds + removes := s.pendingRemoves + s.pendingAdds = nil + s.pendingRemoves = nil + s.batchTimer = nil + s.mu.Unlock() + + if len(adds) == 0 && len(removes) == 0 { + return + } + + // Collapse add/remove deltas into the current primary allowlist state. This + // avoids leaving pf opposite the allowlist when an Add and Remove for the + // same IP land in one batch window. + ipsToSync := make(map[netip.Addr]struct{}, len(adds)+len(removes)) + for _, ip := range adds { + ipsToSync[ip] = struct{}{} + } + for _, ip := range removes { + ipsToSync[ip] = struct{}{} + } + + var tableAdds, tableRemoves []string + for ip := range ipsToSync { + if p.allowList != nil && p.allowList.Contains(ip) { + tableAdds = append(tableAdds, ip.String()) + } else { + tableRemoves = append(tableRemoves, ip.String()) + } + } + + // Apply additions. + if len(tableAdds) > 0 { + // pfctl -t -T add accepts multiple IPs space-separated. + args := append([]string{"-a", pfAnchorName, "-t", pfFirewallTable, "-T", "add"}, tableAdds...) + if out, err := exec.Command("pfctl", args...).CombinedOutput(); err != nil { + p.Warn().Err(err).Str("output", string(out)). + Msgf("Firewall: failed to add %d IPs to pf table", len(tableAdds)) + } else { + p.Debug().Msgf("Firewall: added %d IPs to pf table %s", len(tableAdds), pfFirewallTable) + } + } + + // Apply removals. + if len(tableRemoves) > 0 { + args := append([]string{"-a", pfAnchorName, "-t", pfFirewallTable, "-T", "delete"}, tableRemoves...) + if out, err := exec.Command("pfctl", args...).CombinedOutput(); err != nil { + // Not a hard error - the IP may have already been removed (e.g., by a Flush). + p.Debug().Err(err).Str("output", string(out)). + Msgf("Firewall: failed to remove %d IPs from pf table (may already be gone)", len(tableRemoves)) + } else { + p.Debug().Msgf("Firewall: removed %d IPs from pf table %s", len(tableRemoves), pfFirewallTable) + } + } +} + +// pfFirewallFlushTable removes all entries from the pf firewall table. +// Called on network changes and config reloads before the AllowList is flushed. +func (p *prog) pfFirewallFlushTable() { + if state, ok := p.platformFirewallState.(*pfFirewallState); ok && state != nil { + state.mu.Lock() + if state.batchTimer != nil { + state.batchTimer.Stop() + state.batchTimer = nil + } + state.pendingAdds = nil + state.pendingRemoves = nil + state.mu.Unlock() + } + + out, err := exec.Command("pfctl", "-a", pfAnchorName, "-t", pfFirewallTable, "-T", "flush").CombinedOutput() + if err != nil { + p.Debug().Err(err).Str("output", string(out)).Msg("Firewall: failed to flush pf table (may not exist yet)") + } else { + p.Info().Msg("Firewall: flushed pf table " + pfFirewallTable) + } +} + +// pfFirewallPopulateTable bulk-loads all currently allowed IPs into the pf table. +// Called after anchor rule installation to ensure the table is populated. +func (p *prog) pfFirewallPopulateTable() { + if p.allowList == nil { + return + } + ips := p.allowList.AllowedIPs() + if len(ips) == 0 { + return + } + + ipStrs := make([]string, 0, len(ips)) + for _, ip := range ips { + ipStrs = append(ipStrs, ip.String()) + } + + args := append([]string{"-a", pfAnchorName, "-t", pfFirewallTable, "-T", "add"}, ipStrs...) + if out, err := exec.Command("pfctl", args...).CombinedOutput(); err != nil { + p.Warn().Err(err).Str("output", string(out)). + Msgf("Firewall: failed to populate pf table with %d IPs", len(ips)) + } else { + p.Info().Msgf("Firewall: populated pf table with %d allowed IPs", len(ips)) + } +} + +// firewallApplyExceptionsPlatform mirrors a change to the organization's Allowed +// Destination IP list into the pf exception table, reporting whether pf took it. +// +// An error - including "pf enforcement is not up yet", because the anchor that +// declares the table has not been loaded and pfctl would fail - leaves the +// caller's applied snapshot unadvanced, so the same delta is retried later. +func (p *prog) firewallApplyExceptionsPlatform(added, removed []netip.Prefix) error { + if state, ok := p.platformFirewallState.(*pfFirewallState); !ok || state == nil { + return errors.New("pf firewall enforcement is not initialized") + } + return errors.Join( + p.pfFirewallExceptionTableOp("add", prefixStrings(added)), + p.pfFirewallExceptionTableOp("delete", prefixStrings(removed)), + ) +} + +const ( + // pfExceptionTableOpTimeout bounds one pfctl call against the exception table. + // reconcileDestinations holds destinationsMu across the mirror, and both the + // configuration refresh loop and Firewall Mode teardown contend on that lock, + // so a pfctl that never returns would stall refresh detection of custom_config + // and pin changes along with the teardown itself. Generous enough that a busy + // pf never trips it, short enough that a wedged one is not indefinite. + pfExceptionTableOpTimeout = 30 * time.Second + + // pfExceptionTableOpChunk caps the addresses handed to one pfctl invocation. + // The organization's list is API-supplied and unbounded, and every entry + // becomes an argv element, so a long enough list would exceed ARG_MAX and fail + // as a whole rather than being applied. + pfExceptionTableOpChunk = 500 +) + +// pfFirewallExceptionTableOp runs one pfctl table operation ("add", "delete" or +// "replace") against the exception table. pf table entries are addressed exactly, +// so deleting a network never disturbs a resolved host address inside it. +// +// Long lists are split across invocations. Only the first chunk carries the +// caller's operation: a chunked "replace" would otherwise leave the table holding +// the last chunk alone, each call having discarded what the previous one +// installed, so the chunks after it add to what the replace established. +func (p *prog) pfFirewallExceptionTableOp(op string, entries []string) error { + if len(entries) == 0 { + return nil + } + for _, chunk := range pfExceptionTableChunks(op, entries) { + if err := p.pfFirewallExceptionTableCall(chunk.op, chunk.entries); err != nil { + return err + } + } + return nil +} + +// pfExceptionTableChunk is one pfctl invocation's share of a table operation. +type pfExceptionTableChunk struct { + op string + entries []string +} + +// pfExceptionTableChunks splits a table operation into invocation-sized pieces. +// Only the first piece carries the requested operation; the rest add, so that a +// split "replace" installs the whole set instead of each piece discarding what +// the previous one installed. "add" and "delete" are per-entry operations, so +// splitting them changes nothing. +func pfExceptionTableChunks(op string, entries []string) []pfExceptionTableChunk { + var chunks []pfExceptionTableChunk + for start := 0; start < len(entries); start += pfExceptionTableOpChunk { + end := min(start+pfExceptionTableOpChunk, len(entries)) + chunkOp := op + // Only a replace changes after the first chunk. "add" and "delete" are + // per-entry, and rewriting a later delete chunk as an add would put back + // exactly the destinations the organization withdrew. + if op == "replace" && start > 0 { + chunkOp = "add" + } + chunks = append(chunks, pfExceptionTableChunk{op: chunkOp, entries: entries[start:end]}) + } + return chunks +} + +// pfFirewallExceptionTableCall runs a single pfctl invocation for one chunk. +func (p *prog) pfFirewallExceptionTableCall(op string, entries []string) error { + out, err := pfExceptionTableCommand(op, entries) + if err != nil { + // A delete against a table that does not exist has already achieved what it + // asked for: with no table there is nothing permitting the entry. Treating + // it as a failure would keep the withdrawal pending forever, since no later + // retry can make an absent table deletable. This matches the WFP mirror, + // which tolerates FWP_E_FILTER_NOT_FOUND on delete for the same reason. + if op == "delete" && pfTableMissing(out) { + p.Debug().Int("entries", len(entries)). + Msgf("Firewall: pf table %s does not exist; the allowed destinations it would have held are already not permitted", pfFirewallExceptionTable) + return nil + } + return fmt.Errorf("pfctl -t %s -T %s (%d entries): %w (output: %s)", + pfFirewallExceptionTable, op, len(entries), err, strings.TrimSpace(string(out))) + } + p.Debug().Strs("entries", entries). + Msgf("Firewall: %s %d allowed destinations in pf table %s", pfTableOpPastTense(op), len(entries), pfFirewallExceptionTable) + return nil +} + +// pfExceptionTableCommand runs one pfctl exception-table call under a timeout. +func pfExceptionTableCommand(op string, entries []string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), pfExceptionTableOpTimeout) + defer cancel() + args := append([]string{"-a", pfAnchorName, "-t", pfFirewallExceptionTable, "-T", op}, entries...) + return exec.CommandContext(ctx, "pfctl", args...).CombinedOutput() +} + +// pfTableMissing reports whether pfctl failed because the table is not loaded. +func pfTableMissing(out []byte) bool { + return strings.Contains(strings.ToLower(string(out)), "table does not exist") +} + +// pfTableOpPastTense renders a pfctl table operation for log messages. +func pfTableOpPastTense(op string) string { + switch op { + case "delete": + return "removed" + case "replace": + return "installed" + default: + return "added" + } +} + +// firewallReplaceExceptionsPlatform makes the pf exception table hold exactly +// desired, whatever it held before. +// +// This is what runs when pf enforcement starts, and it must succeed before the +// applied snapshot is established: the table is a persist table that outlives the +// process, so a destination the organization withdrew while ctrld was stopped is +// still in it. Reporting failure is the point - a discarded error here would +// leave that entry bypassing Firewall Mode for the life of the process, with +// nothing pending to say so. +func (p *prog) firewallReplaceExceptionsPlatform(desired []netip.Prefix) error { + if state, ok := p.platformFirewallState.(*pfFirewallState); !ok || state == nil { + return errors.New("pf firewall enforcement is not initialized") + } + entries := prefixStrings(desired) + if len(entries) == 0 { + // pfctl -T replace needs at least one address; emptying is a flush. + out, err := pfExceptionTableCommand("flush", nil) + if err != nil { + if pfTableMissing(out) { + p.Debug().Msgf("Firewall: pf table %s does not exist; nothing is permitted through it", pfFirewallExceptionTable) + return nil + } + return fmt.Errorf("pfctl -t %s -T flush: %w (output: %s)", + pfFirewallExceptionTable, err, strings.TrimSpace(string(out))) + } + p.Debug().Msgf("Firewall: emptied pf table %s", pfFirewallExceptionTable) + return nil + } + return p.pfFirewallExceptionTableOp("replace", entries) +} + +// buildPFFirewallRules generates the pf rules for firewall mode enforcement. +// These rules are appended to the anchor by buildPFAnchorRules() when firewall +// mode is active. +// +// The strategy is: +// - Define table (dynamically populated via pfctl -T add/delete) +// - Block all outbound traffic by default (after DNS intercept rules) +// - Pass outbound to IPs in +// - Pass outbound from ctrld's group (already handled by blanket exemption) +// - Pass loopback, link-local, multicast (already handled by permanent allowlist, +// but explicit pf rules prevent kernel-level blocking before our check) +// +// IMPORTANT: These rules must come AFTER the DNS intercept rules in the anchor +// so that DNS itself still works (DNS is how IPs get into the allowlist). +func buildPFFirewallRules() string { + var rules strings.Builder + + rules.WriteString("\n# --- Firewall Mode: DNS-resolved IP allowlist enforcement ---\n") + rules.WriteString("# Only IPs resolved by ctrld are allowed for outbound connections.\n") + rules.WriteString("# Table is dynamically populated from DNS responses.\n\n") + + // Declare the tables. pfctl -T add/delete operates on these dynamically. + fmt.Fprintf(&rules, "table <%s> persist\n", pfFirewallTable) + fmt.Fprintf(&rules, "table <%s> persist\n\n", pfFirewallExceptionTable) + + // Pass traffic to allowed IPs (both IPv4 and IPv6). + rules.WriteString("# Allow outbound to DNS-resolved IPs.\n") + fmt.Fprintf(&rules, "pass out quick inet proto { tcp, udp } from any to <%s>\n", pfFirewallTable) + fmt.Fprintf(&rules, "pass out quick inet6 proto { tcp, udp } from any to <%s>\n\n", pfFirewallTable) + + // Pass traffic to the organization's allowed destinations. These are reachable + // by literal IP, with no DNS lookup for ctrld to observe, which is the whole + // point of the list; the table is populated from the API's effective set. + rules.WriteString("# Allow outbound to organization allowed destination IPs.\n") + fmt.Fprintf(&rules, "pass out quick inet proto { tcp, udp } from any to <%s>\n", pfFirewallExceptionTable) + fmt.Fprintf(&rules, "pass out quick inet6 proto { tcp, udp } from any to <%s>\n\n", pfFirewallExceptionTable) + + // Allow ICMP/ICMPv6 - needed for path MTU discovery, ping, etc. + rules.WriteString("# Allow ICMP (path MTU discovery, ping, etc.)\n") + rules.WriteString("pass out quick inet proto icmp\n") + rules.WriteString("pass out quick inet6 proto icmp6\n\n") + + // Allow all loopback traffic (safety net - permanent allowlist covers this too). + rules.WriteString("# Allow all loopback traffic.\n") + rules.WriteString("pass out quick on lo0\n") + rules.WriteString("pass in quick on lo0\n\n") + + // Allow RFC1918 and link-local - these are in the permanent allowlist but + // explicit pf rules prevent the block rule below from catching them. + rules.WriteString("# Allow private/link-local ranges (LAN, printers, NAS, mDNS, DHCP).\n") + rules.WriteString("pass out quick inet proto { tcp, udp } from any to 10.0.0.0/8\n") + rules.WriteString("pass out quick inet proto { tcp, udp } from any to 172.16.0.0/12\n") + rules.WriteString("pass out quick inet proto { tcp, udp } from any to 192.168.0.0/16\n") + rules.WriteString("pass out quick inet proto { tcp, udp } from any to 169.254.0.0/16\n") + rules.WriteString("pass out quick inet proto { tcp, udp } from any to 100.64.0.0/10\n") + rules.WriteString("pass out quick inet6 proto { tcp, udp } from any to fe80::/10\n\n") + + // Allow multicast (mDNS, SSDP, etc.). + rules.WriteString("# Allow multicast (mDNS, SSDP, etc.).\n") + rules.WriteString("pass out quick inet proto { tcp, udp } from any to 224.0.0.0/4\n") + rules.WriteString("pass out quick inet6 proto { tcp, udp } from any to ff00::/8\n\n") + + // Allow DHCP (UDP 67/68) - needed for network configuration. + rules.WriteString("# Allow DHCP.\n") + rules.WriteString("pass out quick inet proto udp from any port 68 to any port 67\n\n") + + // Block everything else. This is the enforcement rule. + // "block return" sends TCP RST / ICMP unreachable so apps fail fast instead of timing out. + rules.WriteString("# Block all other outbound traffic (IPs not resolved by ctrld).\n") + rules.WriteString("block return out quick inet proto { tcp, udp } from any to any\n") + rules.WriteString("block return out quick inet6 proto { tcp, udp } from any to any\n") + + return rules.String() +} + +// forwardedSource is a trusted VM/container source subnet. iface is the ingress +// interface for an auto-detected source, used to scope its pf rules to that +// interface so unrelated same-subnet traffic on other interfaces is unaffected. +// iface is empty for an operator-configured source, which matches on the source +// CIDR alone — an explicit opt-in the admin is responsible for. +type forwardedSource struct { + prefix netip.Prefix + iface string +} + +// firewallForwardedSources parses the operator-configured forwarded-workload +// source subnets (service.firewall_forwarded_sources), dropping - with a warning - +// any entry that is not a valid CIDR, or is not IPv4, so one bad line never voids the +// whole set. These augment auto-detection (see forwardedSources) and are the supported +// way to trust generic bridge* stacks (Multipass, Docker Desktop). +// +// The IPv4 restriction is not cosmetic: forwarded DNS can only be redirected to +// ctrld's IPv4 intercept listener, and an IPv6 source would otherwise produce pf rules +// whose address family contradicts their source literal, which makes pfctl reject the +// entire anchor and take DNS interception down with it. +func (p *prog) firewallForwardedSources() []forwardedSource { + sources, rejected := parseForwardedSourceConfig(p.cfg.Service.FirewallForwardedSources) + p.warnRejectedForwardedSources(rejected) + return sources +} + +// rejectedForwardedSource is a configured entry that could not be used, with the +// reason to report to the operator. +type rejectedForwardedSource struct { + value string + reason string +} + +// parseForwardedSourceConfig parses configured entries, returning the usable sources +// and the rejected ones. It is pure and silent: callers decide when a rejection is +// worth logging, because this runs on every anchor build and every watchdog tick. +func parseForwardedSourceConfig(raw []string) ([]forwardedSource, []rejectedForwardedSource) { + if len(raw) == 0 { + return nil, nil + } + out := make([]forwardedSource, 0, len(raw)) + var rejected []rejectedForwardedSource + for _, s := range raw { + pfx, err := netip.ParsePrefix(strings.TrimSpace(s)) + if err != nil { + rejected = append(rejected, rejectedForwardedSource{ + value: s, + reason: "not a valid CIDR (want e.g. 192.168.64.0/24): " + err.Error(), + }) + continue + } + if !pfx.Addr().Is4() { + rejected = append(rejected, rejectedForwardedSource{ + value: s, + reason: "not IPv4 - forwarded-workload DNS interception is IPv4-only (ctrld's intercept listener is IPv4)", + }) + continue + } + out = append(out, forwardedSource{prefix: pfx.Masked()}) + } + return out, rejected +} + +// warnRejectedForwardedSources reports unusable configured entries, but only when the +// set of rejections changes (see forwardedSourceWarnTracker). +func (p *prog) warnRejectedForwardedSources(rejected []rejectedForwardedSource) { + key := rejectedForwardedSourcesKey(rejected) + + forwardedSourceWarnTracker.mu.Lock() + unchanged := forwardedSourceWarnTracker.key == key + forwardedSourceWarnTracker.key = key + forwardedSourceWarnTracker.mu.Unlock() + + if unchanged { + return + } + for _, r := range rejected { + p.Warn().Str("value", r.value). + Msgf("Firewall: ignoring firewall_forwarded_sources entry - %s", r.reason) + } +} + +// rejectedForwardedSourcesKey returns an order-independent signature of a rejection +// set, so re-parsing an unchanged config is recognised as nothing new to report. +func rejectedForwardedSourcesKey(rejected []rejectedForwardedSource) string { + if len(rejected) == 0 { + return "" + } + parts := make([]string, 0, len(rejected)) + for _, r := range rejected { + parts = append(parts, r.value+"|"+r.reason) + } + sort.Strings(parts) + return strings.Join(parts, ",") +} + +// detectForwardedSources auto-discovers VM/NAT networks on the host so the common +// case works with no configuration. +// +// An interface qualifies only if it is up, carries an RFC1918 IPv4 network, AND its +// VM ownership can be proven one of two ways: +// +// - its own name is vendor-specific (hypervisorVMNetPrefixes): Parallels, legacy +// Fusion, VirtualBox host-only; +// - it is a bridge whose member list contains a vendor-specific interface +// (bridgeHasVMMember): every vmnet.framework stack, where the address lives on +// bridge10x and the vendor-named vmenet* member has none. +// +// Ownership proof is the security boundary: a bridge is trusted for what is bridged +// into it, never for its name. The RFC1918 filter is the second boundary - a stack +// presenting a public range is never auto-trusted. Each detected source keeps its +// ingress interface so its pf rules stay scoped to it. +// +// Addresses are checked before membership so the ifconfig call is only made for a +// bridge that could actually qualify; a Thunderbolt bridge with no RFC1918 address +// costs nothing. +func (p *prog) detectForwardedSources() []forwardedSource { + ifaces, err := net.Interfaces() + if err != nil { + p.Warn().Err(err).Msg("Firewall: could not enumerate interfaces for forwarded-source detection") + return nil + } + var out []forwardedSource + for _, ifi := range ifaces { + if ifi.Flags&net.FlagUp == 0 { + continue + } + addrs, err := ifi.Addrs() + if err != nil { + continue + } + sources, reason := forwardedSourcesForIface(ifi.Name, addrs, bridgeMembersFn) + for _, src := range sources { + out = append(out, src) + // Debug, not Info: detection re-runs on every anchor build. The effective + // set is reported once by logForwardedSources at init and on every change. + p.Debug().Str("iface", src.iface).Str("subnet", src.prefix.String()).Str("reason", reason). + Msg("Firewall: auto-detected VM/container network for forwarded DNS") + } + } + return out +} + +// forwardedSourcesForIface decides whether one up interface is an auto-trusted +// forwarded-workload source, and returns its subnets plus why it qualified. +// +// Split out from interface enumeration so the trust decision - which is a security +// boundary - is testable against synthetic interfaces, including the cases that must +// NOT qualify: a Thunderbolt bridge, a public-range VM network, an address-less +// vendor interface. +// +// members is only consulted for a bridge that already has an RFC1918 IPv4 address, so +// no subprocess runs for the address-less or public bridges on a typical host. +func forwardedSourcesForIface(name string, addrs []net.Addr, members func(string) []string) ([]forwardedSource, string) { + vendorNamed := isHypervisorVMNetIface(name) + if !vendorNamed && !isBridgeIface(name) { + return nil, "" + } + prefixes := privateIPv4Prefixes(addrs) + if len(prefixes) == 0 { + return nil, "" + } + reason := "vendor VM interface" + if !vendorNamed { + mem := members(name) + if !bridgeHasVMMember(mem) { + return nil, "" + } + reason = "bridge with VM member " + strings.Join(vmMembers(mem), ",") + } + out := make([]forwardedSource, 0, len(prefixes)) + for _, pfx := range prefixes { + out = append(out, forwardedSource{prefix: pfx, iface: name}) + } + return out, reason +} + +// privateIPv4Prefixes returns the masked RFC1918 IPv4 networks among addrs, skipping +// public and IPv6 addresses. +func privateIPv4Prefixes(addrs []net.Addr) []netip.Prefix { + var out []netip.Prefix + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + pfx, err := netip.ParsePrefix(ipnet.String()) + if err != nil { + continue + } + pfx = pfx.Masked() + if !pfx.Addr().Is4() || !pfx.Addr().IsPrivate() { + continue + } + out = append(out, pfx) + } + return out +} + +// vmMembers returns the vendor VM interfaces among a bridge's members, for logging +// which member made the bridge trusted. +func vmMembers(members []string) []string { + var out []string + for _, m := range members { + if isHypervisorVMNetIface(m) { + out = append(out, m) + } + } + return out +} + +// forwardedSources returns the effective trusted set: auto-detected vendor VM +// networks UNION operator-configured subnets, de-duplicated by prefix (a subnet +// that is both auto-detected and configured keeps the interface-scoped +// auto-detected form). Config augments auto-detection; it never disables it. +func (p *prog) forwardedSources() []forwardedSource { + var out []forwardedSource + seen := make(map[netip.Prefix]struct{}) + for _, group := range [][]forwardedSource{ + p.detectForwardedSources(), + p.firewallForwardedSources(), + } { + for _, src := range group { + if _, dup := seen[src.prefix]; dup { + continue + } + seen[src.prefix] = struct{}{} + out = append(out, src) + } + } + return out +} + +// forwardedSourceSetKey returns a deterministic, order-independent signature of a +// forwarded-source set. reconcileForwardedSources compares it across time to detect +// when VM/container interfaces appear or disappear. +func forwardedSourceSetKey(sources []forwardedSource) string { + parts := make([]string, 0, len(sources)) + for _, s := range sources { + parts = append(parts, s.iface+"|"+s.prefix.String()) + } + sort.Strings(parts) + return strings.Join(parts, ",") +} + +// currentForwardedSources returns the effective forwarded-source set, or nil when +// Firewall Mode is off. +// +// The gate matters because detection is not free: it enumerates interfaces, execs +// ifconfig for each candidate bridge, and re-parses config. Those rules are only ever +// emitted in firewall mode, so doing any of it with firewall mode off is pure waste - +// and it would report unusable config entries on a path where nothing wants them. +// Anchor rebuilds happen on every tunnel change, watchdog restore and VPN DNS update +// regardless of firewall mode, so this is the difference between zero work and work on +// every one of them. +func (p *prog) currentForwardedSources() []forwardedSource { + if !p.firewallModeEnabled() { + return nil + } + return p.forwardedSources() +} + +// buildPFForwardedSourceRulesFor generates the pf rules that make the given +// VM/container (forwarded/NATed) source subnets first-class Firewall Mode clients +// WITHOUT an interface-wide bypass. Returns "" for an empty input so anchor +// behavior is unchanged when there are no sources. +// +// For each source subnet: +// +// - Plaintext DNS (port 53) is force-routed through ctrld's loopback listener +// (route-to lo0, which then hits the existing rdr-on-lo0 redirect). ctrld +// therefore observes and policy-enforces every guest resolution, and the +// resolved IP lands in . The guest's subsequent egress to that +// IP is permitted by the existing "pass out ... to " rule: NAT +// rewrites the guest source to the host, which that rule's "from any" covers. +// - DoT (port 853) to any resolver is blocked so the guest cannot swap in an +// alternate encrypted resolver to escape policy. DoH over 443 is +// indistinguishable from ordinary HTTPS and is a documented limitation. +// +// Auto-detected sources carry their ingress interface and are scoped with +// "on ", so an unrelated interface on the same private range is never +// affected. Configured sources (admin opt-in) match on the source CIDR alone. +// All matches are on the pre-NAT guest source (inbound): after NAT the source is +// the host and could no longer be told apart. This is an explicit, per-subnet +// trust boundary - a direct public IP the guest never resolved through ctrld stays +// blocked, so the guest cannot bypass Control D policy. +// +// IPv4 only, and strictly per address family: ctrld's intercept listener is IPv4, so +// IPv6 guest DNS cannot be redirected here - the anchor's existing +// "block out ... inet6 ... port 53" rule forces guests to fall back to interceptable +// IPv4 DNS, and guest IPv6 egress to an alternate resolver is covered by the blanket +// IPv6 block plus the fact that such a resolver never enters . +// Consequently only IPv4 sources produce rules, and each rule's address family +// matches its source literal: pf rejects a whole anchor over a single +// "inet6 ... from 192.168.x.0/24" mismatch, which would take DNS interception down +// with it. Non-IPv4 sources are skipped here as a backstop; firewallForwardedSources +// already drops them at parse time with a warning. +func buildPFForwardedSourceRulesFor(sources []forwardedSource, listenerIP string) string { + sources = ipv4ForwardedSources(sources) + if len(sources) == 0 { + return "" + } + + var rules strings.Builder + rules.WriteString("\n# --- Firewall Mode: forwarded workload (VM/container) DNS interception ---\n") + rules.WriteString("# VM/container source subnets (auto-detected vendor VM networks + configured).\n") + rules.WriteString("# Their DNS is forced through ctrld so guest resolutions are policy-enforced and\n") + fmt.Fprintf(&rules, "# populate <%s>; egress to allowed IPs is then permitted by the allowlist rule\n", pfFirewallTable) + rules.WriteString("# below. Auto-detected sources are scoped to their ingress interface; this is an\n") + rules.WriteString("# explicit, per-subnet trust boundary - NOT an interface-wide permit.\n\n") + + for _, src := range sources { + cidr := src.prefix.String() + on := "" + label := "configured (source-CIDR scope)" + if src.iface != "" { + on = "on " + src.iface + " " + label = "auto-detected on " + src.iface + } + fmt.Fprintf(&rules, "# %s - %s\n", cidr, label) + // Force guest plaintext DNS (port 53) onto loopback, where the existing + // rdr-on-lo0 rule redirects it to ctrld. Matched inbound (pre-NAT); "quick" + // so it wins over the blanket block that buildPFFirewallRules appends after. + fmt.Fprintf(&rules, "pass in quick %sroute-to lo0 inet proto udp from %s to ! %s port 53\n", on, cidr, listenerIP) + fmt.Fprintf(&rules, "pass in quick %sroute-to lo0 inet proto tcp from %s to ! %s port 53\n", on, cidr, listenerIP) + // Block DoT so the guest cannot escape ctrld via an alternate encrypted + // resolver. DoH over 443 is indistinguishable from HTTPS - documented limitation. + // inet only: the source literal is IPv4, and pf refuses to load an anchor + // containing an inet6 rule with an IPv4 source. + fmt.Fprintf(&rules, "block return in quick %sinet proto { tcp, udp } from %s to any port 853\n\n", on, cidr) + } + + return rules.String() +} + +// ipv4ForwardedSources returns the IPv4 subset of sources. Forwarded-workload DNS +// interception is IPv4-only (the intercept listener is IPv4), and every emitted rule +// must match its source's address family or pfctl rejects the entire anchor. +func ipv4ForwardedSources(sources []forwardedSource) []forwardedSource { + out := make([]forwardedSource, 0, len(sources)) + for _, src := range sources { + if src.prefix.Addr().Is4() { + out = append(out, src) + } + } + return out +} + +// reconcileForwardedSources rebuilds the pf anchor when the effective forwarded- +// source set (auto-detected VM networks + configured) has changed since the last +// build, and drops the pf states of every subnet whose trust changed. +// +// VM/container interfaces appear and disappear at runtime (guest start/stop) and no +// existing path rebuilds an otherwise-intact anchor for that: ensurePFAnchorActive() +// returns early while the rules still exist, checkTunnelInterfaceChanges() tracks +// only tunnel interfaces, and pfInterceptMonitor() rebuilds only after the host +// interception probe fails. Without this, a guest started after ctrld would never be +// trusted, and a stopped guest's subnet would stay trusted until an unrelated +// rebuild. Invoked from the network-change paths and, as a time bound when no event +// fires, from the pf watchdog tick. +// +// A reload failure is not latched: the applied snapshot only advances once pf has +// actually accepted the new anchor, so the next reconcile (at the latest the next +// watchdog tick) retries the same change instead of treating it as done. +func (p *prog) reconcileForwardedSources() { + if !p.firewallModeEnabled() || p.dnsInterceptState == nil { + return + } + state, ok := p.platformFirewallState.(*pfFirewallState) + if !ok || state == nil { + return + } + sources := p.forwardedSources() + reload := func() error { return p.reloadForwardedSourceAnchor(sources) } + gained, lost, changed, err := state.applyForwardedSourceChange(sources, reload) + if !changed { + return + } + if err != nil { + p.Warn().Err(err).Strs("gained_trust", prefixStrings(gained)).Strs("lost_trust", prefixStrings(lost)). + Msg("Firewall: forwarded-source set changed but the pf anchor reload failed — keeping the previously applied set so the next reconcile retries") + return + } + p.Info().Strs("gained_trust", prefixStrings(gained)).Strs("lost_trust", prefixStrings(lost)). + Msg("Firewall: forwarded-source set changed (VM/container start/stop), pf anchor rebuilt") + // Restate the whole effective set, so one log line always answers "what is trusted + // right now" without replaying every earlier transition. + p.logForwardedSources(sources) + // Rules only govern new states, so kill the states of the affected subnets: + // a subnet that lost trust must stop using states created while it was trusted, + // and one that just gained it must have its pre-existing (un-intercepted) DNS + // states re-evaluated instead of running until they expire. The guest simply + // re-establishes the connections under the new rules. Only reached after a + // successful load - killing states against the old anchor would just have them + // recreated under the very rules the change was meant to replace. + p.killForwardedSourceStates(append(gained, lost...)) +} + +// applyForwardedSourceChange performs one reconcile step: compare cur against the +// last applied forwarded-source set and, when it differs, install it via reload. +// It reports which subnets gained and lost trust (trust identity is (subnet, +// interface scope), so a subnet that stays but changes scope appears in both lists), +// whether there was anything to do at all, and reload's error. +// +// The applied snapshot advances ONLY after reload returns nil. A failed write or +// pfctl load therefore leaves the previous set recorded, so the running anchor and +// the snapshot cannot diverge and the change is retried on the next reconcile rather +// than silently dropped. The whole check-reload-record sequence is serialized so +// concurrent callers (watchdog tick vs. network change) cannot both rebuild or +// interleave their snapshot updates. +// +// Split out from reconcileForwardedSources so the guest start/stop lifecycle and the +// failure-then-retry path are deterministically testable without pf or a hypervisor. +func (s *pfFirewallState) applyForwardedSourceChange(cur []forwardedSource, reload func() error) (gained, lost []netip.Prefix, changed bool, err error) { + s.applyForwardedMu.Lock() + defer s.applyForwardedMu.Unlock() + + key := forwardedSourceSetKey(cur) + s.mu.Lock() + applied, appliedKey := s.lastForwardedSources, s.lastForwardedKey + s.mu.Unlock() + + if key == appliedKey { + return nil, nil, false, nil + } + gained = forwardedSubnetsNotIn(cur, applied) + lost = forwardedSubnetsNotIn(applied, cur) + + if err := reload(); err != nil { + return gained, lost, true, err + } + + s.mu.Lock() + s.lastForwardedSources = cur + s.lastForwardedKey = key + s.mu.Unlock() + return gained, lost, true, nil +} + +// forwardedSubnetsNotIn returns the subnets of a whose exact trust entry (subnet + +// interface scope) is absent from b, de-duplicated. +func forwardedSubnetsNotIn(a, b []forwardedSource) []netip.Prefix { + inB := make(map[forwardedSource]struct{}, len(b)) + for _, src := range b { + inB[src] = struct{}{} + } + var out []netip.Prefix + seen := make(map[netip.Prefix]struct{}, len(a)) + for _, src := range a { + if _, ok := inB[src]; ok { + continue + } + if _, dup := seen[src.prefix]; dup { + continue + } + seen[src.prefix] = struct{}{} + out = append(out, src.prefix) + } + return out +} + +// forwardedSourceDescriptions renders the effective trust set for logging, naming each +// subnet's origin so an admin can tell an auto-detected VM network (and the interface +// its rules are scoped to) from an entry they configured. +func forwardedSourceDescriptions(sources []forwardedSource) []string { + out := make([]string, 0, len(sources)) + for _, src := range sources { + if src.iface != "" { + out = append(out, src.prefix.String()+" (auto-detected on "+src.iface+")") + continue + } + out = append(out, src.prefix.String()+" (configured)") + } + return out +} + +// logForwardedSources reports the effective forwarded-workload trust set. Configured +// entries were previously invisible in the log - only auto-detection said anything - +// so an admin had no way to confirm firewall_forwarded_sources took effect. The empty +// case is logged too, with what to do about it, because "no guest DNS interception" +// looks identical to "feature silently did nothing". +func (p *prog) logForwardedSources(sources []forwardedSource) { + if len(sources) == 0 { + p.Info().Msg("Firewall: no forwarded-workload (VM/container) sources — guest DNS is not intercepted. " + + "Auto-detection needs an up interface with an RFC1918 IPv4 address that is either vendor-named " + + "(vnic*, vboxnet*, vmnet*) or a bridge with a VM member (vmenet*); anything else must be listed " + + "in service.firewall_forwarded_sources") + return + } + p.Info().Int("count", len(sources)).Strs("sources", forwardedSourceDescriptions(sources)). + Msg("Firewall: forwarded-workload (VM/container) DNS interception active for these source subnets") +} + +// killForwardedSourceStates drops the pf state entries sourced from the given +// subnets. Targeted (pfctl -k ) rather than a global state flush, so a +// guest starting or stopping never resets unrelated host connections. +func (p *prog) killForwardedSourceStates(prefixes []netip.Prefix) { + for _, pfx := range prefixes { + out, err := exec.Command("pfctl", "-k", pfx.String()).CombinedOutput() + if err != nil { + // Not a hard error - most often there simply are no matching states. + p.Debug().Err(err).Str("subnet", pfx.String()).Str("output", strings.TrimSpace(string(out))). + Msg("Firewall: could not kill pf states for changed forwarded source") + continue + } + p.Info().Str("subnet", pfx.String()). + Msg("Firewall: killed pf states for changed forwarded source") + } +} + +// recordAppliedForwardedSources records sources as the set pf is now enforcing. +// +// Called by the rebuild paths that are not the forwarded-source reconcile itself +// (tunnel change, watchdog restore, VPN DNS exemptions, forced reload, startup): each +// of those installs a full anchor that already contains the current set, so without +// this the next reconcile would compare against a stale snapshot and redo the work. +// +// Takes applyForwardedMu so a record cannot land in the middle of a reconcile's +// compare-reload-record sequence and be overwritten by it, or overwrite it. +func (p *prog) recordAppliedForwardedSources(sources []forwardedSource) { + state, ok := p.platformFirewallState.(*pfFirewallState) + if !ok || state == nil { + return + } + state.applyForwardedMu.Lock() + defer state.applyForwardedMu.Unlock() + + state.mu.Lock() + state.lastForwardedSources = sources + state.lastForwardedKey = forwardedSourceSetKey(sources) + state.mu.Unlock() +} + +// reloadForwardedSourceAnchor rebuilds and reloads the ctrld pf anchor so the given +// forwarded-source rules take effect. Mirrors the anchor reload used by the intercept +// watchdog, but reports failure to the caller: whether pf actually accepted the anchor +// decides whether the new source set may be recorded as applied. +// +// The set is passed in rather than re-detected, so what gets loaded is exactly what +// the caller compared and will record. Re-detecting here could install a set that +// differs from the recorded snapshot if an interface appeared in between. +func (p *prog) reloadForwardedSourceAnchor(sources []forwardedSource) error { + var vpnExemptions []vpnDNSExemption + if p.vpnDNS != nil { + vpnExemptions = p.vpnDNS.CurrentExemptions() + } + rulesStr := p.buildPFAnchorRulesWith(vpnExemptions, sources) + if err := writePFAnchorFile(rulesStr); err != nil { + return fmt.Errorf("write pf anchor %s: %w", pfAnchorFile, err) + } + if out, err := exec.Command("pfctl", "-a", pfAnchorName, "-f", pfAnchorFile).CombinedOutput(); err != nil { + return fmt.Errorf("load pf anchor %s: %w (output: %s)", pfAnchorName, err, strings.TrimSpace(string(out))) + } + return nil +} diff --git a/cmd/cli/firewall_darwin_bridge_test.go b/cmd/cli/firewall_darwin_bridge_test.go new file mode 100644 index 00000000..3c78436f --- /dev/null +++ b/cmd/cli/firewall_darwin_bridge_test.go @@ -0,0 +1,416 @@ +//go:build darwin + +package cli + +import ( + "net" + "net/netip" + "testing" + + "github.com/Control-D-Inc/ctrld/internal/firewall" +) + +// vmnetBridgeIfconfig is ifconfig output for a vmnet.framework bridge: the RFC1918 +// gateway lives here, and the vendor-named vmenet0 is an address-less member. This is +// the shape that name-only detection could never see. +const vmnetBridgeIfconfig = `bridge100: flags=8a63 mtu 1500 + options=3 + ether 5e:cf:7f:9a:1b:64 + inet 192.168.64.1 netmask 0xffffff00 broadcast 192.168.64.255 + Configuration: + id 0:0:0:0:0:0 priority 0 hellotime 0 fwddelay 0 + maxage 0 holdcnt 0 proto stp maxaddr 100 timeout 1200 + root id 0:0:0:0:0:0 priority 0 ifcost 0 port 0 + ipfilter disabled flags 0x0 + member: vmenet0 flags=3 + ifmaxaddr 0 port 22 priority 0 path cost 0 + nd6 options=201 + media: + status: active +` + +// thunderboltBridgeIfconfig is ifconfig output for the Thunderbolt bridge macOS +// creates by default. It can carry an RFC1918 address, and its members are physical +// interfaces - trusting it would force-route unrelated same-subnet traffic. +const thunderboltBridgeIfconfig = `bridge0: flags=8863 mtu 1500 + options=63 + ether 36:12:8a:1f:2b:00 + inet 192.168.10.5 netmask 0xffffff00 broadcast 192.168.10.255 + Configuration: + id 0:0:0:0:0:0 priority 0 hellotime 0 fwddelay 0 + member: en1 flags=3 + ifmaxaddr 0 port 9 priority 0 path cost 0 + member: en2 flags=3 + ifmaxaddr 0 port 10 priority 0 path cost 0 + nd6 options=201 + media: + status: inactive +` + +func TestParseBridgeMembers(t *testing.T) { + tests := []struct { + name string + out string + want []string + }{ + {"vmnet.framework bridge", vmnetBridgeIfconfig, []string{"vmenet0"}}, + {"thunderbolt bridge", thunderboltBridgeIfconfig, []string{"en1", "en2"}}, + {"no members", "bridge2: flags=8822 mtu 1500\n\tether 1a:2b:3c\n", nil}, + {"empty output", "", nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parseBridgeMembers(tc.out) + if len(got) != len(tc.want) { + t.Fatalf("parseBridgeMembers() = %v, want %v", got, tc.want) + } + for i := range got { + if got[i] != tc.want[i] { + t.Errorf("member[%d] = %q, want %q", i, got[i], tc.want[i]) + } + } + }) + } +} + +func TestBridgeHasVMMember(t *testing.T) { + tests := []struct { + name string + members []string + want bool + }{ + {"vmnet.framework member", []string{"vmenet0"}, true}, + {"parallels member", []string{"vnic0"}, true}, + {"mixed with vm member", []string{"en1", "vmenet2"}, true}, + {"physical members only", []string{"en1", "en2"}, false}, + {"no members", nil, false}, + // A name that merely looks bridge-ish proves nothing. + {"bridge member", []string{"bridge1"}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := bridgeHasVMMember(tc.members); got != tc.want { + t.Errorf("bridgeHasVMMember(%v) = %v, want %v", tc.members, got, tc.want) + } + }) + } +} + +func mustAddrs(t *testing.T, cidrs ...string) []net.Addr { + t.Helper() + var out []net.Addr + for _, c := range cidrs { + ip, ipnet, err := net.ParseCIDR(c) + if err != nil { + t.Fatalf("bad test CIDR %q: %v", c, err) + } + out = append(out, &net.IPNet{IP: ip, Mask: ipnet.Mask}) + } + return out +} + +// TestForwardedSourcesForIface covers the trust decision, including every case that +// must NOT be auto-trusted. This is a security boundary: anything that qualifies here +// gets its guest traffic passed to allowed public destinations. +func TestForwardedSourcesForIface(t *testing.T) { + tests := []struct { + name string + iface string + addrs []string + members []string + wantCIDR []string + }{ + { + // The case name-only detection missed: address on the bridge, vendor + // interface bridged into it. This is UTM/Docker/Multipass/Fusion 12.1+. + name: "bridge with vmenet member is trusted", + iface: "bridge100", + addrs: []string{"192.168.64.1/24"}, + members: []string{"vmenet0"}, + wantCIDR: []string{"192.168.64.0/24"}, + }, + { + // The reason membership is required rather than the bridge name. + name: "thunderbolt bridge is not trusted", + iface: "bridge0", + addrs: []string{"192.168.10.5/24"}, + members: []string{"en1", "en2"}, + }, + { + name: "vendor-named interface with its own address is trusted", + iface: "vnic0", + addrs: []string{"10.211.55.2/24"}, + wantCIDR: []string{"10.211.55.0/24"}, + }, + { + // vmenet* under vmnet.framework: up, but no address of its own. + name: "address-less vendor interface yields nothing", + iface: "vmenet0", + addrs: nil, + }, + { + // The RFC1918 boundary: a VM network on a public range is never trusted. + name: "public range on a VM bridge is not trusted", + iface: "bridge100", + addrs: []string{"93.184.216.34/24"}, + members: []string{"vmenet0"}, + }, + { + // Interception is IPv4-only; an IPv6-only VM bridge must not qualify. + name: "ipv6 only is not trusted", + iface: "bridge100", + addrs: []string{"fd00::1/64"}, + members: []string{"vmenet0"}, + }, + { + name: "physical uplink is not trusted", + iface: "en0", + addrs: []string{"192.168.1.20/24"}, + }, + { + name: "vpn tunnel is not trusted", + iface: "utun4", + addrs: []string{"10.2.0.2/24"}, + }, + { + name: "mixed addresses keep only the private ipv4 one", + iface: "bridge101", + addrs: []string{"fd00::1/64", "192.168.105.1/24"}, + members: []string{"vmenet1"}, + wantCIDR: []string{"192.168.105.0/24"}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + members := func(string) []string { return tc.members } + got, reason := forwardedSourcesForIface(tc.iface, mustAddrs(t, tc.addrs...), members) + if len(got) != len(tc.wantCIDR) { + t.Fatalf("got %d sources %v, want %d %v", len(got), got, len(tc.wantCIDR), tc.wantCIDR) + } + for i, want := range tc.wantCIDR { + if got[i].prefix != netip.MustParsePrefix(want) { + t.Errorf("prefix[%d] = %s, want %s", i, got[i].prefix, want) + } + // Rules must be scoped to the interface traffic actually arrives on. + if got[i].iface != tc.iface { + t.Errorf("source %s scoped to %q, want %q", got[i].prefix, got[i].iface, tc.iface) + } + } + if len(got) > 0 && reason == "" { + t.Error("a trusted source must report why it qualified") + } + }) + } +} + +// TestForwardedSourcesForIface_NoMemberLookupWithoutAddress verifies the ifconfig call +// is skipped for a bridge that cannot qualify anyway. Detection runs on every anchor +// build and every watchdog tick, so this keeps a typical host at zero subprocesses. +func TestForwardedSourcesForIface_NoMemberLookupWithoutAddress(t *testing.T) { + called := false + members := func(string) []string { + called = true + return []string{"vmenet0"} + } + + if got, _ := forwardedSourcesForIface("bridge100", nil, members); got != nil { + t.Errorf("address-less bridge must yield nothing, got %v", got) + } + if called { + t.Error("member list must not be queried for a bridge with no RFC1918 address") + } + + // A public-range bridge is equally hopeless, and equally must not exec. + called = false + if got, _ := forwardedSourcesForIface("bridge100", mustAddrs(t, "93.184.216.34/24"), members); got != nil { + t.Errorf("public-range bridge must yield nothing, got %v", got) + } + if called { + t.Error("member list must not be queried for a bridge with no RFC1918 address") + } +} + +// TestParseForwardedSourceConfig covers the parse/reject split: usable entries survive +// alongside bad ones, and each rejection carries a reason to report. +func TestParseForwardedSourceConfig(t *testing.T) { + sources, rejected := parseForwardedSourceConfig([]string{ + "192.168.64.7/24", // host bits get normalized + "not-a-cidr", // malformed + "fd00::/64", // not IPv4 + " 10.0.0.0/8 ", // surrounding space tolerated + }) + + wantSources := map[string]bool{"192.168.64.0/24": true, "10.0.0.0/8": true} + if len(sources) != len(wantSources) { + t.Fatalf("got %d usable sources %v, want %d", len(sources), sources, len(wantSources)) + } + for _, src := range sources { + if !wantSources[src.prefix.String()] { + t.Errorf("unexpected usable prefix %s", src.prefix) + } + if src.iface != "" { + t.Errorf("configured source %s must have no interface scope, got %q", src.prefix, src.iface) + } + } + + if len(rejected) != 2 { + t.Fatalf("got %d rejections %v, want 2", len(rejected), rejected) + } + for _, r := range rejected { + if r.value != "not-a-cidr" && r.value != "fd00::/64" { + t.Errorf("unexpected rejected value %q", r.value) + } + if r.reason == "" { + t.Errorf("rejection of %q carries no reason", r.value) + } + } +} + +func TestParseForwardedSourceConfig_Empty(t *testing.T) { + sources, rejected := parseForwardedSourceConfig(nil) + if sources != nil || rejected != nil { + t.Errorf("empty config must yield nothing, got %v / %v", sources, rejected) + } +} + +// TestRejectedForwardedSourcesKey verifies the signature ignores order, so re-parsing +// an unchanged config is recognised as nothing new, while a changed set is not. +func TestRejectedForwardedSourcesKey(t *testing.T) { + a := []rejectedForwardedSource{{value: "x", reason: "r1"}, {value: "y", reason: "r2"}} + b := []rejectedForwardedSource{{value: "y", reason: "r2"}, {value: "x", reason: "r1"}} + if rejectedForwardedSourcesKey(a) != rejectedForwardedSourcesKey(b) { + t.Error("key must be order-independent") + } + if rejectedForwardedSourcesKey(nil) != "" { + t.Error("no rejections must produce an empty key") + } + c := []rejectedForwardedSource{{value: "x", reason: "r1"}} + if rejectedForwardedSourcesKey(a) == rejectedForwardedSourcesKey(c) { + t.Error("different rejection sets must produce different keys") + } +} + +// resetForwardedSourceWarnTracker clears the process-wide warning dedupe so each test +// starts from "nothing reported yet". +func resetForwardedSourceWarnTracker(t *testing.T) { + t.Helper() + forwardedSourceWarnTracker.mu.Lock() + forwardedSourceWarnTracker.key = "" + forwardedSourceWarnTracker.mu.Unlock() +} + +func trackedRejectionKey() string { + forwardedSourceWarnTracker.mu.Lock() + defer forwardedSourceWarnTracker.mu.Unlock() + return forwardedSourceWarnTracker.key +} + +// TestWarnRejectedForwardedSources_OnlyOnChange verifies a standing bad entry is +// reported once rather than on every watchdog tick, and that a newly-introduced one is +// still reported after a config reload. +// +// The dedupe must not depend on pfFirewallState: no state is installed here, matching +// the window where Firewall Mode is on but pf enforcement is still deferred until +// intercept mode starts. +func TestWarnRejectedForwardedSources_OnlyOnChange(t *testing.T) { + resetForwardedSourceWarnTracker(t) + p := progWithForwardedSources("not-a-cidr", "192.168.64.0/24") + + // First parse reports; the signature is now recorded. + p.firewallForwardedSources() + first := trackedRejectionKey() + if first == "" { + t.Fatal("a rejected entry must be recorded as reported") + } + + // Re-parsing the same config (every anchor build, every 30s tick) must not change + // what is recorded - that is what stops the repeated warning. + for i := 0; i < 5; i++ { + p.firewallForwardedSources() + } + if got := trackedRejectionKey(); got != first { + t.Errorf("recorded rejection key changed on re-parse: %q -> %q", first, got) + } + + // A config reload that introduces a different bad entry must be reported. + p.cfg.Service.FirewallForwardedSources = []string{"also-not-a-cidr"} + p.firewallForwardedSources() + if trackedRejectionKey() == first { + t.Error("a newly-introduced bad entry must be reported, not suppressed") + } + + // Fixing the config clears the recorded set, so a later regression reports again. + p.cfg.Service.FirewallForwardedSources = []string{"192.168.64.0/24"} + p.firewallForwardedSources() + if got := trackedRejectionKey(); got != "" { + t.Errorf("a clean config must clear the recorded rejections, got %q", got) + } +} + +// TestCurrentForwardedSources_GatedOnFirewallMode verifies no detection or config +// parsing happens with Firewall Mode off. Anchor rebuilds run on tunnel changes, +// watchdog restores and VPN DNS updates regardless of firewall mode, so an ungated +// call would enumerate interfaces, exec ifconfig and re-report bad config entries on +// every one of them. +func TestCurrentForwardedSources_GatedOnFirewallMode(t *testing.T) { + resetForwardedSourceWarnTracker(t) + p := progWithForwardedSources("not-a-cidr", "192.168.64.0/24") + + // Firewall mode off: nothing detected, and the bad entry is not even looked at. + if got := p.currentForwardedSources(); got != nil { + t.Errorf("firewall mode off must yield no sources, got %v", got) + } + if got := trackedRejectionKey(); got != "" { + t.Errorf("config must not be parsed with firewall mode off, but a rejection was recorded: %q", got) + } + + // With firewall mode on (allowList present), the configured entry is honoured and + // the bad one reported. + p.allowList = firewall.New() + got := p.currentForwardedSources() + if len(got) != 1 || got[0].prefix != netip.MustParsePrefix("192.168.64.0/24") { + t.Errorf("firewall mode on must yield the configured source, got %v", got) + } + if trackedRejectionKey() == "" { + t.Error("the unusable entry must be reported once firewall mode is on") + } +} + +// TestRecordAppliedForwardedSources verifies a full-anchor rebuild can baseline the +// reconcile snapshot, so the next reconcile does not redo the same change. +func TestRecordAppliedForwardedSources(t *testing.T) { + p := progWithForwardedSources() + state := &pfFirewallState{} + p.platformFirewallState = state + + sources := []forwardedSource{{prefix: netip.MustParsePrefix("192.168.64.0/24"), iface: "bridge100"}} + p.recordAppliedForwardedSources(sources) + + if state.lastForwardedKey != forwardedSourceSetKey(sources) { + t.Errorf("snapshot key = %q, want %q", state.lastForwardedKey, forwardedSourceSetKey(sources)) + } + if len(state.lastForwardedSources) != 1 || state.lastForwardedSources[0] != sources[0] { + t.Errorf("snapshot sources = %v, want %v", state.lastForwardedSources, sources) + } + + // A reconcile against the same set must now find nothing to do: no second rebuild, + // no killed states, no transition logged for something already in effect. + reloads := 0 + _, _, changed, err := state.applyForwardedSourceChange(sources, func() error { reloads++; return nil }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if changed { + t.Error("reconcile treated an already-applied set as a change") + } + if reloads != 0 { + t.Errorf("anchor was rebuilt %d times for an unchanged set, want 0", reloads) + } +} + +// TestRecordAppliedForwardedSources_NoState verifies recording is a no-op when +// firewall mode is off, since the rebuild paths call it unconditionally. +func TestRecordAppliedForwardedSources_NoState(t *testing.T) { + p := progWithForwardedSources() + p.recordAppliedForwardedSources([]forwardedSource{{prefix: netip.MustParsePrefix("10.0.0.0/8")}}) +} diff --git a/cmd/cli/firewall_darwin_test.go b/cmd/cli/firewall_darwin_test.go new file mode 100644 index 00000000..cc754596 --- /dev/null +++ b/cmd/cli/firewall_darwin_test.go @@ -0,0 +1,653 @@ +//go:build darwin + +package cli + +import ( + "errors" + "net/netip" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/firewall" +) + +// progWithForwardedSources builds a prog whose config declares the given +// forwarded-workload source subnets. A logger is attached so the invalid-entry +// warning path is safe to exercise. +func progWithForwardedSources(sources ...string) *prog { + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}, + }} + p.cfg.Service.FirewallForwardedSources = sources + p.logger.Store(mainLog.Load()) + return p +} + +// TestIsHypervisorVMNetIface verifies only vendor-specific VM/NAT interfaces +// qualify for auto-detection - never generic bridges (bridge*, which macOS also +// uses for Thunderbolt/aggregated links), physical uplinks, loopback, or VPN tunnels. +func TestIsHypervisorVMNetIface(t *testing.T) { + tests := []struct { + name string + want bool + }{ + {"vmnet8", true}, // VMware Fusion + {"vmenet0", true}, // Apple Virtualization.framework / UTM + {"vnic0", true}, // Parallels + {"vboxnet0", true}, // VirtualBox + {"bridge0", false}, // generic bridge (Thunderbolt/aggregated) - NOT auto-trusted + {"bridge100", false}, // Multipass/Docker generic bridge - opt-in only + {"en0", false}, // physical uplink + {"lo0", false}, // loopback + {"utun3", false}, // VPN tunnel + {"awdl0", false}, // Apple Wireless Direct Link + } + for _, tt := range tests { + if got := isHypervisorVMNetIface(tt.name); got != tt.want { + t.Errorf("isHypervisorVMNetIface(%q) = %v, want %v", tt.name, got, tt.want) + } + } +} + +// TestBuildPFForwardedSourceRulesFor_Basic verifies the rules emit, for each source: +// a route-to-lo0 redirect of the guest's plaintext DNS (udp+tcp) and a DoT block; +// that auto-detected sources are scoped "on " while configured sources match +// on CIDR alone; and crucially NOT an interface-wide or destination-wide permit that +// would let the guest bypass policy. +func TestBuildPFForwardedSourceRulesFor_Basic(t *testing.T) { + sources := []forwardedSource{ + {prefix: netip.MustParsePrefix("192.168.105.0/24"), iface: "vmnet8"}, // auto-detected + {prefix: netip.MustParsePrefix("10.211.55.0/24")}, // configured (no iface) + } + rules := buildPFForwardedSourceRulesFor(sources, "127.0.0.1") + + wants := []string{ + // Auto-detected: scoped to its ingress interface. + "pass in quick on vmnet8 route-to lo0 inet proto udp from 192.168.105.0/24 to ! 127.0.0.1 port 53", + "pass in quick on vmnet8 route-to lo0 inet proto tcp from 192.168.105.0/24 to ! 127.0.0.1 port 53", + "block return in quick on vmnet8 inet proto { tcp, udp } from 192.168.105.0/24 to any port 853", + // Configured: CIDR-only (admin opt-in), no "on ". + "pass in quick route-to lo0 inet proto udp from 10.211.55.0/24 to ! 127.0.0.1 port 53", + "block return in quick inet proto { tcp, udp } from 10.211.55.0/24 to any port 853", + } + for _, w := range wants { + if !strings.Contains(rules, w) { + t.Errorf("missing rule:\n %s\nin:\n%s", w, rules) + } + } + + // Address family must match the source literal. An "inet6 ... from " + // rule makes pfctl reject the whole anchor, which would take DNS interception + // down with it, so no inet6 rule may name an IPv4 source. + for _, line := range strings.Split(rules, "\n") { + if strings.Contains(line, "inet6") && (strings.Contains(line, "192.168.105.0/24") || strings.Contains(line, "10.211.55.0/24")) { + t.Errorf("inet6 rule with an IPv4 source - pf address-family mismatch:\n %s", line) + } + } + + // Security boundary: every "pass" for a source must be the port-53 redirect; + // no rule may grant a source an unrestricted destination. + for _, line := range strings.Split(rules, "\n") { + if strings.HasPrefix(line, "pass") && !strings.Contains(line, "port 53") { + t.Errorf("forwarded-source pass rule is not scoped to DNS - possible policy bypass:\n %s", line) + } + if strings.HasPrefix(line, "pass") && strings.HasSuffix(strings.TrimSpace(line), "to any") { + t.Errorf("forwarded-source rules must not contain a blanket 'to any' permit:\n %s", line) + } + } +} + +// TestBuildPFForwardedSourceRulesFor_Empty verifies that with no sources the +// builder returns "", leaving anchor behavior unchanged. +func TestBuildPFForwardedSourceRulesFor_Empty(t *testing.T) { + if got := buildPFForwardedSourceRulesFor(nil, "127.0.0.1"); got != "" { + t.Errorf("expected empty output with no sources, got:\n%s", got) + } +} + +// TestFirewallForwardedSources_InvalidDropped verifies a malformed CIDR and a +// non-IPv4 CIDR are dropped (with a warning) without voiding the valid config +// entries, that host bits are normalized to the network address, and that configured +// sources carry no interface. Rejecting IPv6 at parse time is what keeps the emitted +// rules single-family: a mixed-family rule makes pfctl reject the whole anchor. +func TestFirewallForwardedSources_InvalidDropped(t *testing.T) { + p := progWithForwardedSources("192.168.64.7/24", "not-a-cidr", "fd00::/64", "10.0.0.0/8") + got := p.firewallForwardedSources() + + want := map[string]bool{"192.168.64.0/24": true, "10.0.0.0/8": true} + if len(got) != len(want) { + t.Fatalf("got %d sources, want %d: %v", len(got), len(want), got) + } + for _, src := range got { + if !want[src.prefix.String()] { + t.Errorf("unexpected prefix %s (invalid entries should be dropped)", src.prefix) + } + if src.iface != "" { + t.Errorf("configured source %s must have no interface, got %q", src.prefix, src.iface) + } + if !src.prefix.Addr().Is4() { + t.Errorf("non-IPv4 source %s must be dropped (interception is IPv4-only)", src.prefix) + } + } +} + +// TestBuildPFForwardedSourceRulesFor_SkipsNonIPv4 verifies an IPv6 source produces no +// rules even if one reaches the builder, so a stray entry can never introduce a +// mixed-family rule that pfctl would reject the whole anchor over. +func TestBuildPFForwardedSourceRulesFor_SkipsNonIPv4(t *testing.T) { + v6 := forwardedSource{prefix: netip.MustParsePrefix("fd00::/64")} + if got := buildPFForwardedSourceRulesFor([]forwardedSource{v6}, "127.0.0.1"); got != "" { + t.Errorf("IPv6-only source must produce no rules, got:\n%s", got) + } + + v4 := forwardedSource{prefix: netip.MustParsePrefix("192.168.64.0/24"), iface: "vmnet8"} + rules := buildPFForwardedSourceRulesFor([]forwardedSource{v6, v4}, "127.0.0.1") + if strings.Contains(rules, "fd00::") { + t.Errorf("IPv6 source must be skipped in a mixed set:\n%s", rules) + } + if !strings.Contains(rules, "from 192.168.64.0/24 to ! 127.0.0.1 port 53") { + t.Errorf("IPv4 source must still produce its rules:\n%s", rules) + } + for _, line := range strings.Split(rules, "\n") { + if strings.HasPrefix(line, "pass") || strings.HasPrefix(line, "block") { + if strings.Contains(line, "inet6") { + t.Errorf("no inet6 rule may be emitted for IPv4-only sources:\n %s", line) + } + } + } +} + +// TestPFForwardedSourceRules_Syntax runs the real pf parser over the generated rules. +// This is the check string assertions cannot make: pfctl rejects an ENTIRE ruleset +// over one malformed or mixed-address-family rule, so a bad forwarded-source rule +// would take DNS interception down with it rather than just failing to trust a guest. +// +// Two rulesets are parsed: the forwarded-source rules alone (self-contained, so this +// arm is environment-independent) and the full anchor ctrld would load. +func TestPFForwardedSourceRules_Syntax(t *testing.T) { + // lo0 as the auto-detected ingress interface: any interface name parses, and lo0 + // is the one guaranteed to exist on every runner. + sources := []forwardedSource{ + {prefix: netip.MustParsePrefix("192.168.105.0/24"), iface: "lo0"}, // auto-detected, scoped + {prefix: netip.MustParsePrefix("10.211.55.0/24")}, // configured, CIDR-only + } + + t.Run("forwarded rules alone", func(t *testing.T) { + rules := buildPFForwardedSourceRulesFor(sources, "127.0.0.1") + if rules == "" { + t.Fatal("no forwarded-source rules generated") + } + pfctlParseCheck(t, rules) + }) + + t.Run("full anchor", func(t *testing.T) { + p := progWithForwardedSources("192.168.64.0/24", "10.211.55.0/24") + p.allowList = firewall.New() + rules := p.buildPFAnchorRules(nil) + if !strings.Contains(rules, "from 192.168.64.0/24 to ! ") { + t.Fatalf("forwarded-source rules missing from anchor under test:\n%s", rules) + } + pfctlParseCheck(t, stripPFGroupRules(rules)) + }) +} + +// pfctlParseCheck validates a ruleset with the real pf parser in ctrld's anchor +// context, failing the test on any parse error. pfctl needs /dev/pf, so the check +// skips (rather than fails) where the runner cannot open it. +func pfctlParseCheck(t *testing.T, ruleset string) { + t.Helper() + + pfctl, err := exec.LookPath("pfctl") + if err != nil { + t.Skip("pfctl not available:", err) + } + file := filepath.Join(t.TempDir(), "ctrld-rules-test.conf") + if err := os.WriteFile(file, []byte(ruleset), 0600); err != nil { + t.Fatalf("write ruleset under test: %v", err) + } + + // -n parses and validates without loading anything. + out, err := exec.Command(pfctl, "-a", pfAnchorName, "-n", "-f", file).CombinedOutput() + if err == nil { + return + } + msg := strings.TrimSpace(string(out)) + if strings.Contains(msg, "Permission denied") || strings.Contains(msg, "Operation not permitted") || + strings.Contains(msg, "/dev/pf") { + t.Skipf("pfctl cannot open /dev/pf on this runner (%v): %s", err, msg) + } + t.Errorf("pfctl rejected the generated ruleset (%v):\n%s\n--- ruleset ---\n%s", err, msg, ruleset) +} + +// stripPFGroupRules drops rules scoped to ctrld's runtime group. That group is created +// by the installed service (dscl), so on a dev box or CI runner pfctl reports "unknown +// group _ctrld" for them - an environment fact, not a defect in the generated rules. +// Only those lines are removed, so pf's ordering requirement (translation rules before +// filtering rules) still holds for what remains. +func stripPFGroupRules(ruleset string) string { + lines := strings.Split(ruleset, "\n") + kept := make([]string, 0, len(lines)) + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "#") && strings.Contains(trimmed, "group "+pfGroupName) { + continue + } + kept = append(kept, line) + } + return strings.Join(kept, "\n") +} + +// TestForwardedSources_UnionDedup verifies the effective set unions auto-detected +// and configured subnets, de-duplicated by prefix. Auto-detection is +// environment-dependent, so this asserts config entries are always included and +// that duplicate config entries collapse to one - deterministic regardless of host. +func TestForwardedSources_UnionDedup(t *testing.T) { + p := progWithForwardedSources("192.168.199.0/24", "192.168.199.0/24") + got := p.forwardedSources() + + count := 0 + for _, src := range got { + if src.prefix.String() == "192.168.199.0/24" { + count++ + } + } + if count != 1 { + t.Errorf("configured subnet appears %d times, want exactly 1 (union must dedup):\n%v", count, got) + } +} + +// TestForwardedSourceDescriptions verifies the log rendering names each subnet's +// origin, including the interface an auto-detected source is scoped to. Configured +// entries used to be invisible in the log, which left admins unable to confirm +// firewall_forwarded_sources took effect. +func TestForwardedSourceDescriptions(t *testing.T) { + got := forwardedSourceDescriptions([]forwardedSource{ + {prefix: netip.MustParsePrefix("192.168.105.0/24"), iface: "vmenet0"}, + {prefix: netip.MustParsePrefix("192.168.252.0/24")}, + }) + want := []string{ + "192.168.105.0/24 (auto-detected on vmenet0)", + "192.168.252.0/24 (configured)", + } + if !equalStringSets(got, want) { + t.Errorf("descriptions = %v, want %v", got, want) + } + + if got := forwardedSourceDescriptions(nil); len(got) != 0 { + t.Errorf("empty set must render no descriptions, got %v", got) + } +} + +// TestForwardedSourceSetKey verifies the signature is order-independent, distinguishes +// interface scope, and changes when the set changes - the basis for detecting VM +// start/stop at runtime. +func TestForwardedSourceSetKey(t *testing.T) { + a := netip.MustParsePrefix("192.168.64.0/24") + b := netip.MustParsePrefix("10.211.55.0/24") + + // Order-independent. + k1 := forwardedSourceSetKey([]forwardedSource{{prefix: a, iface: "vmnet8"}, {prefix: b}}) + k2 := forwardedSourceSetKey([]forwardedSource{{prefix: b}, {prefix: a, iface: "vmnet8"}}) + if k1 != k2 { + t.Errorf("key must be order-independent: %q vs %q", k1, k2) + } + + // A guest appearing changes the key (empty -> one source). + if forwardedSourceSetKey(nil) == k1 { + t.Error("adding a source must change the key") + } + + // A guest stopping changes the key (two sources -> one). + k3 := forwardedSourceSetKey([]forwardedSource{{prefix: b}}) + if k3 == k1 { + t.Error("removing a source must change the key") + } + + // Same prefix on a different interface is a distinct trust and must differ. + kIface := forwardedSourceSetKey([]forwardedSource{{prefix: a, iface: "vmnet8"}}) + kNoIface := forwardedSourceSetKey([]forwardedSource{{prefix: a}}) + if kIface == kNoIface { + t.Error("interface scope must affect the key") + } +} + +// TestApplyForwardedSourceChange_Lifecycle walks the guest start/stop lifecycle +// deterministically (no pf, no hypervisor): initial build, guest start, no-change +// re-check, second guest start, guest stop, scope change, and back to none. Each +// anchor reload succeeds here. It asserts both halves of the contract - whether the +// anchor needs rebuilding, and which subnets' pf states must be dropped because +// their trust changed. +func TestApplyForwardedSourceChange_Lifecycle(t *testing.T) { + vmA := forwardedSource{prefix: netip.MustParsePrefix("192.168.105.0/24"), iface: "vmnet8"} + vmB := forwardedSource{prefix: netip.MustParsePrefix("10.211.55.0/24"), iface: "vnic0"} + cfgB := forwardedSource{prefix: vmB.prefix} // same subnet, configured (no iface scope) + + steps := []struct { + name string + cur []forwardedSource + wantChanged bool + wantGained []string + wantLost []string + }{ + {name: "initial state, no guests", cur: nil, wantChanged: false}, + { + name: "first guest starts", cur: []forwardedSource{vmA}, + wantChanged: true, wantGained: []string{"192.168.105.0/24"}, + }, + {name: "network change, nothing moved", cur: []forwardedSource{vmA}, wantChanged: false}, + { + // Reordered plus a new guest: order must not register as a change. + name: "second guest starts", cur: []forwardedSource{vmB, vmA}, + wantChanged: true, wantGained: []string{"10.211.55.0/24"}, + }, + { + name: "first guest stops", cur: []forwardedSource{vmB}, + wantChanged: true, wantLost: []string{"192.168.105.0/24"}, + }, + { + // Same subnet, different scope: a distinct trust, so its states must be + // dropped even though the subnet itself neither appeared nor vanished. + name: "guest subnet loses its interface scope", cur: []forwardedSource{cfgB}, + wantChanged: true, + wantGained: []string{"10.211.55.0/24"}, + wantLost: []string{"10.211.55.0/24"}, + }, + { + name: "last guest stops", cur: nil, + wantChanged: true, wantLost: []string{"10.211.55.0/24"}, + }, + {name: "still no guests", cur: nil, wantChanged: false}, + } + + state := &pfFirewallState{} + reloads := 0 + okReload := func() error { reloads++; return nil } + wantReloads := 0 + for _, step := range steps { + gained, lost, changed, err := state.applyForwardedSourceChange(step.cur, okReload) + if err != nil { + t.Fatalf("%s: unexpected reload error: %v", step.name, err) + } + if changed != step.wantChanged { + t.Errorf("%s: changed = %v, want %v", step.name, changed, step.wantChanged) + } + if got := prefixStrings(gained); !equalStringSets(got, step.wantGained) { + t.Errorf("%s: gained trust = %v, want %v", step.name, got, step.wantGained) + } + if got := prefixStrings(lost); !equalStringSets(got, step.wantLost) { + t.Errorf("%s: lost trust = %v, want %v", step.name, got, step.wantLost) + } + // The anchor must be rebuilt exactly on the transitions, never on a re-check. + if step.wantChanged { + wantReloads++ + } + if reloads != wantReloads { + t.Errorf("%s: anchor reloads = %d, want %d", step.name, reloads, wantReloads) + } + } +} + +// TestApplyForwardedSourceChange_FailureThenRetry is the convergence guarantee: a +// failed anchor write/load must NOT advance the applied snapshot, so the very next +// reconcile (at the latest the next watchdog tick) retries the same transition +// instead of seeing the new key and going quiet with the old anchor still installed. +func TestApplyForwardedSourceChange_FailureThenRetry(t *testing.T) { + guest := netip.MustParsePrefix("192.168.105.0/24") + cur := []forwardedSource{{prefix: guest, iface: "vmnet8"}} + + state := &pfFirewallState{} + loadErr := errors.New("pfctl: syntax error") + attempts := 0 + failing := func() error { attempts++; return loadErr } + succeeding := func() error { attempts++; return nil } + + // Attempt 1: guest starts, reload fails. The caller is told what changed (so it + // can log it) but must not treat it as applied. + gained, lost, changed, err := state.applyForwardedSourceChange(cur, failing) + if !changed || !errors.Is(err, loadErr) { + t.Fatalf("failed reload: changed = %v, err = %v, want true / the load error", changed, err) + } + if got := prefixStrings(gained); !equalStringSets(got, []string{guest.String()}) { + t.Errorf("failed reload: gained trust = %v, want %v", got, []string{guest.String()}) + } + if len(lost) != 0 { + t.Errorf("failed reload: lost trust = %v, want none", prefixStrings(lost)) + } + if state.lastForwardedKey != "" || state.lastForwardedSources != nil { + t.Fatalf("failed reload must not advance the applied snapshot, got key %q sources %v", + state.lastForwardedKey, state.lastForwardedSources) + } + + // Attempt 2: nothing else moved, but the change is still pending - it must be + // retried and reported identically, not swallowed. + gained, _, changed, err = state.applyForwardedSourceChange(cur, failing) + if !changed || err == nil { + t.Fatalf("retry after failure: changed = %v, err = %v, want true / an error", changed, err) + } + if got := prefixStrings(gained); !equalStringSets(got, []string{guest.String()}) { + t.Errorf("retry after failure: gained trust = %v, want %v", got, []string{guest.String()}) + } + + // Attempt 3: pf accepts the anchor - now the snapshot advances and the affected + // subnet's states are reported for killing. + gained, _, changed, err = state.applyForwardedSourceChange(cur, succeeding) + if !changed || err != nil { + t.Fatalf("successful reload: changed = %v, err = %v, want true / nil", changed, err) + } + if got := prefixStrings(gained); !equalStringSets(got, []string{guest.String()}) { + t.Errorf("successful reload: gained trust = %v, want %v", got, []string{guest.String()}) + } + if state.lastForwardedKey == "" { + t.Fatal("successful reload must record the applied source set") + } + + // Attempt 4: converged - no further rebuild, and no reload call at all. + before := attempts + if _, _, changed, err := state.applyForwardedSourceChange(cur, succeeding); changed || err != nil { + t.Errorf("after convergence: changed = %v, err = %v, want false / nil", changed, err) + } + if attempts != before { + t.Errorf("after convergence: reload was called %d extra time(s), want 0", attempts-before) + } + + // A failure while *removing* trust must likewise not be latched: the subnet stays + // recorded as applied until pf accepts the anchor without it. + if _, lost, changed, err := state.applyForwardedSourceChange(nil, failing); !changed || err == nil { + t.Errorf("guest stop with failing reload: changed = %v, err = %v, want true / an error", changed, err) + } else if got := prefixStrings(lost); !equalStringSets(got, []string{guest.String()}) { + t.Errorf("guest stop with failing reload: lost trust = %v, want %v", got, []string{guest.String()}) + } + if state.lastForwardedKey == "" { + t.Error("failed removal must keep the previously applied set recorded") + } + if _, lost, _, err := state.applyForwardedSourceChange(nil, succeeding); err != nil { + t.Errorf("guest stop retry: unexpected error %v", err) + } else if got := prefixStrings(lost); !equalStringSets(got, []string{guest.String()}) { + t.Errorf("guest stop retry: lost trust = %v, want %v", got, []string{guest.String()}) + } + if state.lastForwardedKey != "" || state.lastForwardedSources != nil { + t.Errorf("after successful removal the applied set must be empty, got key %q sources %v", + state.lastForwardedKey, state.lastForwardedSources) + } +} + +// TestReconcileForwardedSources_GatedOff verifies the reconcile entry point is inert +// when firewall mode is off or pf state was never initialized, so the network-change +// and watchdog call sites never touch pf outside firewall mode. +func TestReconcileForwardedSources_GatedOff(t *testing.T) { + // Firewall mode off (no allowList) - must return before touching pf state. + off := progWithForwardedSources("192.168.64.0/24") + off.platformFirewallState = &pfFirewallState{} + off.dnsInterceptState = &pfState{anchorFile: pfAnchorFile, anchorName: pfAnchorName} + off.reconcileForwardedSources() + if state := off.platformFirewallState.(*pfFirewallState); state.lastForwardedKey != "" { + t.Errorf("reconcile must not record a source set when firewall mode is off, got %q", state.lastForwardedKey) + } + + // Firewall mode on but pf firewall state not initialized - must not panic. + noState := progWithForwardedSources("192.168.64.0/24") + noState.allowList = firewall.New() + noState.dnsInterceptState = &pfState{anchorFile: pfAnchorFile, anchorName: pfAnchorName} + noState.reconcileForwardedSources() + + // Firewall mode on but intercept inactive - no anchor to rebuild. + noIntercept := progWithForwardedSources("192.168.64.0/24") + noIntercept.allowList = firewall.New() + noIntercept.platformFirewallState = &pfFirewallState{} + noIntercept.reconcileForwardedSources() + if state := noIntercept.platformFirewallState.(*pfFirewallState); state.lastForwardedKey != "" { + t.Errorf("reconcile must not record a source set without intercept, got %q", state.lastForwardedKey) + } +} + +// equalStringSets compares two string slices ignoring order and nil-vs-empty. +func equalStringSets(got, want []string) bool { + if len(got) != len(want) { + return false + } + seen := make(map[string]int, len(got)) + for _, s := range got { + seen[s]++ + } + for _, s := range want { + seen[s]-- + if seen[s] < 0 { + return false + } + } + return true +} + +// TestDetectForwardedSources_OnlyPrivate verifies detection returns only private +// IPv4 vendor-VM subnets, each tagged with a vendor interface. Environment-dependent, +// so it asserts a property rather than an exact set. +func TestDetectForwardedSources_OnlyPrivate(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + for _, src := range p.detectForwardedSources() { + if !src.prefix.Addr().Is4() { + t.Errorf("detected non-IPv4 forwarded source: %s", src.prefix) + } + if !src.prefix.Addr().IsPrivate() { + t.Errorf("detected non-private forwarded source (must never auto-trust public): %s", src.prefix) + } + if !isHypervisorVMNetIface(src.iface) { + t.Errorf("detected source on non-vendor interface %q", src.iface) + } + } +} + +// TestPFBuildAnchorRules_ForwardedSourcesGating verifies the forwarded-source rules +// appear in the full anchor only when firewall mode is active, and when a configured +// source is present it appears before the blanket allowlist block so the redirect +// wins. A configured source makes the "on" case deterministic regardless of host. +func TestPFBuildAnchorRules_ForwardedSourcesGating(t *testing.T) { + // Firewall OFF (no allowList): a configured source must NOT appear. + off := progWithForwardedSources("192.168.64.0/24") + if rules := off.buildPFAnchorRules(nil); strings.Contains(rules, "192.168.64.0/24 to ! ") { + t.Errorf("forwarded-source rules must not be emitted when firewall mode is off:\n%s", rules) + } + + // Firewall ON: allowList present → rules appear, before the blanket block. + on := progWithForwardedSources("192.168.64.0/24") + on.allowList = firewall.New() + rules := on.buildPFAnchorRules(nil) + + fwdIdx := strings.Index(rules, "from 192.168.64.0/24 to ! 127.0.0.1 port 53") + blockIdx := strings.Index(rules, "block return out quick inet proto { tcp, udp } from any to any") + if fwdIdx < 0 { + t.Fatalf("configured forwarded-source redirect missing when firewall mode is on:\n%s", rules) + } + if blockIdx < 0 { + t.Fatalf("blanket firewall block missing:\n%s", rules) + } + if fwdIdx >= blockIdx { + t.Errorf("forwarded-source redirect (%d) must come before the blanket block (%d)", fwdIdx, blockIdx) + } +} + +// TestBuildPFFirewallRulesDeclaresExceptionTable pins the pf side of the +// organization's Allowed Destination IP list. +// +// Every cmd/cli test of the allowed-destination paths stubs the platform mirror, +// so nothing else reaches this generator: the table the mirror populates could +// stop being declared, or lose its pass rules, and the mirror would keep +// reporting success while every approved destination stayed blocked. Both +// families are asserted - a list is not usable if only one of them passes. +func TestBuildPFFirewallRulesDeclaresExceptionTable(t *testing.T) { + rules := buildPFFirewallRules() + + wants := []string{ + // Declared persist, like the dynamic table: pfctl -T add/delete/replace + // against an undeclared table fails, and persist is what keeps the table + // alive while it holds no addresses. + "table <" + pfFirewallExceptionTable + "> persist", + "pass out quick inet proto { tcp, udp } from any to <" + pfFirewallExceptionTable + ">", + "pass out quick inet6 proto { tcp, udp } from any to <" + pfFirewallExceptionTable + ">", + } + for _, want := range wants { + if !strings.Contains(rules, want) { + t.Errorf("missing rule:\n %s\nin:\n%s", want, rules) + } + } + + // The exception table is separate from the dynamic one on purpose: the flushes + // that discard DNS-resolved IPs must leave administratively allowed + // destinations in place. + if pfFirewallExceptionTable == pfFirewallTable { + t.Fatal("the exception table and the dynamic table are the same table; a flush would drop the organization's list") + } +} + +// TestPFExceptionTableChunks covers the argv-length split. The organization's +// list is API-supplied and unbounded, and every entry becomes an argv element, so +// a long enough list would blow past ARG_MAX and fail as a whole. +func TestPFExceptionTableChunks(t *testing.T) { + entries := make([]string, pfExceptionTableOpChunk*2+1) + for i := range entries { + entries[i] = "203.0.113.10/32" + } + + if got := pfExceptionTableChunks("replace", nil); len(got) != 0 { + t.Errorf("chunks for an empty list = %d, want 0", len(got)) + } + + short := pfExceptionTableChunks("replace", entries[:2]) + if len(short) != 1 || short[0].op != "replace" || len(short[0].entries) != 2 { + t.Fatalf("a list that fits was split: %+v", short) + } + + // A split replace must replace once and add the rest. Splitting it into three + // replaces would leave pf holding only the final chunk, with the organization's + // other destinations silently dropped while the mirror reported success. + split := pfExceptionTableChunks("replace", entries) + if len(split) != 3 { + t.Fatalf("chunks = %d, want 3 for %d entries at %d per call", len(split), len(entries), pfExceptionTableOpChunk) + } + if split[0].op != "replace" { + t.Errorf("first chunk op = %q, want replace", split[0].op) + } + for _, chunk := range split[1:] { + if chunk.op != "add" { + t.Errorf("chunk after the first has op %q, want add: a second replace discards the first", chunk.op) + } + } + var total int + for _, chunk := range split { + total += len(chunk.entries) + } + if total != len(entries) { + t.Errorf("chunked entries = %d, want %d: the split dropped entries", total, len(entries)) + } + + // delete is per-entry, so every chunk keeps the operation. + for _, chunk := range pfExceptionTableChunks("delete", entries) { + if chunk.op != "delete" { + t.Errorf("delete chunk op = %q, want delete", chunk.op) + } + } +} diff --git a/cmd/cli/firewall_others.go b/cmd/cli/firewall_others.go new file mode 100644 index 00000000..20d078dd --- /dev/null +++ b/cmd/cli/firewall_others.go @@ -0,0 +1,31 @@ +//go:build !windows && !darwin + +package cli + +import "net/netip" + +// initPlatformFirewall is a no-op on unsupported platforms (Linux, etc.). +// Firewall mode on Linux would require iptables/nftables or eBPF — future work. +func (p *prog) initPlatformFirewall() { + p.Warn().Msg("Firewall: platform enforcement not available on this OS; firewall_mode fails open and only records allowlist stats") +} + +// firewallFlushPlatform is a no-op on unsupported platforms. +func (p *prog) firewallFlushPlatform() {} + +// shutdownPlatformFirewall is a no-op on unsupported platforms. +func (p *prog) shutdownPlatformFirewall() {} + +// firewallApplyExceptionsPlatform succeeds trivially on unsupported platforms. +// Nothing enforces the allowlist here, so the organization's allowed destinations +// need no platform rules and there is nothing that can fail; the in-memory set is +// still maintained for stats and for the Contains() path used by embedders. +func (p *prog) firewallApplyExceptionsPlatform(added, removed []netip.Prefix) error { + return nil +} + +// firewallReplaceExceptionsPlatform succeeds trivially on unsupported platforms, +// for the same reason: there is no platform state to replace. +func (p *prog) firewallReplaceExceptionsPlatform(desired []netip.Prefix) error { + return nil +} diff --git a/cmd/cli/firewall_test.go b/cmd/cli/firewall_test.go new file mode 100644 index 00000000..5b74b79d --- /dev/null +++ b/cmd/cli/firewall_test.go @@ -0,0 +1,668 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net/netip" + "strings" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" + "github.com/Control-D-Inc/ctrld/internal/firewall" +) + +func TestExtractHostFromEndpoint(t *testing.T) { + tests := []struct { + name string + endpoint string + want string + }{ + {name: "https URL", endpoint: "https://dns.controld.com/abcdef", want: "dns.controld.com"}, + {name: "URL with userinfo", endpoint: "https://user:pass@dns.controld.com/abcdef", want: "dns.controld.com"}, + {name: "URL with IPv6 literal", endpoint: "https://[2606:4700:4700::1111]:443/dns-query", want: "2606:4700:4700::1111"}, + {name: "host port", endpoint: "1.2.3.4:53", want: "1.2.3.4"}, + {name: "bare IP", endpoint: "1.2.3.4", want: "1.2.3.4"}, + {name: "DNS stamp", endpoint: "sdns://AgcAAAAAAAAAAA", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := extractHostFromEndpoint(tt.endpoint); got != tt.want { + t.Fatalf("extractHostFromEndpoint(%q) = %q, want %q", tt.endpoint, got, tt.want) + } + }) + } +} + +func TestParseAllowedDestinations(t *testing.T) { + tests := []struct { + name string + entries []string + want []string + wantRejected []string + wantWide []string + }{ + { + name: "bare IPv4 becomes a host prefix", + entries: []string{"203.0.113.10"}, + want: []string{"203.0.113.10/32"}, + }, + { + name: "bare IPv6 becomes a host prefix", + entries: []string{"2606:1a40::1"}, + want: []string{"2606:1a40::1/128"}, + }, + { + name: "CIDRs of both families", + entries: []string{"198.51.100.0/24", "2001:db8::/48"}, + want: []string{"198.51.100.0/24", "2001:db8::/48"}, + }, + { + name: "IPv4-in-IPv6 is unmapped to its IPv4 form", + entries: []string{"::ffff:203.0.113.10", "::ffff:198.51.100.0/120"}, + want: []string{"203.0.113.10/32", "198.51.100.0/24"}, + }, + { + name: "surrounding whitespace is tolerated", + entries: []string{" 203.0.113.10 ", "\t198.51.100.0/24"}, + want: []string{"203.0.113.10/32", "198.51.100.0/24"}, + }, + { + name: "empty entries are skipped without being reported", + entries: []string{"", " ", "203.0.113.10"}, + want: []string{"203.0.113.10/32"}, + }, + { + name: "one bad entry does not void the rest", + entries: []string{"203.0.113.10", "not-an-ip", "198.51.100.0/33", "example.com"}, + want: []string{"203.0.113.10/32"}, + wantRejected: []string{"not-an-ip", "198.51.100.0/33", "example.com"}, + }, + { + name: "no entries", + entries: nil, + }, + { + // A full-range prefix is accepted - the organization is entitled to one - + // but it lets every destination of that family bypass Firewall Mode, so it + // has to be reported rather than disappearing into a count. + name: "full-range prefixes are reported as wide", + // Masking happens in normalizeExceptions, so the parsed form is still + // the entry as sent; the mask is what makes it a full range. + entries: []string{"1.2.3.4/0", "::/0", "203.0.113.10"}, + want: []string{"1.2.3.4/0", "::/0", "203.0.113.10/32"}, + wantWide: []string{"1.2.3.4/0", "::/0"}, + }, + { + // unmapPrefix turns this into 0.0.0.0/0, which the raw entry does not look + // like at all. + name: "an IPv4-mapped full range is reported after unmapping", + entries: []string{"::ffff:0:0/96"}, + want: []string{"0.0.0.0/0"}, + wantWide: []string{"0.0.0.0/0"}, + }, + { + // /8 and /32 are the floors themselves: a whole classical IPv4 network + // and a whole IPv6 RIR allocation are wide, but both are things an + // organization can legitimately mean, so neither is reported. + name: "prefixes at the floor are not reported as wide", + entries: []string{"198.51.100.0/24", "10.0.0.0/8", "2001:db8::/48", "2001:db8::/32"}, + want: []string{"198.51.100.0/24", "10.0.0.0/8", "2001:db8::/48", "2001:db8::/32"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefixes, rejected, wide := parseAllowedDestinations(tt.entries) + if got := strings.Join(prefixStrings(prefixes), ","); got != strings.Join(tt.want, ",") { + t.Errorf("prefixes = %q, want %q", got, strings.Join(tt.want, ",")) + } + if got := strings.Join(rejected, ","); got != strings.Join(tt.wantRejected, ",") { + t.Errorf("rejected = %q, want %q", got, strings.Join(tt.wantRejected, ",")) + } + if got := strings.Join(prefixStrings(wide), ","); got != strings.Join(tt.wantWide, ",") { + t.Errorf("wide = %q, want %q", got, strings.Join(tt.wantWide, ",")) + } + }) + } +} + +// progWithAllowList builds a prog with Firewall Mode's allowlist in place and a +// logger attached, so the allowed-destination paths can be exercised without any +// platform enforcement. +func progWithAllowList() *prog { + p := &prog{allowList: firewall.New()} + p.logger.Store(discardLogger()) + return p +} + +// discardLogger returns a logger that writes nowhere. +// +// Firewall Mode's paths start background workers that log as soon as they run, +// concurrently with the test goroutine. The package-wide test logger writes into +// a shared strings.Builder (see TestMain) that is neither safe for concurrent +// writes nor for a write racing another test's read of it, so tests that spawn +// those workers must not share it. +func discardLogger() *ctrld.Logger { + return &ctrld.Logger{Logger: zap.NewNop()} +} + +// mirrorCall records one attempt to change platform enforcement: a delta, or a +// full replace (replace is true, and added carries the whole desired set). +type mirrorCall struct { + added []string + removed []string + replace bool +} + +// stubMirror replaces both platform mirrors for the duration of a test, +// recording every change they are handed and failing while *failing is true. The +// recorded calls are what proves a rejected change is retried rather than +// forgotten. +func stubMirror(t *testing.T, calls *[]mirrorCall, failing *bool) { + t.Helper() + origMirror, origReplace := firewallMirrorExceptionsFn, firewallReplaceExceptionsFn + t.Cleanup(func() { + firewallMirrorExceptionsFn, firewallReplaceExceptionsFn = origMirror, origReplace + }) + firewallMirrorExceptionsFn = func(_ *prog, added, removed []netip.Prefix) error { + *calls = append(*calls, mirrorCall{added: prefixStrings(added), removed: prefixStrings(removed)}) + if *failing { + return errors.New("platform enforcement rejected the change") + } + return nil + } + firewallReplaceExceptionsFn = func(_ *prog, desired []netip.Prefix) error { + *calls = append(*calls, mirrorCall{added: prefixStrings(desired), replace: true}) + if *failing { + return errors.New("platform enforcement rejected the replacement") + } + return nil + } +} + +func (c mirrorCall) String() string { + kind := "delta" + if c.replace { + kind = "replace" + } + return kind + " added=" + strings.Join(c.added, ",") + " removed=" + strings.Join(c.removed, ",") +} + +// TestAllowedDestinationsRetriedAfterMirrorFailure is the regression guard for +// committing a change in memory that platform enforcement refused: an addition +// that pf/WFP rejected must be retried by the next refresh, even though that +// refresh carries an identical list from the API and so produces no new delta. +// Without a separate applied snapshot, the destination would stay blocked with +// the logs claiming it was applied. +func TestAllowedDestinationsRetriedAfterMirrorFailure(t *testing.T) { + p := progWithAllowList() + var calls []mirrorCall + failing := true + stubMirror(t, &calls, &failing) + + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}} + p.syncAllowedDestinations() + if len(calls) != 1 || strings.Join(calls[0].added, ",") != "203.0.113.10/32" { + t.Fatalf("first refresh: calls = %v", calls) + } + + // An identical refresh must retry the rejected addition. + p.syncAllowedDestinations() + if len(calls) != 2 { + t.Fatalf("identical refresh after a failure did not retry: calls = %v", calls) + } + if strings.Join(calls[1].added, ",") != "203.0.113.10/32" { + t.Fatalf("retry carried the wrong delta: %v", calls[1]) + } + if got := p.pendingDestinations(p.allowList); got != 1 { + t.Fatalf("pendingDestinations = %d, want 1 while the mirror is failing", got) + } + + // Once the platform accepts it, the change stops being retried. + failing = false + p.syncAllowedDestinations() + if len(calls) != 3 { + t.Fatalf("recovery refresh did not reach the mirror: calls = %v", calls) + } + p.syncAllowedDestinations() + if len(calls) != 3 { + t.Fatalf("an applied set was mirrored again: calls = %v", calls) + } + if got := p.pendingDestinations(p.allowList); got != 0 { + t.Fatalf("pendingDestinations = %d, want 0 after a successful mirror", got) + } +} + +// TestAllowedDestinationRemovalRetriedAfterMirrorFailure is the same guarantee +// for the direction that matters more: a withdrawn destination whose removal the +// platform rejected must keep being retried, or it stays permitted for good. +func TestAllowedDestinationRemovalRetriedAfterMirrorFailure(t *testing.T) { + p := progWithAllowList() + var calls []mirrorCall + failing := false + stubMirror(t, &calls, &failing) + + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10", "198.51.100.0/24"}} + p.syncAllowedDestinations() + if len(calls) != 1 { + t.Fatalf("initial apply: calls = %v", calls) + } + + // The organization withdraws one entry and the removal is rejected. + failing = true + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}} + p.syncAllowedDestinations() + if len(calls) != 2 || strings.Join(calls[1].removed, ",") != "198.51.100.0/24" { + t.Fatalf("withdrawal: calls = %v", calls) + } + + // The next refresh sends the same (already reduced) list; the removal must + // still be retried rather than treated as done. + p.syncAllowedDestinations() + if len(calls) != 3 || strings.Join(calls[2].removed, ",") != "198.51.100.0/24" { + t.Fatalf("identical refresh after a failed removal: calls = %v", calls) + } + + failing = false + p.syncAllowedDestinations() + p.syncAllowedDestinations() + if len(calls) != 4 { + t.Fatalf("removal kept being retried after it succeeded: calls = %v", calls) + } + if got := p.pendingDestinations(p.allowList); got != 0 { + t.Fatalf("pendingDestinations = %d, want 0", got) + } +} + +// TestResyncRetriedUntilPlatformAcceptsIt covers enforcement starting over state +// ctrld cannot describe - most importantly a macOS persist pf table that outlived +// the previous run. The whole set is replaced rather than added, and a replace +// the platform rejected must be retried: otherwise a destination the organization +// withdrew while ctrld was stopped stays in that table forever, with nothing +// pending to reveal it. +func TestResyncRetriedUntilPlatformAcceptsIt(t *testing.T) { + p := progWithAllowList() + var calls []mirrorCall + failing := true + stubMirror(t, &calls, &failing) + + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}} + p.markDestinationsForResync() + p.syncAllowedDestinations() + + if len(calls) != 1 || !calls[0].replace { + t.Fatalf("a resync must replace the whole set, not apply a delta: calls = %v", calls) + } + if got := p.pendingDestinations(p.allowList); got == 0 { + t.Fatal("a rejected replace was reported as nothing pending") + } + + // Retried, still as a replace: until it succeeds nothing about what the + // platform holds is known, so a delta would leave stale entries behind. + p.reconcileAllowedDestinations() + if len(calls) != 2 || !calls[1].replace { + t.Fatalf("rejected replace was not retried: calls = %v", calls) + } + + failing = false + p.reconcileAllowedDestinations() + if len(calls) != 3 || !calls[2].replace { + t.Fatalf("recovery did not replace the set: calls = %v", calls) + } + if got := p.pendingDestinations(p.allowList); got != 0 { + t.Fatalf("pendingDestinations = %d after a successful replace, want 0", got) + } + + // Once the platform is known-good, later changes go back to deltas. + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10", "198.51.100.0/24"}} + p.syncAllowedDestinations() + if len(calls) != 4 || calls[3].replace { + t.Fatalf("a later change should be a delta: calls = %v", calls) + } +} + +// TestResyncWithEmptyListRetriedUntilAccepted is the same guarantee for an +// organization with no entries at all: the platform still owes ctrld a flush of +// whatever it inherited, so an empty desired set is not "nothing to do". +func TestResyncWithEmptyListRetriedUntilAccepted(t *testing.T) { + p := progWithAllowList() + var calls []mirrorCall + failing := true + stubMirror(t, &calls, &failing) + + p.rc = &controld.ResolverConfig{} + p.markDestinationsForResync() + p.syncAllowedDestinations() + + if len(calls) != 1 || !calls[0].replace || len(calls[0].added) != 0 { + t.Fatalf("empty list did not ask the platform to empty itself: calls = %v", calls) + } + if got := p.pendingDestinations(p.allowList); got == 0 { + t.Fatal("a rejected flush of an empty set was reported as nothing pending") + } + + p.reconcileAllowedDestinations() + if len(calls) != 2 { + t.Fatalf("rejected flush was not retried: calls = %v", calls) + } + + failing = false + p.reconcileAllowedDestinations() + if got := p.pendingDestinations(p.allowList); got != 0 { + t.Fatalf("pendingDestinations = %d after the flush succeeded, want 0", got) + } +} + +// TestRetiredGenerationDoesNotTouchEnforcement pins the teardown boundary: a +// maintenance worker holds the allowlist and generation of the run that started +// it, and once that generation is over - Firewall Mode turned off, or a reload - +// it must not mirror anything, or it would reinstall permits into enforcement +// that is being removed or now belongs to another run. +func TestRetiredGenerationDoesNotTouchEnforcement(t *testing.T) { + p := progWithAllowList() + var calls []mirrorCall + failing := false + stubMirror(t, &calls, &failing) + + gen := p.startFirewallGeneration() + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}} + p.syncAllowedDestinations() + if len(calls) != 1 { + t.Fatalf("initial apply: calls = %v", calls) + } + + // Firewall Mode goes off: the generation ends and the applied set is dropped, + // which is exactly the state that used to make a stale worker reinstall. + stale := p.allowList + p.retireFirewallDestinations() + p.allowList = nil + + p.reconcileDestinations(stale, gen) + if len(calls) != 1 { + t.Fatalf("a retired generation reached platform enforcement: calls = %v", calls) + } + + // A new generation may act again. + p.allowList = stale + newGen := p.startFirewallGeneration() + p.reconcileDestinations(stale, newGen) + if len(calls) != 2 { + t.Fatalf("the current generation was blocked: calls = %v", calls) + } +} + +// TestSyncAllowedDestinationsFollowsResolverConfig covers what a configuration +// refresh has to deliver: a destination added to the organization's list becomes +// reachable without ctrld having resolved it, and one removed from the list stops +// being reachable on the next refresh. +func TestSyncAllowedDestinationsFollowsResolverConfig(t *testing.T) { + p := progWithAllowList() + direct := netip.MustParseAddr("203.0.113.10") + inRange := netip.MustParseAddr("198.51.100.7") + + // Refresh before the organization has any entries. + p.rc = &controld.ResolverConfig{} + p.syncAllowedDestinations() + if p.allowList.Contains(direct) { + t.Fatalf("%s allowed with an empty organization list", direct) + } + + // The organization adds an address and a CIDR. + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10", "198.51.100.0/24"}} + p.syncAllowedDestinations() + if !p.allowList.Contains(direct) || !p.allowList.Contains(inRange) { + t.Fatalf("allowed destinations not reachable: %s=%v %s=%v", + direct, p.allowList.Contains(direct), inRange, p.allowList.Contains(inRange)) + } + + // The organization removes the CIDR; the remaining entry is untouched. + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}} + p.syncAllowedDestinations() + if p.allowList.Contains(inRange) { + t.Fatalf("%s still allowed after its entry was removed", inRange) + } + if !p.allowList.Contains(direct) { + t.Fatalf("%s should still be allowed", direct) + } + + // The organization clears the list entirely. + p.rc = &controld.ResolverConfig{} + p.syncAllowedDestinations() + if p.allowList.Contains(direct) { + t.Fatalf("%s still allowed after the list was cleared", direct) + } +} + +// TestApplyAllowedDestinationsFirewallModeOff pins that devices with Firewall +// Mode disabled are unaffected: there is no allowlist to apply the list to, and +// the refresh path must not panic on the nil one. +func TestApplyAllowedDestinationsFirewallModeOff(t *testing.T) { + p := &prog{rc: &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}}} + p.logger.Store(discardLogger()) + p.syncAllowedDestinations() + if p.allowList != nil { + t.Fatal("applying allowed destinations created an allowlist while firewall mode is off") + } +} + +// TestSyncFirewallModeAppliesAllowedDestinations covers the startup and reload +// path: turning Firewall Mode on builds a fresh allowlist, which must be seeded +// with the organization's list from the resolver config the run started with - +// otherwise the destinations stay blocked until the next hourly refresh. +func TestSyncFirewallModeAppliesAllowedDestinations(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}} + p.cfg.Service.FirewallMode = "on" + p.logger.Store(discardLogger()) + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10"}} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + p.syncFirewallMode(ctx) + if p.allowList == nil { + t.Fatal("firewall mode on did not create an allowlist") + } + if !p.allowList.Contains(netip.MustParseAddr("203.0.113.10")) { + t.Fatal("allowed destination not applied when firewall mode came up") + } + + // Turning the mode off drops the set with the allowlist, and forgets what + // enforcement was holding so a later re-enable reinstalls everything. + p.cfg.Service.FirewallMode = "off" + p.syncFirewallMode(ctx) + if p.allowList != nil { + t.Fatal("firewall mode off did not clear the allowlist") + } + if got := p.pendingDestinations(p.allowList); got != 0 { + t.Fatalf("pendingDestinations = %d with firewall mode off, want 0", got) + } + + p.cfg.Service.FirewallMode = "on" + p.syncFirewallMode(ctx) + if !p.allowList.Contains(netip.MustParseAddr("203.0.113.10")) { + t.Fatal("allowed destination not re-applied after firewall mode was turned back on") + } +} + +// TestAllowedDestinationsSurviveConcurrentReload drives the interleaving the race +// detector caught: apiConfigReload applies the organization's destinations on the +// refresh goroutine while a config reload replaces - or clears - the allowlist on +// another. +// +// The apply path used to read p.allowList twice, once to check it for nil and +// again to call SetExceptions on it, with a parse in between. A reload that +// turned Firewall Mode off inside that gap left the second read nil, and ctrld +// panicked on a refresh that had nothing wrong with it. +func TestAllowedDestinationsSurviveConcurrentReload(t *testing.T) { + origMirror, origReplace := firewallMirrorExceptionsFn, firewallReplaceExceptionsFn + t.Cleanup(func() { + firewallMirrorExceptionsFn, firewallReplaceExceptionsFn = origMirror, origReplace + }) + firewallMirrorExceptionsFn = func(*prog, []netip.Prefix, []netip.Prefix) error { return nil } + firewallReplaceExceptionsFn = func(*prog, []netip.Prefix) error { return nil } + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(discardLogger()) + p.rc = &controld.ResolverConfig{DestinationIPs: []string{"203.0.113.10", "198.51.100.0/24"}} + p.cfg.Service.FirewallMode = "on" + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.syncFirewallMode(ctx) + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + p.syncAllowedDestinations() + } + }() + + // Firewall Mode off then on is what clears p.allowList and installs a fresh + // one, which is the whole of the reload path this refresh can collide with. + for range 20 { + p.cfg.Service.FirewallMode = "off" + p.syncFirewallMode(ctx) + p.cfg.Service.FirewallMode = "on" + p.syncFirewallMode(ctx) + } + close(stop) + <-done + + // The churn must not have cost the set: the run that is live at the end owes + // the platform exactly what the API last sent. + p.syncAllowedDestinations() + p.destinationsMu.Lock() + applied := prefixStrings(p.appliedDestinations) + p.destinationsMu.Unlock() + if got := strings.Join(applied, ","); got != "198.51.100.0/24,203.0.113.10/32" { + t.Errorf("applied destinations = %q, want the full set after the reload churn", got) + } +} + +// TestAllowedDestinationLogsKeepAddressesOutOfWarnings holds the whole +// allowed-destination path to one policy: Warn and Info carry counts, addresses +// appear only at Debug. +// +// Warn logs are persisted and travel in support bundles exactly like Info ones, +// and both a rejected entry and an accepted one are an organization's network +// topology - "10.0.0.0/33" names a real network as surely as the entry next to +// it. logDestinationChange already followed this rule; the rejection and +// wide-prefix warnings are the paths that can leak around it. +func TestAllowedDestinationLogsKeepAddressesOutOfWarnings(t *testing.T) { + var calls []mirrorCall + failing := false + stubMirror(t, &calls, &failing) + + core, logs := observer.New(zapcore.DebugLevel) + p := progWithAllowList() + p.logger.Store(&ctrld.Logger{Logger: zap.New(core)}) + + const ( + bad = "10.0.0.0/33" // near-miss topology: a real network, a bad mask + wide = "0.0.0.0/0" // accepted, and blankets the whole family + ordinal = "203.0.113.10/32" // an ordinary accepted entry + ) + p.rc = &controld.ResolverConfig{DestinationIPs: []string{bad, wide, "203.0.113.10"}} + p.syncAllowedDestinations() + + var warned, wideWarned bool + for _, entry := range logs.FilterLevelExact(zapcore.WarnLevel).All() { + line := entry.Message + fmt.Sprint(entry.ContextMap()) + for _, addr := range []string{bad, wide, ordinal, "203.0.113.10"} { + if strings.Contains(line, addr) { + t.Errorf("a Warn line carries the address %q, which support bundles then carry too:\n %s", addr, line) + } + } + if strings.Contains(entry.Message, "not a valid IP address or CIDR") { + warned = true + if got := entry.ContextMap()["rejected"]; got != int64(1) { + t.Errorf("rejected count = %v, want 1", got) + } + } + if strings.Contains(entry.Message, "very wide range") { + wideWarned = true + if got := entry.ContextMap()["bits"]; got != int64(0) { + t.Errorf("wide prefix bits = %v, want 0 for a full range", got) + } + } + } + if !warned { + t.Error("an unusable entry was dropped without any warning") + } + if !wideWarned { + t.Error("a full-range destination was accepted without any warning; the bypass would be invisible") + } + + // The values are still recoverable by whoever turns Debug on to look. + var debugged string + for _, entry := range logs.FilterLevelExact(zapcore.DebugLevel).All() { + debugged += entry.Message + fmt.Sprint(entry.ContextMap()) + } + if !strings.Contains(debugged, bad) { + t.Errorf("the rejected entry %q appears in no Debug line, so nothing can diagnose it", bad) + } + if !strings.Contains(debugged, wide) { + t.Errorf("the wide entry %q appears in no Debug line", wide) + } +} + +// TestFirewallPermanentAllowListPermitsControlDEndpoints is the regression guard +// for the Windows lockout described on controld.APIEndpointIPs. +// +// Firewall Mode learns destinations from queries ctrld's own listener answered. +// The addresses asserted here are the ones each endpoint falls back to when DNS +// does not work at all - which is the state a ctrld blocked by its own filters is +// in - so they are exactly the ones no lookup can ever teach it. +func TestFirewallPermanentAllowListPermitsControlDEndpoints(t *testing.T) { + for _, dev := range []bool{false, true} { + t.Run(map[bool]string{false: "prod", true: "dev"}[dev], func(t *testing.T) { + origDev := cdDev + cdDev = dev + t.Cleanup(func() { cdDev = origDev }) + + al := firewall.New() + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initFirewallAllowList(context.Background(), al) + + apiIPs := controld.APIEndpointIPs(dev) + if len(apiIPs) == 0 { + t.Fatal("no ControlD API addresses to permit") + } + + endpoints := map[string][]string{ + // The API transport's direct addresses. + "API": apiIPs, + // The upgrade download server's fallback. performUpgrade runs the + // download in a detached child process, and WFP's block-all filters + // carry no process condition, so the service blocks its own upgrade. + "download server": {downloadServerIp}, + } + for what, ips := range endpoints { + for _, ipStr := range ips { + ip, err := netip.ParseAddr(ipStr) + if err != nil { + t.Fatalf("the %s address %q does not parse: %v", what, ipStr, err) + } + if !al.Contains(ip) { + t.Errorf("the ControlD %s address %s is not permitted; Firewall Mode would block ctrld's own socket to it", what, ip) + } + } + } + }) + } +} diff --git a/cmd/cli/firewall_windows.go b/cmd/cli/firewall_windows.go new file mode 100644 index 00000000..34abb27b --- /dev/null +++ b/cmd/cli/firewall_windows.go @@ -0,0 +1,765 @@ +//go:build windows + +package cli + +import ( + "errors" + "fmt" + "net/netip" + "runtime" + "sync" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +// wfpFirewallState holds the state for WFP-based firewall mode enforcement on Windows. +// When firewall mode is active, we add dynamic WFP permit filters for each allowed IP +// on top of a block-all base filter. +type wfpFirewallState struct { + mu sync.Mutex + + // pendingAdds and pendingRemoves accumulate changes for batched WFP updates. + pendingAdds []netip.Addr + pendingRemoves []netip.Addr + + // batchTimer fires after the accumulation window to flush pending changes. + batchTimer *time.Timer + + // filterMap tracks WFP filter IDs for each dynamically allowed IP so we can remove them. + // Maps IP string → WFP filter ID. + filterMap map[string]uint64 + + // permanentFilterMap tracks permit filters for permanent allowlist entries + // (loopback/private/link-local/listener/upstream IPs). These stay installed + // across dynamic allowlist flushes. + permanentFilterMap map[string]uint64 + + // exceptionFilterMap tracks permit filters for the organization's Allowed + // Destination IP list, keyed by prefix string. Kept apart from filterMap so + // the flushes that discard DNS-resolved IPs leave them installed, and apart + // from permanentFilterMap because entries are removed when the organization + // removes them. Guarded by mu. + // + // Known gap, shared with filterMap and permanentFilterMap: rebuildDNSIntercept + // recreates the WFP engine without re-initializing this state, so after a + // health-monitor repair every ID here refers to a filter in a session that is + // gone. Mirroring then fails for good and allowed_destinations_pending stays + // non-zero until the service restarts. Predates the allowed-destination work + // and wants its own fix - re-initializing platform firewall state as part of + // the rebuild - rather than a patch here. + exceptionFilterMap map[string]uint64 + + // blockFilterIDv4 and blockFilterIDv6 are the base block-all filters. + blockFilterIDv4 uint64 + blockFilterIDv6 uint64 + + // engineHandle is the WFP engine handle from the intercept state. + engineHandle uintptr +} + +const ( + // wfpFirewallBatchInterval is the accumulation window for batching WFP filter updates. + wfpFirewallBatchInterval = 200 * time.Millisecond +) + +// firewallFlushPlatform removes all dynamic WFP permit filters on Windows. +// Called on network changes and config reloads before the in-memory allowlist is flushed. +func (p *prog) firewallFlushPlatform() { + fwState, ok := p.platformFirewallState.(*wfpFirewallState) + if !ok || fwState == nil { + return + } + fwState.flushAll(p) +} + +// shutdownPlatformFirewall removes Windows firewall-mode WFP filters, including +// dynamic permits, permanent permits, and the base block-all filters. +func (p *prog) shutdownPlatformFirewall() { + fwState, ok := p.platformFirewallState.(*wfpFirewallState) + if !ok || fwState == nil { + return + } + fwState.shutdown(p) +} + +// initPlatformFirewall initializes Windows-specific firewall enforcement (WFP filters). +func (p *prog) initPlatformFirewall() { + if fwState, ok := p.platformFirewallState.(*wfpFirewallState); ok && fwState != nil { + // A reload re-enters here with enforcement already up, to install permits + // for permanent entries added since - a newly resolved API address, say - + // which AddPermanent records in memory without any callback that would + // reach WFP. Both populate calls skip what they already hold, so this is a + // refresh rather than a reinstall. + fwState.populatePermanentFilters(p) + // The filter IDs this state holds describe whatever engine session + // installed them, which after a rebuildDNSIntercept is not the live one - + // so ask for a full replace instead of a delta against a snapshot that may + // describe filters that no longer exist. markDestinationsForResync is + // idempotent and cheap, and an unnecessary replace is a no-op the mirrors + // already tolerate. + p.markDestinationsForResync() + p.reconcileAllowedDestinations() + fwState.populateFilters(p) + return + } + + if !hardIntercept || p.dnsInterceptState == nil { + p.Info().Msg("Firewall: WFP enforcement requires hard intercept mode") + return + } + + state, ok := p.dnsInterceptState.(*wfpState) + if !ok || state == nil { + p.Warn().Msg("Firewall: could not access WFP state for firewall enforcement") + return + } + + fwState := &wfpFirewallState{ + filterMap: make(map[string]uint64), + permanentFilterMap: make(map[string]uint64), + exceptionFilterMap: make(map[string]uint64), + engineHandle: state.engineHandle, + } + p.platformFirewallState = fwState + + // Install base block-all outbound filters. We add our firewall filters to the + // SAME sublayer as DNS intercept with carefully chosen weights: + // - Existing DNS permits (localhost): weight 10 (highest, evaluated first) + // - Firewall IP permits: weight 5 (middle) + // - Existing DNS block: weight 1 (blocks non-localhost DNS) + // - Firewall block-all: weight 1 (catch-all for non-DNS) + // + // WFP evaluates higher weights first within a sublayer. DNS permits at 10 + // always win, ensuring DNS resolution works. Firewall IP permits at 5 + // override the block-all for resolved IPs. The existing DNS block at 1 + // and our block-all at 1 are both catch-alls (DNS block has port 53 + // conditions so it only catches DNS; our block-all has no conditions + // so it catches everything else). + if err := p.addWFPFirewallBlockFilters(fwState); err != nil { + p.Error().Err(err).Msg("Firewall: failed to install WFP block-all filters") + return + } + + // Install permits for the permanent allowlist before enabling dynamic updates. + // Without these, the block-all filters would also block ctrld upstreams, + // listener/loopback traffic, LAN ranges, and other permanent exceptions. + fwState.populatePermanentFilters(p) + + // The organization's allowed destinations are applied as soon as the allowlist + // exists, which is before WFP enforcement comes up. This session holds no + // filters of its own yet, so mark the set for a full install and let the + // reconcile put it in - and retry it if WFP refuses. + p.markDestinationsForResync() + p.reconcileAllowedDestinations() + + // Register batch callback. + p.allowList.SetOnBatchChange(func(added []netip.Addr, removed []netip.Addr) { + fwState.mu.Lock() + defer fwState.mu.Unlock() + + if len(added) > 0 { + fwState.pendingAdds = append(fwState.pendingAdds, added...) + } + if len(removed) > 0 { + fwState.pendingRemoves = append(fwState.pendingRemoves, removed...) + } + fwState.scheduleBatchFlush(p) + }) + + // Register individual change callback. + p.allowList.SetOnChange(func(ip netip.Addr, isAdded bool) { + fwState.mu.Lock() + defer fwState.mu.Unlock() + + if isAdded { + fwState.pendingAdds = append(fwState.pendingAdds, ip) + } else { + fwState.pendingRemoves = append(fwState.pendingRemoves, ip) + } + fwState.scheduleBatchFlush(p) + }) + + // DNS responses may have populated the allowlist before platform callbacks + // were registered. Add permit filters for that snapshot so WFP starts with + // the same view as the in-memory allowlist. + fwState.populateFilters(p) + + p.Info().Msg("Firewall: WFP enforcement initialized with block-all base filters") +} + +// scheduleBatchFlush starts or resets the batch timer. Must be called with fwState.mu held. +func (s *wfpFirewallState) scheduleBatchFlush(p *prog) { + if s.batchTimer != nil { + return + } + s.batchTimer = time.AfterFunc(wfpFirewallBatchInterval, func() { + s.flushBatch(p) + }) +} + +// flushBatch applies accumulated WFP filter changes. +func (s *wfpFirewallState) flushBatch(p *prog) { + s.mu.Lock() + defer s.mu.Unlock() + + adds := s.pendingAdds + removes := s.pendingRemoves + s.pendingAdds = nil + s.pendingRemoves = nil + s.batchTimer = nil + + // Collapse add/remove deltas into the current primary allowlist state. This + // avoids leaving WFP opposite the allowlist when an Add and Remove for the + // same IP land in one batch window. + ipsToSync := make(map[netip.Addr]struct{}, len(adds)+len(removes)) + for _, ip := range adds { + ipsToSync[ip] = struct{}{} + } + for _, ip := range removes { + ipsToSync[ip] = struct{}{} + } + + for ip := range ipsToSync { + key := ip.String() + allowed := p.allowList != nil && p.allowList.Contains(ip) + if !allowed { + s.removePermitFilterLocked(p, key) + continue + } + + if _, exists := s.filterMap[key]; exists { + continue // Already has a permit filter. + } + + filterID, err := p.addWFPFirewallPermitFilter(s, ip) + if err != nil { + p.Warn().Err(err).Msgf("Firewall: failed to add WFP permit filter for %s", key) + continue + } + s.filterMap[key] = filterID + p.Debug().Msgf("Firewall: added WFP permit filter for %s (ID: %d)", key, filterID) + } +} + +// removePermitFilterLocked removes one dynamic WFP permit filter. s.mu must be held. +func (s *wfpFirewallState) removePermitFilterLocked(p *prog, key string) { + filterID, ok := s.filterMap[key] + if !ok { + return + } + r1, _, _ := procFwpmFilterDeleteById0.Call(s.engineHandle, uintptr(filterID)) + if r1 != 0 { + p.Debug().Msgf("Firewall: failed to remove WFP filter for %s (HRESULT 0x%x, may already be gone)", key, r1) + } else { + p.Debug().Msgf("Firewall: removed WFP permit filter for %s", key) + } + delete(s.filterMap, key) +} + +// flushAll synchronously removes every dynamic WFP permit filter and clears any +// queued batch work. The block-all filters stay installed. +func (s *wfpFirewallState) flushAll(p *prog) { + s.mu.Lock() + if s.batchTimer != nil { + s.batchTimer.Stop() + s.batchTimer = nil + } + s.pendingAdds = nil + s.pendingRemoves = nil + + filters := make(map[string]uint64, len(s.filterMap)) + for key, filterID := range s.filterMap { + filters[key] = filterID + } + s.filterMap = make(map[string]uint64) + s.mu.Unlock() + + for key, filterID := range filters { + r1, _, _ := procFwpmFilterDeleteById0.Call(s.engineHandle, uintptr(filterID)) + if r1 != 0 { + p.Debug().Msgf("Firewall: failed to remove WFP filter for %s during flush (HRESULT 0x%x, may already be gone)", key, r1) + } else { + p.Debug().Msgf("Firewall: removed WFP permit filter for %s during flush", key) + } + } +} + +// shutdown removes every WFP filter owned by firewall mode. +func (s *wfpFirewallState) shutdown(p *prog) { + s.flushAll(p) + + s.mu.Lock() + permanentFilters := make(map[string]uint64, len(s.permanentFilterMap)) + for key, filterID := range s.permanentFilterMap { + permanentFilters[key] = filterID + } + s.permanentFilterMap = make(map[string]uint64) + + exceptionFilters := make(map[string]uint64, len(s.exceptionFilterMap)) + for key, filterID := range s.exceptionFilterMap { + exceptionFilters[key] = filterID + } + s.exceptionFilterMap = make(map[string]uint64) + + blockIDs := []uint64{s.blockFilterIDv4, s.blockFilterIDv6} + s.blockFilterIDv4 = 0 + s.blockFilterIDv6 = 0 + s.mu.Unlock() + + for key, filterID := range permanentFilters { + if r1, _, _ := procFwpmFilterDeleteById0.Call(s.engineHandle, uintptr(filterID)); r1 != 0 { + p.Debug().Msgf("Firewall: failed to remove permanent WFP filter for %s during shutdown (HRESULT 0x%x, may already be gone)", key, r1) + } + } + for key, filterID := range exceptionFilters { + if r1, _, _ := procFwpmFilterDeleteById0.Call(s.engineHandle, uintptr(filterID)); r1 != 0 { + p.Debug().Msgf("Firewall: failed to remove allowed destination WFP filter for %s during shutdown (HRESULT 0x%x, may already be gone)", key, r1) + } + } + for _, filterID := range blockIDs { + if filterID == 0 { + continue + } + if r1, _, _ := procFwpmFilterDeleteById0.Call(s.engineHandle, uintptr(filterID)); r1 != 0 { + p.Debug().Msgf("Firewall: failed to remove WFP block filter %d during shutdown (HRESULT 0x%x, may already be gone)", filterID, r1) + } + } +} + +// populatePermanentFilters mirrors the in-memory permanent allowlist into WFP +// permit filters so the base block-all rule does not block ctrld itself, local +// network traffic, or upstream resolver endpoints. +func (s *wfpFirewallState) populatePermanentFilters(p *prog) { + if p.allowList == nil { + return + } + addrs, prefixes := p.allowList.PermanentEntries() + + s.mu.Lock() + defer s.mu.Unlock() + + for _, ip := range addrs { + key := "addr:" + ip.String() + if _, exists := s.permanentFilterMap[key]; exists { + continue + } + filterID, err := p.addWFPFirewallPermitFilter(s, ip) + if err != nil { + p.Warn().Err(err).Msgf("Firewall: failed to add permanent WFP permit for %s", ip) + continue + } + s.permanentFilterMap[key] = filterID + p.Debug().Msgf("Firewall: added permanent WFP permit for %s (ID: %d)", ip, filterID) + } + + for _, prefix := range prefixes { + key := "prefix:" + prefix.String() + if _, exists := s.permanentFilterMap[key]; exists { + continue + } + filterID, err := p.addWFPFirewallPermitPrefix(s, prefix) + if err != nil { + p.Warn().Err(err).Msgf("Firewall: failed to add permanent WFP permit for %s", prefix) + continue + } + s.permanentFilterMap[key] = filterID + p.Debug().Msgf("Firewall: added permanent WFP permit for %s (ID: %d)", prefix, filterID) + } +} + +// fwpErrFilterNotFound is FWP_E_FILTER_NOT_FOUND: the filter is already gone, so +// a delete that reports it has achieved what it was asked to do. +const fwpErrFilterNotFound = 0x80320003 + +// firewallApplyExceptionsPlatform mirrors a change to the organization's Allowed +// Destination IP list into WFP permit filters, reporting whether WFP took it. +// +// An error - including "WFP enforcement is not up yet", because there is no +// engine handle to install filters through - leaves the caller's applied snapshot +// unadvanced, so the same delta is retried later. +func (p *prog) firewallApplyExceptionsPlatform(added, removed []netip.Prefix) error { + fwState, ok := p.platformFirewallState.(*wfpFirewallState) + if !ok || fwState == nil { + return errors.New("WFP firewall enforcement is not initialized") + } + return fwState.syncExceptionFilters(p, added, removed) +} + +// firewallReplaceExceptionsPlatform makes WFP hold permit filters for exactly +// desired, whatever it held before, and reports whether it took. +// +// This is what runs when WFP enforcement starts. A fresh session holds nothing, +// but the session can also be re-initialized over state this process installed +// earlier, so anything not in desired is removed rather than assumed absent. +func (p *prog) firewallReplaceExceptionsPlatform(desired []netip.Prefix) error { + fwState, ok := p.platformFirewallState.(*wfpFirewallState) + if !ok || fwState == nil { + return errors.New("WFP firewall enforcement is not initialized") + } + return fwState.syncExceptionFilters(p, desired, fwState.exceptionsNotIn(desired)) +} + +// exceptionsNotIn returns the prefixes WFP currently permits that are absent from +// keep, i.e. the filters a full resync has to remove. +func (s *wfpFirewallState) exceptionsNotIn(keep []netip.Prefix) []netip.Prefix { + kept := make(map[string]struct{}, len(keep)) + for _, prefix := range keep { + kept[prefix.String()] = struct{}{} + } + + s.mu.Lock() + defer s.mu.Unlock() + + var out []netip.Prefix + for key := range s.exceptionFilterMap { + if _, ok := kept[key]; ok { + continue + } + prefix, err := netip.ParsePrefix(key) + if err != nil { + continue + } + out = append(out, prefix) + } + return out +} + +// syncExceptionFilters installs permit filters for added prefixes and removes the +// filters of removed ones. Permits use the same weight as dynamically allowed +// IPs, so they override the base block-all filter while still losing to the +// higher-weighted DNS permits. +// +// A filter whose deletion failed keeps its ID in the map: dropping it would leak +// a permit that no longer belongs to any allowed destination and that nothing +// could ever remove, while the returned error keeps the withdrawal pending so the +// next reconcile retries the same deletion. +func (s *wfpFirewallState) syncExceptionFilters(p *prog, added, removed []netip.Prefix) error { + s.mu.Lock() + defer s.mu.Unlock() + + var errs []error + + for _, prefix := range removed { + key := prefix.String() + filterID, ok := s.exceptionFilterMap[key] + if !ok { + continue + } + if r1, _, _ := procFwpmFilterDeleteById0.Call(s.engineHandle, uintptr(filterID)); r1 != 0 && r1 != fwpErrFilterNotFound { + errs = append(errs, fmt.Errorf("delete WFP permit filter for allowed destination %s: HRESULT 0x%x", key, r1)) + continue + } + delete(s.exceptionFilterMap, key) + p.Debug().Msgf("Firewall: removed WFP permit filter for allowed destination %s", key) + } + + for _, prefix := range added { + key := prefix.String() + if _, exists := s.exceptionFilterMap[key]; exists { + continue + } + filterID, err := p.addWFPFirewallPermitPrefix(s, prefix) + if err != nil { + errs = append(errs, fmt.Errorf("add WFP permit filter for allowed destination %s: %w", key, err)) + continue + } + s.exceptionFilterMap[key] = filterID + p.Debug().Msgf("Firewall: added WFP permit filter for allowed destination %s (ID: %d)", key, filterID) + } + + return errors.Join(errs...) +} + +// populateFilters installs permit filters for IPs already present in the allowlist +// before WFP callbacks were registered. +func (s *wfpFirewallState) populateFilters(p *prog) { + if p.allowList == nil { + return + } + ips := p.allowList.AllowedIPs() + if len(ips) == 0 { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + for _, ip := range ips { + key := ip.String() + if _, exists := s.filterMap[key]; exists { + continue + } + + filterID, err := p.addWFPFirewallPermitFilter(s, ip) + if err != nil { + p.Warn().Err(err).Msgf("Firewall: failed to add initial WFP permit filter for %s", key) + continue + } + s.filterMap[key] = filterID + p.Debug().Msgf("Firewall: added initial WFP permit filter for %s (ID: %d)", key, filterID) + } +} + +// addWFPFirewallBlockFilters installs the base block-all outbound filters. +// These block ALL non-loopback outbound TCP/UDP traffic. Per-IP permit filters +// (added dynamically from the allowlist) override these for resolved IPs. +func (p *prog) addWFPFirewallBlockFilters(fwState *wfpFirewallState) error { + // Block all outbound IPv4 TCP/UDP. + filterName, _ := windows.UTF16PtrFromString("ctrld Firewall Block All IPv4") + filter := fwpmFilter0{ + subLayerKey: ctrldSubLayerGUID, + weight: fwpValue0{ + valueType: fwpUint8, // FWP_UINT8 + value: 1, // Must be lower than DNS permits (10). Firewall IP permits (5) override this. + }, + action: fwpmAction0{ + actionType: fwpActionBlock, // FWP_ACTION_BLOCK + }, + layerKey: fwpmLayerALEAuthConnectV4, + } + filter.displayData.name = filterName + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + fwState.engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + if r1 != 0 { + return fmt.Errorf("FwpmFilterAdd0 (block IPv4) failed: HRESULT 0x%x", r1) + } + fwState.blockFilterIDv4 = filterID + + // Block all outbound IPv6 TCP/UDP. + filterNameV6, _ := windows.UTF16PtrFromString("ctrld Firewall Block All IPv6") + filterV6 := fwpmFilter0{ + subLayerKey: ctrldSubLayerGUID, + weight: fwpValue0{ + valueType: fwpUint8, + value: 1, + }, + action: fwpmAction0{ + actionType: fwpActionBlock, + }, + layerKey: fwpmLayerALEAuthConnectV6, + } + filterV6.displayData.name = filterNameV6 + + var filterIDv6 uint64 + r1, _, _ = procFwpmFilterAdd0.Call( + fwState.engineHandle, + uintptr(unsafe.Pointer(&filterV6)), + 0, + uintptr(unsafe.Pointer(&filterIDv6)), + ) + if r1 != 0 { + if fwState.blockFilterIDv4 != 0 { + deleteResult, _, _ := procFwpmFilterDeleteById0.Call(fwState.engineHandle, uintptr(fwState.blockFilterIDv4)) + if deleteResult != 0 { + p.Debug().Msgf("Firewall: failed to roll back IPv4 block filter %d after IPv6 setup failure (HRESULT 0x%x)", fwState.blockFilterIDv4, deleteResult) + } + fwState.blockFilterIDv4 = 0 + } + return fmt.Errorf("FwpmFilterAdd0 (block IPv6) failed: HRESULT 0x%x", r1) + } + fwState.blockFilterIDv6 = filterIDv6 + + p.Info().Msgf("Firewall: WFP block-all filters installed (v4 ID: %d, v6 ID: %d)", filterID, filterIDv6) + return nil +} + +// addWFPFirewallPermitPrefix adds a WFP permit filter for a permanent CIDR prefix. +func (p *prog) addWFPFirewallPermitPrefix(fwState *wfpFirewallState, prefix netip.Prefix) (uint64, error) { + prefix = prefix.Masked() + if prefix.Addr().Is4() { + return p.addWFPFirewallPermitIPv4Prefix(fwState, prefix) + } + return p.addWFPFirewallPermitIPv6Prefix(fwState, prefix) +} + +func (p *prog) addWFPFirewallPermitIPv4Prefix(fwState *wfpFirewallState, prefix netip.Prefix) (uint64, error) { + addr4 := prefix.Addr().As4() + addr := uint32(addr4[0])<<24 | uint32(addr4[1])<<16 | uint32(addr4[2])<<8 | uint32(addr4[3]) + bits := prefix.Bits() + var mask uint32 + if bits == 0 { + mask = 0 + } else { + mask = ^uint32(0) << uint(32-bits) + } + addrMask := fwpV4AddrAndMask{addr: addr & mask, mask: mask} + + filterName, _ := windows.UTF16PtrFromString("ctrld Firewall Permit " + prefix.String()) + condition := fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + condition.condValue.valueType = fwpV4AddrMask + condition.condValue.value = uint64(uintptr(unsafe.Pointer(&addrMask))) + + filter := fwpmFilter0{ + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 1, + filterCondition: (*fwpmFilterCondition0)(unsafe.Pointer(&condition)), + weight: fwpValue0{ + valueType: fwpUint8, + value: 5, + }, + action: fwpmAction0{ + actionType: fwpActionPermit, + }, + layerKey: fwpmLayerALEAuthConnectV4, + } + filter.displayData.name = filterName + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + fwState.engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(&addrMask) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 (permit IPv4 prefix %s) failed: HRESULT 0x%x", prefix, r1) + } + return filterID, nil +} + +func (p *prog) addWFPFirewallPermitIPv6Prefix(fwState *wfpFirewallState, prefix netip.Prefix) (uint64, error) { + addrMask := fwpV6AddrAndMask{addr: prefix.Addr().As16(), prefixLength: uint8(prefix.Bits())} + + filterName, _ := windows.UTF16PtrFromString("ctrld Firewall Permit " + prefix.String()) + condition := fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + condition.condValue.valueType = fwpV6AddrMask + condition.condValue.value = uint64(uintptr(unsafe.Pointer(&addrMask))) + + filter := fwpmFilter0{ + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 1, + filterCondition: (*fwpmFilterCondition0)(unsafe.Pointer(&condition)), + weight: fwpValue0{ + valueType: fwpUint8, + value: 5, + }, + action: fwpmAction0{ + actionType: fwpActionPermit, + }, + layerKey: fwpmLayerALEAuthConnectV6, + } + filter.displayData.name = filterName + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + fwState.engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(&addrMask) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 (permit IPv6 prefix %s) failed: HRESULT 0x%x", prefix, r1) + } + return filterID, nil +} + +// addWFPFirewallPermitFilter adds a WFP permit filter for a single IP address. +// Returns the filter ID for later removal. +func (p *prog) addWFPFirewallPermitFilter(fwState *wfpFirewallState, ip netip.Addr) (uint64, error) { + ip = ip.Unmap() + + if ip.Is4() { + return p.addWFPFirewallPermitIPv4(fwState, ip) + } + return p.addWFPFirewallPermitIPv6(fwState, ip) +} + +// addWFPFirewallPermitIPv4 adds a WFP permit filter for an IPv4 address. +func (p *prog) addWFPFirewallPermitIPv4(fwState *wfpFirewallState, ip netip.Addr) (uint64, error) { + addr4 := ip.As4() + ipUint32 := uint32(addr4[0])<<24 | uint32(addr4[1])<<16 | uint32(addr4[2])<<8 | uint32(addr4[3]) + + filterName, _ := windows.UTF16PtrFromString("ctrld Firewall Permit " + ip.String()) + + condition := fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + condition.condValue.valueType = fwpUint32 + condition.condValue.value = uint64(ipUint32) + + filter := fwpmFilter0{ + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 1, + filterCondition: (*fwpmFilterCondition0)(unsafe.Pointer(&condition)), + weight: fwpValue0{ + valueType: fwpUint8, + value: 5, // Higher than block-all (1), lower than DNS permits (10). + }, + action: fwpmAction0{ + actionType: fwpActionPermit, // FWP_ACTION_PERMIT + }, + layerKey: fwpmLayerALEAuthConnectV4, + } + filter.displayData.name = filterName + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + fwState.engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 (permit IPv4 %s) failed: HRESULT 0x%x", ip, r1) + } + return filterID, nil +} + +// addWFPFirewallPermitIPv6 adds a WFP permit filter for an IPv6 address. +func (p *prog) addWFPFirewallPermitIPv6(fwState *wfpFirewallState, ip netip.Addr) (uint64, error) { + addr16 := ip.As16() + + filterName, _ := windows.UTF16PtrFromString("ctrld Firewall Permit " + ip.String()) + + condition := fwpmFilterCondition0{ + fieldKey: fwpmConditionIPRemoteAddress, + matchType: fwpMatchEqual, + } + condition.condValue.valueType = fwpByteArray16Type + condition.condValue.value = uint64(uintptr(unsafe.Pointer(&addr16))) + + filter := fwpmFilter0{ + subLayerKey: ctrldSubLayerGUID, + numFilterConds: 1, + filterCondition: (*fwpmFilterCondition0)(unsafe.Pointer(&condition)), + weight: fwpValue0{ + valueType: fwpUint8, + value: 5, // Higher than block-all (1), lower than DNS permits (10). + }, + action: fwpmAction0{ + actionType: fwpActionPermit, + }, + layerKey: fwpmLayerALEAuthConnectV6, + } + filter.displayData.name = filterName + + var filterID uint64 + r1, _, _ := procFwpmFilterAdd0.Call( + fwState.engineHandle, + uintptr(unsafe.Pointer(&filter)), + 0, + uintptr(unsafe.Pointer(&filterID)), + ) + runtime.KeepAlive(addr16) + if r1 != 0 { + return 0, fmt.Errorf("FwpmFilterAdd0 (permit IPv6 %s) failed: HRESULT 0x%x", ip, r1) + } + return filterID, nil +} diff --git a/cmd/cli/force_reload_lifetime_test.go b/cmd/cli/force_reload_lifetime_test.go new file mode 100644 index 00000000..53d662c3 --- /dev/null +++ b/cmd/cli/force_reload_lifetime_test.go @@ -0,0 +1,49 @@ +package cli + +import ( + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +func TestForceFetchingAPIDrainsBeforeRestoration(t *testing.T) { + old := cdUID + cdUID = "testuid" + t.Cleanup(func() { cdUID = old }) + p := &prog{cfg: &ctrld.Config{}, stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), apiForceReloadCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + // Pause the admitted producer after delivery, before it reads cooldown + // configuration. Shutdown must join it even though the consumer is done. + p.mu.Lock() + locked := true + defer func() { + if locked { + p.mu.Unlock() + } + }() + p.forceFetchingAPI("testuid.verify.controld.com") + select { + case <-p.apiForceReloadCh: + case <-time.After(time.Second): + t.Fatal("force-reload producer did not send") + } + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownGate(t, p) + assertNetworkShutdownBlocked(t, closed) + p.mu.Unlock() + locked = false + waitNetworkShutdownSignal(t, closed, "force-reload producer drain") + // Service restoration precedes closing stopCh, so that channel alone + // cannot cancel the producer's send or cooldown while restoration joins it. + if stopRequested(p.stopCh) || stopRequested(p.runAbortCh) { + t.Fatal("network shutdown closed a caller-owned lifetime channel") + } + p.forceFetchingAPI("testuid.verify.controld.com") + select { + case <-p.apiForceReloadCh: + t.Fatal("shutdown admitted a late force-reload producer") + default: + } +} diff --git a/cmd/cli/hostname.go b/cmd/cli/hostname.go index d28435db..67d56f36 100644 --- a/cmd/cli/hostname.go +++ b/cmd/cli/hostname.go @@ -1,14 +1,86 @@ package cli -import "regexp" +import ( + "fmt" + "regexp" + "strings" +) // validHostname reports whether hostname is a valid hostname. // A valid hostname contains 3 -> 64 characters and conform to RFC1123. +// This function validates hostnames to ensure they meet DNS naming standards +// and prevents invalid hostnames from being used in DNS configurations func validHostname(hostname string) bool { hostnameLen := len(hostname) if hostnameLen < 3 || hostnameLen > 64 { return false } + // RFC1123 regex pattern ensures hostnames follow DNS naming conventions + // This prevents issues with DNS resolution and system compatibility validHostnameRfc1123 := regexp.MustCompile(`^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$`) return validHostnameRfc1123.MatchString(hostname) } + +// isHostnameChar reports whether r is part of validHostname's accepted +// charset (letters, digits, hyphen, dot). It does not check position, so a +// hostname can fail validHostname on structure (length, leading/trailing +// hyphen) while every one of its characters passes here. +func isHostnameChar(r rune) bool { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return true + case r == '-' || r == '.': + return true + } + return false +} + +// offendingHostnameChars returns the distinct characters in hostname that +// validHostname's charset does not accept, in first-seen order. Empty when +// every character is accepted - a rejection can still come from structure +// alone (too short, too long, a leading or trailing hyphen). +func offendingHostnameChars(hostname string) string { + seen := make(map[rune]bool) + var bad []rune + for _, r := range hostname { + if isHostnameChar(r) || seen[r] { + continue + } + seen[r] = true + bad = append(bad, r) + } + return string(bad) +} + +// serverFoldedHostnameChars are the characters ControlD's +// DevicesTableModel.formatDeviceName folds to '-' (or strips) when it +// registers a device name server-side. Only '.' can actually reach +// hostnameMayBeFoldedByServer through validateCustomHostnameFlag's guarded +// path: validHostname runs first and already rejects any hostname +// containing a space or a '+' as CUSTOM_HOSTNAME_INVALID, so those two never +// get here from an explicit --custom-hostname value. They stay in this set +// for completeness: a mobile caller can set CustomHostname to an +// OS-derived default directly, without going through +// validateCustomHostnameFlag at all, so a space or '+' can still reach the +// API unvalidated by this client. +const serverFoldedHostnameChars = ". +" + +// hostnameMayBeFoldedByServer reports whether hostname contains a character +// ControlD may fold or strip when it registers the device, so the name ctrld +// accepted may not be the name the dashboard ends up showing. +func hostnameMayBeFoldedByServer(hostname string) bool { + return strings.ContainsAny(hostname, serverFoldedHostnameChars) +} + +// customHostnameFailureMessage names the field, the offending character(s) +// when there are any, and the allowed format for CUSTOM_HOSTNAME_INVALID. +// ctrld's accept/reject rule (validHostname) is unchanged - this only +// explains a rejection that used to be a bare fatal exit. +func customHostnameFailureMessage(hostname string) string { + const allowedFormat = "3-64 characters of letters, digits, hyphens, and dots (RFC1123 hostname format)" + reason := "is not a valid hostname" + if bad := offendingHostnameChars(hostname); bad != "" { + reason = fmt.Sprintf("contains characters a hostname cannot use: %q", bad) + } + return fmt.Sprintf("--custom-hostname (CustomHostname) %q %s; allowed format: %s", hostname, reason, allowedFormat) +} diff --git a/cmd/cli/hostname_test.go b/cmd/cli/hostname_test.go index f44b2313..b096215d 100644 --- a/cmd/cli/hostname_test.go +++ b/cmd/cli/hostname_test.go @@ -33,3 +33,65 @@ func Test_validHostname(t *testing.T) { }) } } + +// TestOffendingHostnameChars pins the characters surfaced in the +// CUSTOM_HOSTNAME_INVALID message, so the failure names what is actually +// wrong instead of a bare "invalid hostname". +func TestOffendingHostnameChars(t *testing.T) { + tests := []struct { + name string + hostname string + want string + }{ + {"single offender", "foo@bar", "@"}, + {"space", "foo bar", " "}, + {"dot is allowed", "foo.bar", ""}, + {"hyphen is allowed", "foo-bar", ""}, + {"distinct offenders in order", "a!b!c#d", "!#"}, + {"structurally invalid but no bad char", strings.Repeat("a", 65), ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := offendingHostnameChars(tc.hostname); got != tc.want { + t.Errorf("offendingHostnameChars(%q) = %q, want %q", tc.hostname, got, tc.want) + } + }) + } +} + +// TestHostnameMayBeFoldedByServer pins the characters ControlD's +// DevicesTableModel.formatDeviceName folds or strips when it registers a +// device, so a ctrld-accepted name using one gets a heads-up notice instead +// of silently registering under a different name. +func TestHostnameMayBeFoldedByServer(t *testing.T) { + tests := []struct { + name string + hostname string + want bool + }{ + {"dot", "foo.bar", true}, + {"space", "foo bar", true}, + {"plus", "foo+bar", true}, + {"plain", "foobar", false}, + {"hyphen only", "foo-bar", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hostnameMayBeFoldedByServer(tc.hostname); got != tc.want { + t.Errorf("hostnameMayBeFoldedByServer(%q) = %v, want %v", tc.hostname, got, tc.want) + } + }) + } +} + +// TestCustomHostnameFailureMessage pins the message shape the T6 contract +// requires: the flag/field name, the offending character(s), and the +// allowed format. +func TestCustomHostnameFailureMessage(t *testing.T) { + msg := customHostnameFailureMessage("foo@bar") + for _, want := range []string{"--custom-hostname", "CustomHostname", "@", "allowed format"} { + if !strings.Contains(msg, want) { + t.Errorf("customHostnameFailureMessage() = %q, want it to contain %q", msg, want) + } + } +} diff --git a/cmd/cli/http_log.go b/cmd/cli/http_log.go new file mode 100644 index 00000000..c794cf00 --- /dev/null +++ b/cmd/cli/http_log.go @@ -0,0 +1,172 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "sync" +) + +// HTTP log server endpoint constants +const ( + httpLogEndpointPing = "/ping" + httpLogEndpointLogs = "/logs" + httpLogEndpointExit = "/exit" +) + +// httpLogClient sends logs to an HTTP server via POST requests. +// This replaces the logConn functionality with HTTP-based communication. +type httpLogClient struct { + baseURL string + client *http.Client +} + +// newHTTPLogClient creates a new HTTP log client +func newHTTPLogClient(sockPath string) *httpLogClient { + return &httpLogClient{ + baseURL: "http://unix", + client: &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sockPath) + }, + }, + }, + } +} + +// Write sends log data to the HTTP server via POST request +func (hlc *httpLogClient) Write(b []byte) (int, error) { + // Send log data via HTTP POST to /logs endpoint + resp, err := hlc.client.Post(hlc.baseURL+httpLogEndpointLogs, "text/plain", bytes.NewReader(b)) + if err != nil { + // Ignore errors to prevent log pollution, just like the original logConn + return len(b), nil + } + resp.Body.Close() + return len(b), nil +} + +// Ping tests if the HTTP log server is available +func (hlc *httpLogClient) Ping() error { + resp, err := hlc.client.Get(hlc.baseURL + httpLogEndpointPing) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +// Close sends exit signal to the HTTP server +func (hlc *httpLogClient) Close() error { + // Send exit signal via HTTP POST with empty body + resp, err := hlc.client.Post(hlc.baseURL+httpLogEndpointExit, "text/plain", bytes.NewReader([]byte{})) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +// GetLogs retrieves all collected logs from the HTTP server +func (hlc *httpLogClient) GetLogs() ([]byte, error) { + resp, err := hlc.client.Get(hlc.baseURL + httpLogEndpointLogs) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNoContent { + return []byte{}, nil + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +// httpLogServer starts an HTTP server listening on unix socket to collect logs from runCmd. +func httpLogServer(sockPath string, stopLogCh chan struct{}) error { + addr, err := net.ResolveUnixAddr("unix", sockPath) + if err != nil { + return fmt.Errorf("invalid log sock path: %w", err) + } + + ln, err := net.ListenUnix("unix", addr) + if err != nil { + return fmt.Errorf("could not listen log socket: %w", err) + } + defer ln.Close() + + // Create a log writer to store all logs + logWriter := newLogWriter() + + // Use a sync.Once to ensure channel is only closed once + var channelClosed sync.Once + + mux := http.NewServeMux() + mux.HandleFunc(httpLogEndpointPing, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + }) + + mux.HandleFunc(httpLogEndpointLogs, func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + // POST /logs - Store log data + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusBadRequest) + return + } + + // Store log data in log writer + logWriter.Write(body) + + w.WriteHeader(http.StatusOK) + + case http.MethodGet: + // GET /logs - Retrieve all logs + // Get all logs from the log writer + logWriter.mu.Lock() + logs := logWriter.buf.Bytes() + logWriter.mu.Unlock() + + if len(logs) == 0 { + w.WriteHeader(http.StatusNoContent) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write(logs) + + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } + }) + + mux.HandleFunc(httpLogEndpointExit, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Close the stop channel to signal completion (only once) + channelClosed.Do(func() { + close(stopLogCh) + }) + w.WriteHeader(http.StatusOK) + }) + + server := &http.Server{Handler: mux} + return server.Serve(ln) +} diff --git a/cmd/cli/http_log_test.go b/cmd/cli/http_log_test.go new file mode 100644 index 00000000..ad664d49 --- /dev/null +++ b/cmd/cli/http_log_test.go @@ -0,0 +1,747 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/net/nettest" +) + +func unixDomainSocketPath(t *testing.T) string { + t.Helper() + sockPath, err := nettest.LocalPath() + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + return sockPath +} + +func TestHTTPLogServer(t *testing.T) { + sockPath := unixDomainSocketPath(t) + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + serverErr := make(chan error, 1) + go func() { + serverErr <- httpLogServer(sockPath, stopLogCh) + }() + + // Wait a bit for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP client + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sockPath) + }, + }, + } + + t.Run("Ping endpoint", func(t *testing.T) { + resp, err := client.Get("http://unix" + httpLogEndpointPing) + if err != nil { + t.Fatalf("Failed to ping server: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + }) + + t.Run("Ping endpoint wrong method", func(t *testing.T) { + resp, err := client.Post("http://unix"+httpLogEndpointPing, "text/plain", bytes.NewReader([]byte("test"))) + if err != nil { + t.Fatalf("Failed to send POST to ping: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("Expected status 405, got %d", resp.StatusCode) + } + }) + + t.Run("Log endpoint", func(t *testing.T) { + testLog := "test log message" + resp, err := client.Post("http://unix"+httpLogEndpointLogs, "text/plain", bytes.NewReader([]byte(testLog))) + if err != nil { + t.Fatalf("Failed to send log: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + // Check if log was stored by retrieving it + logsResp, err := client.Get("http://unix" + httpLogEndpointLogs) + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + defer logsResp.Body.Close() + + if logsResp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200 for logs, got %d", logsResp.StatusCode) + } + + body, err := io.ReadAll(logsResp.Body) + if err != nil { + t.Fatalf("Failed to read logs: %v", err) + } + + if !strings.Contains(string(body), testLog) { + t.Errorf("Expected log '%s' not found in stored logs", testLog) + } + }) + + t.Run("Log endpoint wrong method", func(t *testing.T) { + // Test unsupported method (PUT) on /logs endpoint + req, err := http.NewRequest("PUT", "http://unix"+httpLogEndpointLogs, bytes.NewReader([]byte("test"))) + if err != nil { + t.Fatalf("Failed to create PUT request: %v", err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to send PUT to logs: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("Expected status 405, got %d", resp.StatusCode) + } + }) + + t.Run("Exit endpoint", func(t *testing.T) { + resp, err := client.Post("http://unix"+httpLogEndpointExit, "text/plain", bytes.NewReader([]byte{})) + if err != nil { + t.Fatalf("Failed to send exit: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + // Check if channel is closed by trying to read from it + select { + case _, ok := <-stopLogCh: + if ok { + t.Error("Expected channel to be closed, but it's still open") + } + case <-time.After(1 * time.Second): + t.Error("Timeout waiting for channel closure") + } + }) + + t.Run("Exit endpoint wrong method", func(t *testing.T) { + resp, err := client.Get("http://unix" + httpLogEndpointExit) + if err != nil { + t.Fatalf("Failed to send GET to exit: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("Expected status 405, got %d", resp.StatusCode) + } + }) + + t.Run("Multiple log messages", func(t *testing.T) { + logs := []string{"log1", "log2", "log3"} + + for _, log := range logs { + resp, err := client.Post("http://unix"+httpLogEndpointLogs, "text/plain", bytes.NewReader([]byte(log+"\n"))) + if err != nil { + t.Fatalf("Failed to send log '%s': %v", log, err) + } + resp.Body.Close() + } + + // Check if all logs were stored by retrieving them + logsResp, err := client.Get("http://unix" + httpLogEndpointLogs) + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + defer logsResp.Body.Close() + + if logsResp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200 for logs, got %d", logsResp.StatusCode) + } + + body, err := io.ReadAll(logsResp.Body) + if err != nil { + t.Fatalf("Failed to read logs: %v", err) + } + + logContent := string(body) + for i, expectedLog := range logs { + if !strings.Contains(logContent, expectedLog) { + t.Errorf("Log %d: expected '%s' not found in stored logs", i, expectedLog) + } + } + }) + + t.Run("Large log message", func(t *testing.T) { + largeLog := strings.Repeat("a", 1024*10) // 10KB log message + resp, err := client.Post("http://unix"+httpLogEndpointLogs, "text/plain", bytes.NewReader([]byte(largeLog))) + if err != nil { + t.Fatalf("Failed to send large log: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + // Check if large log was stored by retrieving it + logsResp, err := client.Get("http://unix" + httpLogEndpointLogs) + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + defer logsResp.Body.Close() + + if logsResp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200 for logs, got %d", logsResp.StatusCode) + } + + body, err := io.ReadAll(logsResp.Body) + if err != nil { + t.Fatalf("Failed to read logs: %v", err) + } + + if !strings.Contains(string(body), largeLog) { + t.Error("Large log message was not stored correctly") + } + }) + + // Clean up + os.Remove(sockPath) +} + +func TestHTTPLogServerInvalidSocketPath(t *testing.T) { + // Test with invalid socket path + invalidPath := "/invalid/path/that/does/not/exist.sock" + stopLogCh := make(chan struct{}) + + err := httpLogServer(invalidPath, stopLogCh) + if err == nil { + t.Error("Expected error for invalid socket path") + } + + if !strings.Contains(err.Error(), "could not listen log socket") { + t.Errorf("Expected 'could not listen log socket' error, got: %v", err) + } +} + +func TestHTTPLogServerSocketInUse(t *testing.T) { + // Create a temporary socket path + sockPath := unixDomainSocketPath(t) + defer os.Remove(sockPath) + + // Create the first server + stopLogCh1 := make(chan struct{}) + serverErr1 := make(chan error, 1) + go func() { + serverErr1 <- httpLogServer(sockPath, stopLogCh1) + }() + + // Wait for first server to start + time.Sleep(100 * time.Millisecond) + + // Try to create a second server on the same socket + stopLogCh2 := make(chan struct{}) + err := httpLogServer(sockPath, stopLogCh2) + if err == nil { + t.Error("Expected error when socket is already in use") + } + + if !strings.Contains(err.Error(), "could not listen log socket") { + t.Errorf("Expected 'could not listen log socket' error, got: %v", err) + } +} + +func TestHTTPLogServerConcurrentRequests(t *testing.T) { + // Create a temporary socket path + sockPath := unixDomainSocketPath(t) + defer os.Remove(sockPath) + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + serverErr := make(chan error, 1) + go func() { + serverErr <- httpLogServer(sockPath, stopLogCh) + }() + + // Wait for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP client + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sockPath) + }, + }, + } + + // Send concurrent requests + numRequests := 10 + done := make(chan bool, numRequests) + + for i := 0; i < numRequests; i++ { + go func(i int) { + defer func() { done <- true }() + + logMsg := fmt.Sprintf("concurrent log %d", i) + resp, err := client.Post("http://unix"+httpLogEndpointLogs, "text/plain", bytes.NewReader([]byte(logMsg))) + if err != nil { + t.Errorf("Failed to send concurrent log %d: %v", i, err) + return + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200 for request %d, got %d", i, resp.StatusCode) + } + }(i) + } + + // Wait for all requests to complete + for i := 0; i < numRequests; i++ { + select { + case <-done: + // Request completed + case <-time.After(5 * time.Second): + t.Errorf("Timeout waiting for concurrent request %d", i) + } + } + + // Check if all logs were stored by retrieving them + logsResp, err := client.Get("http://unix" + httpLogEndpointLogs) + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + defer logsResp.Body.Close() + + if logsResp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200 for logs, got %d", logsResp.StatusCode) + } + + body, err := io.ReadAll(logsResp.Body) + if err != nil { + t.Fatalf("Failed to read logs: %v", err) + } + + logContent := string(body) + // Verify all logs were stored + for i := 0; i < numRequests; i++ { + expectedLog := fmt.Sprintf("concurrent log %d", i) + if !strings.Contains(logContent, expectedLog) { + t.Errorf("Log '%s' was not stored", expectedLog) + } + } +} + +func TestHTTPLogServerErrorHandling(t *testing.T) { + // Create a temporary socket path + sockPath := unixDomainSocketPath(t) + defer os.Remove(sockPath) + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + serverErr := make(chan error, 1) + go func() { + serverErr <- httpLogServer(sockPath, stopLogCh) + }() + + // Wait for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP client + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sockPath) + }, + }, + } + + t.Run("Invalid request body", func(t *testing.T) { + // Test with malformed request - this will fail at HTTP level, not server level + // The server will return 400 Bad Request for invalid body + resp, err := client.Post("http://unix"+httpLogEndpointLogs, "text/plain", strings.NewReader("")) + if err != nil { + t.Fatalf("Failed to send request: %v", err) + } + defer resp.Body.Close() + + // Empty body should still be processed successfully + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + }) +} + +func BenchmarkHTTPLogServer(b *testing.B) { + // Create a temporary socket path + tmpDir := b.TempDir() + sockPath := filepath.Join(tmpDir, "bench.sock") + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + go func() { + httpLogServer(sockPath, stopLogCh) + }() + + // Wait for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP client + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sockPath) + }, + }, + } + + // Benchmark log sending + b.ResetTimer() + for i := 0; i < b.N; i++ { + logMsg := fmt.Sprintf("benchmark log %d", i) + resp, err := client.Post("http://unix"+httpLogEndpointLogs, "text/plain", bytes.NewReader([]byte(logMsg))) + if err != nil { + b.Fatalf("Failed to send log: %v", err) + } + resp.Body.Close() + } + + // Clean up + os.Remove(sockPath) +} + +func TestHTTPLogClient(t *testing.T) { + // Create a temporary socket path + sockPath := unixDomainSocketPath(t) + defer os.Remove(sockPath) + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + serverErr := make(chan error, 1) + go func() { + serverErr <- httpLogServer(sockPath, stopLogCh) + }() + + // Wait for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP log client + client := newHTTPLogClient(sockPath) + + t.Run("Ping server", func(t *testing.T) { + err := client.Ping() + if err != nil { + t.Errorf("Ping failed: %v", err) + } + }) + + t.Run("Write logs", func(t *testing.T) { + testLog := "test log message from client" + n, err := client.Write([]byte(testLog)) + if err != nil { + t.Errorf("Write failed: %v", err) + } + if n != len(testLog) { + t.Errorf("Expected to write %d bytes, wrote %d", len(testLog), n) + } + + // Check if log was stored by retrieving it + logs, err := client.GetLogs() + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + + if !strings.Contains(string(logs), testLog) { + t.Errorf("Expected log '%s' not found in stored logs", testLog) + } + }) + + t.Run("Close client", func(t *testing.T) { + err := client.Close() + if err != nil { + t.Errorf("Close failed: %v", err) + } + + // Check if channel is closed (signaling completion) + select { + case _, ok := <-stopLogCh: + if ok { + t.Error("Expected channel to be closed, but it's still open") + } + case <-time.After(1 * time.Second): + t.Error("Timeout waiting for channel closure") + } + }) +} + +func TestHTTPLogClientServerUnavailable(t *testing.T) { + // Create client with non-existent socket + sockPath := "/non/existent/socket.sock" + client := newHTTPLogClient(sockPath) + + t.Run("Ping unavailable server", func(t *testing.T) { + err := client.Ping() + if err == nil { + t.Error("Expected ping to fail for unavailable server") + } + }) + + t.Run("Write to unavailable server", func(t *testing.T) { + testLog := "test log message" + n, err := client.Write([]byte(testLog)) + if err != nil { + t.Errorf("Write should not return error (ignores errors): %v", err) + } + if n != len(testLog) { + t.Errorf("Expected to write %d bytes, wrote %d", len(testLog), n) + } + }) + + t.Run("Close unavailable server", func(t *testing.T) { + err := client.Close() + if err == nil { + t.Error("Expected close to fail for unavailable server") + } + }) +} + +func BenchmarkHTTPLogClient(b *testing.B) { + // Create a temporary socket path + tmpDir := b.TempDir() + sockPath := filepath.Join(tmpDir, "bench.sock") + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + go func() { + httpLogServer(sockPath, stopLogCh) + }() + + // Wait for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP log client + client := newHTTPLogClient(sockPath) + + // Benchmark client writes + b.ResetTimer() + for i := 0; i < b.N; i++ { + logMsg := fmt.Sprintf("benchmark write %d", i) + client.Write([]byte(logMsg)) + } + + // Clean up + os.Remove(sockPath) +} + +func TestHTTPLogServerWithLogWriter(t *testing.T) { + // Create a temporary socket path + sockPath := unixDomainSocketPath(t) + defer os.Remove(sockPath) + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + serverErr := make(chan error, 1) + go func() { + serverErr <- httpLogServer(sockPath, stopLogCh) + }() + + // Wait a bit for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP client + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sockPath) + }, + }, + } + + t.Run("Store and retrieve logs", func(t *testing.T) { + // Send multiple log messages + logs := []string{"log message 1", "log message 2", "log message 3"} + + for _, log := range logs { + resp, err := client.Post("http://unix"+httpLogEndpointLogs, "text/plain", bytes.NewReader([]byte(log+"\n"))) + if err != nil { + t.Fatalf("Failed to send log '%s': %v", log, err) + } + resp.Body.Close() + } + + // Retrieve all logs + resp, err := client.Get("http://unix" + httpLogEndpointLogs) + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("Failed to read logs response: %v", err) + } + + logContent := string(body) + for _, log := range logs { + if !strings.Contains(logContent, log) { + t.Errorf("Expected log '%s' not found in retrieved logs", log) + } + } + }) + + t.Run("Empty logs endpoint", func(t *testing.T) { + // Create a new server for this test + sockPath2 := unixDomainSocketPath(t) + stopLogCh2 := make(chan struct{}) + + go func() { + httpLogServer(sockPath2, stopLogCh2) + }() + time.Sleep(100 * time.Millisecond) + + client2 := &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", sockPath2) + }, + }, + } + + resp, err := client2.Get("http://unix" + httpLogEndpointLogs) + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent { + t.Errorf("Expected status 204, got %d", resp.StatusCode) + } + + os.Remove(sockPath2) + }) + + t.Run("Channel closure on exit", func(t *testing.T) { + // Send exit signal + resp, err := client.Post("http://unix"+httpLogEndpointExit, "text/plain", bytes.NewReader([]byte{})) + if err != nil { + t.Fatalf("Failed to send exit: %v", err) + } + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + // Check if channel is closed by trying to read from it + select { + case _, ok := <-stopLogCh: + if ok { + t.Error("Expected channel to be closed, but it's still open") + } + case <-time.After(1 * time.Second): + t.Error("Timeout waiting for channel closure") + } + }) +} + +func TestHTTPLogClientGetLogs(t *testing.T) { + // Create a temporary socket path + sockPath := unixDomainSocketPath(t) + defer os.Remove(sockPath) + + // Create log channel + stopLogCh := make(chan struct{}) + + // Start HTTP log server in a goroutine + go func() { + httpLogServer(sockPath, stopLogCh) + }() + + // Wait a bit for server to start + time.Sleep(100 * time.Millisecond) + + // Create HTTP log client + client := newHTTPLogClient(sockPath) + + t.Run("Get logs from client", func(t *testing.T) { + // Send some logs + testLogs := []string{"client log 1", "client log 2", "client log 3"} + for _, log := range testLogs { + client.Write([]byte(log + "\n")) + } + + // Retrieve logs using client method + logs, err := client.GetLogs() + if err != nil { + t.Fatalf("Failed to get logs: %v", err) + } + + logContent := string(logs) + for _, log := range testLogs { + if !strings.Contains(logContent, log) { + t.Errorf("Expected log '%s' not found in retrieved logs", log) + } + } + }) + + t.Run("Get empty logs", func(t *testing.T) { + // Create a new client for empty logs test + sockPath2 := unixDomainSocketPath(t) + stopLogCh2 := make(chan struct{}) + + go func() { + httpLogServer(sockPath2, stopLogCh2) + }() + time.Sleep(100 * time.Millisecond) + + client2 := newHTTPLogClient(sockPath2) + logs, err := client2.GetLogs() + if err != nil { + t.Fatalf("Failed to get empty logs: %v", err) + } + + if len(logs) != 0 { + t.Errorf("Expected empty logs, got %d bytes", len(logs)) + } + + os.Remove(sockPath2) + }) +} diff --git a/cmd/cli/intercept_listener_linux_test.go b/cmd/cli/intercept_listener_linux_test.go new file mode 100644 index 00000000..c93a4653 --- /dev/null +++ b/cmd/cli/intercept_listener_linux_test.go @@ -0,0 +1,100 @@ +//go:build linux + +package cli + +import ( + "net" + "strconv" + "testing" + + ctrld "github.com/Control-D-Inc/ctrld" +) + +// heldListener holds a port on one IP for both UDP and TCP, the two binds +// tryUpdateListenerConfigIntercept makes. +type heldListener struct { + port int + udp net.PacketConn + tcp net.Listener +} + +func (h *heldListener) Close() { + if h.udp != nil { + h.udp.Close() + h.udp = nil + } + if h.tcp != nil { + h.tcp.Close() + h.tcp = nil + } +} + +// holdListenerPort binds a port on ip for both UDP and TCP and keeps it bound +// until the returned listener is closed. The test cleanup closes it too. +func holdListenerPort(t *testing.T, ip string) *heldListener { + t.Helper() + for range 20 { + udp, err := net.ListenPacket("udp4", net.JoinHostPort(ip, "0")) + if err != nil { + t.Fatal(err) + } + port := udp.LocalAddr().(*net.UDPAddr).Port + tcp, err := net.Listen("tcp4", net.JoinHostPort(ip, strconv.Itoa(port))) + if err != nil { + udp.Close() + continue + } + h := &heldListener{port: port, udp: udp, tcp: tcp} + t.Cleanup(h.Close) + return h + } + t.Fatalf("no port free for both UDP and TCP on %s", ip) + return nil +} + +func explicitListenerConfig(ip string, port int) *ctrld.Config { + return &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: ip, Port: port}}} +} + +// Socket-only Linux control for an explicit intercept listener: an address the +// configuration sets binds unchanged, and once occupied it fails rather than +// silently falling back to another address. Native macOS PF delivery is a +// separate QA gate. +// +// The ports are the test's own, so the result does not depend on what else the +// host runs. An occupied port stays held by the test for the whole check. A +// free one has to be released before the check, which binds it itself, so it +// is released immediately before. +func TestExplicitInterceptListenerBinding(t *testing.T) { + captureDebugMainLog(t) + for _, ip := range []string{"127.0.0.1", "127.0.0.2"} { + t.Run(ip+"/free", func(t *testing.T) { + held := holdListenerPort(t, ip) + cfg := explicitListenerConfig(ip, held.port) + held.Close() + changed, ok := tryUpdateListenerConfigIntercept(cfg, func() {}, false) + if !ok || changed { + t.Fatalf("custom listener did not bind unchanged: changed=%v ok=%v", changed, ok) + } + }) + for _, occupied := range []string{"udp", "tcp", "udp+tcp"} { + t.Run(ip+"/occupied "+occupied, func(t *testing.T) { + held := holdListenerPort(t, ip) + switch occupied { + case "udp": + held.tcp.Close() + held.tcp = nil + case "tcp": + held.udp.Close() + held.udp = nil + } + cfg := explicitListenerConfig(ip, held.port) + changed, ok := tryUpdateListenerConfigIntercept(cfg, func() {}, false) + if ok || changed || cfg.Listener["0"].IP != ip || cfg.Listener["0"].Port != held.port { + t.Fatalf("occupied explicit listener silently fell back: changed=%v ok=%v listener=%s:%d", + changed, ok, cfg.Listener["0"].IP, cfg.Listener["0"].Port) + } + }) + } + } +} diff --git a/cmd/cli/intercept_mode.go b/cmd/cli/intercept_mode.go new file mode 100644 index 00000000..f4ed935e --- /dev/null +++ b/cmd/cli/intercept_mode.go @@ -0,0 +1,215 @@ +package cli + +import ( + "errors" + "fmt" + "runtime" + "strconv" + "strings" + + "github.com/kardianos/service" + + "github.com/Control-D-Inc/ctrld" +) + +// interceptByDefault reports whether this host intercepts DNS when no mode is +// selected: the supported desktop platforms, macOS and Windows workstation +// editions. Windows Server is excluded: a server, a domain controller running +// its own DNS server in particular, keeps interface DNS mode unless intercept +// is selected explicitly, which still works there. Elsewhere intercept is not +// supported at all. It is a variable so tests pin both behaviors on any host. +var interceptByDefault = interceptByDefaultOn(runtime.GOOS, ctrld.IsDesktopPlatform()) + +// interceptByDefaultOn is interceptByDefault for a given GOOS. iOS builds the +// darwin files, so ctrld.IsDesktopPlatform reports it as a desktop, but it has +// no pf to intercept with. +func interceptByDefaultOn(goos string, desktop bool) bool { + return desktop && goos != "ios" +} + +// interceptBlockedByOSUpstream reports whether intercept mode cannot serve an +// OS-resolver upstream on this platform. On macOS, pf redirects ctrld's own +// port-53 queries to such an upstream back to ctrld. It is a variable so tests +// pin both behaviors on any host. +var interceptBlockedByOSUpstream = runtime.GOOS == "darwin" || runtime.GOOS == "ios" + +// interceptConfigBlocker reports why cfg cannot run in intercept mode on this +// platform, or nil. An explicit selection of intercept mode fails on it in +// validateDNSIntercept; the default does not select intercept mode for it. +func interceptConfigBlocker(cfg *ctrld.Config) error { + if !interceptBlockedByOSUpstream || cfg == nil { + return nil + } + for name, uc := range cfg.Upstream { + if uc != nil && (uc.Type == ctrld.ResolverTypeOS || uc.Type == "") { + return fmt.Errorf("dns intercept: upstream %q uses OS resolver (port 53) which would create "+ + "a redirect loop with pf. Use DoH upstreams (--proto doh) with dns-intercept mode", name) + } + } + return nil +} + +// Where the effective intercept mode came from, for logs and status output. +const ( + interceptModeSourceFlag = "flag" + interceptModeSourceConfig = "config" + interceptModeSourceDefault = "default" +) + +// interceptModeSource records where setDNS took the effective mode from. setDNS +// replaces interceptMode with the resolved value, so the flag alone no longer +// tells an explicit selection from the platform default afterwards. +var interceptModeSource string + +// runningAsService reports whether this process runs as the installed service, +// started by the OS service manager, rather than interactively. It is a +// variable so tests pin both behaviors. +var runningAsService = func() bool { return !service.Interactive() } + +// interceptModeDefaultFlag marks a service that "ctrld start" installed with +// no intercept mode selected: the service applies the platform default +// (defaultInterceptMode). It is recorded as one --intercept-mode-default=true +// argument. Every ctrld run since 2023 ignores unknown flags, so the marker is +// safe for a downgraded binary, and the inline value keeps an older parser +// from taking the next argument as its value. +const interceptModeDefaultFlag = "--intercept-mode-default" + +// interceptModeDefault is set by the hidden --intercept-mode-default flag of +// "ctrld run". +var interceptModeDefault bool + +// errInterceptDefaultLegacyService is why an installed service that records +// no intercept mode keeps interface DNS mode. +var errInterceptDefaultLegacyService = errors.New("the service records no intercept mode, so it was installed " + + "before dns intercept mode became the default; keeping interface DNS mode") + +// platformDefaultInterceptMode is the mode a new installation, or a foreground +// run, uses for cfg when neither --intercept-mode nor service.intercept_mode +// selects one: "dns" on the supported desktop platforms, and none (interface +// DNS mode) elsewhere, so Windows Server keeps its DNS setup and an +// unsupported platform never enters an intercept start that is known to fail. +// +// The default also steps back to interface DNS mode for a config that cannot +// run in intercept mode, and returns the reason: a config that worked in +// interface DNS mode must not start failing because the default changed. +func platformDefaultInterceptMode(cfg *ctrld.Config) (mode string, blocked error) { + if !interceptByDefault { + return "", nil + } + if err := interceptConfigBlocker(cfg); err != nil { + return "", err + } + return "dns", nil +} + +// defaultInterceptMode is the mode this process uses for cfg when neither +// --intercept-mode nor service.intercept_mode selects one. It ranks below both, +// so a mode in a Control D custom config or a --base64_config still wins. +// +// An installed service gets the platform default only if "ctrld start" +// marked it as a fresh install with --intercept-mode-default +// (installInterceptMode). A service with neither a mode nor the marker was +// installed before dns intercept mode became the default, in what was then +// Standard mode. It keeps interface DNS mode: an upgrade, whichever path +// performs it, must not silently switch an existing Standard install to +// intercept mode. +// +// The default is decided here, against the config the service runs with, +// which in Control D mode is only known once the service fetched it. +func defaultInterceptMode(cfg *ctrld.Config) (mode string, blocked error) { + if interceptByDefault && runningAsService() && !interceptModeDefault { + return "", errInterceptDefaultLegacyService + } + return platformDefaultInterceptMode(cfg) +} + +// interceptModeFromArgs returns the --intercept-mode value in a service's +// arguments, in either the "--intercept-mode value" or the +// "--intercept-mode=value" form, and whether the flag is present. +func interceptModeFromArgs(args []string) (mode string, ok bool) { + for i, arg := range args { + switch { + case arg == "--intercept-mode": + if i+1 < len(args) { + return args[i+1], true + } + return "", true + case strings.HasPrefix(arg, "--intercept-mode="): + return strings.TrimPrefix(arg, "--intercept-mode="), true + } + } + return "", false +} + +// interceptDefaultFromArgs reports whether a service's arguments carry the +// --intercept-mode-default marker, as "--intercept-mode-default" or +// "--intercept-mode-default=". +func interceptDefaultFromArgs(args []string) bool { + for _, arg := range args { + switch { + case arg == interceptModeDefaultFlag: + return true + case strings.HasPrefix(arg, interceptModeDefaultFlag+"="): + on, err := strconv.ParseBool(strings.TrimPrefix(arg, interceptModeDefaultFlag+"=")) + return err == nil && on + } + } + return false +} + +// installInterceptMode decides what "ctrld start" records in the arguments of +// the service it installs when the command line selects no mode: an explicit +// mode, the --intercept-mode-default marker (markDefault), or nothing. +// +// Recording is what lets an installed service tell a fresh install from one +// that predates the dns default (defaultInterceptMode): +// +// - a fresh install: the marker. The service applies the platform default +// itself, below service.intercept_mode and against the config it fetches, +// so a mode or an os upstream in a Control D custom config still counts. +// - a reinstall over an existing service: keep what its arguments record, +// a mode or the marker. If they record neither, it was installed before the +// default and ran in Standard mode, so record "off". existingArgs is nil +// when they could not be read; that also records "off", so a read failure +// cannot switch a Standard install to intercept mode. A reinstall whose +// config sets service.intercept_mode records nothing: the config decides. +// +// Platforms without the dns default record nothing, as before. +func installInterceptMode(flag string, cfg *ctrld.Config, installed bool, existingArgs []string) (mode string, markDefault bool) { + if flag != "" || !interceptByDefault { + return "", false + } + if !installed { + return "", true + } + if mode, ok := interceptModeFromArgs(existingArgs); ok && validInterceptMode(mode) { + return mode, false + } + if interceptDefaultFromArgs(existingArgs) { + return "", true + } + if cfg != nil && cfg.Service.InterceptMode != "" { + return "", false + } + return "off", false +} + +// resolveInterceptMode applies the intercept mode precedence: an explicit +// --intercept-mode value, then service.intercept_mode, then the platform +// default for cfg. "off" at either explicit level is a selection like "dns" or +// "hard", and stops the lookup: it selects interface DNS mode even where the +// default is "dns". +func resolveInterceptMode(flag string, cfg *ctrld.Config) (mode, source string) { + configured := "" + if cfg != nil { + configured = cfg.Service.InterceptMode + } + switch { + case flag != "": + return flag, interceptModeSourceFlag + case configured != "": + return configured, interceptModeSourceConfig + } + mode, _ = defaultInterceptMode(cfg) + return mode, interceptModeSourceDefault +} diff --git a/cmd/cli/intercept_mode_config_test.go b/cmd/cli/intercept_mode_config_test.go new file mode 100644 index 00000000..d62757e0 --- /dev/null +++ b/cmd/cli/intercept_mode_config_test.go @@ -0,0 +1,74 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestUpdateConfigInterceptMode(t *testing.T) { + tests := []struct { + name string + current string + mode string + want string + wantUpdated bool + }{ + // "off" is kept as "off": an unset value now means the platform + // default, which is "dns" on Windows and macOS (issue #589). + {name: "off replaces persisted dns", current: "dns", mode: "off", want: "off", wantUpdated: true}, + {name: "off replaces persisted hard", current: "hard", mode: "off", want: "off", wantUpdated: true}, + {name: "off persists over unset", current: "", mode: "off", want: "off", wantUpdated: true}, + {name: "off already persisted", current: "off", mode: "off", want: "off"}, + {name: "dns replaces persisted off", current: "off", mode: "dns", want: "dns", wantUpdated: true}, + {name: "empty flag preserves config", current: "hard", mode: "", want: "hard"}, + {name: "empty flag leaves unset alone", current: "", mode: "", want: ""}, + {name: "new mode persists", current: "", mode: "dns", want: "dns", wantUpdated: true}, + {name: "same mode is unchanged", current: "dns", mode: "dns", want: "dns"}, + {name: "invalid mode is ignored", current: "dns", mode: "bogus", want: "dns"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &ctrld.Config{} + cfg.Service.InterceptMode = tt.current + if updated := updateConfigInterceptMode(cfg, tt.mode); updated != tt.wantUpdated { + t.Fatalf("updateConfigInterceptMode() updated = %v, want %v", updated, tt.wantUpdated) + } + if cfg.Service.InterceptMode != tt.want { + t.Fatalf("service.intercept_mode = %q, want %q", cfg.Service.InterceptMode, tt.want) + } + }) + } +} + +func TestConfiguredInterceptMode(t *testing.T) { + oldInterceptMode := interceptMode + t.Cleanup(func() { interceptMode = oldInterceptMode }) + + tests := []struct { + name string + flag string + config string + supported bool + want string + }{ + {name: "empty flag falls back to config", flag: "", config: "dns", supported: true, want: "dns"}, + {name: "explicit off is final", flag: "off", config: "dns", supported: true, want: "off"}, + {name: "explicit hard wins over config", flag: "hard", config: "dns", supported: true, want: "hard"}, + {name: "config off is final over the default", flag: "", config: "off", supported: true, want: "off"}, + {name: "unset uses the dns default", flag: "", config: "", supported: true, want: "dns"}, + {name: "unset on an unsupported platform stays off", flag: "", config: "", supported: false, want: ""}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + useInterceptByDefault(t, tc.supported) + p := &prog{cfg: &ctrld.Config{}} + p.cfg.Service.InterceptMode = tc.config + interceptMode = tc.flag + if got := p.configuredInterceptMode(); got != tc.want { + t.Fatalf("configuredInterceptMode() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/intercept_mode_test.go b/cmd/cli/intercept_mode_test.go new file mode 100644 index 00000000..62ac4d0d --- /dev/null +++ b/cmd/cli/intercept_mode_test.go @@ -0,0 +1,384 @@ +package cli + +import ( + "testing" + + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld" +) + +// useInterceptByDefault pins whether this host intercepts when no mode is +// selected, so the default is tested the same way on every CI host. It also +// pins the process to a foreground run; useRunningAsService changes that. +func useInterceptByDefault(t *testing.T, supported bool) { + t.Helper() + old, oldService := interceptByDefault, runningAsService + interceptByDefault = supported + runningAsService = func() bool { return false } + t.Cleanup(func() { interceptByDefault, runningAsService = old, oldService }) +} + +// useRunningAsService pins whether this process runs as the installed service. +// Call it after useInterceptByDefault. +func useRunningAsService(t *testing.T, asService bool) { + t.Helper() + old := runningAsService + runningAsService = func() bool { return asService } + t.Cleanup(func() { runningAsService = old }) +} + +// useInterceptBlockedByOSUpstream pins whether an OS-resolver upstream rules +// out intercept mode, as it does on macOS. +func useInterceptBlockedByOSUpstream(t *testing.T, blocked bool) { + t.Helper() + old := interceptBlockedByOSUpstream + interceptBlockedByOSUpstream = blocked + t.Cleanup(func() { interceptBlockedByOSUpstream = old }) +} + +// useInterceptModeState restores the package intercept state a test changes. +func useInterceptModeState(t *testing.T) { + t.Helper() + mode, source, intercept, hard := interceptMode, interceptModeSource, dnsIntercept, hardIntercept + t.Cleanup(func() { + interceptMode, interceptModeSource, dnsIntercept, hardIntercept = mode, source, intercept, hard + }) +} + +// The full precedence table (issue #589): --intercept-mode, then +// service.intercept_mode, then the platform default. "off" at either explicit +// level is final, even over a persisted "dns". +func TestResolveInterceptModePrecedence(t *testing.T) { + for _, tc := range []struct { + flag, config string + supported bool + want, source string + }{ + // Supported desktop platforms (macOS, Windows workstation). + {flag: "", config: "", supported: true, want: "dns", source: interceptModeSourceDefault}, + {flag: "", config: "dns", supported: true, want: "dns", source: interceptModeSourceConfig}, + {flag: "", config: "hard", supported: true, want: "hard", source: interceptModeSourceConfig}, + {flag: "", config: "off", supported: true, want: "off", source: interceptModeSourceConfig}, + {flag: "dns", config: "", supported: true, want: "dns", source: interceptModeSourceFlag}, + {flag: "hard", config: "dns", supported: true, want: "hard", source: interceptModeSourceFlag}, + {flag: "off", config: "", supported: true, want: "off", source: interceptModeSourceFlag}, + {flag: "off", config: "dns", supported: true, want: "off", source: interceptModeSourceFlag}, + {flag: "off", config: "hard", supported: true, want: "off", source: interceptModeSourceFlag}, + {flag: "dns", config: "off", supported: true, want: "dns", source: interceptModeSourceFlag}, + // Windows Server and unsupported platforms keep interface DNS mode + // by default; explicit selections still mean what they say, and + // intercept works on Windows Server when selected. + {flag: "", config: "", supported: false, want: "", source: interceptModeSourceDefault}, + {flag: "", config: "dns", supported: false, want: "dns", source: interceptModeSourceConfig}, + {flag: "off", config: "dns", supported: false, want: "off", source: interceptModeSourceFlag}, + {flag: "hard", config: "", supported: false, want: "hard", source: interceptModeSourceFlag}, + } { + useInterceptByDefault(t, tc.supported) + cfg := &ctrld.Config{} + cfg.Service.InterceptMode = tc.config + mode, source := resolveInterceptMode(tc.flag, cfg) + if mode != tc.want || source != tc.source { + t.Errorf("flag=%q config=%q supported=%v: got (%q, %q), want (%q, %q)", + tc.flag, tc.config, tc.supported, mode, source, tc.want, tc.source) + } + } +} + +// iOS builds the darwin files and counts as a desktop platform there, but it +// has no pf: it must never intercept by default. +func TestInterceptByDefaultOn(t *testing.T) { + for _, tc := range []struct { + goos string + desktop bool + want bool + }{ + {goos: "darwin", desktop: true, want: true}, + {goos: "windows", desktop: true, want: true}, + {goos: "windows", desktop: false, want: false}, // Windows Server + {goos: "ios", desktop: true, want: false}, + {goos: "android", desktop: false, want: false}, + {goos: "linux", desktop: false, want: false}, + } { + if got := interceptByDefaultOn(tc.goos, tc.desktop); got != tc.want { + t.Errorf("interceptByDefaultOn(%q, %v) = %v, want %v", tc.goos, tc.desktop, got, tc.want) + } + } +} + +// On macOS an OS-resolver upstream cannot run in intercept mode, because pf +// redirects ctrld's own port-53 queries to it back to ctrld. The default then +// keeps interface DNS mode, where such a config works; an explicit selection +// still selects intercept mode and fails on it, as before. +func TestDefaultInterceptModeStepsBackForOSUpstream(t *testing.T) { + useInterceptByDefault(t, true) + withUpstream := func(typ string) *ctrld.Config { + return &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: typ}}} + } + for _, tc := range []struct { + name string + blockedByOS bool + cfg *ctrld.Config + flag string + want string + wantBlocked bool + }{ + {name: "doh upstream on macOS", blockedByOS: true, cfg: withUpstream(ctrld.ResolverTypeDOH), want: "dns"}, + {name: "os upstream on macOS", blockedByOS: true, cfg: withUpstream(ctrld.ResolverTypeOS), want: "", wantBlocked: true}, + {name: "untyped upstream on macOS", blockedByOS: true, cfg: withUpstream(""), want: "", wantBlocked: true}, + {name: "os upstream on Windows", blockedByOS: false, cfg: withUpstream(ctrld.ResolverTypeOS), want: "dns"}, + {name: "explicit dns with os upstream on macOS", blockedByOS: true, cfg: withUpstream(ctrld.ResolverTypeOS), flag: "dns", want: "dns"}, + } { + t.Run(tc.name, func(t *testing.T) { + useInterceptBlockedByOSUpstream(t, tc.blockedByOS) + if mode, _ := resolveInterceptMode(tc.flag, tc.cfg); mode != tc.want { + t.Fatalf("resolveInterceptMode = %q, want %q", mode, tc.want) + } + if tc.flag == "" { + if _, blocked := defaultInterceptMode(tc.cfg); (blocked != nil) != tc.wantBlocked { + t.Fatalf("default blocked = %v, want blocked = %v", blocked, tc.wantBlocked) + } + } + }) + } +} + +// The listener binding strategy follows the same precedence, so a macOS start +// that intercepts by default also binds the way intercept mode needs. +func TestListenerInterceptModeUsesPlatformDefault(t *testing.T) { + useInterceptModeState(t) + interceptMode = "" + cfg := &ctrld.Config{} + + useInterceptByDefault(t, true) + if got := listenerInterceptMode(cfg); got != "dns" { + t.Errorf("supported, unset: listenerInterceptMode = %q, want dns", got) + } + cfg.Service.InterceptMode = "off" + if got := listenerInterceptMode(cfg); got != "off" { + t.Errorf("supported, config off: listenerInterceptMode = %q, want off", got) + } + cfg.Service.InterceptMode = "" + useInterceptByDefault(t, false) + if got := listenerInterceptMode(cfg); got != "" { + t.Errorf("unsupported, unset: listenerInterceptMode = %q, want empty", got) + } +} + +// setDNS starts intercept mode by default on the supported desktop platforms, +// and never by default anywhere else: on Windows Server an unset mode keeps +// interface DNS mode, and on an unsupported platform it must not even try, so +// nothing logs an intercept-start failure. +func TestSetDNSInterceptDefault(t *testing.T) { + for _, tc := range []struct { + name string + flag, config string + supported bool + wantIntercept bool + wantSource string + }{ + {name: "supported, unset", supported: true, wantIntercept: true, wantSource: interceptModeSourceDefault}, + {name: "supported, explicit off", flag: "off", supported: true, wantSource: interceptModeSourceFlag}, + {name: "supported, explicit off over persisted dns", flag: "off", config: "dns", supported: true, wantSource: interceptModeSourceFlag}, + {name: "supported, persisted off", config: "off", supported: true, wantSource: interceptModeSourceConfig}, + {name: "supported, persisted hard", config: "hard", supported: true, wantIntercept: true, wantSource: interceptModeSourceConfig}, + {name: "not a desktop platform, unset", supported: false, wantSource: interceptModeSourceDefault}, + {name: "not a desktop platform, explicit dns", flag: "dns", supported: false, wantIntercept: true, wantSource: interceptModeSourceFlag}, + } { + t.Run(tc.name, func(t *testing.T) { + useInterceptModeState(t) + useInterceptByDefault(t, tc.supported) + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}) + cfg.Service.InterceptMode = tc.config + interceptMode, interceptModeSource, dnsIntercept, hardIntercept = tc.flag, "", false, false + + h.run(t) + + if got := h.interceptCalls > 0; got != tc.wantIntercept { + t.Fatalf("intercept start attempted = %v, want %v", got, tc.wantIntercept) + } + if interceptModeSource != tc.wantSource { + t.Fatalf("interceptModeSource = %q, want %q", interceptModeSource, tc.wantSource) + } + }) + } +} + +func TestInterceptModeFromArgs(t *testing.T) { + for _, tc := range []struct { + args []string + want string + wantOK bool + }{ + {args: nil}, + {args: []string{"/usr/local/bin/ctrld", "run", "--cd=abc"}}, + {args: []string{"/usr/local/bin/ctrld", "run", "--intercept-mode", "dns", "--cd=abc"}, want: "dns", wantOK: true}, + {args: []string{"ctrld.exe", "run", "--intercept-mode=hard"}, want: "hard", wantOK: true}, + {args: []string{"ctrld", "run", "--intercept-mode", "off"}, want: "off", wantOK: true}, + {args: []string{"ctrld", "run", "--intercept-mode"}, want: "", wantOK: true}, + } { + got, ok := interceptModeFromArgs(tc.args) + if got != tc.want || ok != tc.wantOK { + t.Errorf("interceptModeFromArgs(%q) = (%q, %v), want (%q, %v)", tc.args, got, ok, tc.want, tc.wantOK) + } + } +} + +// useInterceptModeDefault pins the hidden --intercept-mode-default flag. +func useInterceptModeDefault(t *testing.T, on bool) { + t.Helper() + old := interceptModeDefault + interceptModeDefault = on + t.Cleanup(func() { interceptModeDefault = old }) +} + +func TestInterceptDefaultFromArgs(t *testing.T) { + for _, tc := range []struct { + args []string + want bool + }{ + {args: nil}, + {args: []string{"ctrld", "run", "--cd=abc"}}, + {args: []string{"ctrld", "run", "--intercept-mode-default=true"}, want: true}, + {args: []string{"ctrld", "run", "--intercept-mode-default"}, want: true}, + {args: []string{"ctrld", "run", "--intercept-mode-default=false"}}, + {args: []string{"ctrld", "run", "--intercept-mode=dns"}}, + } { + if got := interceptDefaultFromArgs(tc.args); got != tc.want { + t.Errorf("interceptDefaultFromArgs(%q) = %v, want %v", tc.args, got, tc.want) + } + } + // The marker must not read as --intercept-mode, and the reverse. + if _, ok := interceptModeFromArgs([]string{"ctrld", "run", "--intercept-mode-default=true"}); ok { + t.Error("the marker read as --intercept-mode") + } +} + +// "ctrld start" records what a service it installs gets when no mode is +// selected. A fresh install gets the --intercept-mode-default marker, so the +// service applies the default itself, below service.intercept_mode. A +// reinstall keeps the existing service's mode or marker, and an existing +// service that records neither was a Standard install, so it is pinned to +// "off" (#589). +func TestInstallInterceptMode(t *testing.T) { + legacyArgs := []string{"/usr/local/bin/ctrld", "run", "--cd=abc", "--config=/etc/controld/ctrld.toml"} + with := func(extra ...string) []string { return append(append([]string{}, legacyArgs...), extra...) } + withConfigMode := func(mode string) *ctrld.Config { + cfg := &ctrld.Config{} + cfg.Service.InterceptMode = mode + return cfg + } + for _, tc := range []struct { + name string + desktop bool + flag string + cfg *ctrld.Config + installed bool + existingArgs []string + wantMode string + wantMarker bool + }{ + {name: "fresh install is marked for the default", desktop: true, cfg: &ctrld.Config{}, wantMarker: true}, + {name: "fresh install with a config mode is still marked", desktop: true, cfg: withConfigMode("hard"), wantMarker: true}, + {name: "explicit flag records nothing more", desktop: true, flag: "off", cfg: &ctrld.Config{}}, + {name: "fresh install on Windows Server", desktop: false, cfg: &ctrld.Config{}}, + {name: "existing Standard service is pinned off", desktop: true, cfg: &ctrld.Config{}, installed: true, existingArgs: legacyArgs, wantMode: "off"}, + {name: "unreadable existing service is pinned off", desktop: true, cfg: &ctrld.Config{}, installed: true, existingArgs: nil, wantMode: "off"}, + {name: "existing service with a config mode is left to the config", desktop: true, cfg: withConfigMode("dns"), installed: true, existingArgs: legacyArgs}, + {name: "existing default service keeps the marker", desktop: true, cfg: &ctrld.Config{}, installed: true, + existingArgs: with("--intercept-mode-default=true"), wantMarker: true}, + {name: "existing dns service keeps dns", desktop: true, cfg: &ctrld.Config{}, installed: true, + existingArgs: with("--intercept-mode", "dns"), wantMode: "dns"}, + {name: "existing hard service keeps hard", desktop: true, cfg: &ctrld.Config{}, installed: true, + existingArgs: with("--intercept-mode=hard"), wantMode: "hard"}, + {name: "existing off service keeps off", desktop: true, cfg: &ctrld.Config{}, installed: true, + existingArgs: with("--intercept-mode", "off"), wantMode: "off"}, + {name: "existing service on Windows Server records nothing", desktop: false, cfg: &ctrld.Config{}, installed: true, existingArgs: legacyArgs}, + } { + t.Run(tc.name, func(t *testing.T) { + useInterceptByDefault(t, tc.desktop) + mode, marker := installInterceptMode(tc.flag, tc.cfg, tc.installed, tc.existingArgs) + if mode != tc.wantMode || marker != tc.wantMarker { + t.Fatalf("installInterceptMode = (%q, marker %v), want (%q, marker %v)", mode, marker, tc.wantMode, tc.wantMarker) + } + }) + } +} + +// How an installed service resolves its mode when its arguments select none. +// +// - The legacy upgrade case (#589): a Standard service installed before the +// dns default has neither a mode nor the marker. Whatever upgrades its +// binary, ctrld upgrade from an older release included, the new binary +// runs it with the same arguments, and it must stay in interface DNS mode. +// - A service "ctrld start" marked as a fresh install applies the default, +// but below service.intercept_mode: a mode from a Control D custom config +// or a --base64_config wins. +// - The default is decided against the config the service runs with, so an +// os upstream there steps it back, with that reason, not the legacy one. +// - A foreground run uses the default. +func TestInstalledServiceInterceptModeResolution(t *testing.T) { + osUpstream := map[string]*ctrld.UpstreamConfig{"0": {Type: ctrld.ResolverTypeOS}} + for _, tc := range []struct { + name string + asService bool + marker bool + configMode string + upstreams map[string]*ctrld.UpstreamConfig + wantMode string + wantIntercept bool + wantReason error // checked when the default is not applied + }{ + {name: "upgraded legacy Standard service", asService: true, wantMode: "", wantReason: errInterceptDefaultLegacyService}, + {name: "fresh default service", asService: true, marker: true, wantMode: "dns", wantIntercept: true}, + {name: "fresh default service, config selects hard", asService: true, marker: true, configMode: "hard", wantMode: "hard", wantIntercept: true}, + {name: "fresh default service, config selects off", asService: true, marker: true, configMode: "off", wantMode: "off"}, + {name: "fresh default service, os upstream", asService: true, marker: true, upstreams: osUpstream, wantMode: ""}, + {name: "foreground run", asService: false, wantMode: "dns", wantIntercept: true}, + } { + t.Run(tc.name, func(t *testing.T) { + useInterceptModeState(t) + useInterceptByDefault(t, true) + useInterceptBlockedByOSUpstream(t, true) + useRunningAsService(t, tc.asService) + useInterceptModeDefault(t, tc.marker) + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}) + cfg.Service.InterceptMode = tc.configMode + cfg.Upstream = tc.upstreams + interceptMode, interceptModeSource, dnsIntercept, hardIntercept = "", "", false, false + + if got := listenerInterceptMode(&cfg); got != tc.wantMode { + t.Fatalf("listenerInterceptMode = %q, want %q", got, tc.wantMode) + } + h.run(t) + if got := h.interceptCalls > 0; got != tc.wantIntercept { + t.Fatalf("intercept start attempted = %v, want %v", got, tc.wantIntercept) + } + if tc.configMode == "" && tc.wantMode == "" { + _, reason := defaultInterceptMode(&cfg) + if tc.wantReason != nil && reason != tc.wantReason { + t.Fatalf("default reason = %v, want %v", reason, tc.wantReason) + } + if tc.wantReason == nil && (reason == nil || reason == errInterceptDefaultLegacyService) { + t.Fatalf("default reason = %v, want the os upstream reason", reason) + } + } + }) + } +} + +// The marker is a hidden flag of "ctrld run", recorded as one argument. +func TestRunCmdInterceptModeDefaultFlag(t *testing.T) { + useInterceptModeDefault(t, false) + cmd := InitRunCmd(&cobra.Command{}) + f := cmd.Flags().Lookup("intercept-mode-default") + if f == nil || !f.Hidden { + t.Fatalf("--intercept-mode-default flag = %+v, want a hidden flag", f) + } + if err := cmd.ParseFlags([]string{"--intercept-mode-default=true"}); err != nil { + t.Fatal(err) + } + if !interceptModeDefault { + t.Fatal("--intercept-mode-default=true did not set the marker") + } +} diff --git a/cmd/cli/intercept_probe.go b/cmd/cli/intercept_probe.go new file mode 100644 index 00000000..36e5707e --- /dev/null +++ b/cmd/cli/intercept_probe.go @@ -0,0 +1,68 @@ +package cli + +// Interception probe registry. +// +// A probe sends a DNS query for a unique synthetic domain through the OS resolver and +// waits for ctrld's own handler to receive it. That is the only way to tell "the rules are +// present" from "the rules are actually redirecting packets", and both the macOS pf path +// and the Windows NRPT path use it. +// +// Each attempt registers its own domain, so overlapping probes cannot cancel each other, +// and deregistration only removes the entry it owns. + +// registerInterceptProbe registers domain and returns the channel it will be signalled on +// plus the function that removes the registration. +// +//lint:ignore U1000 used on darwin (pf probes) and windows (NRPT probes) +func (p *prog) registerInterceptProbe(domain string) (<-chan struct{}, func()) { + ch := make(chan struct{}, 1) + + p.interceptProbeMu.Lock() + current, _ := p.interceptProbes.Load().(map[string]chan struct{}) + next := make(map[string]chan struct{}, len(current)+1) + for k, v := range current { + next[k] = v + } + next[domain] = ch + p.interceptProbes.Store(next) + p.interceptProbeMu.Unlock() + + return ch, func() { + p.interceptProbeMu.Lock() + defer p.interceptProbeMu.Unlock() + current, _ := p.interceptProbes.Load().(map[string]chan struct{}) + // Only drop the entry while it is still this attempt's channel. A later probe + // that reused the domain owns the slot now, and clearing it would make that one + // wait out its timeout for a query it already received. + if existing, ok := current[domain]; !ok || existing != ch { + return + } + next := make(map[string]chan struct{}, len(current)) + for k, v := range current { + if k != domain { + next[k] = v + } + } + p.interceptProbes.Store(next) + } +} + +// signalInterceptProbe reports whether domain is a pending probe, signalling its waiter +// when it is. Called from the DNS handler for every query, so the common case is a nil or +// empty map and no allocation. +func (p *prog) signalInterceptProbe(domain string) bool { + probes, _ := p.interceptProbes.Load().(map[string]chan struct{}) + if len(probes) == 0 { + return false + } + ch, ok := probes[domain] + if !ok { + return false + } + select { + case ch <- struct{}{}: + default: + // Buffered channel already holds a signal: the waiter has what it needs. + } + return true +} diff --git a/cmd/cli/intercept_target.go b/cmd/cli/intercept_target.go new file mode 100644 index 00000000..7fd3498c --- /dev/null +++ b/cmd/cli/intercept_target.go @@ -0,0 +1,15 @@ +package cli + +// publishInterceptTarget mirrors the loopback DNS target that ctrld wrote into +// a network service. Callers hold interceptDNSTargetMu. +func (p *prog) publishInterceptTarget(target string) { + p.interceptTargetMirror.Store(target) +} + +// interceptTargetSnapshot reads the mirrored loopback DNS target. It takes no +// lock, so a report of the network never waits on the networksetup calls that +// the target mutex covers. The value is empty while ctrld holds no service. +func (p *prog) interceptTargetSnapshot() string { + target, _ := p.interceptTargetMirror.Load().(string) + return target +} diff --git a/cmd/cli/internal_domains.go b/cmd/cli/internal_domains.go new file mode 100644 index 00000000..e058ddfd --- /dev/null +++ b/cmd/cli/internal_domains.go @@ -0,0 +1,762 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/netip" + "slices" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" +) + +// internalDomainUpstreamPrefix names the upstreams generated for organization +// Internal Domains that select explicit resolvers. It must not collide with +// the numeric key used for the Control D upstream. The key only names the +// upstream: what identifies it as generated, and in which mode, is +// UpstreamConfig.InternalDomain, which no configuration file can set. +const internalDomainUpstreamPrefix = "internal_" + +// Values of UpstreamConfig.InternalDomain on a generated upstream. +const ( + // internalDomainUpstreamFallback serves an Internal Domain in explicit + // resolver with network fallback mode. + internalDomainUpstreamFallback = "fallback" + // internalDomainUpstreamOnly serves an Internal Domain in explicit + // resolver only mode: its answer or failure is final. + internalDomainUpstreamOnly = "only" +) + +// internalDomainUpstreamName is the Name given to every generated Internal +// Domain upstream. It is only a display label: what identifies an upstream as +// generated is UpstreamConfig.InternalDomain, which no configuration can set. +const internalDomainUpstreamName = "Internal Domain resolver" + +// internalDomainsSummary counts the outcome of applying Internal Domains. It +// carries no domain names and no resolver addresses: info and warning logs +// report these counts, and the values themselves stay at debug level. +type internalDomainsSummary struct { + domains int // domains that produced rules + osMode int // domains routed to the OS/default resolver + explicit int // domains routed to configured resolvers + only int // explicit domains with no network fallback + resolvers int // explicit resolver upstreams generated + skipped int // entries dropped: unusable domain, no usable resolver, or duplicate + preempted int // domains already covered by a higher precedence rule +} + +// empty reports whether nothing was applied and nothing was dropped, which is +// the shape of a config whose split_dns list was absent or empty. +func (s internalDomainsSummary) empty() bool { + return s.domains == 0 && s.skipped == 0 && s.preempted == 0 +} + +// internalDomainLabelOK reports whether label satisfies the ASCII hostname +// contract the API validates against: 1-63 characters, letters, digits and +// hyphens only, and no leading or trailing hyphen. Punycode labels are plain +// LDH and pass unchanged. +func internalDomainLabelOK(label string) bool { + if len(label) == 0 || len(label) > 63 { + return false + } + if label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for i := 0; i < len(label); i++ { + c := label[i] + switch { + case c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-': + default: + return false + } + } + return true +} + +// normalizeInternalDomain canonicalizes a domain to the form policy rules are +// matched in, or returns "" if it is not a domain ctrld will route. +// +// Canonicalization is limited to what cannot change which names match: +// lowercasing, surrounding whitespace, the root dot, and a leading "*." (the +// caller adds the wildcard rule itself). What remains is validated as an ASCII +// hostname suffix rather than screened against a list of bad characters, +// because a denylist admits whatever it forgot: an embedded newline, quote, +// colon or control character would otherwise reach a generated rule. +// +// An invalid entry is rejected rather than repaired. Deleting the offending +// characters would produce a different suffix from the one that was +// configured, and routing a private domain to the wrong place is worse than +// not routing it. Single-label names stay supported. +// +// The rule matches the API's own validation, so a value the dashboard accepted +// passes here too. +func normalizeInternalDomain(domain string) string { + d := strings.ToLower(strings.TrimSpace(domain)) + d = strings.TrimPrefix(d, "*.") + d = strings.TrimSuffix(d, ".") + if d == "" || len(d) > 253 { + return "" + } + for _, label := range strings.Split(d, ".") { + if !internalDomainLabelOK(label) { + return "" + } + } + return d +} + +// internalDomainEndpoint converts a resolver to an endpoint ctrld can dial. +// A bare IPv4 or IPv6 address gets the default DNS port; and address that +// already carries a port keeps it. Anything else is rejected, so a malformed +// entry cannot silently become a hostname lookup. +func internalDomainEndpoint(resolver string) (string, bool) { + s := strings.TrimSpace(resolver) + if s == "" { + return "", false + } + if addr, err := netip.ParseAddr(s); err == nil { + return net.JoinHostPort(addr.String(), "53"), true + } + host, port, err := net.SplitHostPort(s) + if err != nil { + return "", false + } + addr, err := netip.ParseAddr(host) + if err != nil { + return "", false + } + p, err := strconv.Atoi(port) + if err != nil || p < 1 || p > 65535 { + return "", false + } + return net.JoinHostPort(addr.String(), port), true +} + +// internalDomainResolution is how an Internal Domain is resolved. +type internalDomainResolution int + +const ( + // internalDomainViaOS is the "network default" mode: the endpoint's + // OS/default resolver. + internalDomainViaOS internalDomainResolution = iota + // internalDomainViaResolvers is the configured resolvers first, then the + // active network and VPN resolvers. The default explicit selection. + internalDomainViaResolvers + // internalDomainViaResolversOnly is the configured resolvers and nothing + // else. + internalDomainViaResolversOnly +) + +// internalDomainMode reports how entry is resolved, and whether ctrld +// understands its mode at all. +// +// Mode is authoritative when the API sends it. "os" ignores any resolver +// addresses the entry still carries, so switching a domain back to the OS +// resolver cannot be undone by addresses a previous selection left behind; +// the explicit modes stay explicit even when the list they point at turns out +// to be unusable, rather than silently becoming OS resolution. An empty mode +// is a deployment that predates the field, where the list is the only signal, +// and an explicit list gets the default explicit mode. +// +// An unrecognized mode is not guessed at: routing a private domain by guess is +// the disclosure this feature exists to prevent. It fails closed instead. With +// resolvers it is resolved as explicit resolver only, through the resolvers the +// administrator chose and nothing else, so a wire value this build does not +// know, such as a renamed strict mode, cannot send the domain to the Control D +// upstream or the network. Without resolvers there is nothing to fail closed +// to, so the entry is reported as unknown and dropped. +func internalDomainMode(entry controld.SplitDNS) (resolution internalDomainResolution, known bool) { + switch strings.ToLower(strings.TrimSpace(entry.Mode)) { + case controld.SplitDNSModeOS: + return internalDomainViaOS, true + case controld.SplitDNSModeResolvers: + return internalDomainViaResolvers, true + case controld.SplitDNSModeResolversOnly: + return internalDomainViaResolversOnly, true + case "": + if len(entry.Resolvers) > 0 { + return internalDomainViaResolvers, true + } + return internalDomainViaOS, true + default: + if len(entry.Resolvers) > 0 { + return internalDomainViaResolversOnly, true + } + return internalDomainViaOS, false + } +} + +// internalDomainModeRecognized reports whether mode is one this build knows, +// as opposed to one internalDomainMode resolves by failing closed. +func internalDomainModeRecognized(mode string) bool { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "", controld.SplitDNSModeOS, controld.SplitDNSModeResolvers, controld.SplitDNSModeResolversOnly: + return true + } + return false +} + +// internalDomainRouting returns how an entry is resolved and the endpoints it +// routes to, or a reason it cannot be used. No endpoints and no reason means +// OS resolution. +// +// Generation and refresh comparison both go through here, so the two can never +// disagree about what an entry means. +func internalDomainRouting(entry controld.SplitDNS) (resolution internalDomainResolution, endpoints []string, reason string) { + resolution, known := internalDomainMode(entry) + if !known { + return resolution, nil, "unknown_mode" + } + if resolution == internalDomainViaOS { + return resolution, nil, "" + } + endpoints = make([]string, 0, len(entry.Resolvers)) + for _, resolver := range entry.Resolvers { + if endpoint, ok := internalDomainEndpoint(resolver); ok { + endpoints = append(endpoints, endpoint) + } + } + // The administrator selected explicit resolvers and none of them parsed. + // Falling back to the OS resolver would turn that selection into a + // different one, so the entry is dropped and the domain keeps its normal + // path instead. + if len(endpoints) == 0 { + return resolution, nil, "no_usable_resolver" + } + return resolution, endpoints, "" +} + +// internalDomainRuleExists reports whether the listener policy already routes +// source. Internal Domains never overwrite a rule that is already there, which +// is what gives Magic Folder excludes and endpoint custom configuration +// precedence over them. +func internalDomainRuleExists(lc *ctrld.ListenerConfig, source string) bool { + if lc.Policy == nil { + return false + } + for _, rule := range lc.Policy.Rules { + if _, ok := rule[source]; ok { + return true + } + } + return false +} + +// internalDomainSpecificity ranks a normalized domain by label count, which is +// how much of a name it pins down. "z.example.com" ranks above "example.com". +func internalDomainSpecificity(domain string) int { + if domain == "" { + return 0 + } + return strings.Count(domain, ".") + 1 +} + +// orderInternalDomainsBySpecificity returns entries most-specific first. +// +// upstreamFor matches policy rules in slice order and takes the first hit, so +// generation order is routing precedence. The API sorts Internal Domains +// bytewise by domain, which puts a parent suffix ahead of its children: +// appending in that order would let "*.example.com" swallow every query a +// configured "z.example.com" rule was meant to answer, and the administrator +// never chose that priority. Overlapping suffixes are permitted, so the more +// specific one has to win. +// +// The sort is stable, so entries that are equally specific keep API order and +// the first of two identical domains still wins the duplicate check. +func orderInternalDomainsBySpecificity(entries []controld.SplitDNS) []controld.SplitDNS { + ordered := make([]controld.SplitDNS, len(entries)) + copy(ordered, entries) + sort.SliceStable(ordered, func(i, j int) bool { + a, b := normalizeInternalDomain(ordered[i].Domain), normalizeInternalDomain(ordered[j].Domain) + if sa, sb := internalDomainSpecificity(a), internalDomainSpecificity(b); sa != sb { + return sa > sb + } + return a < b + }) + return ordered +} + +// applyInternalDomains adds a policy rule per organization Internal Domain to +// every listener in cfg, and the upstreams those rules need. +// +// Each domain contributes two rules, the domain itself and "*." + domain, which +// is how a suffix and all of its subdomains are matched. An entry with no +// resolvers routes to the OS/default resolver by way of an empty upstream list, +// the same representation Magic Folder excludes use. An entry with resolvers +// gets one generated upstream per address, listed in configured order so the +// later ones act as failover for the first. Those upstreams carry the mode in +// UpstreamConfig.InternalDomain. +// +// cfg is rebuilt from the API response on every fetch, so this function is the +// only writer of internal_* upstreams: a removed domain leaves nothing behind. +func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) internalDomainsSummary { + var summary internalDomainsSummary + if len(entries) == 0 || len(cfg.Listener) == 0 { + return summary + } + if cfg.Upstream == nil { + cfg.Upstream = make(map[string]*ctrld.UpstreamConfig) + } + seen := make(map[string]struct{}, len(entries)) + for _, entry := range orderInternalDomainsBySpecificity(entries) { + domain := normalizeInternalDomain(entry.Domain) + if domain == "" { + mainLog.Load().Debug().Msgf("internal domains: skipping unusable domain %q", entry.Domain) + summary.skipped++ + continue + } + if _, dup := seen[domain]; dup { + mainLog.Load().Debug().Msgf("internal domains: skipping duplicate domain %q", domain) + summary.skipped++ + continue + } + + // Routing is resolved, and preemption decided, before anything is + // written: an entry that cannot be used, or a domain already covered by + // a higher precedence rule, must not leave a generated upstream behind + // with no rule referring to it. + resolution, endpoints, reason := internalDomainRouting(entry) + if reason != "" { + mainLog.Load().Debug().Msgf("internal domains: skipping %q: %s", domain, reason) + summary.skipped++ + continue + } + if !internalDomainModeRecognized(entry.Mode) { + mainLog.Load().Debug().Msgf("internal domains: %q has unrecognized mode %q; using its resolvers only", domain, entry.Mode) + } + seen[domain] = struct{}{} + + sources := []string{domain, "*." + domain} + free := make(map[*ctrld.ListenerConfig][]string, len(cfg.Listener)) + anyFree := false + for _, lc := range cfg.Listener { + for _, source := range sources { + if internalDomainRuleExists(lc, source) { + mainLog.Load().Debug().Msgf("internal domains: %q already routed by a higher precedence rule", source) + continue + } + free[lc] = append(free[lc], source) + anyFree = true + } + } + if !anyFree { + summary.preempted++ + continue + } + + marker := internalDomainUpstreamFallback + if resolution == internalDomainViaResolversOnly { + marker = internalDomainUpstreamOnly + } + targets := make([]string, 0, len(endpoints)) + for _, endpoint := range endpoints { + key := internalDomainUpstreamPrefix + strconv.Itoa(summary.resolvers) + cfg.Upstream[key] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: endpoint, + Timeout: 5000, + InternalDomain: marker, + } + targets = append(targets, upstreamPrefix+key) + summary.resolvers++ + } + for lc, sources := range free { + if lc.Policy == nil { + lc.Policy = &ctrld.ListenerPolicyConfig{} + } + for _, source := range sources { + lc.Policy.Rules = append(lc.Policy.Rules, ctrld.Rule{source: targets}) + } + } + summary.domains++ + if len(targets) == 0 { + summary.osMode++ + mainLog.Load().Debug().Msgf("internal domains: %q routed to the OS resolver", domain) + } else { + summary.explicit++ + if resolution == internalDomainViaResolversOnly { + summary.only++ + } + mainLog.Load().Debug().Msgf("internal domains: %q routed to %v", domain, targets) + } + } + return summary +} + +// isGeneratedInternalDomainUpstream reports whether uc is an upstream +// applyInternalDomains created. +// +// Generated Internal Domain resolvers are part of the managed Control D +// configuration, not upstreams the endpoint operator chose, so eligibility +// checks that ask "is this a plain single-upstream Control D install" must not +// count them. Neither the key nor any other configurable field is proof of +// that: a local config or an endpoint custom configuration can name an upstream +// anything and give it any type and name. Only the InternalDomain marker is, +// because ctrld sets it and no configuration file can. +func isGeneratedInternalDomainUpstream(uc *ctrld.UpstreamConfig) bool { + return uc != nil && uc.InternalDomain != "" +} + +// internalDomainUpstream returns the generated Internal Domain upstream an +// "upstream." reference names, or nil if it names any other upstream. +func (p *prog) internalDomainUpstream(upstream string) *ctrld.UpstreamConfig { + if p.cfg == nil { + return nil + } + uc := p.cfg.Upstream[strings.TrimPrefix(upstream, upstreamPrefix)] + if !isGeneratedInternalDomainUpstream(uc) { + return nil + } + return uc +} + +// isInternalDomainUpstream reports whether an upstream reference names a +// generated Internal Domain resolver, in either explicit mode. +func (p *prog) isInternalDomainUpstream(upstream string) bool { + return p.internalDomainUpstream(upstream) != nil +} + +// internalDomainExplicitUpstreams reports whether every upstream serving a +// query is a configured Internal Domain resolver, in either explicit mode. +// +// proxy() uses this to try the configured resolvers before anything else, and +// to keep such a query off the OS-resolver catch-all when they fail: that +// catch-all may reach public nameservers, and a private name must not be sent +// somewhere it was not meant to go. In fallback mode proxy() tries the +// network resolvers instead (see internalDomainFallbackUpstreams). It also +// keeps an unreachable internal resolver from triggering the endpoint-wide +// recovery flow, which exists for the loss of general DNS, not for one +// unavailable internal server. +func (p *prog) internalDomainExplicitUpstreams(upstreams []string) bool { + if len(upstreams) == 0 { + return false + } + for _, upstream := range upstreams { + if !p.isInternalDomainUpstream(upstream) { + return false + } + } + return true +} + +// internalDomainFallbackUpstreams reports whether upstreams are the configured +// resolvers of an Internal Domain in explicit-with-network-fallback mode, whose +// failure, SERVFAIL or NXDOMAIN hands the query to the active network and VPN +// resolvers. +func (p *prog) internalDomainFallbackUpstreams(upstreams []string) bool { + if len(upstreams) == 0 { + return false + } + for _, upstream := range upstreams { + uc := p.internalDomainUpstream(upstream) + if uc == nil || uc.InternalDomain != internalDomainUpstreamFallback { + return false + } + } + return true +} + +// internalDomainFallbackRcode reports whether a configured resolver's answer +// sends a fallback-mode query on to the next resolver. SERVFAIL means the +// resolver could not answer, and NXDOMAIN may only mean it does not know a +// name the local network's resolver does. REFUSED and NOTIMP mean it will not +// serve this endpoint, as an organization resolver with an access list answers +// an endpoint off the organization network, which is closer to unreachable +// than to an answer. Any other answer is final, including an empty NOERROR: +// the name exists, it just has no record of that type. +func internalDomainFallbackRcode(rcode int) bool { + switch rcode { + case dns.RcodeServerFailure, dns.RcodeNameError, dns.RcodeRefused, dns.RcodeNotImplemented: + return true + } + return false +} + +// internalDomainsEqual reports whether two Internal Domains lists would produce +// the same routing. Domains are compared in canonical form and independently of +// the order the API returned them, so an unchanged list does not trigger a +// reload on every refresh. Resolver order is significant: it is the order the +// administrator chose, and it decides which resolver is tried first. The mode +// is significant too, including a switch between the two explicit modes. +func internalDomainsEqual(a, b []controld.SplitDNS) bool { + type route struct { + domain string + resolution internalDomainResolution + endpoints []string + } + canonical := func(entries []controld.SplitDNS) []route { + out := make([]route, 0, len(entries)) + for _, entry := range entries { + domain := normalizeInternalDomain(entry.Domain) + if domain == "" { + continue + } + // Entries that cannot be used route nothing, so they cannot make + // an otherwise unchanged list look changed. + resolution, endpoints, reason := internalDomainRouting(entry) + if reason != "" { + continue + } + out = append(out, route{domain: domain, resolution: resolution, endpoints: endpoints}) + } + sort.Slice(out, func(i, j int) bool { return out[i].domain < out[j].domain }) + return out + } + x, y := canonical(a), canonical(b) + if len(x) != len(y) { + return false + } + for i := range x { + if x[i].domain != y[i].domain || x[i].resolution != y[i].resolution || !slices.Equal(x[i].endpoints, y[i].endpoints) { + return false + } + } + return true +} + +// internalDomainFailureReason classifies a resolver failure for logging. The +// error itself names the endpoint that could not be reached, so only this +// classification may appear above debug level. +func internalDomainFailureReason(err error) string { + switch { + case err == nil: + return "no_answer" + case errors.Is(err, syscall.ECONNREFUSED): + return "refused" + case errors.Is(err, syscall.EADDRNOTAVAIL): + return "source_unavailable" + case ctrldnet.IsUnreachable(err): + return "unreachable" + } + var netErr net.Error + if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &netErr) && netErr.Timeout()) { + return "timeout" + } + return "error" +} + +// logUpstreamProbeFailure reports a background probe failure against the +// upstream that the reference upstream names. +// +// Loop checking and recovery checking run on timers, so for a generated +// Internal Domain resolver they would publish the organization's private +// address without any user ever querying the domain. The name and the +// endpoint of any upstream are operator text that can hold a token, so both +// stay on the debug line. The leveled line names the upstream by its bounded +// name, so the outcome stays visible while the operator text does not. A +// generated Internal Domain resolver also trades the address-bearing error +// for a classification. +func (p *prog) logUpstreamProbeFailure(upstream string, uc *ctrld.UpstreamConfig, err error, level func() *ctrld.LogEvent, format string, args ...any) { + detail := append([]any{}, args...) + p.Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) + event := level().Str("upstream", journalUpstreamName(upstream)) + if !isGeneratedInternalDomainUpstream(uc) { + event.Err(err).Msgf(format, detail...) + return + } + event.Str("failure", internalDomainFailureReason(err)). + Msgf(format+" for an Internal Domain resolver", detail...) +} + +// betterInternalDomainNegative returns whichever of two negative answers a +// fallback-mode query should end with: NXDOMAIN over SERVFAIL, since a +// resolver that says the name does not exist has answered, and otherwise the +// first one seen. +func betterInternalDomainNegative(current, candidate *dns.Msg) *dns.Msg { + switch { + case candidate == nil: + return current + case current == nil: + return candidate + case candidate.Rcode == dns.RcodeNameError && current.Rcode != dns.RcodeNameError: + return candidate + } + return current +} + +// internalDomainVPNUpstream returns the upstream the network fallback uses for +// a VPN DNS server. A seam: VPN DNS servers are dialed on port 53, which a test +// fixture cannot listen on. +var internalDomainVPNUpstream = func(m *vpnDNSManager, server string) *ctrld.UpstreamConfig { + return m.upstreamConfigFor(server) +} + +// internalDomainOSResolve resolves msg through the OS resolver for the network +// fallback. A seam, so a test sees the context the fallback passes: that +// context is what keeps the private name off public nameservers. +var internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + resolver, err := ctrld.NewResolver(ctx, osUpstreamConfig) + if err != nil { + return nil, err + } + resolveCtx, cancel := osUpstreamConfig.Context(ctx) + defer cancel() + return resolver.Resolve(resolveCtx, msg) +} + +// resolveInternalDomainOnNetwork is the network fallback of an Internal Domain +// in explicit-with-network-fallback mode. proxy() runs it once the configured +// resolvers have failed, or answered SERVFAIL or NXDOMAIN. +// +// It asks only resolvers that the endpoint's active network or VPN provides, +// in this order: VPN DNS servers whose domains match the query, then VPN DNS +// servers with no domains (both only in dns-intercept mode, where ctrld tracks +// them), then the OS resolver's LAN nameservers. The OS resolver is asked as a +// LAN-only query, so the private name never reaches a public nameserver, not +// even one DHCP supplied, nor ctrld's own public fallback. With no LAN +// nameserver the OS step sends nothing. +// +// The first NOERROR answer is returned as answer, including an empty one, +// which is final. Otherwise negative is the best SERVFAIL or NXDOMAIN answer +// seen. While Windows is serving retained VPN DNS state, a VPN transport +// failure stops the fallback before the OS resolver, as it does for VPN split +// routing. +func (p *prog) resolveInternalDomainOnNetwork(ctx context.Context, msg *dns.Msg) (answer, negative *dns.Msg) { + domain := msg.Question[0].Name + resolve := func(ctx context.Context, uc *ctrld.UpstreamConfig) (*dns.Msg, error) { + resolver, err := ctrld.NewResolver(ctx, uc) + if err != nil { + return nil, err + } + resolveCtx, cancel := uc.Context(ctx) + defer cancel() + return resolver.Resolve(resolveCtx, msg) + } + // accept reports whether a network answer is final, and keeps a negative + // one for the end. + accept := func(source string, m *dns.Msg) bool { + if !sameQuestion(msg, m) { + ctrld.Log(ctx, p.Debug(), "internal domain fallback: discarding answer from %s: question mismatch", source) + return false + } + if m.Rcode == dns.RcodeSuccess { + ctrld.Log(ctx, p.Debug(), "internal domain fallback: %s answered %s", source, domain) + return true + } + ctrld.Log(ctx, p.Debug(), "internal domain fallback: %s answered %s for %s", + source, dns.RcodeToString[m.Rcode], domain) + if internalDomainFallbackRcode(m.Rcode) { + negative = betterInternalDomainNegative(negative, m) + } + return false + } + + if dnsIntercept && p.vpnDNS != nil { + var servers []string + for _, server := range slices.Concat(p.vpnDNS.UpstreamForDomain(domain), p.vpnDNS.DomainlessServers()) { + if !slices.Contains(servers, server) { + servers = append(servers, server) + } + } + var gotAnswer, gotTransportFailure bool + for _, server := range servers { + m, err := resolve(ctx, internalDomainVPNUpstream(p.vpnDNS, server)) + if m == nil { + gotTransportFailure = true + ctrld.Log(ctx, p.Debug().Err(err), "internal domain fallback: VPN DNS server %s failed", server) + continue + } + gotAnswer = true + p.vpnDNS.VPNDNSReachable() + if accept("VPN DNS server "+server, m) { + return m, nil + } + } + if !gotAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, servers) { + return nil, negative + } + } + + m, err := internalDomainOSResolve(ctrld.LanOnlyQueryCtx(ctx), msg) + if m == nil { + ctrld.Log(ctx, p.Debug().Err(err), "internal domain fallback: OS resolver LAN nameservers failed") + return nil, negative + } + if accept("OS resolver", m) { + return m, nil + } + return nil, negative +} + +// internalDomainProbeInterval is how often, at most, a down fallback-mode +// Internal Domain resolver is re-checked in the background. +const internalDomainProbeInterval = 30 * time.Second + +// internalDomainProber re-checks down fallback-mode Internal Domain resolvers +// in the background, and marks one up again when it answers. +// +// proxy() skips such a resolver while it is down, so that an endpoint away +// from the organization network goes to the network fallback without waiting +// on it. A generated resolver comes back up only by answering a query: it is +// excluded from recovery, and the loop checker leaves down upstreams alone. So +// without this re-check, a resolver that went down would stay skipped until +// the next reload, even once the endpoint is back on the organization network. +// +// The zero value is ready to use. +type internalDomainProber struct { + mu sync.Mutex + last map[string]time.Time + + // now and start are seams, so a test controls the clock and runs the + // re-check itself. nil means time.Now and a new goroutine. + now func() time.Time + start func(func()) +} + +// probe re-checks upstream in the background, at most once per +// internalDomainProbeInterval, by sending it a copy of msg. The query goes to +// the organization's own resolver, which it was meant for anyway. Any DNS +// answer proves the resolver reachable, the same as on the query path, and +// marks it up in um; a failure leaves it down until the next re-check. +func (pr *internalDomainProber) probe(um *upstreamMonitor, upstream string, uc *ctrld.UpstreamConfig, msg *dns.Msg) { + now := time.Now + if pr.now != nil { + now = pr.now + } + pr.mu.Lock() + if last, ok := pr.last[upstream]; ok && now().Sub(last) < internalDomainProbeInterval { + pr.mu.Unlock() + return + } + if pr.last == nil { + pr.last = make(map[string]time.Time) + } + pr.last[upstream] = now() + pr.mu.Unlock() + + query := msg.Copy() + run := func() { + var answer *dns.Msg + // The resolver address is private: keep transport errors that name it + // at debug, as the query path does. + ctx := ctrld.PrivateResolverCtx(ctrld.LoggerCtx(context.Background(), mainLog.Load())) + resolver, err := ctrld.NewResolver(ctx, uc) + if err == nil { + resolveCtx, cancel := uc.Context(ctx) + answer, err = resolver.Resolve(resolveCtx, query) + cancel() + } + if answer == nil { + mainLog.Load().Debug().Err(err).Msgf("internal domains: %s is still down", upstream) + return + } + mainLog.Load().Debug().Msgf("internal domains: %s answered the re-check", upstream) + um.noteSuccess(upstream) + } + if pr.start != nil { + pr.start(run) + return + } + go run() +} diff --git a/cmd/cli/internal_domains_fallback_test.go b/cmd/cli/internal_domains_fallback_test.go new file mode 100644 index 00000000..6f6e8b80 --- /dev/null +++ b/cmd/cli/internal_domains_fallback_test.go @@ -0,0 +1,587 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// startRcodeDNSFixture is startDNSFixture answering every question with rcode. +// A NOERROR answer carries an A record unless empty is set, which gives the +// name-exists-but-no-record-of-that-type answer. +func startRcodeDNSFixture(t *testing.T, rcode int, empty bool) *dnsFixture { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := &dnsFixture{addr: pc.LocalAddr().String()} + started := make(chan struct{}) + server := &dns.Server{ + PacketConn: pc, + NotifyStartedFunc: func() { close(started) }, + Handler: dns.HandlerFunc(func(w dns.ResponseWriter, msg *dns.Msg) { + f.mu.Lock() + if len(msg.Question) > 0 { + f.qs = append(f.qs, msg.Question[0].Name) + } + f.mu.Unlock() + reply := new(dns.Msg) + reply.SetRcode(msg, rcode) + if rcode == dns.RcodeSuccess && !empty && len(msg.Question) > 0 { + rr, _ := dns.NewRR(msg.Question[0].Name + " 60 IN A 10.9.9.9") + reply.Answer = append(reply.Answer, rr) + } + _ = w.WriteMsg(reply) + }), + } + go func() { _ = server.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("DNS fixture did not start") + } + t.Cleanup(func() { _ = server.Shutdown() }) + return f +} + +func deadLoopbackAddr(t *testing.T) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := pc.LocalAddr().String() + _ = pc.Close() + return addr +} + +// Explicit resolver with network fallback is the default explicit mode: the +// configured resolver is asked first, and a timeout, an unreachable resolver, +// SERVFAIL or NXDOMAIN hands the query to the network's resolver. A valid +// empty answer is final, and "explicit resolver only" never falls back. +func TestInternalDomainsNetworkFallback(t *testing.T) { + for _, tc := range []struct { + name string + mode string + // internal is the configured resolver's rcode; -1 is unreachable. + internal int + internalEmpty bool + network int + wantNetwork bool + wantRcode int + wantAnswer bool + }{ + {name: "nxdomain falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "servfail falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "unreachable falls back", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "absent mode falls back", mode: "", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "empty answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, internalEmpty: true, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess}, + {name: "refused falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "notimp falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNotImplemented, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "refused and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "formerr is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeFormatError, network: dns.RcodeSuccess, wantRcode: dns.RcodeFormatError}, + {name: "only mode keeps refused", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantRcode: dns.RcodeRefused}, + {name: "unknown mode fails closed", mode: "explicit_only", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError}, + {name: "nobody knows the name", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "configured nxdomain beats network servfail", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeServerFailure, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "unreachable and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "only mode keeps nxdomain", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError}, + {name: "only mode keeps servfail", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure}, + {name: "only mode fails when unreachable", mode: controld.SplitDNSModeResolversOnly, internal: -1, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure}, + } { + t.Run(tc.name, func(t *testing.T) { + network := startRcodeDNSFixture(t, tc.network, false) + useOSResolverFixture(t, network) + + var internal *dnsFixture + resolver := deadLoopbackAddr(t) + if tc.internal >= 0 { + internal = startRcodeDNSFixture(t, tc.internal, tc.internalEmpty) + resolver = internal.addr + } + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: tc.mode, Resolvers: []string{resolver}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if internal != nil { + if got := internal.questions(); len(got) != 1 { + t.Errorf("configured resolver questions = %v, want it asked first", got) + } + } + if got := network.questions(); (len(got) > 0) != tc.wantNetwork { + t.Errorf("network resolver questions = %v, want asked = %v", got, tc.wantNetwork) + } + if res == nil || res.answer == nil { + t.Fatalf("no answer") + } + if res.answer.Rcode != tc.wantRcode { + t.Errorf("rcode = %s, want %s", dns.RcodeToString[res.answer.Rcode], dns.RcodeToString[tc.wantRcode]) + } + if got := len(res.answer.Answer) > 0; got != tc.wantAnswer { + t.Errorf("answer records = %v, want records = %v", res.answer.Answer, tc.wantAnswer) + } + }) + } +} + +// The configured resolvers are all tried, in order, before the network: an +// NXDOMAIN from the first does not skip the second. +func TestInternalDomainsNetworkFallbackTriesEveryConfiguredResolverFirst(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + first := startRcodeDNSFixture(t, dns.RcodeNameError, false) + second := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{first.addr, second.addr}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if len(first.questions()) != 1 || len(second.questions()) != 1 { + t.Errorf("configured resolvers asked %v, %v; want both once", first.questions(), second.questions()) + } + if got := network.questions(); len(got) != 0 { + t.Errorf("network resolver asked %v although a configured resolver answered", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the second resolver's NOERROR", res) + } +} + +// The fallback is no reason to log the private resolver address above debug, +// or the domain: the summary-only REPLY line carries neither. +func TestInternalDomainsNetworkFallbackLogsNoPrivateValues(t *testing.T) { + // The shared log output is console-encoded on master; capture JSON so the + // level of each line can be checked. + buf := captureInternalDomainsLogs(t) + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(ctrld.LoggerCtx(context.Background(), p.logger.Load()), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + before := len(buf.String()) + p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + out := buf.String()[before:] + + if !strings.Contains(out, "internal domain network fallback") { + t.Fatalf("expected the fallback REPLY line, got: %s", out) + } + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + if strings.Contains(line, internal.addr) || strings.Contains(line, "corp.example") { + t.Errorf("private value disclosed above debug level: %s", line) + } + } +} + +// internalDomainUpstreamsProg returns a prog whose config has one legacy +// upstream per key, each carrying the given InternalDomain marker. +func internalDomainUpstreamsProg(markers map[string]string) *prog { + cfg := &ctrld.Config{Upstream: make(map[string]*ctrld.UpstreamConfig, len(markers))} + for key, marker := range markers { + cfg.Upstream[key] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: "10.0.0.53:53", + InternalDomain: marker, + } + } + return &prog{cfg: cfg} +} + +func TestInternalDomainFallbackUpstreams(t *testing.T) { + p := internalDomainUpstreamsProg(map[string]string{ + "internal_0": internalDomainUpstreamFallback, + "internal_1": internalDomainUpstreamFallback, + "internal_2": internalDomainUpstreamOnly, + "0": "", + }) + for _, tc := range []struct { + in []string + want bool + }{ + {in: nil, want: false}, + {in: []string{"upstream.internal_0", "upstream.internal_1"}, want: true}, + {in: []string{"upstream.internal_2"}, want: false}, + {in: []string{"upstream.internal_0", "upstream.internal_2"}, want: false}, + {in: []string{"upstream.internal_0", "upstream.0"}, want: false}, + {in: []string{upstreamOS}, want: false}, + } { + if got := p.internalDomainFallbackUpstreams(tc.in); got != tc.want { + t.Errorf("internalDomainFallbackUpstreams(%v) = %v, want %v", tc.in, got, tc.want) + } + } + if !p.internalDomainExplicitUpstreams([]string{"upstream.internal_2"}) { + t.Error("an explicit-only upstream must still count as an Internal Domain resolver") + } +} + +// Both explicit modes generate internal_ upstreams; the mode is carried by +// the InternalDomain marker. Switching between the modes is a routing change +// the next refresh must apply. +func TestInternalDomainsExplicitModes(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + {Domain: "b.example.com", Mode: " Resolvers_Only ", Resolvers: []string{"10.0.0.54"}}, + }) + if summary.explicit != 2 || summary.only != 1 || summary.resolvers != 2 { + t.Fatalf("summary = %+v", summary) + } + for _, tc := range []struct{ domain, target, marker string }{ + {"a.example.com", "upstream.internal_0", internalDomainUpstreamFallback}, + {"b.example.com", "upstream.internal_1", internalDomainUpstreamOnly}, + } { + if targets, _ := ruleTargets(t, cfg, tc.domain); strings.Join(targets, ",") != tc.target { + t.Errorf("%s targets = %v, want %s", tc.domain, targets, tc.target) + } + uc := cfg.Upstream[strings.TrimPrefix(tc.target, upstreamPrefix)] + if uc == nil || uc.InternalDomain != tc.marker { + t.Errorf("%s upstream = %+v, want marker %q", tc.domain, uc, tc.marker) + } + } + + fallback := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}} + only := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"10.0.0.53"}}} + legacy := []controld.SplitDNS{{Domain: "a.example.com", Resolvers: []string{"10.0.0.53"}}} + if internalDomainsEqual(fallback, only) { + t.Error("switching to explicit resolver only must be detected as a change") + } + if !internalDomainsEqual(fallback, legacy) { + t.Error("an absent mode with resolvers is the default explicit mode") + } +} + +// An upstream a configuration defines is an ordinary upstream, whatever it is +// named and however closely it copies a generated one: a local ctrld.toml that +// names an upstream internal_, gives it the legacy type and the generated +// name, must still get the OS-resolver catch-all when it fails. +func TestInternalDomainsUserDefinedLookalikeIsOrdinaryUpstream(t *testing.T) { + osFixture := startDNSFixture(t) + useOSResolverFixture(t, osFixture) + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + cfg.Upstream["internal_0"] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadLoopbackAddr(t), + Timeout: 500, + } + cfg.Listener["0"].Policy.Rules = append(cfg.Listener["0"].Policy.Rules, + ctrld.Rule{"*.corp.example": []string{"upstream.internal_0"}}) + p := newInternalDomainsProg(t, cfg) + if p.isInternalDomainUpstream("upstream.internal_0") || isGeneratedInternalDomainUpstream(cfg.Upstream["internal_0"]) { + t.Fatal("a user-defined upstream was recognized as a generated Internal Domain resolver") + } + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := osFixture.questions(); len(got) != 1 { + t.Errorf("OS resolver questions = %v, want the ordinary catch all", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the OS resolver's NOERROR", res) + } +} + +// captureInternalDomainProbes makes p's background re-checks run only when the +// test calls them, and returns the ones scheduled so far. +func captureInternalDomainProbes(p *prog) *[]func() { + var probes []func() + p.internalDomainProbes.start = func(run func()) { probes = append(probes, run) } + return &probes +} + +// The off-site case that fallback mode exists for: the organization resolver +// is unreachable and the network answers. Once the monitor reports the +// resolver down, a query must go to the network without waiting on the +// resolver's timeout, and a background re-check must bring it back once it +// answers again. +func TestInternalDomainsDownResolverIsSkippedAndReChecked(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + blackhole := blackholeUDPAddr(t) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{blackhole}}}) + p := newInternalDomainsProg(t, cfg) + probes := captureInternalDomainProbes(p) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ask := func() (*proxyResponse, time.Duration) { + t.Helper() + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + start := time.Now() + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + return res, time.Since(start) + } + + const upstream = upstreamPrefix + internalDomainUpstreamPrefix + "0" + p.um.mu.Lock() + p.um.markDown(upstream, maxFailureRequest, "immediate") + p.um.mu.Unlock() + + for i := 0; i < 2; i++ { + res, elapsed := ask() + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatalf("query %d: answer = %v, want the network's NOERROR", i+1, res) + } + // The blackholed resolver would cost its 2s read timeout. + if elapsed > time.Second { + t.Fatalf("query %d took %v: it waited on a resolver that is down", i+1, elapsed) + } + } + if got := len(*probes); got != 1 { + t.Fatalf("re-checks scheduled = %d, want one per interval", got) + } + + // The resolver is still unreachable: the re-check leaves it down. + (*probes)[0]() + if !p.um.isDown(upstream) { + t.Fatal("a failed re-check marked the resolver up") + } + + // Back on the organization network: the next re-check, once the interval + // has passed, finds it answering and marks it up, so queries use it again. + internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + cfg.Upstream["internal_0"].Endpoint = internal.addr + later := time.Now().Add(internalDomainProbeInterval) + p.internalDomainProbes.now = func() time.Time { return later } + ask() + if got := len(*probes); got != 2 { + t.Fatalf("re-checks scheduled = %d, want a second one after the interval", got) + } + (*probes)[1]() + if p.um.isDown(upstream) { + t.Fatal("the resolver answered the re-check but is still down") + } + before := len(network.questions()) + if res, _ := ask(); res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatalf("answer = %v, want the resolver's NOERROR", res) + } + if got := internal.questions(); len(got) != 2 { + t.Errorf("resolver questions = %v, want the re-check and the query", got) + } + if got := len(network.questions()); got != before { + t.Errorf("network asked again although the resolver answered") + } +} + +// Explicit resolver only has no fallback to skip to, so a down resolver is +// still asked: skipping it would only turn its answer into a SERVFAIL. +func TestInternalDomainsOnlyModeDoesNotSkipDownResolver(t *testing.T) { + useOSResolverFixture(t, startDNSFixture(t)) + internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + probes := captureInternalDomainProbes(p) + + p.um.mu.Lock() + p.um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"0", maxFailureRequest, "immediate") + p.um.mu.Unlock() + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := internal.questions(); len(got) != 1 { + t.Errorf("resolver questions = %v, want it asked although it is down", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the resolver's NOERROR", res) + } + if len(*probes) != 0 { + t.Errorf("explicit resolver only scheduled %d re-checks", len(*probes)) + } +} + +// fallbackTestQuery sends one query for host.corp.example through proxy(). +func fallbackTestQuery(t *testing.T, p *prog, cfg *ctrld.Config) *proxyResponse { + t.Helper() + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + return p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) +} + +// The OS step of the fallback is what keeps the private name off public +// nameservers, and it does so through the context it passes. A test that +// swaps the OS resolver for a single-endpoint fixture would pass with any +// context, so this one sees the context itself and runs the step through a +// real OS resolver whose pool holds a LAN and a public nameserver. +func TestInternalDomainsNetworkFallbackOSStepIsLanOnly(t *testing.T) { + lan := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + pool := ctrld.NewResolverWithNameserver([]string{lan.addr, "192.0.2.53:53"}) + var lanOnly []bool + prev := internalDomainOSResolve + internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + isLanOnly, _ := ctx.Value(ctrld.LanOnlyQueryCtxKey{}).(bool) + lanOnly = append(lanOnly, isLanOnly) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return pool.Resolve(ctx, msg) + } + t.Cleanup(func() { internalDomainOSResolve = prev }) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + + res := fallbackTestQuery(t, p, cfg) + if len(lanOnly) != 1 || !lanOnly[0] { + t.Fatalf("OS step contexts LAN-only = %v, want one LAN-only query", lanOnly) + } + if got := lan.questions(); len(got) != 1 { + t.Errorf("LAN nameserver questions = %v, want the fallback query", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the LAN nameserver's NOERROR", res) + } +} + +// useVPNFallbackFixtures routes the fallback's VPN DNS servers to fixtures: +// real VPN DNS servers are dialed on port 53. +func useVPNFallbackFixtures(t *testing.T, servers map[string]string) { + t.Helper() + prev := internalDomainVPNUpstream + internalDomainVPNUpstream = func(_ *vpnDNSManager, server string) *ctrld.UpstreamConfig { + return &ctrld.UpstreamConfig{Name: "VPN DNS", Type: ctrld.ResolverTypeLegacy, Endpoint: servers[server], Timeout: 1000} + } + t.Cleanup(func() { internalDomainVPNUpstream = prev }) +} + +// The fallback asks VPN DNS servers that match the domain, then domain-less +// ones, then the OS resolver, and stops at the first NOERROR. +func TestInternalDomainsNetworkFallbackVPNOrder(t *testing.T) { + for _, tc := range []struct { + name string + matched, domainless int + wantMatched, wantDomless int + wantOS bool + }{ + {name: "matching VPN server answers", matched: dns.RcodeSuccess, domainless: dns.RcodeSuccess, wantMatched: 1}, + {name: "then the domain-less one", matched: dns.RcodeNameError, domainless: dns.RcodeSuccess, wantMatched: 1, wantDomless: 1}, + {name: "then the OS resolver", matched: dns.RcodeNameError, domainless: dns.RcodeRefused, wantMatched: 1, wantDomless: 1, wantOS: true}, + } { + t.Run(tc.name, func(t *testing.T) { + useDNSIntercept(t) + osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, osFixture) + matched := startRcodeDNSFixture(t, tc.matched, false) + domainless := startRcodeDNSFixture(t, tc.domainless, false) + useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": matched.addr, "10.8.0.2": domainless.addr}) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + p.vpnDNS = &vpnDNSManager{ + routes: map[string][]string{"corp.example": {"10.8.0.1"}}, + domainlessServers: []string{"10.8.0.2"}, + logger: &p.logger, + } + + res := fallbackTestQuery(t, p, cfg) + if got := len(matched.questions()); got != tc.wantMatched { + t.Errorf("matching VPN server asked %d times, want %d", got, tc.wantMatched) + } + if got := len(domainless.questions()); got != tc.wantDomless { + t.Errorf("domain-less VPN server asked %d times, want %d", got, tc.wantDomless) + } + if got := len(osFixture.questions()) > 0; got != tc.wantOS { + t.Errorf("OS resolver asked = %v, want %v", got, tc.wantOS) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want NOERROR", res) + } + }) + } +} + +// While Windows is serving retained VPN DNS state, a VPN transport failure +// stops the fallback before the OS resolver, as it does for VPN split routing. +func TestInternalDomainsNetworkFallbackStopsOnRetainedVPNTransportFailure(t *testing.T) { + useDNSIntercept(t) + prevSettling := vpnDNSSettlingEnabled + vpnDNSSettlingEnabled = true + t.Cleanup(func() { vpnDNSSettlingEnabled = prevSettling }) + osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, osFixture) + useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": deadLoopbackAddr(t)}) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + p.vpnDNS = &vpnDNSManager{ + routes: map[string][]string{"corp.example": {"10.8.0.1"}}, + retainedAfterEmptyDiscovery: true, + logger: &p.logger, + } + + res := fallbackTestQuery(t, p, cfg) + if got := osFixture.questions(); len(got) != 0 { + t.Errorf("OS resolver asked %v while retained VPN DNS state is active", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeNameError { + t.Errorf("answer = %v, want the configured resolver's NXDOMAIN", res) + } +} diff --git a/cmd/cli/internal_domains_review_test.go b/cmd/cli/internal_domains_review_test.go new file mode 100644 index 00000000..196ae1d3 --- /dev/null +++ b/cmd/cli/internal_domains_review_test.go @@ -0,0 +1,651 @@ +package cli + +import ( + "context" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// dnsFixture is a loopback DNS server that records the questions it is asked, +// so a test can prove where a query actually went on the wire rather than +// inferring it from an rcode. +type dnsFixture struct { + addr string + mu sync.Mutex + qs []string +} + +func (f *dnsFixture) questions() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.qs...) +} + +func startDNSFixture(t *testing.T) *dnsFixture { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := &dnsFixture{addr: pc.LocalAddr().String()} + started := make(chan struct{}) + server := &dns.Server{ + PacketConn: pc, + NotifyStartedFunc: func() { close(started) }, + Handler: dns.HandlerFunc(func(w dns.ResponseWriter, msg *dns.Msg) { + f.mu.Lock() + if len(msg.Question) > 0 { + f.qs = append(f.qs, msg.Question[0].Name) + } + f.mu.Unlock() + reply := new(dns.Msg) + reply.SetRcode(msg, dns.RcodeSuccess) + _ = w.WriteMsg(reply) + }), + } + go func() { _ = server.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("DNS fixture did not start") + } + t.Cleanup(func() { _ = server.Shutdown() }) + return f +} + +// useOSResolverFixture points the OS-resolver boundary at a loopback server for +// the duration of the test, so "went to the OS resolver" is observable. +func useOSResolverFixture(t *testing.T, f *dnsFixture) { + t.Helper() + prev := osUpstreamConfig + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "OS resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: f.addr, + Timeout: 2000, + } + t.Cleanup(func() { osUpstreamConfig = prev }) +} + +func useDNSIntercept(t *testing.T) { + t.Helper() + prev := dnsIntercept + dnsIntercept = true + t.Cleanup(func() { dnsIntercept = prev }) +} + +func newInternalDomainsProg(t *testing.T, cfg *ctrld.Config) *prog { + t.Helper() + p := &prog{cfg: cfg} + p.logger.Store(mainLog.Load()) + p.um = newUpstreamMonitor(cfg, p.logger.Load()) + p.lanLoopGuard = newLoopGuard() + p.ptrLoopGuard = newLoopGuard() + return p +} + +// The DNS-intercept recovery bypass forwards everything to the OS/DHCP resolver +// while general DNS is broken. An Internal Domain with explicit resolvers must +// not take part: its resolvers may be perfectly healthy, and the private name +// must not reach the network's DNS. +func TestInternalDomainsExplicitResolverSurvivesRecoveryBypass(t *testing.T) { + osFixture := startDNSFixture(t) + internalFixture := startDNSFixture(t) + useOSResolverFixture(t, osFixture) + useDNSIntercept(t) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Resolvers: []string{internalFixture.addr}}, + }) + p := newInternalDomainsProg(t, cfg) + p.recoveryBypass.Store(true) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ask := func(name string) *proxyResponse { + t.Helper() + ctx := context.WithValue(ctrld.LoggerCtx(context.Background(), p.logger.Load()), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", strings.TrimSuffix(name, ".")) + return p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname(name, dns.TypeA), ufr: ufr}) + } + + res := ask("host.corp.example.") + if got := internalFixture.questions(); len(got) != 1 || got[0] != "host.corp.example." { + t.Errorf("internal resolver questions = %v, want the query to reach it", got) + } + if got := osFixture.questions(); len(got) != 0 { + t.Errorf("recovery bypass leaked an Internal Domain query to the OS resolver: %v", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the internal resolver's NOERROR", res) + } + + // Control: an ordinary query still takes the bypass. + if res := ask("www.example.net."); res == nil || res.answer == nil { + t.Errorf("ordinary query during recovery bypass = %v", res) + } + if got := osFixture.questions(); len(got) != 1 || got[0] != "www.example.net." { + t.Errorf("OS resolver questions = %v, want the ordinary query to take the bypass", got) + } +} + +// In "explicit resolver only" mode, when the configured resolvers are +// unreachable the query must fail rather than reach the OS resolver, and the +// failure must be observable on the wire as an attempt against the configured +// address. +func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) { + osFixture := startDNSFixture(t) + useOSResolverFixture(t, osFixture) + useDNSIntercept(t) + + // A loopback port nothing listens on: the dial fails fast and locally. + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := pc.LocalAddr().String() + _ = pc.Close() + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{dead}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(ctrld.LoggerCtx(context.Background(), p.logger.Load()), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := osFixture.questions(); len(got) != 0 { + t.Errorf("unreachable internal resolver leaked to the OS resolver: %v", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Errorf("answer = %v, want SERVFAIL", res) + } +} + +// The API returns Internal Domains sorted bytewise by domain, which puts a +// parent suffix ahead of its children. Generation order is routing precedence, +// so the more specific suffix has to win regardless of API order. +func TestInternalDomainsOverlappingSuffixPrecedence(t *testing.T) { + for _, tc := range []struct { + name string + entries []controld.SplitDNS + want map[string][]string + }{ + { + name: "parent os, child explicit", + entries: []controld.SplitDNS{ + {Domain: "example.com"}, + {Domain: "z.example.com", Resolvers: []string{"10.0.0.2"}}, + }, + want: map[string][]string{ + "z.example.com": {"upstream.internal_0"}, + "host.z.example.com": {"upstream.internal_0"}, + "a.b.z.example.com": {"upstream.internal_0"}, + "example.com": {}, + "other.example.com": {}, + "notz.example.com": {}, + "z.example.com.evil.gg": {"upstream.0"}, + "example.org": {"upstream.0"}, + }, + }, + { + name: "parent explicit, child os", + entries: []controld.SplitDNS{ + {Domain: "example.com", Resolvers: []string{"10.0.0.2"}}, + {Domain: "z.example.com"}, + }, + want: map[string][]string{ + "z.example.com": {}, + "host.z.example.com": {}, + "example.com": {"upstream.internal_0"}, + "other.example.com": {"upstream.internal_0"}, + "example.org": {"upstream.0"}, + }, + }, + { + name: "three levels, API order", + entries: []controld.SplitDNS{ + {Domain: "example.com", Resolvers: []string{"10.0.0.1"}}, + {Domain: "y.example.com"}, + {Domain: "x.y.example.com", Resolvers: []string{"10.0.0.3"}}, + }, + want: map[string][]string{ + "host.x.y.example.com": {"upstream.internal_0"}, + "x.y.example.com": {"upstream.internal_0"}, + "other.y.example.com": {}, + "y.example.com": {}, + "other.example.com": {"upstream.internal_1"}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, tc.entries) + p := newInternalDomainsProg(t, cfg) + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + for domain, want := range tc.want { + ufr := p.upstreamFor(context.Background(), "0", cfg.Listener["0"], addr, "", domain) + if strings.Join(ufr.upstreams, ",") != strings.Join(want, ",") { + t.Errorf("upstreamFor(%q) = %v, want %v", domain, ufr.upstreams, want) + } + } + }) + } +} + +// Ordering is a generation concern; it must not make an unchanged list look +// changed at refresh time. +func TestInternalDomainsOrderingIsRefreshStable(t *testing.T) { + apiOrder := []controld.SplitDNS{ + {Domain: "example.com"}, + {Domain: "z.example.com", Resolvers: []string{"10.0.0.2"}}, + } + if !internalDomainsEqual(apiOrder, orderInternalDomainsBySpecificity(apiOrder)) { + t.Error("reordering for specificity must not read as a configuration change") + } + first, second := internalDomainsTestConfig(), internalDomainsTestConfig() + applyInternalDomains(first, apiOrder) + applyInternalDomains(second, orderInternalDomainsBySpecificity(apiOrder)) + if len(first.Listener["0"].Policy.Rules) != len(second.Listener["0"].Policy.Rules) { + t.Error("generation must not depend on the order the API returned") + } +} + +// Generated Internal Domain upstreams carry the organization's private resolver +// addresses. The production setup path must not publish them above debug. +func TestInternalDomainsSetupKeepsResolverAddressesOutOfInfoLogs(t *testing.T) { + buf := captureInternalDomainsLogs(t) + const secret = "10.77.88.99" + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Resolvers: []string{secret}}, + }) + // Drop the Control D upstream: setupUpstream would try to bootstrap it. + delete(cfg.Upstream, "0") + + p := newInternalDomainsProg(t, cfg) + before := len(buf.String()) + p.setupUpstream(cfg) + out := buf.String()[before:] + + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + if strings.Contains(line, secret) { + t.Errorf("resolver address disclosed above debug level: %s", line) + } + } + if !strings.Contains(out, secret) { + t.Errorf("expected the address at debug level for diagnosis, got: %s", out) + } +} + +// A transport error names the endpoint it could not reach. For an Internal +// Domain upstream that endpoint is private, so only a classification may appear +// at error level. +func TestInternalDomainsResolverFailureLogsNoAddress(t *testing.T) { + buf := captureInternalDomainsLogs(t) + // The default mode falls back to the network after the failure; keep that + // on loopback. + useOSResolverFixture(t, startDNSFixture(t)) + + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := pc.LocalAddr().String() + _ = pc.Close() + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{dead}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(ctrld.LoggerCtx(context.Background(), p.logger.Load()), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + before := len(buf.String()) + p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + out := buf.String()[before:] + + host, _, _ := net.SplitHostPort(dead) + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + if strings.Contains(line, dead) || strings.Contains(line, host+":") { + t.Errorf("resolver address disclosed above debug level: %s", line) + } + } + if !strings.Contains(out, `"failure":`) { + t.Errorf("expected a classified failure at error level, got: %s", out) + } + if !strings.Contains(out, dead) { + t.Error("private resolver address must remain available at debug") + } + + // Master's transport-level diagnostic must stay at error for ordinary + // legacy upstreams; only generated Internal Domain queries opt into privacy. + // Generated upstreams are identified by their InternalDomain marker, not + // by name, so the control is an ordinary upstream to the same endpoint. + ordinary := &ctrld.UpstreamConfig{Name: "ordinary", Type: ctrld.ResolverTypeLegacy, Endpoint: dead, Timeout: 5000} + ordinary.Init(ctx) + before = len(buf.String()) + p.queryUpstream(ctx, &proxyRequest{msg: newDnsMsgWithHostname("public.example.", dns.TypeA)}, + "upstream.0", ordinary) + ordinaryError := false + for _, line := range strings.Split(buf.String()[before:], "\n") { + if strings.Contains(line, `"level":"error"`) && strings.Contains(line, "Legacy request failed") && strings.Contains(line, dead) { + ordinaryError = true + } + } + if !ordinaryError { + t.Error("ordinary legacy resolver lost its error-level transport diagnostic") + } +} + +// selfUninstallEligibility runs the production setup path and reports whether +// the REFUSED-triggered deletion check stays available. +func selfUninstallEligibility(t *testing.T, p *prog, cfg *ctrld.Config) bool { + t.Helper() + p.setupUpstream(cfg) + return p.canSelfUninstall.Load() +} + +// Generated Internal Domain resolvers are part of the managed configuration. +// Counting them as independent upstreams made an ordinary managed install look +// custom, which silently disabled the REFUSED-triggered deletion check. +func TestInternalDomainsKeepSelfUninstallEligibility(t *testing.T) { + for _, tc := range []struct { + name string + entries []controld.SplitDNS + extra map[string]*ctrld.UpstreamConfig + want bool + }{ + {name: "no internal domains", want: true}, + { + name: "os mode", + entries: []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeOS}}, + want: true, + }, + { + name: "explicit mode", + entries: []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}}, + want: true, + }, + { + name: "several explicit domains", + entries: []controld.SplitDNS{ + {Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53", "10.0.0.54"}}, + {Domain: "lab.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.55"}}, + }, + want: true, + }, + { + // A genuinely custom multi-upstream install keeps its safeguard. + name: "a real second upstream", + extra: map[string]*ctrld.UpstreamConfig{"1": {Endpoint: "https://dns.example.net/x", Type: ctrld.ResolverTypeDOH, BootstrapIP: "192.0.2.1"}}, + want: false, + }, + { + // Only the shape applyInternalDomains produces is discounted, so a + // custom config cannot buy eligibility by naming an upstream + // "internal_0". + name: "an upstream merely named like a generated one", + extra: map[string]*ctrld.UpstreamConfig{"internal_0": {Name: "mine", Endpoint: "https://dns.example.net/x", Type: ctrld.ResolverTypeDOH, BootstrapIP: "192.0.2.1"}}, + want: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, tc.entries) + for k, uc := range tc.extra { + cfg.Upstream[k] = uc + } + p := newInternalDomainsProg(t, cfg) + if got := selfUninstallEligibility(t, p, cfg); got != tc.want { + t.Errorf("canSelfUninstall = %v, want %v (upstreams: %d)", got, tc.want, len(cfg.Upstream)) + } + }) + } +} + +// Eligibility is recomputed on every setup, so a reload that adds a real second +// upstream withdraws it and one that removes Internal Domains keeps it. The old +// code only ever raised the flag, which made startup and refresh disagree. +func TestSelfUninstallEligibilityRecomputedAcrossReloads(t *testing.T) { + explicit := []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}} + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, explicit) + p := newInternalDomainsProg(t, cfg) + if !selfUninstallEligibility(t, p, cfg) { + t.Fatal("fresh start with explicit Internal Domains must stay eligible") + } + + // Reload: Internal Domains removed. Still a plain managed install. + removed := internalDomainsTestConfig() + if !selfUninstallEligibility(t, p, removed) { + t.Error("removing Internal Domains must keep eligibility") + } + + // Reload: a genuinely custom second upstream appears. + custom := internalDomainsTestConfig() + custom.Upstream["1"] = &ctrld.UpstreamConfig{Endpoint: "https://dns.example.net/x", Type: ctrld.ResolverTypeDOH, BootstrapIP: "192.0.2.1"} + if selfUninstallEligibility(t, p, custom) { + t.Error("a real second upstream must withdraw eligibility, not stay latched on") + } + + // Reload: back to the managed config with Internal Domains. + again := internalDomainsTestConfig() + applyInternalDomains(again, explicit) + if !selfUninstallEligibility(t, p, again) { + t.Error("eligibility must come back with the managed config") + } +} + +// The visible consequence: with explicit Internal Domains configured, a REFUSED +// answer is still counted toward the deletion check instead of being ignored. +// Counting alone performs no uninstall; that stays gated on the API confirming +// the device is gone. +func TestInternalDomainsKeepRefusedQueryCounting(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + }) + p := newInternalDomainsProg(t, cfg) + p.setupUpstream(cfg) + + refused := new(dns.Msg) + refused.SetRcode(newDnsMsgWithHostname("host.corp.example.", dns.TypeA), dns.RcodeRefused) + p.doSelfUninstall(&proxyResponse{answer: refused, refused: false}) + + p.selfUninstallMu.Lock() + count, checking := p.refusedQueryCount, p.checkingSelfUninstall + p.selfUninstallMu.Unlock() + if count != 1 { + t.Errorf("refusedQueryCount = %d, want 1", count) + } + if checking { + t.Error("a single REFUSED answer must not start a device-status check") + } +} + +// logLinesAboveDebug returns the log lines emitted since mark that a support +// bundle reader would see at info level or above. +func logLinesAboveDebug(buf *syncBuffer, mark int) []string { + var out []string + for _, line := range strings.Split(strings.TrimSpace(buf.String()[mark:]), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + out = append(out, line) + } + return out +} + +// deadUDPAddr returns a loopback address with nothing listening, so a dial +// fails fast and locally with connection refused. +func deadUDPAddr(t *testing.T) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := pc.LocalAddr().String() + _ = pc.Close() + return addr +} + +// blackholeUDPAddr returns a loopback address that accepts packets and never +// answers, so an exchange fails with a timeout rather than a refusal. +func blackholeUDPAddr(t *testing.T) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pc.Close() }) + return pc.LocalAddr().String() +} + +// internalDomainsProbeConfig generates an explicit Internal Domain pointing at +// addr, with a short timeout so a blackholed probe fails quickly. +func internalDomainsProbeConfig(t *testing.T, addr string) *ctrld.Config { + t.Helper() + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{addr}}, + }) + uc := cfg.Upstream["internal_0"] + if uc == nil { + t.Fatal("no internal upstream generated") + } + uc.Timeout = 300 + uc.Init(ctrld.LoggerCtx(context.Background(), mainLog.Load())) + return cfg +} + +// The loop checker runs on a one-minute ticker and probes every upstream whose +// address is local, which every generated Internal Domain resolver is. No user +// query is involved, so without the privacy treatment it publishes the +// organization's resolver addresses on its own schedule. +func TestInternalDomainsLoopCheckHidesResolverAddress(t *testing.T) { + buf := captureInternalDomainsLogs(t) + for _, tc := range []struct { + name string + addr func(*testing.T) string + }{ + {"connection refused", deadUDPAddr}, + {"timeout", blackholeUDPAddr}, + } { + t.Run(tc.name, func(t *testing.T) { + addr := tc.addr(t) + host, _, _ := net.SplitHostPort(addr) + cfg := internalDomainsProbeConfig(t, addr) + p := newInternalDomainsProg(t, cfg) + p.loop = make(map[string]bool) + // Only the generated upstream is local, so it is the only one probed. + delete(cfg.Upstream, "0") + + mark := len(buf.String()) + p.checkDnsLoop() + above := logLinesAboveDebug(buf, mark) + + for _, line := range above { + if strings.Contains(line, addr) || strings.Contains(line, host+":") { + t.Errorf("loop check disclosed the resolver address: %s", line) + } + } + if len(above) == 0 { + t.Error("the failure must stay visible above debug") + } + var classified bool + for _, line := range above { + if strings.Contains(line, `"failure":`) { + classified = true + } + } + if !classified { + t.Errorf("expected a classified failure, got: %v", above) + } + if !strings.Contains(buf.String()[mark:], addr) { + t.Error("the address must remain at debug level for diagnosis") + } + }) + } +} + +// Preventing an Internal Domain query from starting recovery does not keep its +// resolver out of recovery started for another reason: buildRecoveryUpstreams +// takes every non-OS upstream. +func TestInternalDomainsRecoveryCheckHidesResolverAddress(t *testing.T) { + buf := captureInternalDomainsLogs(t) + for _, tc := range []struct { + name string + addr func(*testing.T) string + }{ + {"connection refused", deadUDPAddr}, + {"timeout", blackholeUDPAddr}, + } { + t.Run(tc.name, func(t *testing.T) { + addr := tc.addr(t) + host, _, _ := net.SplitHostPort(addr) + cfg := internalDomainsProbeConfig(t, addr) + p := newInternalDomainsProg(t, cfg) + + upstreams := p.buildRecoveryUpstreams(RecoveryReasonNetworkChange) + uc, selected := upstreams["upstream.internal_0"] + if !selected { + t.Fatalf("recovery upstreams = %v, want the Internal Domain resolver included", upstreams) + } + + mark := len(buf.String()) + if err := p.checkUpstreamOnce("upstream.internal_0", uc, &upstreamFailureLog{}); err == nil { + t.Fatal("expected the probe to fail") + } + above := logLinesAboveDebug(buf, mark) + + for _, line := range above { + if strings.Contains(line, addr) || strings.Contains(line, host+":") { + t.Errorf("recovery check disclosed the resolver address: %s", line) + } + } + var classified bool + for _, line := range above { + if strings.Contains(line, `"failure":`) { + classified = true + } + } + if !classified { + t.Errorf("expected a classified failure above debug, got: %v", above) + } + if !strings.Contains(buf.String()[mark:], addr) { + t.Error("the address must remain at debug level for diagnosis") + } + }) + } +} diff --git a/cmd/cli/internal_domains_test.go b/cmd/cli/internal_domains_test.go new file mode 100644 index 00000000..8ac5a077 --- /dev/null +++ b/cmd/cli/internal_domains_test.go @@ -0,0 +1,662 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" + + "github.com/miekg/dns" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// captureInternalDomainsLogs uses a test-local JSON sink so counts and privacy +// assertions inspect structured fields at their actual levels, independent of +// TestMain's console encoder and any previous test replacing mainLog. Call it +// before constructing a prog: its logger and monitor retain the selected logger. +func captureInternalDomainsLogs(t *testing.T) *syncBuffer { + t.Helper() + buf := &syncBuffer{} + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(buf), + zap.DebugLevel, + ) + old := mainLog.Load() + mainLog.Store(&ctrld.Logger{Logger: zap.New(core)}) + t.Cleanup(func() { mainLog.Store(old) }) + return buf +} + +func internalDomainsTestConfig() *ctrld.Config { + return &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {Policy: &ctrld.ListenerPolicyConfig{Name: "My Policy"}}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Endpoint: "https://dns.controld.com/test", Type: ctrld.ResolverTypeDOH, BootstrapIP: "127.0.0.1"}}, + Network: map[string]*ctrld.NetworkConfig{"0": {Name: "Network 0", Cidrs: []string{"0.0.0.0/0"}}}, + } +} + +// ruleTargets returns the targets of the rule for source, and whether it exists. +func ruleTargets(t *testing.T, cfg *ctrld.Config, source string) ([]string, bool) { + t.Helper() + for _, rule := range cfg.Listener["0"].Policy.Rules { + if targets, ok := rule[source]; ok { + return targets, true + } + } + return nil, false +} + +func TestNormalizeInternalDomain(t *testing.T) { + long63 := strings.Repeat("a", 63) + for _, tc := range []struct{ name, in, want string }{ + // Canonicalization that cannot change which names match. + {"plain", "aws.example.com", "aws.example.com"}, + {"case and root dot", " AWS.Example.COM. ", "aws.example.com"}, + {"leading wildcard", "*.aws.example.com", "aws.example.com"}, + {"surrounding control whitespace", "\r\nCORP.EXAMPLE.\r\n", "corp.example"}, + {"single label", "intranet", "intranet"}, + {"punycode", "xn--80ak6aa92e.com", "xn--80ak6aa92e.com"}, + {"digits and hyphens", "dc-01.site2.example.com", "dc-01.site2.example.com"}, + {"max label", long63 + ".example.com", long63 + ".example.com"}, + + // Rejected: an empty or absent name. + {"empty", "", ""}, + {"whitespace only", " ", ""}, + {"root only", ".", ""}, + {"wildcard only", "*.", ""}, + + // Rejected: empty labels. A leading dot is not repaired, because the + // repaired value is a different suffix from the configured one. + {"leading dot", ".aws.example.com.", ""}, + {"double dot", "aws..example.com", ""}, + {"trailing double dot", "aws.example.com..", ""}, + + // Rejected: characters outside the ASCII hostname contract. A denylist + // would have to name every one of these; the positive rule does not. + {"space", "has space.com", ""}, + {"embedded wildcard", "we*rd.com", ""}, + {"slash", "a/b.com", ""}, + {"newline", "aws\nevil.example.com", ""}, + {"carriage return", "aws\revil.example.com", ""}, + {"nul", "aws\x00evil.example.com", ""}, + {"escape", "aws\x1bevil.example.com", ""}, + {"delete", "aws\x7fevil.example.com", ""}, + {"quote", "aws\"evil.example.com", ""}, + {"colon", "aws:53.example.com", ""}, + {"underscore", "_msdcs.example.com", ""}, + {"non-ascii", "exämple.com", ""}, + + // Rejected: hyphen placement and length. + {"leading hyphen", "-aws.example.com", ""}, + {"trailing hyphen", "aws-.example.com", ""}, + {"label too long", strings.Repeat("a", 64) + ".example.com", ""}, + {"name too long", strings.Repeat("a.", 127) + "example.com", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := normalizeInternalDomain(tc.in); got != tc.want { + t.Errorf("normalizeInternalDomain(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// A rejected domain is counted and produces no rule, rather than being repaired +// into a suffix nobody configured. +func TestApplyInternalDomainsRejectsMalformedDomains(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws\nevil.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + {Domain: "ok.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.54"}}, + }) + if summary.domains != 1 || summary.skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + for _, rule := range cfg.Listener["0"].Policy.Rules { + for source := range rule { + if strings.ContainsAny(source, "\n\r\x00") { + t.Errorf("control character reached a policy rule: %q", source) + } + if source != "ok.example.com" && source != "*.ok.example.com" { + t.Errorf("unexpected rule source %q", source) + } + } + } +} + +func TestInternalDomainEndpoint(t *testing.T) { + for _, tc := range []struct { + in string + want string + ok bool + }{ + {"10.0.0.10", "10.0.0.10:53", true}, + {" 10.0.0.11 ", "10.0.0.11:53", true}, + {"2001:db8::1", "[2001:db8::1]:53", true}, + {"10.0.0.10:5353", "10.0.0.10:5353", true}, + {"[2001:db8::1]:5353", "[2001:db8::1]:5353", true}, + {"dns.internal.example.com", "", false}, + {"10.0.0.256", "", false}, + {"", "", false}, + {"10.0.0.10:0", "", false}, + {"10.0.0.10:70000", "", false}, + {"10.0.0.10:dns", "", false}, + } { + got, ok := internalDomainEndpoint(tc.in) + if got != tc.want || ok != tc.ok { + t.Errorf("internalDomainEndpoint(%q) = %q, %v; want %q, %v", tc.in, got, ok, tc.want, tc.ok) + } + } +} + +func TestApplyInternalDomainsOSResolverMode(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example.com"}}) + + if summary.domains != 1 || summary.osMode != 1 || summary.explicit != 0 || summary.resolvers != 0 { + t.Fatalf("summary = %+v", summary) + } + for _, source := range []string{"corp.example.com", "*.corp.example.com"} { + targets, ok := ruleTargets(t, cfg, source) + if !ok { + t.Fatalf("no rule for %q", source) + } + if len(targets) != 0 { + t.Errorf("rule %q targets = %v, want empty (OS resolver)", source, targets) + } + } + if len(cfg.Upstream) != 1 { + t.Errorf("upstreams = %v, want only the Control D upstream", cfg.Upstream) + } +} + +func TestApplyInternalDomainsExplicitResolverMode(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "2001:db8::11"}}, + }) + + if summary.domains != 1 || summary.explicit != 1 || summary.osMode != 0 || summary.resolvers != 2 { + t.Fatalf("summary = %+v", summary) + } + want := []string{"upstream.internal_0", "upstream.internal_1"} + for _, source := range []string{"aws.example.com", "*.aws.example.com"} { + targets, ok := ruleTargets(t, cfg, source) + if !ok { + t.Fatalf("no rule for %q", source) + } + if strings.Join(targets, ",") != strings.Join(want, ",") { + t.Errorf("rule %q targets = %v, want %v (administrator order)", source, targets, want) + } + } + if got := cfg.Upstream["internal_0"]; got == nil || got.Endpoint != "10.0.0.10:53" || got.Type != ctrld.ResolverTypeLegacy { + t.Errorf("internal_0 = %+v", got) + } + if got := cfg.Upstream["internal_1"]; got == nil || got.Endpoint != "[2001:db8::11]:53" { + t.Errorf("internal_1 = %+v", got) + } +} + +func TestApplyInternalDomainsMalformedResolvers(t *testing.T) { + t.Run("drops only the malformed address", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"not-an-ip", "10.0.0.10"}}, + }) + if summary.domains != 1 || summary.resolvers != 1 || summary.skipped != 0 { + t.Fatalf("summary = %+v", summary) + } + targets, _ := ruleTargets(t, cfg, "aws.example.com") + if len(targets) != 1 || targets[0] != "upstream.internal_0" { + t.Errorf("targets = %v", targets) + } + }) + + t.Run("drops the entry when no address is usable", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"not-an-ip", ""}}, + }) + if summary.domains != 0 || summary.skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + // Falling back to the OS resolver here would silently replace the + // administrator's explicit selection with a different one. + if _, ok := ruleTargets(t, cfg, "aws.example.com"); ok { + t.Error("entry with no usable resolver must not produce a rule") + } + if len(cfg.Upstream) != 1 { + t.Errorf("upstreams = %v, want no generated upstream", cfg.Upstream) + } + }) + + t.Run("drops an unusable domain", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{{Domain: " ", Resolvers: []string{"10.0.0.10"}}}) + if summary.domains != 0 || summary.skipped != 1 || len(cfg.Upstream) != 1 { + t.Fatalf("summary = %+v, upstreams = %v", summary, cfg.Upstream) + } + }) + + t.Run("drops a duplicate domain", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + {Domain: "AWS.Example.com.", Resolvers: []string{"10.0.0.99"}}, + }) + if summary.domains != 1 || summary.skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + targets, _ := ruleTargets(t, cfg, "aws.example.com") + if len(targets) != 1 || cfg.Upstream["internal_0"].Endpoint != "10.0.0.10:53" { + t.Errorf("first entry must win: targets = %v", targets) + } + }) +} + +// Magic Folder excludes and endpoint custom configuration are written to the +// policy before Internal Domains are applied, and must keep their routing. +func TestApplyInternalDomainsPrecedence(t *testing.T) { + t.Run("an existing rule is never overwritten", func(t *testing.T) { + cfg := internalDomainsTestConfig() + cfg.Listener["0"].Policy.Rules = []ctrld.Rule{{"aws.example.com": []string{}}} + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + }) + targets, _ := ruleTargets(t, cfg, "aws.example.com") + if len(targets) != 0 { + t.Errorf("exclude rule was overwritten: targets = %v", targets) + } + // The wildcard was still free, so the domain is only partly preempted. + if wildcard, ok := ruleTargets(t, cfg, "*.aws.example.com"); !ok || len(wildcard) != 1 { + t.Errorf("wildcard rule = %v, %v", wildcard, ok) + } + if summary.domains != 1 { + t.Errorf("summary = %+v", summary) + } + }) + + t.Run("a fully covered domain is reported as preempted", func(t *testing.T) { + cfg := internalDomainsTestConfig() + cfg.Listener["0"].Policy.Rules = []ctrld.Rule{ + {"aws.example.com": []string{"upstream.0"}}, + {"*.aws.example.com": []string{"upstream.0"}}, + } + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + }) + if summary.domains != 0 || summary.preempted != 1 { + t.Fatalf("summary = %+v", summary) + } + if len(cfg.Listener["0"].Policy.Rules) != 2 { + t.Errorf("rules = %v, want no addition", cfg.Listener["0"].Policy.Rules) + } + }) +} + +// A configured suffix must route itself and its subdomains, and nothing else. +func TestApplyInternalDomainsSuffixMatching(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + {Domain: "corp.example.com"}, + }) + p := newInternalDomainsProg(t, cfg) + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + domain string + want []string + }{ + {"aws.example.com", []string{"upstream.internal_0"}}, + {"host.aws.example.com", []string{"upstream.internal_0"}}, + {"a.b.c.aws.example.com", []string{"upstream.internal_0"}}, + {"corp.example.com", []string{}}, + {"host.corp.example.com", []string{}}, + {"example.com", []string{"upstream.0"}}, + {"notaws.example.com", []string{"upstream.0"}}, + {"aws.example.com.evil.test", []string{"upstream.0"}}, + } { + ufr := p.upstreamFor(context.Background(), "0", cfg.Listener["0"], addr, "", tc.domain) + if strings.Join(ufr.upstreams, ",") != strings.Join(tc.want, ",") { + t.Errorf("upstreamFor(%q) = %v, want %v", tc.domain, ufr.upstreams, tc.want) + } + } +} + +// A refresh rebuilds the generated config, so a removal must leave no rule and +// no upstream behind. +func TestApplyInternalDomainsAuthoritativeReplacement(t *testing.T) { + first := internalDomainsTestConfig() + applyInternalDomains(first, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + {Domain: "old.example.com", Resolvers: []string{"10.0.0.20"}}, + }) + if len(first.Upstream) != 3 { + t.Fatalf("upstreams = %v", first.Upstream) + } + + // The refresh path regenerates cfg from the API response rather than + // mutating the previous one. + second := internalDomainsTestConfig() + summary := applyInternalDomains(second, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.99"}}, + }) + if summary.domains != 1 || summary.resolvers != 1 { + t.Fatalf("summary = %+v", summary) + } + if _, ok := ruleTargets(t, second, "old.example.com"); ok { + t.Error("removed domain left a stale rule") + } + if len(second.Upstream) != 2 { + t.Errorf("upstreams = %v, want the Control D upstream plus one internal", second.Upstream) + } + if second.Upstream["internal_0"].Endpoint != "10.0.0.99:53" { + t.Errorf("resolver change not applied: %+v", second.Upstream["internal_0"]) + } +} + +func TestApplyInternalDomainsEmpty(t *testing.T) { + for _, entries := range [][]controld.SplitDNS{nil, {}} { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, entries) + if !summary.empty() { + t.Errorf("summary = %+v, want empty", summary) + } + if len(cfg.Listener["0"].Policy.Rules) != 0 || len(cfg.Upstream) != 1 { + t.Errorf("absent split_dns changed the config: rules=%v upstreams=%v", + cfg.Listener["0"].Policy.Rules, cfg.Upstream) + } + } +} + +func TestInternalDomainsEqual(t *testing.T) { + base := []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + {Domain: "corp.example.com"}, + } + for _, tc := range []struct { + name string + next []controld.SplitDNS + want bool + }{ + {"unchanged", base, true}, + {"reordered domains", []controld.SplitDNS{ + {Domain: "corp.example.com"}, + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + }, true}, + {"equivalent spelling", []controld.SplitDNS{ + {Domain: "AWS.Example.com.", Resolvers: []string{" 10.0.0.10 ", "10.0.0.11:53"}}, + {Domain: "*.corp.example.com"}, + }, true}, + {"resolver order changed", []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.11", "10.0.0.10"}}, + {Domain: "corp.example.com"}, + }, false}, + {"resolver changed", []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.99"}}, + {Domain: "corp.example.com"}, + }, false}, + {"domain removed", base[:1], false}, + {"domain added", append(append([]controld.SplitDNS{}, base...), controld.SplitDNS{Domain: "new.example.com"}), false}, + {"mode changed to explicit", []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + {Domain: "corp.example.com", Resolvers: []string{"10.0.0.30"}}, + }, false}, + {"emptied", nil, false}, + } { + if got := internalDomainsEqual(base, tc.next); got != tc.want { + t.Errorf("%s: internalDomainsEqual = %v, want %v", tc.name, got, tc.want) + } + } + if !internalDomainsEqual(nil, []controld.SplitDNS{}) { + t.Error("nil and empty lists must compare equal") + } +} + +func TestInternalDomainExplicitUpstreams(t *testing.T) { + p := internalDomainUpstreamsProg(map[string]string{ + "internal_0": internalDomainUpstreamFallback, + "internal_1": internalDomainUpstreamOnly, + "0": "", + }) + for _, tc := range []struct { + in []string + want bool + }{ + {[]string{"upstream.internal_0"}, true}, + {[]string{"upstream.internal_0", "upstream.internal_1"}, true}, + {[]string{"upstream.internal_0", "upstream.0"}, false}, + {[]string{"upstream.0"}, false}, + {[]string{upstreamOS}, false}, + {nil, false}, + {[]string{}, false}, + } { + if got := p.internalDomainExplicitUpstreams(tc.in); got != tc.want { + t.Errorf("internalDomainExplicitUpstreams(%v) = %v, want %v", tc.in, got, tc.want) + } + } +} + +// Info and warning logs must not carry organization domain names or resolver +// addresses; those stay at debug level. +func TestInternalDomainsSummaryLogsCountsOnly(t *testing.T) { + buf := captureInternalDomainsLogs(t) + logger := mainLog.Load() + logInternalDomainsSummary(logger, internalDomainsSummary{ + domains: 2, osMode: 1, explicit: 1, resolvers: 3, skipped: 1, + }) + out := buf.String() + for _, want := range []string{`"domains":2`, `"os_resolver":1`, `"explicit_resolver":1`, `"resolvers":3`, `"skipped":1`} { + if !strings.Contains(out, want) { + t.Errorf("log %q missing %q", out, want) + } + } + if !strings.Contains(out, `"level":"warn"`) { + t.Errorf("a summary with skipped entries must warn: %q", out) + } + + mark := len(buf.String()) + logInternalDomainsSummary(logger, internalDomainsSummary{domains: 1, osMode: 1}) + if !strings.Contains(buf.String()[mark:], `"level":"info"`) { + t.Errorf("a clean summary must be info: %q", buf.String()) + } +} + +// An explicit-resolver Internal Domain must not be handed to VPN DNS split +// routing: the administrator named those resolvers, and VPN suffixes are +// auto-detected. In "explicit resolver only" mode, when the named resolvers +// are unreachable the query fails instead of reaching the OS resolver. +func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) { + buf := captureInternalDomainsLogs(t) + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"127.0.0.1:1"}}, + }) + + p := newInternalDomainsProg(t, cfg) + p.vpnDNS = &vpnDNSManager{routes: map[string][]string{"example.com": {"127.0.0.1:1"}}} + + prevIntercept := dnsIntercept + dnsIntercept = true + t.Cleanup(func() { dnsIntercept = prevIntercept }) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(ctrld.LoggerCtx(context.Background(), p.logger.Load()), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.aws.example.com") + if !p.internalDomainExplicitUpstreams(ufr.upstreams) { + t.Fatalf("upstreams = %v, want only internal resolvers", ufr.upstreams) + } + + // Only this call's tail describes what proxy() did here. + before := len(buf.String()) + // A trailing dot: without it the name is not a valid wire question, and the + // failure would prove nothing about whether a resolver was contacted. + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.aws.example.com.", dns.TypeA), ufr: ufr}) + out := buf.String()[before:] + + if strings.Contains(out, "VPN DNS route matched") { + t.Errorf("VPN split routing overrode an explicit Internal Domain: %s", out) + } + if strings.Contains(out, "Attempting query to OS resolver as a retry catch all") { + t.Errorf("unreachable internal resolvers leaked to the OS resolver: %s", out) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Errorf("answer = %v, want SERVFAIL", res) + } +} + +// A domain that produces no rule must produce no upstream either, or the +// generated config would carry an internal_* upstream nothing refers to. +func TestApplyInternalDomainsLeaveNoOrphanUpstreams(t *testing.T) { + cfg := internalDomainsTestConfig() + cfg.Listener["0"].Policy.Rules = []ctrld.Rule{ + {"aws.example.com": []string{"upstream.0"}}, + {"*.aws.example.com": []string{"upstream.0"}}, + } + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + {Domain: "ok.example.com", Resolvers: []string{"10.0.0.20"}}, + }) + if summary.preempted != 1 || summary.domains != 1 || summary.resolvers != 1 { + t.Fatalf("summary = %+v", summary) + } + for key, uc := range cfg.Upstream { + if !strings.HasPrefix(key, internalDomainUpstreamPrefix) { + continue + } + if uc.Endpoint != "10.0.0.20:53" { + t.Errorf("orphan upstream %s = %+v", key, uc) + } + } + if len(cfg.Upstream) != 2 { + t.Errorf("upstreams = %v, want the Control D upstream plus one internal", cfg.Upstream) + } +} + +// Mode is what the administrator selected, so it decides the routing. The list +// of resolvers is only read when the mode calls for it. +func TestApplyInternalDomainsModeIsAuthoritative(t *testing.T) { + for _, tc := range []struct { + name string + entry controld.SplitDNS + wantTargets []string + wantRule bool + wantUpstreams int + }{ + { + // The administrator switched back to the OS resolver; addresses the + // previous selection left behind must not resurrect themselves. + name: "os mode ignores leftover resolvers", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeOS, Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{}, + wantRule: true, + wantUpstreams: 1, + }, + { + // Explicit with nothing to be explicit about is not OS resolution. + name: "resolvers mode with an empty list is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers}, + wantRule: false, + wantUpstreams: 1, + }, + { + name: "resolvers mode with only malformed addresses is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"nope"}}, + wantRule: false, + wantUpstreams: 1, + }, + { + name: "resolvers mode routes explicitly", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{"upstream.internal_0"}, + wantRule: true, + wantUpstreams: 2, + }, + { + name: "mode is matched case and space insensitively", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: " OS ", Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{}, + wantRule: true, + wantUpstreams: 1, + }, + { + // A wire value this build does not know fails closed to the + // resolvers the administrator chose, never to the Control D upstream. + name: "unknown mode with resolvers fails closed", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday", Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{"upstream.internal_0"}, + wantRule: true, + wantUpstreams: 2, + }, + { + // With no resolvers there is nothing to fail closed to. + name: "unknown mode without resolvers is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday"}, + wantRule: false, + wantUpstreams: 1, + }, + { + // A deployment that predates the field: the list is the only signal. + name: "absent mode infers explicit", + entry: controld.SplitDNS{Domain: "a.example.com", Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{"upstream.internal_0"}, + wantRule: true, + wantUpstreams: 2, + }, + { + name: "absent mode infers os", + entry: controld.SplitDNS{Domain: "a.example.com"}, + wantTargets: []string{}, + wantRule: true, + wantUpstreams: 1, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{tc.entry}) + targets, ok := ruleTargets(t, cfg, "a.example.com") + if ok != tc.wantRule { + t.Fatalf("rule present = %v, want %v", ok, tc.wantRule) + } + if ok && strings.Join(targets, ",") != strings.Join(tc.wantTargets, ",") { + t.Errorf("targets = %v, want %v", targets, tc.wantTargets) + } + if len(cfg.Upstream) != tc.wantUpstreams { + t.Errorf("upstreams = %v, want %d", cfg.Upstream, tc.wantUpstreams) + } + }) + } +} + +// A mode change is a routing change even when the resolver list is untouched, +// so it has to reach the endpoint on the next refresh. +func TestInternalDomainsEqualTracksMode(t *testing.T) { + explicit := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}} + os := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeOS, Resolvers: []string{"10.0.0.53"}}} + if internalDomainsEqual(explicit, os) { + t.Error("switching to the OS resolver must be detected as a change") + } + if internalDomainsEqual(os, explicit) { + t.Error("switching to explicit resolvers must be detected as a change") + } + // An unusable entry routes nothing, so it must not look like a change. + unusable := []controld.SplitDNS{ + {Domain: "a.example.com", Mode: controld.SplitDNSModeOS, Resolvers: []string{"10.0.0.53"}}, + {Domain: "b.example.com", Mode: "forward-someday"}, + } + if !internalDomainsEqual(os, unusable) { + t.Error("an entry ctrld cannot use must not read as a configuration change") + } +} diff --git a/cmd/cli/internal_log_shutdown_test.go b/cmd/cli/internal_log_shutdown_test.go new file mode 100644 index 00000000..eea3ee00 --- /dev/null +++ b/cmd/cli/internal_log_shutdown_test.go @@ -0,0 +1,51 @@ +package cli + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestFinishRunClosesInternalFilesWithoutServiceStop(t *testing.T) { + p := &prog{ + cfg: &ctrld.Config{}, runDone: make(chan struct{}), runAbortCh: make(chan struct{}), + dnsWatcherStopCh: make(chan struct{}), + internalLogWriter: newLogWriter(), internalJournalWriter: newSmallLogWriter(), + } + p.logger.Store(mainLog.Load()) + var files []*os.File + var paths []string + for i, writer := range []*logWriter{p.internalLogWriter, p.internalJournalWriter} { + path := filepath.Join(t.TempDir(), []string{"debug.log", "journal.log"}[i]) + if err := writer.setLogFile(path, logBudget{maxSize: 4096, backups: 1}); err != nil { + t.Fatal(err) + } + t.Cleanup(writer.closeLogFile) + files = append(files, writer.rotating().f) + paths = append(paths, path) + } + p.onStopped = []func(){func() { + for _, writer := range []*logWriter{p.internalLogWriter, p.internalJournalWriter} { + if _, err := writer.Write([]byte("final cleanup\n")); err != nil { + t.Error(err) + } + } + }} + close(p.runDone) + p.finishRun() + for i, f := range files { + if _, err := f.Write([]byte("unexpected write\n")); !errors.Is(err, os.ErrClosed) { + t.Errorf("file %d is not closed: %v", i, err) + } + b, err := os.ReadFile(paths[i]) + if err != nil { + t.Fatal(err) + } + if string(b) != "final cleanup\n" { + t.Errorf("file %d lost final cleanup: %q", i, b) + } + } +} diff --git a/cmd/cli/library.go b/cmd/cli/library.go index 7847dd7f..906e3ac1 100644 --- a/cmd/cli/library.go +++ b/cmd/cli/library.go @@ -9,6 +9,7 @@ import ( // AppCallback provides hooks for injecting certain functionalities // from mobile platforms to main ctrld cli. +// This allows mobile applications to customize behavior without modifying core CLI code type AppCallback struct { HostName func() string LanIp func() string @@ -17,6 +18,7 @@ type AppCallback struct { } // AppConfig allows overwriting ctrld cli flags from mobile platforms. +// This provides a clean interface for mobile apps to configure ctrld behavior type AppConfig struct { CdUID string ProvisionID string @@ -27,18 +29,29 @@ type AppConfig struct { LogPath string } +// Network and HTTP configuration constants const ( + // defaultHTTPTimeout provides reasonable timeout for HTTP operations + // This prevents hanging requests while allowing sufficient time for network delays defaultHTTPTimeout = 30 * time.Second - defaultMaxRetries = 3 - downloadServerIp = "23.171.240.151" + + // defaultMaxRetries provides retry attempts for failed HTTP requests + // This improves reliability in unstable network conditions + defaultMaxRetries = 3 + + // downloadServerIp is the fallback IP for download operations + // This ensures downloads work even when DNS resolution fails + downloadServerIp = "23.171.240.151" ) // httpClientWithFallback returns an HTTP client configured with timeout and IPv4 fallback +// This ensures reliable HTTP operations by preferring IPv4 and handling timeouts gracefully func httpClientWithFallback(timeout time.Duration) *http.Client { return &http.Client{ Timeout: timeout, Transport: &http.Transport{ // Prefer IPv4 over IPv6 + // This improves compatibility with networks that have IPv6 issues DialContext: (&net.Dialer{ Timeout: 10 * time.Second, KeepAlive: 30 * time.Second, @@ -49,9 +62,15 @@ func httpClientWithFallback(timeout time.Duration) *http.Client { } // doWithRetry performs an HTTP request with retries +// This improves reliability by automatically retrying failed requests with exponential backoff func doWithRetry(req *http.Request, maxRetries int, ip string) (*http.Response, error) { + return doWithRetryClient(httpClientWithFallback(defaultHTTPTimeout), req, maxRetries, ip) +} + +// doWithRetryClient is doWithRetry with an injectable client, so the retry and +// error-composition behaviour can be tested without real network access. +func doWithRetryClient(client *http.Client, req *http.Request, maxRetries int, ip string) (*http.Response, error) { var lastErr error - client := httpClientWithFallback(defaultHTTPTimeout) var ipReq *http.Request if ip != "" { ipReq = req.Clone(req.Context()) @@ -60,32 +79,40 @@ func doWithRetry(req *http.Request, maxRetries int, ip string) (*http.Response, } for attempt := 0; attempt < maxRetries; attempt++ { if attempt > 0 { - time.Sleep(time.Second * time.Duration(attempt+1)) // Exponential backoff + // Linear backoff reduces server load and improves success rate + time.Sleep(time.Second * time.Duration(attempt+1)) } resp, err := client.Do(req) if err == nil { return resp, nil } + // Keep the hostname attempt's error: it carries the diagnosis (on Windows, + // a local firewall denying the socket shows up here as WSAEACCES), while the + // direct-ip fallback often fails for an unrelated reason such as an + // unreachable IPv6 route. + attemptErr := err if ipReq != nil { - mainLog.Load().Warn().Err(err).Msgf("dial to %q failed", req.Host) - mainLog.Load().Warn().Msgf("fallback to direct IP to download prod version: %q", ip) - resp, err = client.Do(ipReq) - if err == nil { + mainLog.Load().Warn().Err(err).Msgf("Dial to %q failed", req.Host) + mainLog.Load().Warn().Msgf("Fallback to direct ip to download prod version: %q", ip) + resp, fallbackErr := client.Do(ipReq) + if fallbackErr == nil { return resp, nil } + attemptErr = fmt.Errorf("%w; fallback to direct ip %s failed: %w", attemptErr, ip, fallbackErr) } - lastErr = err - mainLog.Load().Debug().Err(err). + lastErr = attemptErr + mainLog.Load().Debug().Err(attemptErr). Str("method", req.Method). Str("url", req.URL.String()). Msgf("HTTP request attempt %d/%d failed", attempt+1, maxRetries) } - return nil, fmt.Errorf("failed after %d attempts to %s %s: %v", maxRetries, req.Method, req.URL, lastErr) + return nil, fmt.Errorf("failed after %d attempts to %s %s: %w", maxRetries, req.Method, req.URL, lastErr) } // Helper for making GET requests with retries +// This provides a simplified interface for common GET operations with built-in retry logic func getWithRetry(url string, ip string) (*http.Response, error) { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { diff --git a/cmd/cli/library_retry_test.go b/cmd/cli/library_retry_test.go new file mode 100644 index 00000000..e2f06787 --- /dev/null +++ b/cmd/cli/library_retry_test.go @@ -0,0 +1,242 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "syscall" + "testing" + + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// wsaEACCES is WSAEACCES (10013): "An attempt was made to access a socket in a way +// forbidden by its access permissions." This is what Windows reports when a WFP +// filter denies the connect. Used as a plain errno so the test runs everywhere. +const wsaEACCES = syscall.Errno(10013) + +// denyingRoundTripper denies the hostname attempt with firstErr and the direct-ip +// attempt with fbErr, the shape seen during the Firewall Mode incident: the +// hostname attempt was denied by ctrld's own stale block-all filters, while the +// direct-ip fallback failed on an unreachable IPv6 route. +type denyingRoundTripper struct { + hostname string + firstErr error + fbErr error +} + +func (rt *denyingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Host == rt.hostname { + return nil, &net.OpError{Op: "dial", Net: "tcp4", Err: rt.firstErr} + } + return nil, &net.OpError{Op: "dial", Net: "tcp6", Err: rt.fbErr} +} + +func TestDoWithRetryPreservesHostnameError(t *testing.T) { + const hostname = "dl.controld.dev" + req, err := http.NewRequest(http.MethodGet, "https://"+hostname+"/windows-amd64/ctrld-client.exe", nil) + if err != nil { + t.Fatal(err) + } + rt := &denyingRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + + _, err = doWithRetryClient(&http.Client{Transport: rt}, req, 1, "23.171.240.151") + if err == nil { + t.Fatal("expected doWithRetry to fail when both attempts are denied") + } + if !errors.Is(err, wsaEACCES) { + t.Errorf("hostname-attempt error (WSAEACCES) was lost, got: %v", err) + } + if !errors.Is(err, syscall.EHOSTUNREACH) { + t.Errorf("fallback error was lost, got: %v", err) + } +} + +// composedAttemptErrors builds the error shape the two-attempt paths return: each +// attempt's *url.Error (as produced by http.Client.Do) wrapped by a single fmt.Errorf +// with two %w verbs, hostname attempt first. Mirrors doWithFallback in +// internal/controld and doWithRetryClient above. +func composedAttemptErrors(first, fallback error) error { + attempt := func(network string, cause error) error { + return &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.OpError{Op: "dial", Net: network, Err: cause}, + } + } + return fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", + attempt("tcp4", first), "147.185.34.1", attempt("tcp6", fallback)) +} + +// TestComposedFallbackErrorRetryClassification pins which attempt decides whether +// preflight keeps retrying. +// +// Reporting both attempt errors is not purely diagnostic: processCDFlags decides +// retryability with errUrlNetworkError, which uses errors.As, and errors.As is +// order-sensitive - it returns the *first* matching error in the tree. Composing the +// hostname attempt first therefore hands the retry predicate the hostname failure, +// where previously only the fallback's error survived to be classified. +// +// The consequence is deliberate: a locally denied socket (WSAEACCES, a firewall +// blocking ctrld) is no longer treated as a transient network error, so preflight fails +// fast and reports instead of backing off - the incident logged 256 retry cycles +// against filters that were never going to clear on their own. The boot case that +// justifies the indefinite retry, a network unreachable on both attempts, is preserved. +// +// If the wrap order is ever reversed, this test fails rather than silently restoring +// indefinite retries against a host that is actively refusing. +func TestComposedFallbackErrorRetryClassification(t *testing.T) { + tests := []struct { + name string + hostname error + fallback error + wantRetryable bool + }{ + { + // The incident's pair: denied locally, IPv6 route unusable. + name: "denied socket then unreachable fallback fails fast", + hostname: wsaEACCES, + fallback: syscall.EHOSTUNREACH, + wantRetryable: false, + }, + { + // Boot with no network yet: must still retry indefinitely. + name: "network unreachable on both attempts still retries", + hostname: syscall.ENETUNREACH, + fallback: syscall.ENETUNREACH, + wantRetryable: true, + }, + { + name: "connection refused still retries", + hostname: syscall.ECONNREFUSED, + fallback: syscall.EHOSTUNREACH, + wantRetryable: true, + }, + { + name: "permission denied on both attempts fails fast", + hostname: syscall.EACCES, + fallback: syscall.EACCES, + wantRetryable: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := composedAttemptErrors(tc.hostname, tc.fallback) + if got := errUrlNetworkError(err); got != tc.wantRetryable { + t.Errorf("errUrlNetworkError() = %v, want %v", got, tc.wantRetryable) + } + // Both attempts remain reportable regardless of classification. + if !errors.Is(err, tc.hostname) { + t.Error("hostname attempt error was lost") + } + if !errors.Is(err, tc.fallback) { + t.Error("fallback attempt error was lost") + } + }) + } +} + +// TestUnresolvedHostnameDefersToFallbackAttempt covers the asymmetric pair. +// +// Only the hostname attempt resolves DNS, and Go marks a *net.DNSError as temporary only +// for socket failures that reached the server - so a SERVFAIL or "no such host" answer is +// not temporary. At boot behind a captive portal, or before a router's forwarder is up, +// that is exactly how the hostname attempt fails while the network is merely not ready. +// Before the composed error existed only the fallback decided, so this pair retried; +// classifying the hostname attempt alone would fail it fast and reach Fatal. +// +// A name-resolution failure therefore carries no verdict: the fallback attempt decides. +// The locally-denied case above still fails fast, because a denied socket is definitive. +func TestUnresolvedHostnameDefersToFallbackAttempt(t *testing.T) { + dnsFailure := &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.DNSError{Err: "server misbehaving", Name: "api.controld.com", IsTemporary: false}, + } + attempt := func(cause error) error { + return &url.Error{ + Op: "Post", + URL: "https://api.controld.com/utility", + Err: &net.OpError{Op: "dial", Net: "tcp6", Err: cause}, + } + } + + retryable := fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", + dnsFailure, "147.185.34.1", attempt(syscall.ECONNREFUSED)) + if !errUrlNetworkError(retryable) { + t.Error("an unresolved hostname with a retryable fallback must keep retrying: at boot the network is simply not up yet") + } + + denied := fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", + dnsFailure, "147.185.34.1", attempt(wsaEACCES)) + if errUrlNetworkError(denied) { + t.Error("an unresolved hostname with a denied fallback must fail fast: nothing here clears on its own") + } + + // A resolution failure alone still says nothing, so it must not be read as retryable. + if errUrlNetworkError(dnsFailure) { + t.Error("a bare name-resolution failure must not be classified as retryable") + } +} + +// TestDoWithFallbackClassificationEndToEnd drives the real composition in +// internal/controld through the real predicate, instead of asserting a hand-written copy +// of its error shape against another hand-written copy. A change to either side's format +// string or wrap order is caught here. +func TestDoWithFallbackClassificationEndToEnd(t *testing.T) { + const hostname = "api.controld.com" + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + rt := &denyingRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + + _, gotErr := controld.DoWithFallbackForTest(context.Background(), &http.Client{Transport: rt}, req, "147.185.34.1") + if gotErr == nil { + t.Fatal("expected both attempts to fail") + } + if errUrlNetworkError(gotErr) { + t.Errorf("the real composed error was classified as retryable: %v", gotErr) + } + if !errors.Is(gotErr, wsaEACCES) || !errors.Is(gotErr, syscall.EHOSTUNREACH) { + t.Errorf("the real composed error lost an attempt: %v", gotErr) + } +} + +// TestDoWithRetryComposesHostnameAttemptFirst anchors the ordering assumption above to +// the real composition, so a reordering of the wrap in doWithRetryClient is caught here +// and not only in the hand-built shape. +func TestDoWithRetryComposesHostnameAttemptFirst(t *testing.T) { + const hostname = "dl.controld.dev" + req, err := http.NewRequest(http.MethodGet, "https://"+hostname+"/windows-amd64/ctrld-client.exe", nil) + if err != nil { + t.Fatal(err) + } + rt := &denyingRoundTripper{hostname: hostname, firstErr: wsaEACCES, fbErr: syscall.EHOSTUNREACH} + + _, gotErr := doWithRetryClient(&http.Client{Transport: rt}, req, 1, "23.171.240.151") + if gotErr == nil { + t.Fatal("expected both attempts to fail") + } + + // errors.As must reach the hostname attempt first: that is what the retry + // predicate classifies. + var opErr *net.OpError + if !errors.As(gotErr, &opErr) { + t.Fatalf("no net.OpError in the chain: %v", gotErr) + } + if !errors.Is(opErr.Err, wsaEACCES) { + t.Errorf("first OpError in the chain is %v, want the hostname attempt (%v)", opErr.Err, wsaEACCES) + } +} diff --git a/cmd/cli/listener_5353_darwin_test.go b/cmd/cli/listener_5353_darwin_test.go new file mode 100644 index 00000000..d63472f0 --- /dev/null +++ b/cmd/cli/listener_5353_darwin_test.go @@ -0,0 +1,69 @@ +//go:build darwin + +package cli + +import ( + "fmt" + ctrld "github.com/Control-D-Inc/ctrld" + "strings" + "testing" +) + +func TestPFScopedNameserverAddress(t *testing.T) { + for _, tc := range []struct{ server, host, family string }{ + {"[fe80::1%en0]:53", "fe80::1", "inet6"}, + {"2001:db8::53", "2001:db8::53", "inet6"}, + {"192.0.2.53:53", "192.0.2.53", "inet"}, + } { + host, family, ok := pfNameserverAddress(tc.server) + if !ok || host != tc.host || family != tc.family { + t.Fatalf("PF nameserver %q: %q %q %v", tc.server, host, family, ok) + } + } + if _, _, ok := pfNameserverAddress("not an address"); ok { + t.Fatal("invalid PF address accepted") + } +} + +func TestPFRecoveryScopedNameserverRules(t *testing.T) { + captureDebugMainLog(t) + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.2", Port: 5353}}}} + p.logger.Store(mainLog.Load()) + // Recovery converts OS-discovered addresses into this exemption list. + rules := p.buildPFAnchorRulesForTunnels([]vpnDNSExemption{ + {Server: "fe80::1%en0"}, {Server: "fe80::1%en1"}, {Server: "192.0.2.53"}, + }, nil) + want := "pass out quick on ! lo0 inet6 proto { udp, tcp } from any to fe80::1 port 53 group " + pfGroupName + if strings.Count(rules, want) != 1 { + t.Fatalf("scoped recovery exemption missing or duplicated: %s", rules) + } + if strings.Contains(rules, "%en0") || strings.Contains(rules, "%en1") || strings.Contains(rules, "inet proto { udp, tcp } from any to fe80:") { + t.Fatal("socket zone or IPv4 family leaked into recovery PF rules") + } + if !strings.Contains(rules, "inet proto { udp, tcp } from any to 192.0.2.53 port 53 group "+pfGroupName) { + t.Fatal("IPv4 recovery exemption changed") + } +} + +func TestCustomListener5353PFRules(t *testing.T) { + captureDebugMainLog(t) + for _, ip := range []string{"127.0.0.1", "127.0.0.2", "127.0.0.53", "192.0.2.10"} { + t.Run(ip, func(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: ip, Port: 5353}}}} + p.logger.Store(mainLog.Load()) + rules := p.buildPFAnchorRulesForTunnels(nil, []string{"utun9"}) + for _, proto := range []string{"udp", "tcp"} { + want := fmt.Sprintf("rdr on lo0 inet proto %s from any to ! %s port 53 -> %s port 5353", proto, ip, ip) + if !strings.Contains(rules, want) { + t.Errorf("missing redirect %s", want) + } + } + if strings.Contains(rules, "port 5354") { + t.Fatal("explicit port replaced with automatic fallback") + } + if p.interceptDNSTargetValue() == ip { + t.Fatal("target equals rdr exclusion: no port translation") + } + }) + } +} diff --git a/cmd/cli/listener_5353_target_darwin_test.go b/cmd/cli/listener_5353_target_darwin_test.go new file mode 100644 index 00000000..8259e599 --- /dev/null +++ b/cmd/cli/listener_5353_target_darwin_test.go @@ -0,0 +1,41 @@ +//go:build darwin + +package cli + +import ( + "errors" + "slices" + "testing" +) + +func TestNativeCLATCustomListener5353Lifecycle(t *testing.T) { + for _, ip := range []string{"127.0.0.1", "127.0.0.2", "127.0.0.53", "192.0.2.10"} { + t.Run(ip, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("DHCPv4 intentionally absent") + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = ip + p.cfg.Listener["0"].Port = 5353 + target := p.interceptDNSTargetValue() + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.dns["Wi-Fi"], []string{target}) || len(h.setCalls) != 1 { + t.Fatalf("target was not installed idempotently: %v writes %v", h.dns, h.setCalls) + } + if target == ip { + t.Fatal("OS DNS would hit port53 at listener instead of PF port5353 redirect") + } + h.dhcpErr = nil + h.dhcp = []string{"192.0.2.53"} + h.nativeCLAT = false + p.ensureInterceptDNSTarget([]string{"192.0.2.53"}) + if len(h.dns["Wi-Fi"]) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("DHCP return failed to remove target: %+v", h.dns) + } + if p.cfg.Listener["0"].IP != ip || p.cfg.Listener["0"].Port != 5353 { + t.Fatal("target lifecycle changed configured listener") + } + }) + } +} diff --git a/cmd/cli/listener_5353_test.go b/cmd/cli/listener_5353_test.go new file mode 100644 index 00000000..611b4437 --- /dev/null +++ b/cmd/cli/listener_5353_test.go @@ -0,0 +1,34 @@ +package cli + +import ( + ctrld "github.com/Control-D-Inc/ctrld" + "testing" +) + +func TestCustomListener5353Contract(t *testing.T) { + for _, tc := range []struct{ ip, target string }{ + {"127.0.0.1", "127.0.0.53"}, {"127.0.0.2", "127.0.0.53"}, + {"127.0.0.53", "127.0.0.54"}, {"192.0.2.10", "127.0.0.53"}, + } { + t.Run(tc.ip, func(t *testing.T) { + if !isExplicitInterceptListener(tc.ip, 5353) { + t.Fatal("5353 was not explicit") + } + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: tc.ip, Port: 5353}}}} + if got := p.interceptDNSTargetValue(); got != tc.target { + t.Fatalf("target=%s want %s", got, tc.target) + } + current := p.cfg.Listener + generated := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + preserveBoundListeners(generated, current) + if generated["0"].IP != tc.ip || generated["0"].Port != 5353 { + t.Fatalf("generated reload lost bound listener: %+v", generated["0"]) + } + explicit := map[string]*ctrld.ListenerConfig{"0": {IP: tc.ip, Port: 5353}} + preserveBoundListeners(explicit, map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}) + if explicit["0"].IP != tc.ip || explicit["0"].Port != 5353 { + t.Fatalf("explicit reload normalized: %+v", explicit["0"]) + } + }) + } +} diff --git a/cmd/cli/listener_intercept_mode_test.go b/cmd/cli/listener_intercept_mode_test.go new file mode 100644 index 00000000..b762432a --- /dev/null +++ b/cmd/cli/listener_intercept_mode_test.go @@ -0,0 +1,34 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestListenerInterceptModeExplicitOff(t *testing.T) { + oldIntercept := interceptMode + t.Cleanup(func() { interceptMode = oldIntercept }) + + cfg := &ctrld.Config{} + cfg.Service.InterceptMode = "dns" + + tests := []struct { + name string + flag string + want string + }{ + {name: "explicit off is final", flag: "off", want: "off"}, + {name: "empty flag falls back to config", flag: "", want: "dns"}, + {name: "explicit dns wins over config", flag: "dns", want: "dns"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + interceptMode = tc.flag + if got := listenerInterceptMode(cfg); got != tc.want { + t.Fatalf("listenerInterceptMode() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/listener_shutdown_test.go b/cmd/cli/listener_shutdown_test.go new file mode 100644 index 00000000..ae002839 --- /dev/null +++ b/cmd/cli/listener_shutdown_test.go @@ -0,0 +1,296 @@ +package cli + +import ( + "context" + "errors" + "io" + "net" + "os" + "os/exec" + "runtime" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func TestStopRestorationFailureStillStopsWorkers(t *testing.T) { + old := deAllocateIPFn + t.Cleanup(func() { deAllocateIPFn = old }) + want := errors.New("deallocation failed") + calls := 0 + deAllocateIPFn = func(string) error { calls++; return want } + p := &prog{ + cfg: &ctrld.Config{Service: ctrld.ServiceConfig{AllocateIP: true}, Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1"}}}, + waitCh: make(chan struct{}), stopCh: make(chan struct{}), runDone: make(chan struct{}), + runAbortCh: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), + pinCodeValidCh: make(chan struct{}, 1), + } + p.logger.Store(mainLog.Load()) + // Accept the stop even if another test configured a deactivation pin. + p.pinCodeValidCh <- struct{}{} + require.NoError(t, p.Start(nil)) + require.ErrorIs(t, p.Stop(nil), want) + require.True(t, stopRequested(p.stopCh)) + awaitShutdown(t, p.runDone) + p.finishRun() + require.ErrorIs(t, p.restoreOSState(), want, "the first error must survive sync.Once") + require.Equal(t, 1, calls, "never invoke the real IP deallocator") +} + +func awaitShutdown(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("shutdown did not complete") + } +} + +func TestFinishRunWaitsBeyondTimeout(t *testing.T) { + c, peer := net.Pipe() + defer peer.Close() + p := &prog{cfg: &ctrld.Config{}, runAbortCh: make(chan struct{}), runDone: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), logConn: c} + p.logger.Store(mainLog.Load()) + done := make(chan struct{}) + go func() { p.finishRun(); close(done) }() + select { + case <-done: + t.Fatal("released resources while the run was alive") + case <-time.After(shutdownTimeout + 50*time.Millisecond): + } + close(p.runDone) + awaitShutdown(t, done) + _, err := peer.Write([]byte("x")) + require.ErrorIs(t, err, io.ErrClosedPipe) +} + +// Each case runs in a subprocess because run initializes package-global config +// and logging. No service, interface DNS or network-monitor setup is permitted. +func TestRunListenerShutdown(t *testing.T) { + const env = "CTRLD_TEST_LISTENER_SHUTDOWN" + mode := os.Getenv(env) + if mode == "" { + for _, mode := range []string{"stop-before-ready", "abort-before-ready", "drain", "reload", "reload-before-ready", "stop-during-hook", "stop-during-post", "wire-udp", "wire-tcp", "api-drain"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunListenerShutdown$", "-test.v") + cmd.Env = append(os.Environ(), env+"="+mode) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", out) + }) + } + return + } + defer func() { + if t.Failed() { + t.Log(logOutput.String()) + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + t.Log(string(buf[:n])) + } + }() + isMobile = func() bool { return true } + cdUID = "" + iface = "" + apiEntered, apiCanceled, apiRelease := make(chan struct{}), make(chan struct{}), make(chan struct{}) + if mode == "api-drain" { + cdUID = "testuid" + fetchResolverConfigFn = func(ctx context.Context, _ *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + close(apiEntered) + <-ctx.Done() + close(apiCanceled) + <-apiRelease + return nil, ctx.Err() + } + } + var monitorCalls, postCalls atomic.Int32 + monitorNetworkChangesFn = func(*prog, context.Context) error { monitorCalls.Add(1); return nil } + postRunFn = func(*prog) { postCalls.Add(1) } + dir := t.TempDir() + cfg = ctrld.Config{Service: ctrld.ServiceConfig{LogPath: dir + "/ctrld.log"}, Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1"}}} + p := &prog{ + cfg: &cfg, waitCh: make(chan struct{}), stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), + runDone: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), apiReloadCh: make(chan *ctrld.Config), + reloadDoneCh: make(chan struct{}, 1), + apiForceReloadCh: make(chan struct{}), + } + p.logger.Store(mainLog.Load()) + logConnection := &closeCountingLogConn{} + p.logConn = logConnection + entered, drain, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + var hooks atomic.Int32 + p.onStarted = []func(){func() { hooks.Add(1) }} + mutationEntered, mutationRelease, restored := make(chan struct{}), make(chan struct{}), make(chan struct{}) + stopDuringMutation := mode == "stop-during-hook" || mode == "stop-during-post" + if stopDuringMutation { + mutation := func() { + close(mutationEntered) + <-mutationRelease + select { + case <-restored: + t.Error("startup mutation ran after restoration") + default: + } + } + p.onStopped = []func(){func() { close(restored) }} + if mode == "stop-during-hook" { + p.onStarted = []func(){mutation} + } else { + postRunFn = func(*prog) { mutation(); postCalls.Add(1) } + } + } + wire := mode == "wire-udp" || mode == "wire-tcp" + if wire { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + cfg.Listener["0"].Port = ln.Addr().(*net.TCPAddr).Port + require.NoError(t, ln.Close()) + // The real handler blocks in its semaphore while both server Shutdown + // calls drain. Use an internal query so no external upstream is needed. + serveDNSFn = func(p *prog, ctx context.Context, n string) error { + p.sema = &shutdownSemaphore{entered: entered, resume: release} + return p.serveDNS(ctx, n) + } + } else { + serveDNSFn = func(p *prog, _ context.Context, _ string) error { + close(entered) + if mode == "drain" || mode == "reload" || mode == "api-drain" || stopDuringMutation { + p.started <- struct{}{} + } + select { + case <-p.stopCh: + case <-p.runAbortCh: + } + close(drain) + <-release + return nil + } + } + close(p.waitCh) + require.NoError(t, p.Start(nil)) + var queryDone chan error + if wire { + require.Eventually(t, func() bool { return postCalls.Load() == 1 }, time.Second, time.Millisecond) + proto := mode[len("wire-"):] + queryDone = make(chan error, 1) + go func() { + m := new(dns.Msg) + m.SetQuestion(selfCheckInternalTestDomain+".", dns.TypeA) + answer, _, err := (&dns.Client{Net: proto, Timeout: 5 * time.Second}).Exchange(m, net.JoinHostPort("127.0.0.1", strconv.Itoa(cfg.Listener["0"].Port))) + if err == nil && answer.Rcode != dns.RcodeSuccess { + err = errors.New("internal query failed") + } + queryDone <- err + }() + } + awaitShutdown(t, entered) + if mode == "api-drain" { + select { + case p.apiForceReloadCh <- struct{}{}: + case <-time.After(time.Second): + t.Fatal("production run did not start the API reload worker") + } + awaitShutdown(t, apiEntered) + } + if mode == "reload" || mode == "reload-before-ready" { + if mode == "reload" { + require.Eventually(t, func() bool { return postCalls.Load() == 1 }, time.Second, time.Millisecond) + } + configPath = dir + "/reload.toml" + p.apiReloadCh <- &ctrld.Config{ + Service: cfg.Service, + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}, + Network: map[string]*ctrld.NetworkConfig{"0": {Cidrs: []string{"127.0.0.0/8"}}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: ctrld.ResolverTypeDOT, Endpoint: "example.test:853", BootstrapIP: "127.0.0.1"}}, + } + if mode == "reload-before-ready" { + // Give the reload a chance to reach the old run's cancellation path. + select { + case <-p.reloadDoneCh: + t.Fatal("reload skipped one-time startup") + case <-time.After(30 * time.Millisecond): + } + p.started <- struct{}{} + } + select { + case <-p.reloadDoneCh: + case <-time.After(3 * time.Second): + t.Fatal("reload waited for a persistent listener to stop") + } + select { + case <-drain: + t.Fatal("reload stopped the persistent listener") + default: + } + require.EqualValues(t, 1, hooks.Load()) + require.EqualValues(t, 1, postCalls.Load()) + } + if stopDuringMutation { + awaitShutdown(t, mutationEntered) + stopDone := make(chan struct{}) + go func() { _ = p.Stop(nil); close(stopDone) }() + select { + case <-restored: + t.Fatal("restoration raced a startup OS mutation") + case <-time.After(30 * time.Millisecond): + } + close(mutationRelease) + awaitShutdown(t, stopDone) + awaitShutdown(t, restored) + require.False(t, p.startOSState(func() { t.Error("startup permitted after restoration") })) + } else if mode != "abort-before-ready" { + close(p.stopCh) + } + finished := make(chan struct{}) + go func() { p.finishRun(); close(finished) }() + awaitShutdown(t, p.runDone) + if !wire { + awaitShutdown(t, drain) + } + select { + case <-finished: + t.Fatal("finishRun returned before the listener/handler drained") + case <-time.After(30 * time.Millisecond): + } + unblock() + if mode == "api-drain" { + awaitShutdown(t, apiCanceled) + select { + case <-finished: + t.Fatal("finishRun returned before the API reload worker") + case <-time.After(30 * time.Millisecond): + } + close(apiRelease) + } + awaitShutdown(t, finished) + require.EqualValues(t, 1, logConnection.closes.Load(), "startup and shutdown must share log-connection ownership") + if wire { + require.NoError(t, <-queryDone, "active DNS query must finish before resource retirement") + } + if mode == "stop-before-ready" || mode == "abort-before-ready" { + require.Zero(t, hooks.Load(), "startup hooks must not run after cancellation") + require.Zero(t, postCalls.Load()) + } + require.EqualValues(t, 1, monitorCalls.Load(), "real network monitor must not be reached") +} + +type shutdownSemaphore struct { + entered chan struct{} + resume <-chan struct{} + once sync.Once +} + +func (s *shutdownSemaphore) acquire() { s.once.Do(func() { close(s.entered) }); <-s.resume } +func (*shutdownSemaphore) release() {} diff --git a/cmd/cli/log_budget.go b/cmd/cli/log_budget.go new file mode 100644 index 00000000..551dfd2d --- /dev/null +++ b/cmd/cli/log_budget.go @@ -0,0 +1,41 @@ +package cli + +import ( + "github.com/Control-D-Inc/ctrld" +) + +// journalLogFileName is the journal file next to the debug log in the ctrld +// home directory. +const journalLogFileName = "ctrld-journal.log" + +// logBudgets is the disk space of the two streams. The debug stream takes the +// larger share, so a log covers more days. The journal holds the retained +// lines only, so it stays small. +func logBudgets() (debug, journal logBudget) { + return logBudget{maxSize: 10 << 20, backups: 4}, logBudget{maxSize: 2 << 20, backups: 2} +} + +const ( + // logMaxSizeMBLimit and logMaxBackupsLimit bound the two keys. The log + // file opens before ctrld validates its configuration, so a key outside + // these bounds reaches this code: a size near the largest integer never + // rotates, and a count of one million backups renames one million files. + // The validation rules are the first line of defense, and these bounds + // are the second. + logMaxSizeMBLimit = 1024 + logMaxBackupsLimit = 64 +) + +// debugLogBudget starts from the default. The two keys size the debug stream +// only. The journal budget stays fixed. A key outside its bounds keeps the +// default. +func debugLogBudget(svc *ctrld.ServiceConfig) logBudget { + budget, _ := logBudgets() + if mb := svc.LogMaxSizeMB; mb >= 1 && mb <= logMaxSizeMBLimit { + budget.maxSize = int64(mb) << 20 + } + if count := svc.LogMaxBackups; count != nil && *count >= 0 && *count <= logMaxBackupsLimit { + budget.backups = *count + } + return budget +} diff --git a/cmd/cli/log_budget_apply_test.go b/cmd/cli/log_budget_apply_test.go new file mode 100644 index 00000000..79a359b3 --- /dev/null +++ b/cmd/cli/log_budget_apply_test.go @@ -0,0 +1,142 @@ +package cli + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +// setupDebugBudgetTest seeds a debug file with four backups, which is the +// default count, and puts back the two limit keys after the test. +func setupDebugBudgetTest(t *testing.T) string { + t.Helper() + dir := setupLogSetupTest(t) + origSize, origBackups := cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups + t.Cleanup(func() { + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = origSize, origBackups + }) + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = 0, nil + debugPath := filepath.Join(dir, logFileName) + for i := 1; i <= 4; i++ { + if err := os.WriteFile(debugPath+"."+strconv.Itoa(i), []byte("old run\n"), 0o600); err != nil { + t.Fatal(err) + } + } + return debugPath +} + +// assertDebugBudget checks the limits of the open debug file and the backups +// that stay on disk. +func assertDebugBudget(t *testing.T, p *prog, debugPath string, want logBudget) { + t.Helper() + debugWriter, _ := p.internalWriters() + rf := debugWriter.rotating() + if rf == nil { + t.Fatal("the debug file is not open") + } + rf.mu.Lock() + got := rf.budget + rf.mu.Unlock() + if got != want { + t.Fatalf("debug budget = %+v, want %+v", got, want) + } + for i := 1; i <= 4; i++ { + _, err := os.Stat(debugPath + "." + strconv.Itoa(i)) + if i <= want.backups && err != nil { + t.Fatalf("backup %d is gone: %v", i, err) + } + if i > want.backups && !os.IsNotExist(err) { + t.Fatalf("backup %d stays although the limit is %d backups (stat err = %v)", i, want.backups, err) + } + } +} + +// Test_initLogging_appliesTheLimitsOfTheFetchedConfig drives a first start: +// the debug file opens with the local config, and the API preflight then +// writes the limit keys. The second logging setup must apply them at once, so +// no second restart is necessary. +func Test_initLogging_appliesTheLimitsOfTheFetchedConfig(t *testing.T) { + debugPath := setupDebugBudgetTest(t) + cdUID = "test-uid" + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + defaultBudget, _ := logBudgets() + assertDebugBudget(t, p, debugPath, defaultBudget) + + oneBackup := 1 + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = 20, &oneBackup + p.initLogging(false) + + assertDebugBudget(t, p, debugPath, logBudget{maxSize: 20 << 20, backups: 1}) +} + +// Test_applyDebugLogBudget_takesTheLimitsOfARefreshedConfig drives a managed +// config refresh, which applies a new config with no restart. +func Test_applyDebugLogBudget_takesTheLimitsOfARefreshedConfig(t *testing.T) { + debugPath := setupDebugBudgetTest(t) + cdUID = "test-uid" + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + noBackups := 0 + refreshed := ctrld.ServiceConfig{LogMaxSizeMB: 5, LogMaxBackups: &noBackups} + p.applyDebugLogBudget(debugLogBudget(&refreshed)) + + assertDebugBudget(t, p, debugPath, logBudget{maxSize: 5 << 20, backups: 0}) +} + +// Test_applyDebugLogBudget_takesTheLimitsForLogPath checks the log_path file, +// which follows the same two keys. ctrld does not prune its backups, because +// log_path can name a directory that holds the files of other programs. +func Test_applyDebugLogBudget_takesTheLimitsForLogPath(t *testing.T) { + setupDebugBudgetTest(t) + logPath := filepath.Join(t.TempDir(), "operator.log") + cfg.Service.LogPath = logPath + p := &prog{cfg: &cfg} + p.initLogging(true) + + want := logBudget{maxSize: 3 << 20, backups: 2} + p.applyDebugLogBudget(want) + + rf := logPathFile.Load() + rf.mu.Lock() + got := rf.budget + rf.mu.Unlock() + if got != want { + t.Fatalf("log_path budget = %+v, want %+v", got, want) + } +} + +// Test_initLogging_reportsTheRotationOfTheStartHeader drives a restart that +// finds the journal close to its limit. The header of the new run rotates the +// file, and the rotation event must reach the files, because support reads it +// to learn where the history of the last run went. +func Test_initLogging_reportsTheRotationOfTheStartHeader(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + _, journalBudget := logBudgets() + journalPath := filepath.Join(dir, journalLogFileName) + oldRun := strings.Repeat("o", int(journalBudget.maxSize)-10) + "\n" + if err := os.WriteFile(journalPath, []byte(oldRun), 0o600); err != nil { + t.Fatal(err) + } + p := &prog{cfg: &cfg} + + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + if backup := rotatingFileTestContent(t, journalPath+".1"); backup != oldRun { + t.Fatalf("the journal backup holds %d bytes, want the %d bytes of the old run", len(backup), len(oldRun)) + } + for _, path := range []string{journalPath, filepath.Join(dir, logFileName)} { + if content := rotatingFileTestContent(t, path); !strings.Contains(content, "Log rotated") { + t.Fatalf("%s holds no rotation event: %s", filepath.Base(path), content) + } + } +} diff --git a/cmd/cli/log_budget_test.go b/cmd/cli/log_budget_test.go new file mode 100644 index 00000000..fc8f5773 --- /dev/null +++ b/cmd/cli/log_budget_test.go @@ -0,0 +1,45 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func Test_logBudgets(t *testing.T) { + debug, journalStream := logBudgets() + if want := (logBudget{maxSize: 10 << 20, backups: 4}); debug != want { + t.Errorf("debug budget = %+v, want %+v", debug, want) + } + if want := (logBudget{maxSize: 2 << 20, backups: 2}); journalStream != want { + t.Errorf("journal budget = %+v, want %+v", journalStream, want) + } +} + +func Test_debugLogBudget(t *testing.T) { + noBackup, oneBackup := 0, 1 + negativeBackups, tooManyBackups, mostBackups := -1, logMaxBackupsLimit+1, logMaxBackupsLimit + for _, tc := range []struct { + name string + svc ctrld.ServiceConfig + want logBudget + }{ + {"both keys set", ctrld.ServiceConfig{LogMaxSizeMB: 20, LogMaxBackups: &oneBackup}, logBudget{maxSize: 20 << 20, backups: 1}}, + {"size zero keeps the default size", ctrld.ServiceConfig{LogMaxBackups: &oneBackup}, logBudget{maxSize: 10 << 20, backups: 1}}, + {"absent backups keep the default count", ctrld.ServiceConfig{LogMaxSizeMB: 20}, logBudget{maxSize: 20 << 20, backups: 4}}, + {"zero backups keep no backup file", ctrld.ServiceConfig{LogMaxBackups: &noBackup}, logBudget{maxSize: 10 << 20, backups: 0}}, + {"empty config", ctrld.ServiceConfig{}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"negative size keeps the default size", ctrld.ServiceConfig{LogMaxSizeMB: -1}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"size above the limit keeps the default size", ctrld.ServiceConfig{LogMaxSizeMB: logMaxSizeMBLimit + 1}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"the largest size the key takes", ctrld.ServiceConfig{LogMaxSizeMB: logMaxSizeMBLimit}, logBudget{maxSize: logMaxSizeMBLimit << 20, backups: 4}}, + {"negative backups keep the default count", ctrld.ServiceConfig{LogMaxBackups: &negativeBackups}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"backups above the limit keep the default count", ctrld.ServiceConfig{LogMaxBackups: &tooManyBackups}, logBudget{maxSize: 10 << 20, backups: 4}}, + {"the largest backup count the key takes", ctrld.ServiceConfig{LogMaxBackups: &mostBackups}, logBudget{maxSize: 10 << 20, backups: logMaxBackupsLimit}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := debugLogBudget(&tc.svc); got != tc.want { + t.Errorf("debug budget = %+v, want %+v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/log_file.go b/cmd/cli/log_file.go new file mode 100644 index 00000000..3dda04ee --- /dev/null +++ b/cmd/cli/log_file.go @@ -0,0 +1,440 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// errLogFileNotOpen tells a caller that the file is closed, so a write that +// returns no bytes never looks like a success. +var errLogFileNotOpen = errors.New("log file is not open") + +// retryOpenInterval is the wait between two attempts to open a log file that +// could not open. A file error is often short-lived, and one attempt for each +// line would cost a system call for every line. +var retryOpenInterval = time.Minute + +// logBudget is the disk space of one log stream: the size of the current +// file and the number of numbered backups that follow it. +type logBudget struct { + maxSize int64 + backups int +} + +// logRotation describes the file that a rotation moved away. A set err means +// that the file could not move, so the stream continues in the same file. +type logRotation struct { + file string + bytesWritten int64 + firstWrite time.Time + lastWrite time.Time + backups int + err error +} + +// rotatingFile writes one log file and rotates it by size with numbered +// backups. It holds the header bytes, so every file it opens starts with +// the header, and it records the write times that the rotation event needs. +type rotatingFile struct { + mu sync.Mutex + path string + budget logBudget + open func(string, int) (*os.File, error) + rename func(oldpath, newpath string) error + f *os.File + size int64 + firstWrite time.Time + lastWrite time.Time + header []byte + openErr error + onRotate func(logRotation) + // headerWritten tells whether the file that is open now already holds the + // header. A start wires its logging more than once, and each file takes + // one header. + headerWritten bool + // rotateFailedAt is the size at which a rotation could not move the file. + // The next try waits for another budget of data, so a directory that + // refuses the move does not cost one attempt per line. + rotateFailedAt int64 + // retryOpenAt is the time from which a closed file may open again. + retryOpenAt time.Time + // rotatedInFlight keeps onRotate from reporting a rotation that the + // rotation event itself caused. + rotatedInFlight atomic.Bool +} + +// newRotatingFile opens path in append mode and counts what it already holds, +// so a restart continues the current file instead of starting a new one. +func newRotatingFile(path string, budget logBudget, open func(string, int) (*os.File, error)) (*rotatingFile, error) { + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return nil, fmt.Errorf("creating log directory: %w", err) + } + rf := &rotatingFile{path: path, budget: budget, open: open} + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + return nil, fmt.Errorf("opening log file: %w", err) + } + st, err := f.Stat() + if err != nil { + f.Close() + return nil, fmt.Errorf("stat log file: %w", err) + } + rf.f = f + rf.size = st.Size() + return rf, nil +} + +// openFile opens the file through the platform helper, which holds the flags +// that each platform needs: a share mode on Windows, and a refusal to follow +// a symlink elsewhere. +func (rf *rotatingFile) openFile(flags int) (*os.File, error) { + if rf.open == nil { + return openLogFile(rf.path, flags) + } + return rf.open(rf.path, flags) +} + +func (rf *rotatingFile) renameFile(oldpath, newpath string) error { + if rf.rename == nil { + return os.Rename(oldpath, newpath) + } + return rf.rename(oldpath, newpath) +} + +// writeReporting appends p and reports a rotation that happened first. It +// never calls onRotate, because the caller must log the rotation outside of +// the lock. +func (rf *rotatingFile) writeReporting(p []byte) (int, *logRotation, error) { + rf.mu.Lock() + defer rf.mu.Unlock() + + var rotated *logRotation + if rf.needsRotation(len(p)) { + var err error + if rotated, err = rf.rotateLocked(); err != nil { + return 0, rotated, err + } + } + n, err := rf.writeLocked(p) + if err != nil { + return n, rotated, err + } + rf.noteEventWrite() + return n, rotated, nil +} + +// Write makes the file an io.Writer for the logger. It reports the rotation +// after the lock is free, so the rotation event can go back into this file. +func (rf *rotatingFile) Write(p []byte) (int, error) { + n, rotated, err := rf.writeReporting(p) + rf.reportRotation(rotated) + return n, err +} + +// reportRotation calls onRotate for a rotation. It must run with rf.mu free. +func (rf *rotatingFile) reportRotation(rotated *logRotation) { + if rotated == nil || !rf.rotatedInFlight.CompareAndSwap(false, true) { + return + } + defer rf.rotatedInFlight.Store(false) + rf.mu.Lock() + onRotate := rf.onRotate + rf.mu.Unlock() + if onRotate != nil { + onRotate(*rotated) + } +} + +// setHeader stores a copy of the header bytes for the next file. +func (rf *rotatingFile) setHeader(b []byte) { + header := make([]byte, len(b)) + copy(header, b) + rf.mu.Lock() + defer rf.mu.Unlock() + rf.header = header +} + +// writeHeader appends the stored header to the current file, once for each +// file that the writer opens. A restart uses it to mark the point where the +// new run continues an old file. A header that makes the file cross its +// budget goes to a new file, which the rotation starts with the header. +func (rf *rotatingFile) writeHeader() error { + rotated, err := rf.writeHeaderReporting() + rf.reportRotation(rotated) + return err +} + +// writeHeaderReporting writes the header and reports a rotation that the header +// caused. It never calls onRotate, because the caller must log the rotation +// outside of the lock, and at start the logger does not reach the files yet. +func (rf *rotatingFile) writeHeaderReporting() (*logRotation, error) { + rf.mu.Lock() + defer rf.mu.Unlock() + if rf.headerWritten || len(rf.header) == 0 { + return nil, nil + } + if !rf.needsRotation(len(rf.header)) { + return nil, rf.writeHeaderLocked() + } + rotated, err := rf.rotateLocked() + if err != nil || rf.headerWritten { + return rotated, err + } + // The file could not move, so the run continues in the old file, and the + // header must still mark the point where the run starts. + return rotated, rf.writeHeaderLocked() +} + +// setBudget takes the disk space of a later config. The file stays open, so a +// changed key costs no second header. +func (rf *rotatingFile) setBudget(budget logBudget) { + rf.mu.Lock() + defer rf.mu.Unlock() + rf.budget = budget +} + +// rotateNow rotates the file although the budget is not spent. +func (rf *rotatingFile) rotateNow() (*logRotation, error) { + rf.mu.Lock() + defer rf.mu.Unlock() + return rf.rotateLocked() +} + +// paths returns the files that exist, oldest first, for the log readers. +func (rf *rotatingFile) paths() []string { + rf.mu.Lock() + defer rf.mu.Unlock() + files := make([]string, 0, rf.budget.backups+1) + for i := rf.budget.backups; i >= 1; i-- { + backup := rf.backupPath(i) + if _, err := os.Stat(backup); err == nil { + files = append(files, backup) + } + } + if _, err := os.Stat(rf.path); err == nil { + files = append(files, rf.path) + } + return files +} + +func (rf *rotatingFile) currentPath() string { + return rf.path +} + +func (rf *rotatingFile) close() error { + rf.mu.Lock() + defer rf.mu.Unlock() + if rf.f == nil { + return nil + } + err := rf.f.Close() + rf.f = nil + return err +} + +func (rf *rotatingFile) backupPath(index int) string { + return fmt.Sprintf("%s.%d", rf.path, index) +} + +// rotatingSuffix marks the file that a rotation with no backup moves away +// before it opens the new file. The old bytes stay behind the readers that +// hold them open, and the file goes as soon as the new file is open. +const rotatingSuffix = ".rotating" + +func (rf *rotatingFile) rotatingPath() string { + return rf.path + rotatingSuffix +} + +// rotatedPath names the file that the next rotation leaves behind. +func (rf *rotatingFile) rotatedPath() string { + if rf.budget.backups == 0 { + return rf.path + } + return rf.backupPath(1) +} + +// needsRotation reports whether p makes the file cross its budget. A closed +// file never rotates, because a rotation would shift the backups away for a +// write that cannot land. +func (rf *rotatingFile) needsRotation(n int) bool { + if rf.f == nil || rf.budget.maxSize <= 0 { + return false + } + return rf.size+int64(n) > rf.rotateFailedAt+rf.budget.maxSize +} + +// rotateLocked moves the current file away and opens an empty one in its +// place. It must run with rf.mu held. +func (rf *rotatingFile) rotateLocked() (*logRotation, error) { + record := &logRotation{ + file: rf.rotatedPath(), + bytesWritten: rf.size, + firstWrite: rf.firstWrite, + lastWrite: rf.lastWrite, + backups: rf.budget.backups, + } + if rf.f != nil { + rf.f.Close() + rf.f = nil + } + if err := rf.shiftBackups(); err != nil { + record.err = err + return record, rf.keepFileAfterFailedShift() + } + + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_TRUNC) + if err != nil { + rf.size = 0 + return record, rf.noteOpenFailure(err) + } + if rf.budget.backups == 0 { + // The readers that hold the old file keep their bytes; the name goes. + _ = os.Remove(rf.rotatingPath()) + } + rf.f = f + rf.openErr = nil + rf.size = 0 + rf.rotateFailedAt = 0 + rf.firstWrite = time.Time{} + rf.lastWrite = time.Time{} + rf.headerWritten = false + return record, rf.writeHeaderLocked() +} + +// keepFileAfterFailedShift opens the file that could not move. The append flag +// keeps the lines that are already in it, which a truncating open would drop +// although no backup holds them. It must run with rf.mu held. +func (rf *rotatingFile) keepFileAfterFailedShift() error { + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + rf.size = 0 + return rf.noteOpenFailure(err) + } + rf.f = f + rf.openErr = nil + rf.rotateFailedAt = rf.size + return nil +} + +// noteOpenFailure keeps the error of a failed open and holds the next attempt +// back. It must run with rf.mu held. +func (rf *rotatingFile) noteOpenFailure(err error) error { + rf.openErr = fmt.Errorf("reopening log file: %w", err) + rf.retryOpenAt = time.Now().Add(retryOpenInterval) + return rf.openErr +} + +// shiftBackups moves each backup one number up and drops the oldest one. A +// missing backup is normal here, so those moves are best effort: a rotation +// must continue even when one backup cannot move. The move of the current +// file decides the rotation, so its error goes back to the caller. With no +// backup allowed, the current file still moves away, because a truncating +// reopen would change the bytes under an open upload reader. +func (rf *rotatingFile) shiftBackups() error { + if rf.budget.backups == 0 { + return rf.moveCurrentFile(rf.rotatingPath()) + } + for i := rf.budget.backups - 1; i >= 1; i-- { + _ = rf.renameFile(rf.backupPath(i), rf.backupPath(i+1)) + } + return rf.moveCurrentFile(rf.backupPath(1)) +} + +// moveCurrentFile renames the current file. A file that is not there is not +// an error, because a fresh path has nothing to move. +func (rf *rotatingFile) moveCurrentFile(to string) error { + if err := rf.renameFile(rf.path, to); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// pruneNumberedBackups removes the backups of path above keep. A lowered +// log_max_backups would otherwise leave old files that no reader lists. It +// reads the directory, because the numbers can have gaps. Only a name with an +// exact number after the path goes, because the directory can hold the files +// of another writer. +func pruneNumberedBackups(path string, keep int) { + dir := filepath.Dir(path) + entries, err := os.ReadDir(dir) + if err != nil { + return + } + prefix := filepath.Base(path) + "." + for _, entry := range entries { + suffix, found := strings.CutPrefix(entry.Name(), prefix) + if !found { + continue + } + index, err := strconv.Atoi(suffix) + if err != nil || index <= keep { + continue + } + os.Remove(filepath.Join(dir, entry.Name())) + } +} + +// writeHeaderLocked must run with rf.mu held. +func (rf *rotatingFile) writeHeaderLocked() error { + if len(rf.header) == 0 { + return nil + } + rf.headerWritten = true + _, err := rf.writeLocked(rf.header) + return err +} + +// writeLocked appends to the open file and counts the bytes. It must run +// with rf.mu held. +func (rf *rotatingFile) writeLocked(p []byte) (int, error) { + if rf.f == nil { + if rf.openErr == nil { + return 0, errLogFileNotOpen + } + if err := rf.retryOpenLocked(); err != nil { + return 0, err + } + } + n, err := rf.f.Write(p) + rf.size += int64(n) + return n, err +} + +// retryOpenLocked opens a file again that could not open. A file whose open +// failed once must not stay closed for the life of the process, because the +// cause is often a full disk or a directory that comes back. It must run with +// rf.mu held. +func (rf *rotatingFile) retryOpenLocked() error { + if time.Now().Before(rf.retryOpenAt) { + return rf.openErr + } + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + return rf.noteOpenFailure(err) + } + rf.f = f + rf.openErr = nil + rf.size = 0 + if st, err := f.Stat(); err == nil { + rf.size = st.Size() + } + rf.headerWritten = false + return rf.writeHeaderLocked() +} + +// noteEventWrite keeps the times that the rotation event reports. A header +// is not an event, so it does not move these times. +func (rf *rotatingFile) noteEventWrite() { + now := time.Now() + if rf.firstWrite.IsZero() { + rf.firstWrite = now + } + rf.lastWrite = now +} diff --git a/cmd/cli/log_file_test.go b/cmd/cli/log_file_test.go new file mode 100644 index 00000000..c5133a49 --- /dev/null +++ b/cmd/cli/log_file_test.go @@ -0,0 +1,622 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "sync/atomic" + "testing" + "time" +) + +const rotatingFileTestLineSize = 1024 + +// rotatingFileTestLine returns a line of a fixed size that names its +// generation, so a test can tell the content of one file from another. +func rotatingFileTestLine(generation, seq int) []byte { + head := fmt.Sprintf("gen=%d seq=%d ", generation, seq) + return []byte(head + strings.Repeat("x", rotatingFileTestLineSize-len(head)-1) + "\n") +} + +func newTestRotatingFile(t *testing.T, budget logBudget) *rotatingFile { + t.Helper() + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + return rf +} + +func rotatingFileTestContent(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func writeRotatingFileTestLine(t *testing.T, rf *rotatingFile, generation, seq int) *logRotation { + t.Helper() + line := rotatingFileTestLine(generation, seq) + n, rotated, err := rf.writeReporting(line) + if err != nil { + t.Fatalf("write gen=%d seq=%d: %v", generation, seq, err) + } + if n != len(line) { + t.Fatalf("write gen=%d seq=%d wrote %d bytes, want %d", generation, seq, n, len(line)) + } + return rotated +} + +func Test_rotatingFileKeepsNumberedBackups(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 4}) + rotations := 0 + for generation := 1; generation <= 7; generation++ { + for seq := 1; seq <= 4; seq++ { + if writeRotatingFileTestLine(t, rf, generation, seq) != nil { + rotations++ + } + } + } + if rotations != 6 { + t.Fatalf("rotations = %d, want 6", rotations) + } + + generationOf := map[string]int{"": 7, ".1": 6, ".2": 5, ".3": 4, ".4": 3} + for suffix, generation := range generationOf { + content := rotatingFileTestContent(t, rf.path+suffix) + if want := fmt.Sprintf("gen=%d seq=1 ", generation); !strings.HasPrefix(content, want) { + t.Fatalf("file %q starts with %.16q, want %q", rf.path+suffix, content, want) + } + if newer := fmt.Sprintf("gen=%d ", generation+1); strings.Contains(content, newer) { + t.Fatalf("file %q holds %q", rf.path+suffix, newer) + } + } + if _, err := os.Stat(rf.path + ".5"); !os.IsNotExist(err) { + t.Fatalf("stat %s.5 = %v, want a not exist error", rf.path, err) + } +} + +func Test_rotatingFileHeaderStartsEveryFile(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 1}) + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + if rf.size != int64(len(header)) { + t.Fatalf("size = %d, want %d", rf.size, len(header)) + } + if got := strings.Count(rotatingFileTestContent(t, rf.path), string(header)); got != 1 { + t.Fatalf("header count in the first file = %d, want 1", got) + } + + rotations := 0 + for seq := 1; seq <= 5; seq++ { + if writeRotatingFileTestLine(t, rf, 1, seq) != nil { + rotations++ + } + } + if rotations != 1 { + t.Fatalf("rotations = %d, want 1", rotations) + } + if !strings.HasPrefix(rotatingFileTestContent(t, rf.path+".1"), string(header)) { + t.Fatal("the backup file does not start with the header") + } + content := rotatingFileTestContent(t, rf.path) + if !strings.HasPrefix(content, string(header)) { + t.Fatal("the new file does not start with the header") + } + if got := strings.Count(content, string(header)); got != 1 { + t.Fatalf("header count in the new file = %d, want 1", got) + } + if want := int64(len(header) + 2*rotatingFileTestLineSize); rf.size != want { + t.Fatalf("size = %d, want %d", rf.size, want) + } +} + +func Test_rotatingFileRotationRecord(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 2}) + if rf.currentPath() != rf.path { + t.Fatalf("currentPath = %q, want %q", rf.currentPath(), rf.path) + } + start := time.Now() + for seq := 1; seq <= 3; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + sizeBeforeRotation := rf.size + + record, err := rf.rotateNow() + if err != nil { + t.Fatalf("rotateNow: %v", err) + } + if record == nil { + t.Fatal("rotateNow returned no record") + } + if want := rf.path + ".1"; record.file != want { + t.Fatalf("record.file = %q, want %q", record.file, want) + } + if record.backups != 2 { + t.Fatalf("record.backups = %d, want 2", record.backups) + } + if record.bytesWritten != sizeBeforeRotation { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, sizeBeforeRotation) + } + st, err := os.Stat(record.file) + if err != nil { + t.Fatalf("stat %s: %v", record.file, err) + } + if st.Size() != record.bytesWritten { + t.Fatalf("%s holds %d bytes, record.bytesWritten = %d", record.file, st.Size(), record.bytesWritten) + } + if record.firstWrite.Before(start) { + t.Fatalf("record.firstWrite = %v, want at or after %v", record.firstWrite, start) + } + if record.lastWrite.Before(record.firstWrite) { + t.Fatalf("record.lastWrite = %v is before record.firstWrite = %v", record.lastWrite, record.firstWrite) + } + + want := []string{rf.path + ".1", rf.path} + if got := rf.paths(); !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } + writeRotatingFileTestLine(t, rf, 2, 1) + if _, err := rf.rotateNow(); err != nil { + t.Fatalf("second rotateNow: %v", err) + } + want = []string{rf.path + ".2", rf.path + ".1", rf.path} + if got := rf.paths(); !slices.Equal(got, want) { + t.Fatalf("paths after two rotations = %v, want %v", got, want) + } +} + +func Test_rotatingFileZeroBackupsTruncatesInPlace(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 2048, backups: 0}) + var record *logRotation + for seq := 1; seq <= 3; seq++ { + if rotated := writeRotatingFileTestLine(t, rf, 1, seq); rotated != nil { + record = rotated + } + } + if record == nil { + t.Fatal("no rotation happened") + } + if record.file != rf.path { + t.Fatalf("record.file = %q, want %q", record.file, rf.path) + } + if record.backups != 0 { + t.Fatalf("record.backups = %d, want 0", record.backups) + } + if want := int64(2 * rotatingFileTestLineSize); record.bytesWritten != want { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, want) + } + if _, err := os.Stat(rf.path + ".1"); !os.IsNotExist(err) { + t.Fatalf("stat %s.1 = %v, want a not exist error", rf.path, err) + } + if content := rotatingFileTestContent(t, rf.path); strings.Contains(content, "seq=1 ") { + t.Fatal("the truncated file still holds the old content") + } + if got, want := rf.paths(), []string{rf.path}; !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func Test_rotatingFileWriteCallsOnRotate(t *testing.T) { + t.Run("one call per rotation", func(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 4}) + var rotations atomic.Int64 + rf.onRotate = func(logRotation) { rotations.Add(1) } + for generation := 1; generation <= 7; generation++ { + for seq := 1; seq <= 4; seq++ { + if _, err := rf.Write(rotatingFileTestLine(generation, seq)); err != nil { + t.Fatalf("Write: %v", err) + } + } + } + if got := rotations.Load(); got != 6 { + t.Fatalf("onRotate calls = %d, want 6", got) + } + }) + + t.Run("reentrant callback does not deadlock", func(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 1}) + const reentrantLine = "written from onRotate\n" + var rotations atomic.Int64 + rf.onRotate = func(logRotation) { + rotations.Add(1) + if _, err := rf.Write([]byte(reentrantLine)); err != nil { + t.Errorf("reentrant Write: %v", err) + } + } + done := make(chan struct{}) + go func() { + defer close(done) + for seq := 1; seq <= 5; seq++ { + if _, err := rf.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Errorf("Write: %v", err) + return + } + } + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Write did not return, a reentrant onRotate deadlocks it") + } + if got := rotations.Load(); got != 1 { + t.Fatalf("onRotate calls = %d, want 1", got) + } + if !strings.Contains(rotatingFileTestContent(t, rf.path), reentrantLine) { + t.Fatal("the new file does not hold the reentrant line") + } + }) +} + +func Test_rotatingFileCountsExistingSize(t *testing.T) { + path := filepath.Join(t.TempDir(), "logs", "ctrld.log") + budget := logBudget{maxSize: 8192, backups: 1} + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + for seq := 1; seq <= 3; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + if err := rf.close(); err != nil { + t.Fatalf("close: %v", err) + } + + reopened, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile on an existing file: %v", err) + } + t.Cleanup(func() { _ = reopened.close() }) + if want := int64(3 * rotatingFileTestLineSize); reopened.size != want { + t.Fatalf("size = %d, want %d", reopened.size, want) + } + record, err := reopened.rotateNow() + if err != nil { + t.Fatalf("rotateNow: %v", err) + } + if want := int64(3 * rotatingFileTestLineSize); record.bytesWritten != want { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, want) + } + st, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + if st.Size() != 0 { + t.Fatalf("the new file holds %d bytes, want 0", st.Size()) + } +} + +func Test_rotatingFileKeepsReopenError(t *testing.T) { + reopenErr := errors.New("reopen refused") + var opens atomic.Int64 + open := func(path string, flags int) (*os.File, error) { + if opens.Add(1) > 1 { + return nil, reopenErr + } + return os.OpenFile(path, flags, 0600) + } + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 1024, backups: 1}, open) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + + writeRotatingFileTestLine(t, rf, 1, 1) + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 2)); !errors.Is(err, reopenErr) { + t.Fatalf("write after a failed reopen = %v, want %v", err, reopenErr) + } + if rf.f != nil { + t.Fatal("the file is still open after a failed reopen") + } + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 3)); !errors.Is(err, reopenErr) { + t.Fatalf("later write = %v, want %v", err, reopenErr) + } + + oversized := make([]byte, 2*rotatingFileTestLineSize) + _, rotated, err := rf.writeReporting(oversized) + if !errors.Is(err, reopenErr) { + t.Fatalf("oversized write = %v, want %v", err, reopenErr) + } + if rotated != nil { + t.Fatalf("a closed file reported a rotation of %q", rotated.file) + } + if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) { + t.Fatalf("stat %s.2 = %v, want a not exist error: a closed file shifted the backups", path, err) + } +} + +func Test_rotatingFileRetriesAFailedOpen(t *testing.T) { + origInterval := retryOpenInterval + retryOpenInterval = 10 * time.Millisecond + t.Cleanup(func() { retryOpenInterval = origInterval }) + + openErr := errors.New("open refused") + opens := 0 + open := func(path string, flags int) (*os.File, error) { + opens++ + if opens == 2 { + return nil, openErr + } + return os.OpenFile(path, flags, 0600) + } + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: rotatingFileTestLineSize, backups: 1}, open) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + + writeRotatingFileTestLine(t, rf, 1, 1) + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 2)); !errors.Is(err, openErr) { + t.Fatalf("write after a failed reopen = %v, want %v", err, openErr) + } + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 3)); !errors.Is(err, openErr) { + t.Fatalf("write inside the retry interval = %v, want %v", err, openErr) + } + if opens != 2 { + t.Fatalf("opens = %d, want 2: a write inside the retry interval opened the file again", opens) + } + + time.Sleep(2 * retryOpenInterval) + writeRotatingFileTestLine(t, rf, 1, 4) + + content := rotatingFileTestContent(t, path) + if !strings.HasPrefix(content, string(header)) { + t.Fatalf("the reopened file starts with %.40q, want the header", content) + } + if !strings.Contains(content, "gen=1 seq=4 ") { + t.Fatalf("the reopened file misses the line that followed the retry: %.80q", content) + } +} + +func Test_pruneNumberedBackups(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, logFileName) + numbered := []string{path + ".1", path + ".2", path + ".3", path + ".4"} + // A prune must keep every file that another writer owns, because log_path + // can name a file in a directory full of them. + foreign := []string{path + ".bak", filepath.Join(dir, journalLogFileName+".3"), filepath.Join(dir, "syslog.5")} + for _, name := range append(slices.Clone(numbered), foreign...) { + if err := os.WriteFile(name, []byte("old\n"), 0600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + pruneNumberedBackups(path, 1) + + if _, err := os.Stat(numbered[0]); err != nil { + t.Fatalf("the backup inside the count went away: %v", err) + } + for _, name := range numbered[1:] { + if _, err := os.Stat(name); !os.IsNotExist(err) { + t.Fatalf("%s still exists after a prune that keeps one backup (err %v)", name, err) + } + } + for _, name := range foreign { + if _, err := os.Stat(name); err != nil { + t.Fatalf("the prune removed %s, which it does not own: %v", name, err) + } + } +} + +func Test_rotatingFileKeepsFilesAboveTheCount(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}) + late := rf.backupPath(3) + if err := os.WriteFile(late, []byte("late\n"), 0600); err != nil { + t.Fatalf("write the late backup: %v", err) + } + + for seq := 1; seq <= 3; seq++ { + if _, err := rf.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write %d: %v", seq, err) + } + } + + if _, err := os.Stat(late); err != nil { + t.Fatalf("a rotation removed %s, which sits above the backup count: %v", late, err) + } + if _, err := os.Stat(rf.backupPath(1)); err != nil { + t.Fatalf("the rotation left no .1 file: %v", err) + } +} + +func Test_rotatingFileDoesNotFollowASymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the Windows opener takes no symlink flag") + } + dir := t.TempDir() + planted := filepath.Join(dir, "planted.log") + if err := os.WriteFile(planted, []byte("planted\n"), 0600); err != nil { + t.Fatalf("write the planted file: %v", err) + } + path := filepath.Join(dir, "ctrld.log") + if err := os.Symlink(planted, path); err != nil { + t.Fatalf("symlink: %v", err) + } + + rf, err := newRotatingFile(path, logBudget{maxSize: 4096, backups: 1}, nil) + + if err == nil { + _ = rf.close() + t.Fatal("newRotatingFile followed the symlink") + } + if got := rotatingFileTestContent(t, planted); got != "planted\n" { + t.Fatalf("the planted file = %q, want it untouched", got) + } +} + +func Test_rotatingFileKeepsContentWhenRenameFails(t *testing.T) { + renameErr := errors.New("rename refused") + renameFails := true + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + rf.rename = func(oldpath, newpath string) error { + if renameFails { + return renameErr + } + return os.Rename(oldpath, newpath) + } + + for seq := 1; seq <= 2; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + record := writeRotatingFileTestLine(t, rf, 1, 3) + if record == nil { + t.Fatal("the write past the budget reported no rotation") + } + if !errors.Is(record.err, renameErr) { + t.Fatalf("record.err = %v, want %v", record.err, renameErr) + } + content := rotatingFileTestContent(t, path) + for seq := 1; seq <= 3; seq++ { + if want := fmt.Sprintf("gen=1 seq=%d ", seq); !strings.Contains(content, want) { + t.Fatalf("the file lost %q after a failed rename", want) + } + } + if _, err := os.Stat(path + ".1"); !os.IsNotExist(err) { + t.Fatalf("stat %s.1 = %v, want a not exist error", path, err) + } + if writeRotatingFileTestLine(t, rf, 1, 4) != nil { + t.Fatal("the next write rotated again instead of waiting for another budget") + } + + renameFails = false + var second *logRotation + for seq := 5; seq <= 6; seq++ { + if rotated := writeRotatingFileTestLine(t, rf, 1, seq); rotated != nil { + second = rotated + } + } + if second == nil { + t.Fatal("no rotation happened after the rename worked again") + } + if second.err != nil { + t.Fatalf("record.err = %v, want no error", second.err) + } + if _, err := os.Stat(path + ".1"); err != nil { + t.Fatalf("stat %s.1 after a working rename: %v", path, err) + } +} + +// Test_zeroBackupRotationKeepsTheOpenReader opens an upload reader over the +// current file, then rotates with no backup. The reader must return the bytes +// and the size that it advertised, and the current file holds the new +// generation only. +func Test_zeroBackupRotationKeepsTheOpenReader(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 1 << 20, backups: 0}) + if _, err := rf.Write([]byte("old generation!\n")); err != nil { + t.Fatal(err) + } + var upload logParts + upload.addBoundedFiles(rf.paths(), 0) + t.Cleanup(upload.close) + + if _, err := rf.rotateNow(); err != nil { + t.Fatalf("rotate: %v", err) + } + if _, err := rf.Write([]byte("new\n")); err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(io.MultiReader(upload.readers...)) + if err != nil { + t.Fatal(err) + } + if upload.size != 16 || string(got) != "old generation!\n" { + t.Fatalf("reader gave %q (%d bytes advertised), want the old generation of 16 bytes", got, upload.size) + } + if content := rotatingFileTestContent(t, rf.currentPath()); content != "new\n" { + t.Fatalf("current file = %q, want the new generation only", content) + } + if _, err := os.Stat(rf.currentPath() + rotatingSuffix); !os.IsNotExist(err) { + t.Fatalf("the moved file is still on disk: %v", err) + } +} + +// Test_rotatingFileHeaderStaysInTheBudget drives a restart that finds a file +// close to its limit. The header of the new run must go to a new file, so no +// file grows over its budget. +func Test_rotatingFileHeaderStaysInTheBudget(t *testing.T) { + budget := logBudget{maxSize: 4096, backups: 1} + path := filepath.Join(t.TempDir(), "ctrld-journal.log") + oldRun := strings.Repeat("o", int(budget.maxSize)-10) + "\n" + if err := os.WriteFile(path, []byte(oldRun), 0o600); err != nil { + t.Fatal(err) + } + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + var rotations []logRotation + rf.onRotate = func(record logRotation) { rotations = append(rotations, record) } + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + + if backup := rotatingFileTestContent(t, path+".1"); backup != oldRun { + t.Fatalf("the backup holds %d bytes, want the %d bytes of the old run only", len(backup), len(oldRun)) + } + if current := rotatingFileTestContent(t, path); current != string(header) { + t.Fatalf("the current file holds %q, want the header only", current) + } + if len(rotations) != 1 || rotations[0].bytesWritten != int64(len(oldRun)) { + t.Fatalf("rotations = %+v, want one event for the old file", rotations) + } +} + +// Test_rotatingFileHeaderAfterAFailedRotation covers a restart whose rotation +// cannot move the old file. The run continues in that file, and the header +// must still mark the point where the run starts. +func Test_rotatingFileHeaderAfterAFailedRotation(t *testing.T) { + renameErr := errors.New("rename refused") + budget := logBudget{maxSize: 4096, backups: 1} + path := filepath.Join(t.TempDir(), "ctrld-journal.log") + oldRun := strings.Repeat("o", int(budget.maxSize)-10) + "\n" + if err := os.WriteFile(path, []byte(oldRun), 0o600); err != nil { + t.Fatal(err) + } + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + rf.rename = func(string, string) error { return renameErr } + var rotations []logRotation + rf.onRotate = func(record logRotation) { rotations = append(rotations, record) } + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + + if current := rotatingFileTestContent(t, path); current != oldRun+string(header) { + t.Fatalf("the current file holds %d bytes, want the old run and one header", len(current)) + } + if len(rotations) != 1 || !errors.Is(rotations[0].err, renameErr) { + t.Fatalf("rotations = %+v, want one failed rotation", rotations) + } +} diff --git a/cmd/cli/log_header.go b/cmd/cli/log_header.go new file mode 100644 index 00000000..3de514f5 --- /dev/null +++ b/cmd/cli/log_header.go @@ -0,0 +1,225 @@ +package cli + +import ( + "bytes" + "net" + "os" + "runtime" + "slices" + "strconv" + "sync" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" +) + +// processStartTime is the time this process started. A reader compares it with +// the time of the header line to tell a restart from a rotation. +var processStartTime = time.Now() + +const ( + // logHeaderMessage is the message of the header line. Log tools select the + // header by this message. + logHeaderMessage = "Log header" + + // interceptModeOff is the mode word for a ctrld that does not intercept. + interceptModeOff = "off" + + // logHeaderTriggerSend marks the header that a log request renders. It + // tells a reader that the line came from an upload, not from a file open. + logHeaderTriggerSend = "send" + + // upstreamTypeUnspecified names an upstream that has no type in the config. + // ctrld resolves the type at start, so the header must not claim one. + upstreamTypeUnspecified = "unspecified" +) + +// headerOSVersion reads the OS version once. The read starts a subprocess on +// macOS, and a header refresh follows every network change. +var headerOSVersion = sync.OnceValue(osVersion) + +// logHeaderInput holds every value of a header line. The caller collects the +// values, so the renderer stays free of locks and of system calls. +type logHeaderInput struct { + Version string + Commit string + OS string + Arch string + PID int + StartTime time.Time + InterceptMode string + Listeners []string + UpstreamCount int + UpstreamTypes []string + ResolverUID string + LogFiles []string + Network networkSnapshot + Trigger string +} + +// renderLogHeader renders the header as one log line. A file writer stores the +// bytes and writes them itself, so the header reaches a file that the logger +// never fills. +func renderLogHeader(in logHeaderInput) []byte { + var buf bytes.Buffer + // The console sits above info on a default start, and a log file must hold + // its header at every level, so the header takes a core of its own. The + // line is JSON like the journal, because log tools read its fields. + logger := &ctrld.Logger{Logger: zap.New(zapcore.NewCore(newJournalEncoder(), zapcore.AddSync(&buf), zapcore.DebugLevel))} + event := journal(logger.Info()). + Str("version", in.Version). + Str("commit", in.Commit). + Str("os", in.OS). + Str("arch", in.Arch). + Int("pid", in.PID). + Str("start_time", in.StartTime.Format(time.RFC3339)). + Str("intercept_mode", in.InterceptMode). + Strs("listeners", in.Listeners). + Int("upstream_count", in.UpstreamCount). + Strs("upstream_types", in.UpstreamTypes). + Str("resolver_uid", in.ResolverUID). + Strs("log_files", in.LogFiles). + Dict("network", snapshotDict(in.Network)) + if in.Trigger != "" { + event = event.Str("trigger", in.Trigger) + } + event.Msg(logHeaderMessage) + + // The header is the first line of every file and of every upload, and it + // bypasses the writers, so it passes the journal redaction here. + return redactRetainedLine(buf.Bytes()) +} + +// logHeaderInput collects every header value of the running program. +func (p *prog) logHeaderInput() logHeaderInput { + return p.logHeaderInputWith(p.networkSnapshot()) +} + +// logHeaderInputWith collects every header value around one network snapshot. +func (p *prog) logHeaderInputWith(network networkSnapshot) logHeaderInput { + in := logHeaderInput{ + Version: curVersion(), + Commit: commit, + OS: headerOSVersion(), + Arch: runtime.GOARCH, + PID: os.Getpid(), + StartTime: processStartTime, + // The resolver UID is a secret that the provisioning code strips from + // every artifact, so the header keeps its first characters only. + ResolverUID: redactToken(cdUID), + LogFiles: p.logHeaderFiles(), + Network: network, + } + p.headerConfigInto(&in) + return in +} + +// headerConfigInto reads the config under p.mu, because a reload replaces the +// config under that lock while log view and log send render a header. +func (p *prog) headerConfigInto(in *logHeaderInput) { + p.mu.Lock() + defer p.mu.Unlock() + logHeaderFromConfig(p.cfg, in) +} + +// sendLogHeader renders the header that leads one upload. The stored state can +// be older than the request, and support reads this line to learn the network +// of the moment the user asked. +func (p *prog) sendLogHeader() []byte { + in := p.logHeaderInputWith(buildNetworkSnapshot(p.freshSnapshotInputs())) + in.Trigger = logHeaderTriggerSend + return renderLogHeader(in) +} + +// logHeaderFiles names the files that this run writes, debug first. +func (p *prog) logHeaderFiles() []string { + files := make([]string, 0, 3) + for _, rf := range p.openLogFiles() { + files = append(files, rf.currentPath()) + } + return files +} + +// refreshLogHeader renders the header and hands the bytes to every open file. +// The caller must hold no writer lock, because the render reads the config +// and the network. +func (p *prog) refreshLogHeader() { + header := renderLogHeader(p.logHeaderInput()) + for _, rf := range p.openLogFiles() { + rf.setHeader(header) + } +} + +// writeLogHeaders puts the header at the append point of each open file. A +// header that cannot land leaves the file usable, so the error only earns a +// warning. It returns the rotations that the headers caused, because the +// logger does not reach the files yet and cannot report them. +func (p *prog) writeLogHeaders() []logRotation { + var rotations []logRotation + for _, rf := range p.openLogFiles() { + rotated, err := rf.writeHeaderReporting() + if rotated != nil { + rotations = append(rotations, *rotated) + } + if err != nil { + mainLog.Load().Warn().Err(err).Msg("Could not write log header") + } + } + return rotations +} + +// logHeaderFromConfig fills the header fields that the config holds. It keeps +// every upstream endpoint, name, and bootstrap IP out of them, because a token +// can hide in each of them. +func logHeaderFromConfig(cfg *ctrld.Config, in *logHeaderInput) { + if cfg == nil { + in.InterceptMode = interceptModeOff + return + } + in.InterceptMode = headerInterceptMode(cfg) + in.Listeners = listenerAddresses(cfg) + in.UpstreamCount = len(cfg.Upstream) + in.UpstreamTypes = upstreamTypeWords(cfg) +} + +// headerInterceptMode names the mode for the header. An unset mode means that +// ctrld does not intercept, which reads as off. +func headerInterceptMode(cfg *ctrld.Config) string { + if mode := listenerInterceptMode(cfg); mode != "" { + return mode + } + return interceptModeOff +} + +func listenerAddresses(cfg *ctrld.Config) []string { + addresses := make([]string, 0, len(cfg.Listener)) + for _, listener := range cfg.Listener { + if listener == nil { + continue + } + addresses = append(addresses, net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port))) + } + slices.Sort(addresses) + return addresses +} + +func upstreamTypeWords(cfg *ctrld.Config) []string { + words := make([]string, 0, len(cfg.Upstream)) + for _, upstream := range cfg.Upstream { + if upstream == nil { + continue + } + word := upstream.Type + if word == "" { + word = upstreamTypeUnspecified + } + if !slices.Contains(words, word) { + words = append(words, word) + } + } + slices.Sort(words) + return words +} diff --git a/cmd/cli/log_header_test.go b/cmd/cli/log_header_test.go new file mode 100644 index 00000000..e64c3d46 --- /dev/null +++ b/cmd/cli/log_header_test.go @@ -0,0 +1,648 @@ +package cli + +import ( + "encoding/json" + "net" + "net/netip" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// logHeaderTestSecret is the string that must never reach a header line. +const logHeaderTestSecret = "secret-token" + +// logHeaderTestLine is the parsed form of a header line. The json tags name +// the fields that a log tool reads, so a renamed field fails the test. +type logHeaderTestLine struct { + Level string `json:"level"` + Time string `json:"time"` + Journal bool `json:"journal"` + Message string `json:"message"` + Version string `json:"version"` + Commit string `json:"commit"` + OS string `json:"os"` + Arch string `json:"arch"` + PID int `json:"pid"` + StartTime string `json:"start_time"` + InterceptMode string `json:"intercept_mode"` + Listeners []string `json:"listeners"` + UpstreamCount int `json:"upstream_count"` + UpstreamTypes []string `json:"upstream_types"` + ResolverUID string `json:"resolver_uid"` + LogFiles []string `json:"log_files"` + Network struct { + DefaultRouteV4 string `json:"default_route_v4"` + DefaultRouteV6 string `json:"default_route_v6"` + GatewayV4 string `json:"gateway_v4"` + GatewayV6 string `json:"gateway_v6"` + HaveV4 bool `json:"have_v4"` + HaveV6 bool `json:"have_v6"` + Interfaces []struct { + Name string `json:"name"` + Class string `json:"class"` + Up bool `json:"up"` + IPs []string `json:"ips"` + HardwarePort string `json:"hardware_port"` + Service string `json:"service"` + } `json:"interfaces"` + Resolvers []string `json:"resolvers"` + SourceIPv4 string `json:"source_ipv4"` + SourceIPv6 string `json:"source_ipv6"` + InterceptTarget string `json:"intercept_target"` + LinkType string `json:"link_type"` + Tethered bool `json:"tethered"` + } `json:"network"` + Trigger string `json:"trigger"` +} + +func logHeaderTestInput() logHeaderInput { + return logHeaderInput{ + Version: "1.5.8", + Commit: "abc1234", + OS: "darwin 15.6", + Arch: "arm64", + PID: 4242, + StartTime: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + InterceptMode: "dns", + Listeners: []string{"127.0.0.1:53", "127.0.0.1:5354"}, + UpstreamCount: 2, + UpstreamTypes: []string{"doh", "os"}, + ResolverUID: redactToken("abcd1234"), + LogFiles: []string{"/tmp/ctrld.log", "/tmp/ctrld-journal.log"}, + Network: networkSnapshot{ + DefaultRouteV4: "en0", + GatewayV4: "192.0.2.1", + HaveV4: true, + Interfaces: []snapshotInterface{ + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + }, + Resolvers: []string{"192.0.2.1:53"}, + LinkType: "wifi", + }, + } +} + +// parseLogHeader checks that the render is one line and returns it twice: as +// values, and as raw fields for the tests that ask whether a field is there. +func parseLogHeader(t *testing.T, line []byte) (logHeaderTestLine, map[string]json.RawMessage) { + t.Helper() + if len(line) == 0 { + t.Fatal("renderLogHeader returned no bytes") + } + if line[len(line)-1] != '\n' { + t.Fatalf("header does not end with a newline: %q", line) + } + if count := strings.Count(string(line), "\n"); count != 1 { + t.Fatalf("header holds %d newlines, want 1: %q", count, line) + } + var parsed logHeaderTestLine + if err := json.Unmarshal(line, &parsed); err != nil { + t.Fatalf("parse header %q: %v", line, err) + } + fields := map[string]json.RawMessage{} + if err := json.Unmarshal(line, &fields); err != nil { + t.Fatalf("parse header fields %q: %v", line, err) + } + return parsed, fields +} + +func Test_renderLogHeaderHoldsEveryField(t *testing.T) { + in := logHeaderTestInput() + + parsed, fields := parseLogHeader(t, renderLogHeader(in)) + + checks := []struct { + field string + got any + want any + }{ + {"level", parsed.Level, "info"}, + {"journal", parsed.Journal, true}, + {"message", parsed.Message, "Log header"}, + {"version", parsed.Version, in.Version}, + {"commit", parsed.Commit, in.Commit}, + {"os", parsed.OS, in.OS}, + {"arch", parsed.Arch, in.Arch}, + {"pid", parsed.PID, in.PID}, + {"start_time", parsed.StartTime, in.StartTime.Format(time.RFC3339)}, + {"intercept_mode", parsed.InterceptMode, in.InterceptMode}, + {"listeners", parsed.Listeners, in.Listeners}, + {"upstream_count", parsed.UpstreamCount, in.UpstreamCount}, + {"upstream_types", parsed.UpstreamTypes, in.UpstreamTypes}, + // The resolver UID is a secret, so the header keeps the first four + // characters only. + {"resolver_uid", parsed.ResolverUID, "abcd***"}, + {"log_files", parsed.LogFiles, in.LogFiles}, + {"network.default_route_v4", parsed.Network.DefaultRouteV4, in.Network.DefaultRouteV4}, + {"network.default_route_v6", parsed.Network.DefaultRouteV6, in.Network.DefaultRouteV6}, + {"network.gateway_v4", parsed.Network.GatewayV4, in.Network.GatewayV4}, + {"network.have_v4", parsed.Network.HaveV4, in.Network.HaveV4}, + {"network.have_v6", parsed.Network.HaveV6, in.Network.HaveV6}, + {"network.resolvers", parsed.Network.Resolvers, in.Network.Resolvers}, + {"network.link_type", parsed.Network.LinkType, in.Network.LinkType}, + {"network.tethered", parsed.Network.Tethered, in.Network.Tethered}, + } + for _, c := range checks { + if !reflect.DeepEqual(c.got, c.want) { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } + + if len(parsed.Network.Interfaces) != 1 { + t.Fatalf("network.interfaces holds %d entries, want 1", len(parsed.Network.Interfaces)) + } + iface := parsed.Network.Interfaces[0] + if iface.Name != "en0" || iface.Class != "hardware" || !iface.Up || iface.HardwarePort != "Wi-Fi" { + t.Errorf("network.interfaces[0] = %+v, want the hardware port en0 up on Wi-Fi", iface) + } + if !reflect.DeepEqual(iface.IPs, []string{"192.0.2.10/24"}) { + t.Errorf("network.interfaces[0].ips = %v, want 192.0.2.10/24", iface.IPs) + } + if parsed.Time == "" { + t.Error("time is empty") + } + if _, ok := fields["trigger"]; ok { + t.Errorf("trigger is present for an empty Trigger: %s", fields["trigger"]) + } +} + +func Test_renderLogHeaderAddsTriggerWhenSet(t *testing.T) { + in := logHeaderTestInput() + in.Trigger = "send" + + parsed, _ := parseLogHeader(t, renderLogHeader(in)) + + if parsed.Trigger != "send" { + t.Errorf("trigger = %q, want %q", parsed.Trigger, "send") + } +} + +// Test_renderLogHeaderIgnoresConsoleLevel guards the promise that every log +// file starts with a header. The console core sits above info on a default +// start, and the header takes a core of its own. +func Test_renderLogHeaderIgnoresConsoleLevel(t *testing.T) { + parsed, _ := parseLogHeader(t, renderLogHeader(logHeaderTestInput())) + + if parsed.Level != "info" { + t.Errorf("level = %q, want %q", parsed.Level, "info") + } + if parsed.Message != "Log header" { + t.Errorf("message = %q, want %q", parsed.Message, "Log header") + } +} + +func Test_logHeaderFromConfigKeepsUpstreamSecretsOut(t *testing.T) { + old := interceptMode + t.Cleanup(func() { interceptMode = old }) + interceptMode = "" + + cfg := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: "127.0.0.1", Port: 53}, + logHeaderTestSecret + "-l": {IP: "10.0.0.1", Port: 5354}, + }, + Upstream: map[string]*ctrld.UpstreamConfig{ + "0": { + Name: logHeaderTestSecret + "-name", + Type: "doh", + Endpoint: "https://dns.example/" + logHeaderTestSecret, + BootstrapIP: "192.0.2.53", + }, + "1": {Endpoint: "https://dns.example/" + logHeaderTestSecret}, + }, + } + cfg.Service.InterceptMode = "dns" + + in := logHeaderTestInput() + logHeaderFromConfig(cfg, &in) + + if in.InterceptMode != "dns" { + t.Errorf("intercept mode = %q, want %q", in.InterceptMode, "dns") + } + if want := []string{"10.0.0.1:5354", "127.0.0.1:53"}; !reflect.DeepEqual(in.Listeners, want) { + t.Errorf("listeners = %v, want %v", in.Listeners, want) + } + if in.UpstreamCount != 2 { + t.Errorf("upstream count = %d, want 2", in.UpstreamCount) + } + if want := []string{"doh", "unspecified"}; !reflect.DeepEqual(in.UpstreamTypes, want) { + t.Errorf("upstream types = %v, want %v", in.UpstreamTypes, want) + } + + line := renderLogHeader(in) + if strings.Contains(string(line), logHeaderTestSecret) { + t.Errorf("header holds %q: %s", logHeaderTestSecret, line) + } +} + +// An unset intercept mode reads as the platform default: "dns" where ctrld can +// intercept (issue #589), and off (interface DNS mode) elsewhere. +func Test_logHeaderFromConfigReportsPlatformDefaultWhenUnset(t *testing.T) { + old := interceptMode + t.Cleanup(func() { interceptMode = old }) + interceptMode = "" + + for _, tc := range []struct { + supported bool + want string + }{ + {supported: false, want: "off"}, + {supported: true, want: "dns"}, + } { + useInterceptByDefault(t, tc.supported) + var in logHeaderInput + logHeaderFromConfig(&ctrld.Config{}, &in) + if in.InterceptMode != tc.want { + t.Errorf("supported=%v: intercept mode = %q, want %q", tc.supported, in.InterceptMode, tc.want) + } + } +} + +// Test_renderLogHeaderShowsTheSnapshotOfAState renders the header of one +// network state, so a reader of the file sees every interface of the host in +// one order. +func Test_renderLogHeaderShowsTheSnapshotOfAState(t *testing.T) { + stubHeaderSnapshotSources(t) + state := &netmon.State{ + DefaultRouteInterface: "en0", + HaveV4: true, + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "awdl0": {Interface: &net.Interface{Name: "awdl0"}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en0": {netip.MustParsePrefix("192.0.2.10/24"), netip.MustParsePrefix("2001:db8::1/64")}, + "awdl0": {netip.MustParsePrefix("fe80::2/64")}, + }, + } + + in := logHeaderTestInput() + in.Network = buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%en0", Interface: "en0"}, + Meta: interfaceMetaFor, + }) + parsed, _ := parseLogHeader(t, renderLogHeader(in)) + + if parsed.Network.DefaultRouteV4 != "en0" || parsed.Network.DefaultRouteV6 != "en0" { + t.Errorf("default route = %q, %q, want en0 for both families", parsed.Network.DefaultRouteV4, parsed.Network.DefaultRouteV6) + } + if len(parsed.Network.Interfaces) != 2 { + t.Fatalf("network.interfaces holds %d entries, want 2", len(parsed.Network.Interfaces)) + } + if parsed.Network.Interfaces[0].Name != "awdl0" || parsed.Network.Interfaces[0].Class != "airdrop" || parsed.Network.Interfaces[0].Up { + t.Errorf("network.interfaces[0] = %+v, want awdl0 down in the airdrop class", parsed.Network.Interfaces[0]) + } + if got := parsed.Network.Interfaces[1].IPs; !reflect.DeepEqual(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { + t.Errorf("network.interfaces[1].ips = %v", got) + } +} + +// setupLogHeaderWiringTest puts ctrld in cd mode with a temporary home +// directory, so initInternalLogging writes its files there. Every global that +// the logging setup touches goes back when the test ends. +func setupLogHeaderWiringTest(t *testing.T) string { + t.Helper() + stubHeaderSnapshotSources(t) + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog := mainLog.Load() + origLogPathFile := logPathFile.Load() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + logPathFile.Store(origLogPathFile) + }) + + homedir = t.TempDir() + cdUID = "header-wiring-uid" + silent, verbose = false, 0 + mainLog.Store(ctrld.NopLogger) + logPathFile.Store(nil) + return homedir +} + +// startLogHeaderWiringProg runs the internal logging setup and closes both +// files when the test ends. +func startLogHeaderWiringProg(t *testing.T) *prog { + t.Helper() + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + return p +} + +// logHeaderFileLines parses every JSON line of a log file. It returns line 1 +// and the number of header lines, so a test can tell a missing header from a +// repeated one. The debug stream writes console lines, which hold no header, +// so they do not count. +func logHeaderFileLines(t *testing.T, path string) (logHeaderTestLine, int) { + t.Helper() + content := strings.TrimRight(rotatingFileTestContent(t, path), "\n") + if content == "" { + t.Fatalf("%s is empty", path) + } + var firstLine logHeaderTestLine + headers := 0 + for i, line := range strings.Split(content, "\n") { + if i > 0 && !strings.HasPrefix(line, "{") { + continue + } + var parsed logHeaderTestLine + if err := json.Unmarshal([]byte(line), &parsed); err != nil { + t.Fatalf("parse line %d of %s (%q): %v", i+1, path, line, err) + } + if i == 0 { + firstLine = parsed + } + if parsed.Message == logHeaderMessage { + headers++ + } + } + return firstLine, headers +} + +func Test_logHeaderWiring_writesHeaderOnStart(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + + startLogHeaderWiringProg(t) + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + firstLine, headers := logHeaderFileLines(t, path) + if firstLine.Message != logHeaderMessage { + t.Errorf("line 1 of %s = %q, want %q", path, firstLine.Message, logHeaderMessage) + } + if !firstLine.Journal { + t.Errorf("line 1 of %s carries no journal marker", path) + } + if want := redactToken(cdUID); firstLine.ResolverUID != want { + t.Errorf("resolver_uid in %s = %q, want %q", path, firstLine.ResolverUID, want) + } + if headers != 1 { + t.Errorf("%s holds %d header lines, want 1", path, headers) + } + } +} + +func Test_logHeaderInput_keepsSecretsOut(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + origInterceptMode := interceptMode + t.Cleanup(func() { interceptMode = origInterceptMode }) + interceptMode = "" + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + logHeaderTestSecret + "-listener": {IP: "127.0.0.1", Port: 53}, + }, + Upstream: map[string]*ctrld.UpstreamConfig{ + "0": { + Name: logHeaderTestSecret + "-name", + Type: "doh", + Endpoint: "https://dns.example/" + logHeaderTestSecret, + BootstrapIP: "192.0.2.53", + Domain: logHeaderTestSecret + ".example", + }, + }, + }} + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + + if line := renderLogHeader(p.logHeaderInput()); strings.Contains(string(line), logHeaderTestSecret) { + t.Errorf("the header holds %q: %s", logHeaderTestSecret, line) + } + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if content := rotatingFileTestContent(t, path); strings.Contains(content, logHeaderTestSecret) { + t.Errorf("%s holds %q: %s", path, logHeaderTestSecret, content) + } + } +} + +func Test_logHeaderWiring_secondInitAddsNoHeader(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + p := startLogHeaderWiringProg(t) + + p.initInternalLogging(nil) + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if _, headers := logHeaderFileLines(t, path); headers != 1 { + t.Errorf("%s holds %d header lines after a second start, want 1", path, headers) + } + } +} + +func Test_logHeaderWiring_headerLeadsRotatedFile(t *testing.T) { + captureMainLog(t) + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + header := string(renderLogHeader(logHeaderInput{Version: "v1"})) + + rf := lw.rotating() + rf.setHeader([]byte(header)) + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + linesPerFile := int(testLogWriterBudget.maxSize) / rotatingFileTestLineSize + for seq := 1; seq <= linesPerFile+1; seq++ { + if _, err := lw.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write line %d: %v", seq, err) + } + } + + for _, name := range []string{path + ".1", path} { + if content := rotatingFileTestContent(t, name); !strings.HasPrefix(content, header) { + t.Errorf("%s starts with %q, want the header", name, strings.SplitN(content, "\n", 2)[0]) + } + } +} + +func Test_logHeaderWiring_restartAppendsHeader(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + path := filepath.Join(dir, logFileName) + const seed = `{"level":"info","message":"line of the run before"}` + if err := os.WriteFile(path, []byte(seed+"\n"), 0o600); err != nil { + t.Fatalf("seed %s: %v", path, err) + } + + startLogHeaderWiringProg(t) + + lines := strings.Split(strings.TrimRight(rotatingFileTestContent(t, path), "\n"), "\n") + if len(lines) < 2 { + t.Fatalf("%s holds %d lines, want the seed line and the header", path, len(lines)) + } + if lines[0] != seed { + t.Errorf("line 1 of %s = %q, want the seed line", path, lines[0]) + } + var header logHeaderTestLine + if err := json.Unmarshal([]byte(lines[1]), &header); err != nil { + t.Fatalf("parse line 2 of %s (%q): %v", path, lines[1], err) + } + if header.Message != logHeaderMessage { + t.Errorf("line 2 of %s = %q, want %q", path, header.Message, logHeaderMessage) + } + if _, headers := logHeaderFileLines(t, path); headers != 1 { + t.Errorf("%s holds %d header lines, want 1", path, headers) + } +} + +func Test_logHeaderWiring_usesLastNetworkState(t *testing.T) { + stubHeaderSnapshotSources(t) + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + Interface: map[string]netmon.Interface{"en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en7": {netip.MustParsePrefix("192.0.2.7/24")}}, + }) + + if got := p.logHeaderInput().Network.DefaultRouteV4; got != "en7" { + t.Errorf("default_route_v4 = %q, want %q", got, "en7") + } + if got := p.logHeaderInput().Network.GatewayV4; got != "192.0.2.1" { + t.Errorf("gateway_v4 = %q, want %q", got, "192.0.2.1") + } + + p.lastNetworkState.Store(nil) + readNetworkSourceStateFn = func() (*netmon.State, error) { + return &netmon.State{ + Interface: map[string]netmon.Interface{"en9": {Interface: &net.Interface{Name: "en9", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en9": {netip.MustParsePrefix("192.0.2.9/24")}}, + }, nil + } + + network := p.logHeaderInput().Network + if len(network.Interfaces) != 1 || network.Interfaces[0].Name != "en9" || !network.Interfaces[0].Up { + t.Fatalf("interfaces = %+v, want one up en9", network.Interfaces) + } + if !network.HaveV4 { + t.Error("have_v4 = false, want true for a global unicast v4 address") + } + if network.HaveV6 { + t.Error("have_v6 = true, want false without a v6 address") + } +} + +// stubHeaderSnapshotSources keeps the header snapshot away from the host. One +// render reads the hardware ports, the route table, and the NAT64 class, and no +// test may run the commands behind them. +func stubHeaderSnapshotSources(t *testing.T) { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotVirtualSet(t) +} + +// stubLogHeaderNetworkRead makes the fresh network read return one up +// interface, so a header test never depends on the host network. +func stubLogHeaderNetworkRead(t *testing.T, name string) { + t.Helper() + stubHeaderSnapshotSources(t) + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { + return &netmon.State{ + Interface: map[string]netmon.Interface{name: {Interface: &net.Interface{Name: name, Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{name: {netip.MustParsePrefix("192.0.2.9/24")}}, + }, nil + } +} + +func Test_sendLogHeader_usesFreshNetworkAndTrigger(t *testing.T) { + stubHeaderSnapshotSources(t) + stubLogHeaderNetworkRead(t, "en9") + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + Interface: map[string]netmon.Interface{"en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en7": {netip.MustParsePrefix("192.0.2.7/24")}}, + }) + + parsed, _ := parseLogHeader(t, p.sendLogHeader()) + + if parsed.Trigger != "send" { + t.Errorf("trigger = %q, want %q", parsed.Trigger, "send") + } + if parsed.Message != logHeaderMessage { + t.Errorf("message = %q, want %q", parsed.Message, logHeaderMessage) + } + names := make([]string, 0, len(parsed.Network.Interfaces)) + for _, iface := range parsed.Network.Interfaces { + names = append(names, iface.Name) + } + if want := []string{"en9"}; !reflect.DeepEqual(names, want) { + t.Fatalf("network.interfaces = %v, want %v from the fresh read", names, want) + } +} + +// Test_logHeaderInputReadsTheConfigUnderTheLock renders headers beside the +// config swap of a reload. The race detector fails the test when the render +// reads the config outside the lock. +func Test_logHeaderInputReadsTheConfigUnderTheLock(t *testing.T) { + stubLogHeaderNetworkRead(t, "en0") + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: "doh"}}, + }} + reloaded := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: "doq"}, "1": {Type: "legacy"}}, + } + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + p.mu.Lock() + *p.cfg = *reloaded + p.mu.Unlock() + } + }() + for i := 0; i < 300; i++ { + if in := p.logHeaderInput(); len(in.Listeners) != 1 { + t.Fatalf("listeners = %v, want one", in.Listeners) + } + } + close(stop) + <-done +} + +// Test_renderLogHeaderPassesTheSink puts a provisioning secret where the +// header does not expect one. The header is the first line of every file and +// of every upload, so it must pass the same redaction as every other line. +func Test_renderLogHeaderPassesTheSink(t *testing.T) { + origUID := cdUID + cdUID = "abcd1234SECRETUID" + t.Cleanup(func() { cdUID = origUID }) + + line := string(renderLogHeader(logHeaderInput{Version: "1.0", LogFiles: []string{"/var/log/" + cdUID + "/ctrld.log"}})) + + if strings.Contains(line, cdUID) { + t.Fatalf("the header holds the secret: %s", line) + } +} diff --git a/cmd/cli/log_journal.go b/cmd/cli/log_journal.go new file mode 100644 index 00000000..7f998add --- /dev/null +++ b/cmd/cli/log_journal.go @@ -0,0 +1,146 @@ +package cli + +import ( + "io" + "regexp" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// journalField marks an event for the retained journal stream. Both packages +// share the name, so one core keeps the lines of both. +const journalField = ctrld.JournalField + +// retainedURLPath matches the part of a URL that follows the host, for every +// scheme: a DoH, DoH3, or DoQ endpoint carries its token in the path. The +// class stops the match at the characters that end a value inside a log line. +var retainedURLPath = regexp.MustCompile(`([a-z][a-z0-9+.-]*://[^/?"\\\s]+)[/?][^"\\\s]*`) + +// retainedControlDHost matches a Control D resolver host. The first label of +// the host is the resolver ID, which a DoT or DoQ endpoint carries in place of +// a path. +var retainedControlDHost = regexp.MustCompile(`([A-Za-z0-9-]+)(\.dns\.controld\.(?:com|dev))`) + +// redactRetainedLine strips the secrets of one line that the journal keeps. +// Support downloads the journal whole, and any error text can carry a DoH URL +// with the resolver path and the packed query of the user. +func redactRetainedLine(p []byte) []byte { + stripped := retainedURLPath.ReplaceAll(p, []byte(`${1}/[redacted]`)) + stripped = retainedControlDHost.ReplaceAll(stripped, []byte(`[redacted]${2}`)) + return []byte(redactSecrets(string(stripped), journalSecrets()...)) +} + +// journalSecrets names the values that no retained line may hold. The client +// ID stays out: it is a device label, not a secret. redactSecrets drops the +// values that are too short to redact. +func journalSecrets() []string { + uid, _ := controld.ParseRawUID(cdUID) + return []string{cdUID, cdOrg, uid} +} + +// journal marks an event for the retained journal stream. The event keeps +// its own level, so an info event stays info on the console. +func journal(e *ctrld.LogEvent) *ctrld.LogEvent { + return ctrld.Journal(e) +} + +// journalTimeLayout is the time format of a journal line: RFC 3339 with +// milliseconds. +const journalTimeLayout = "2006-01-02T15:04:05.000Z07:00" + +// newJournalEncoder is the encoder of the journal and of the header line: one +// JSON object for each line, with the field names that the log tools read. +// The debug stream keeps its console layout for the operator who reads it on +// the host. +func newJournalEncoder() zapcore.Encoder { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout(journalTimeLayout) + encoderConfig.LevelKey = "level" + encoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder + encoderConfig.MessageKey = "message" + return zapcore.NewJSONEncoder(encoderConfig) +} + +// journalCore writes the lines that the journal keeps and drops the rest, so +// the journal holds a small history that survives a restart. It keeps every +// line at warn level or above, and every line that carries the journal field. +// Notice shares its level value with warn, so a notice line stays too. +type journalCore struct { + zapcore.LevelEnabler + encoder zapcore.Encoder + out zapcore.WriteSyncer + // marked is set when With bound the journal field, so every line of that + // logger belongs to the journal. + marked bool +} + +var _ zapcore.Core = (*journalCore)(nil) + +// newJournalCore makes the core of the journal stream. It accepts every level, +// because a debug line with the journal field belongs to the journal too. +func newJournalCore(w io.Writer) *journalCore { + return &journalCore{ + LevelEnabler: zapcore.DebugLevel, + encoder: newJournalEncoder(), + out: zapcore.AddSync(w), + } +} + +func (c *journalCore) With(fields []zapcore.Field) zapcore.Core { + clone := &journalCore{ + LevelEnabler: c.LevelEnabler, + encoder: c.encoder.Clone(), + out: c.out, + marked: c.marked || hasJournalField(fields), + } + for _, field := range fields { + field.AddTo(clone.encoder) + } + return clone +} + +func (c *journalCore) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.Enabled(entry.Level) { + return checked.AddCore(entry, c) + } + return checked +} + +// Write keeps or drops one line. A dropped line is not an error. +func (c *journalCore) Write(entry zapcore.Entry, fields []zapcore.Field) error { + if !c.retains(entry, fields) { + return nil + } + buf, err := c.encoder.EncodeEntry(entry, fields) + if err != nil { + return err + } + defer buf.Free() + _, err = c.out.Write(redactRetainedLine(buf.Bytes())) + return err +} + +func (c *journalCore) Sync() error { + return c.out.Sync() +} + +// retains reads the level first, because a compare costs less than a scan of +// the fields. +func (c *journalCore) retains(entry zapcore.Entry, fields []zapcore.Field) bool { + return c.marked || entry.Level >= zapcore.WarnLevel || hasJournalField(fields) +} + +// hasJournalField reports whether fields carry the journal marker. +func hasJournalField(fields []zapcore.Field) bool { + for _, field := range fields { + if field.Key == journalField && field.Type == zapcore.BoolType && field.Integer == 1 { + return true + } + } + return false +} diff --git a/cmd/cli/log_journal_test.go b/cmd/cli/log_journal_test.go new file mode 100644 index 00000000..1cdb5ad6 --- /dev/null +++ b/cmd/cli/log_journal_test.go @@ -0,0 +1,292 @@ +package cli + +import ( + "bytes" + "path/filepath" + "strings" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" +) + +func Test_journalSetsMarker(t *testing.T) { + var buf bytes.Buffer + logger := newTestJSONLogger(&buf) + journal(logger.Info()).Msg("marked") + if !bytes.Contains(buf.Bytes(), journalMarker) { + t.Fatalf("journal event lacks the marker: %s", buf.String()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"level":"info"`)) { + t.Fatalf("journal event changed its level: %s", buf.String()) + } +} + +// newJournalTestLogger returns a logger whose only core is the journal core. +func newJournalTestLogger(buf *bytes.Buffer) *ctrld.Logger { + return &ctrld.Logger{Logger: zap.New(newJournalCore(buf))} +} + +func Test_journalCoreKeepsWarningsAndMarkedLines(t *testing.T) { + for _, tc := range []struct { + name string + log func(logger *ctrld.Logger) + kept bool + }{ + {"info with the marker", func(logger *ctrld.Logger) { journal(logger.Info()).Msg("state") }, true}, + {"info without the marker", func(logger *ctrld.Logger) { logger.Info().Msg("state") }, false}, + {"warn without the marker", func(logger *ctrld.Logger) { logger.Warn().Msg("slow") }, true}, + {"debug with the marker", func(logger *ctrld.Logger) { journal(logger.Debug()).Msg("state") }, true}, + // Notice shares its level value with warn, so the journal keeps it. + {"notice without the marker", func(logger *ctrld.Logger) { logger.Notice().Msg("note") }, true}, + {"error without the marker", func(logger *ctrld.Logger) { logger.Error().Msg("failed") }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + tc.log(newJournalTestLogger(&buf)) + if kept := buf.Len() > 0; kept != tc.kept { + t.Fatalf("kept = %v, want %v, line: %q", kept, tc.kept, buf.String()) + } + }) + } +} + +// Test_journalCoreWriteKeepsMarkedEntryOnly drives the core directly, which +// is the path that zap takes for every line. +func Test_journalCoreWriteKeepsMarkedEntryOnly(t *testing.T) { + var buf bytes.Buffer + core := newJournalCore(&buf) + entry := zapcore.Entry{Level: zapcore.InfoLevel, Message: "state"} + + if err := core.Write(entry, []zapcore.Field{zap.Bool(journalField, true)}); err != nil { + t.Fatalf("write of a marked entry: %v", err) + } + if !bytes.Contains(buf.Bytes(), journalMarker) { + t.Fatalf("journal holds %q, want the marked line", buf.String()) + } + marked := buf.Len() + + if err := core.Write(entry, nil); err != nil { + t.Fatalf("write of a plain entry: %v", err) + } + if buf.Len() != marked { + t.Fatalf("journal kept an unmarked line: %q", buf.String()) + } +} + +// Test_journalCoreWithBindsTheMarker proves that a logger with the field +// bound keeps every line, because With moves the field out of Write. +func Test_journalCoreWithBindsTheMarker(t *testing.T) { + var buf bytes.Buffer + logger := newJournalTestLogger(&buf).Bool(journalField, true) + + logger.Info().Msg("state") + + if !bytes.Contains(buf.Bytes(), journalMarker) { + t.Fatalf("journal holds %q, want the bound line", buf.String()) + } +} + +// journalDoHError has the shape of a resolver error text: the DoH URL carries +// the resolver path and the packed query. +const journalDoHError = `Get "https://dns.example/org-v1-SECRET0123?dns=pOgBAAAB": dial tcp` + +// journalRedactedURL is what the sink keeps of journalDoHError. +const journalRedactedURL = "https://dns.example/[redacted]" + +// journalTestLogger returns a logger that writes through the journal sink. +func journalTestLogger(t *testing.T) (*bytes.Buffer, *ctrld.Logger) { + t.Helper() + var buf bytes.Buffer + return &buf, newJournalTestLogger(&buf) +} + +func Test_journalCoreRedactsAKeptLine(t *testing.T) { + buf, logger := journalTestLogger(t) + + logger.Warn().Str("error", journalDoHError).Msg("upstream failed") + + line := buf.String() + if !strings.Contains(line, journalRedactedURL) { + t.Fatalf("the sink kept the URL path: %s", line) + } + for _, secret := range []string{"SECRET0123", "pOgBAAAB"} { + if strings.Contains(line, secret) { + t.Fatalf("the sink kept %q: %s", secret, line) + } + } +} + +func Test_journalCoreRedactsTheProvisionSecrets(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "org-v1-uidofthisdevice" + buf, logger := journalTestLogger(t) + + logger.Warn().Str("detail", "device "+cdUID+" is gone").Msg("provision failed") + + line := buf.String() + if strings.Contains(line, cdUID) { + t.Fatalf("the sink kept the resolver uid: %s", line) + } + if !strings.Contains(line, "[redacted]") { + t.Fatalf("the sink wrote no redaction mark: %s", line) + } +} + +func Test_journalCoreDropsADebugLineWithAURL(t *testing.T) { + buf, logger := journalTestLogger(t) + + logger.Debug().Str("error", journalDoHError).Msg("upstream failed") + + if buf.Len() != 0 { + t.Fatalf("the sink kept a debug line: %s", buf.String()) + } +} + +func Test_journalCoreRedactsAMarkedInfoLine(t *testing.T) { + buf, logger := journalTestLogger(t) + + journal(logger.Info()).Str("error", journalDoHError).Msg("upstream failed") + + line := buf.String() + if !strings.Contains(line, journalRedactedURL) { + t.Fatalf("the sink kept the URL path: %s", line) + } + if !strings.Contains(line, `"level":"info"`) { + t.Fatalf("the sink changed the level: %s", line) + } +} + +// startInternalLogging drives the real cd mode setup and returns the program +// and the directory that holds the debug file and the journal file. +func startInternalLogging(t *testing.T) (*prog, string) { + t.Helper() + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog := mainLog.Load() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + }) + if origMainLog == nil { + mainLog.Store(ctrld.NopLogger) + } + + dir := t.TempDir() + homedir = dir + cdUID = "test-uid" + silent = false + verbose = 0 + stubHeaderSnapshotSources(t) + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initInternalLogging(nil) + // Stop logs through the prog logger, as initLogging wires it. + p.logger.Store(mainLog.Load()) + t.Cleanup(p.closeInternalLogs) + return p, dir +} + +// Test_initInternalLogging_redactsTheJournalFile drives a cd mode run: support +// downloads the journal file, so it holds the redacted URL, while the debug +// file stays raw for the developer who reads it on the host. +func Test_initInternalLogging_redactsTheJournalFile(t *testing.T) { + _, dir := startInternalLogging(t) + + mainLog.Load().Warn().Str("error", journalDoHError).Msg("upstream failed") + + journalText := rotatingFileTestContent(t, filepath.Join(dir, journalLogFileName)) + if !strings.Contains(journalText, journalRedactedURL) { + t.Fatalf("journal file misses the redacted URL: %s", journalText) + } + if strings.Contains(journalText, "SECRET0123") { + t.Fatalf("journal file kept the resolver path: %s", journalText) + } + + debugText := rotatingFileTestContent(t, filepath.Join(dir, logFileName)) + if !strings.Contains(debugText, "org-v1-SECRET0123?dns=pOgBAAAB") { + t.Fatalf("debug file lost the raw URL: %s", debugText) + } +} + +// Test_stopClosesTheInternalLogFiles proves that a stopped service leaves no +// open handle on the debug file and the journal file. +func Test_stopClosesTheInternalLogFiles(t *testing.T) { + origPin := cdDeactivationPin.Load() + t.Cleanup(func() { cdDeactivationPin.Store(origPin) }) + cdDeactivationPin.Store(defaultDeactivationPin) + + p, _ := startInternalLogging(t) + p.stopCh = make(chan struct{}) + p.dnsWatcherStopCh = make(chan struct{}) + for _, writer := range []*logWriter{p.internalLogWriter, p.internalJournalWriter} { + if writer.rotating() == nil { + t.Fatal("the setup opened no file") + } + } + + if err := p.Stop(nil); err != nil { + t.Fatalf("Stop: %v", err) + } + + for name, writer := range map[string]*logWriter{ + logFileName: p.internalLogWriter, + journalLogFileName: p.internalJournalWriter, + } { + if writer.rotating() != nil { + t.Fatalf("%s stayed open after the stop", name) + } + } +} + +// Test_redactRetainedLineKeepsAShortClientID puts a two-letter client ID in +// the resolver UID. The keys of a JSON line hold the same letters, and a +// redaction of the client ID would break the line. +func Test_redactRetainedLineKeepsAShortClientID(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "uid12345678/os" + + line := string(redactRetainedLine([]byte(`{"level":"info","os":"darwin","resolver":"uid12345678/os"}`))) + + if !strings.Contains(line, `"os":"darwin"`) { + t.Fatalf("the redaction changed the os key: %s", line) + } + if strings.Contains(line, "uid12345678") { + t.Fatalf("the redaction kept the resolver uid: %s", line) + } +} + +// Test_redactRetainedLineStripsEverySchemePath covers the endpoints that carry +// a token in their path with a scheme other than https. +func Test_redactRetainedLineStripsEverySchemePath(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {`Get "quic://dns.example/org-v1-SECRET0123?dns=pOgBAAAB": timeout`, `Get "quic://dns.example/[redacted]": timeout`}, + {`h3://dns.example/SECRET0123 failed`, `h3://dns.example/[redacted] failed`}, + {`tls://dns.example:853 failed`, `tls://dns.example:853 failed`}, + {`tls://abcdef12.dns.controld.com:853 failed`, `tls://[redacted].dns.controld.com:853 failed`}, + {`quic://abcdef12.dns.controld.dev failed`, `quic://[redacted].dns.controld.dev failed`}, + {`https://freedns.controld.com/p1 failed`, `https://freedns.controld.com/[redacted] failed`}, + } { + if got := string(redactRetainedLine([]byte(tc.in))); got != tc.want { + t.Errorf("redactRetainedLine(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// Test_redactRetainedLineKeepsAShortResolverUID puts a two-letter resolver UID +// in place. The keys of a JSON line hold the same letters, and a redaction of +// the UID would break every retained line and the header. +func Test_redactRetainedLineKeepsAShortResolverUID(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "os" + + const in = `{"level":"info","os":"darwin","message":"loss of service"}` + if got := string(redactRetainedLine([]byte(in))); got != in { + t.Fatalf("redactRetainedLine(%q) = %q, want the line unchanged", in, got) + } +} diff --git a/cmd/cli/log_path.go b/cmd/cli/log_path.go new file mode 100644 index 00000000..b7156459 --- /dev/null +++ b/cmd/cli/log_path.go @@ -0,0 +1,71 @@ +package cli + +import ( + "os" + "sync/atomic" +) + +// logPathFile is the open log_path file. The header renderer and the log +// readers need it, and both run outside this file. +var logPathFile atomic.Pointer[rotatingFile] + +// pendingLogPathRotation holds the rotation of a start-time backup. The +// logger reaches no file at that point, so initLogging reports it later. +var pendingLogPathRotation atomic.Pointer[logRotation] + +// openLogPathWriter opens the log_path file of this run and stores it for the +// header renderer and the log readers. It returns nil when no log_path is set. +// A start calls it more than once, so a call for the file that is already open +// keeps that file: a second open would put a second header in it. +func openLogPathWriter(doBackup bool) *rotatingFile { + logFilePath := normalizeLogFilePath(cfg.Service.LogPath) + if logFilePath == "" { + return nil + } + budget := debugLogBudget(&cfg.Service) + rf := logPathFile.Load() + if rf == nil || rf.currentPath() != logFilePath { + rf = newLogPathFile(logFilePath, budget) + } else { + rf.setBudget(budget) + } + if doBackup { + backupLogPathFile(rf) + } + return rf +} + +// newLogPathFile opens path and puts it in place of the file of an earlier +// call. A ctrld that cannot write the log file its operator asked for leaves +// no record of the run, so the open failure ends the process. +func newLogPathFile(path string, budget logBudget) *rotatingFile { + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + mainLog.Load().Error().Msgf("failed to create log file: %v", err) + os.Exit(1) + } + rf.onRotate = emitLogRotated + if old := logPathFile.Swap(rf); old != nil { + _ = old.close() + } + return rf +} + +// backupLogPathFile moves the file of an earlier run away, so no run +// overwrites another. +func backupLogPathFile(rf *rotatingFile) { + if !logFileHasContent(rf.currentPath()) { + return + } + record, err := rf.rotateNow() + if err != nil { + mainLog.Load().Error().Msgf("could not backup old log file: %v", err) + return + } + pendingLogPathRotation.Store(record) +} + +func logFileHasContent(path string) bool { + st, err := os.Stat(path) + return err == nil && st.Size() > 0 +} diff --git a/cmd/cli/log_path_rotation_test.go b/cmd/cli/log_path_rotation_test.go new file mode 100644 index 00000000..cb876326 --- /dev/null +++ b/cmd/cli/log_path_rotation_test.go @@ -0,0 +1,178 @@ +package cli + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +const ( + logPathTestBackups = 2 + logPathTestBudgetMB = 1 + logPathTestLineSize = 4 * 1024 + logPathTestLineCount = 300 +) + +// setupLogPathTest points log_path at a temporary file and puts every global +// that initLoggingWithBackup touches back when the test ends. The globals are +// the reason why no test of this package calls t.Parallel. +// +// The temporary directory comes first: its own cleanup removes the directory, +// and a cleanup runs before every cleanup that was added earlier, so the file +// must close before that. +func setupLogPathTest(t *testing.T) string { + t.Helper() + dir := t.TempDir() + origLogPath, origLogLevel := cfg.Service.LogPath, cfg.Service.LogLevel + origMaxSizeMB, origMaxBackups := cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups + origSilent, origVerbose := silent, verbose + origConsoleWriter := consoleWriter + origMainLog := mainLog.Load() + origLogPathFile, origPendingRotation := logPathFile.Load(), pendingLogPathRotation.Load() + t.Cleanup(func() { + if rf := logPathFile.Load(); rf != nil && rf != origLogPathFile { + _ = rf.close() + } + logPathFile.Store(origLogPathFile) + pendingLogPathRotation.Store(origPendingRotation) + cfg.Service.LogPath, cfg.Service.LogLevel = origLogPath, origLogLevel + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = origMaxSizeMB, origMaxBackups + silent, verbose = origSilent, origVerbose + consoleWriter = origConsoleWriter + mainLog.Store(origMainLog) + }) + + backups := logPathTestBackups + cfg.Service.LogPath = filepath.Join(dir, "ctrld.log") + cfg.Service.LogLevel = "info" + cfg.Service.LogMaxSizeMB = logPathTestBudgetMB + cfg.Service.LogMaxBackups = &backups + silent, verbose = false, 0 + consoleWriter = newHumanReadableZapCore(io.Discard, ctrld.NoticeLevel) + mainLog.Store(ctrld.NopLogger) + logPathFile.Store(nil) + pendingLogPathRotation.Store(nil) + return cfg.Service.LogPath +} + +func Test_logPathBackupChainOnStart(t *testing.T) { + path := setupLogPathTest(t) + if err := os.WriteFile(path, []byte("run1\n"), 0o600); err != nil { + t.Fatalf("seed log file: %v", err) + } + + initLoggingWithBackup(true) + if _, err := logPathFile.Load().Write([]byte("run2\n")); err != nil { + t.Fatalf("write run2: %v", err) + } + initLoggingWithBackup(true) + + newest := rotatingFileTestContent(t, path+".1") + if !strings.Contains(newest, "run2") { + t.Fatalf("%s.1 = %q, want the run2 line", path, newest) + } + if strings.Contains(newest, "run1") { + t.Fatalf("%s.1 = %q, want no run1 line", path, newest) + } + oldest := rotatingFileTestContent(t, path+".2") + if !strings.Contains(oldest, "run1") { + t.Fatalf("%s.2 = %q, want the run1 line", path, oldest) + } + current := rotatingFileTestContent(t, path) + if strings.Contains(current, "run1") || strings.Contains(current, "run2") { + t.Fatalf("%s = %q, want neither run line", path, current) + } +} + +func Test_logPathRotatesBySize(t *testing.T) { + path := setupLogPathTest(t) + initLoggingWithBackup(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) { + t.Fatalf("%s.2 exists although the lines fill one backup (stat err = %v)", path, err) + } + want := []string{path + ".1", path} + if got := logPathFile.Load().paths(); !slices.Equal(got, want) { + t.Fatalf("paths() = %v, want %v", got, want) + } + + rotatedLines := 0 + for _, logLine := range strings.Split(rotatingFileTestContent(t, path), "\n") { + if !strings.Contains(logLine, "Log rotated") { + continue + } + rotatedLines++ + if !hasJournalMarker(logLine) { + t.Fatalf("Log rotated line %q carries no journal marker", logLine) + } + } + if rotatedLines != 1 { + t.Fatalf("Log rotated lines in %s = %d, want 1", path, rotatedLines) + } +} + +func Test_logPathHeaderLeadsEveryFile(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en0") + p := &prog{cfg: &cfg} + + // A start wires its logging twice: once for the config it reads from + // disk, and once more for the config that the API sends. + p.initLogging(true) + p.initLogging(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + for _, name := range []string{path + ".1", path} { + firstLine, headers := logHeaderFileLines(t, name) + if firstLine.Message != logHeaderMessage { + t.Errorf("line 1 of %s = %q, want %q", name, firstLine.Message, logHeaderMessage) + } + if headers != 1 { + t.Errorf("%s holds %d header lines, want 1", name, headers) + } + } +} + +func Test_logPathStartBackupLogsRotation(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en0") + if err := os.WriteFile(path, []byte(`{"level":"info","message":"run1"}`+"\n"), 0o600); err != nil { + t.Fatalf("seed log file: %v", err) + } + p := &prog{cfg: &cfg} + + p.initLogging(true) + + rotatedLines := 0 + for _, logLine := range strings.Split(rotatingFileTestContent(t, path), "\n") { + if !strings.Contains(logLine, "Log rotated") { + continue + } + rotatedLines++ + var file string + if err := json.Unmarshal(parseLogLineFields(t, logLine)["file"], &file); err != nil { + t.Fatalf("parse the file field of %q: %v", logLine, err) + } + if file != path+".1" { + t.Errorf("file = %q, want %q", file, path+".1") + } + } + if rotatedLines != 1 { + t.Fatalf("Log rotated lines in %s = %d, want 1", path, rotatedLines) + } +} diff --git a/cmd/cli/log_reader.go b/cmd/cli/log_reader.go new file mode 100644 index 00000000..912b65d7 --- /dev/null +++ b/cmd/cli/log_reader.go @@ -0,0 +1,366 @@ +package cli + +import ( + "bytes" + "errors" + "io" + "os" + "strings" +) + +// logSendDebugBudget bounds the debug bytes of one upload. The journal is +// always whole, so an upload stays small and keeps the retained stream. A test +// lowers it, because a few small files must cross it. +var logSendDebugBudget int64 = 10 << 20 + +// lineScanBufferSize bounds the memory that the search for a line boundary +// needs, whatever the length of the line. +const lineScanBufferSize = 32 * 1024 + +var ( + errInternalLogEmpty = errors.New("internal log is empty") + errLogFileEmpty = errors.New("log file is empty") +) + +type logReader struct { + r io.ReadCloser + size int64 +} + +// logFilePart is the part of one open log file that an upload carries. An +// offset of 0 means the whole file. size is the size at the open, so a write +// after the selection stays out of the upload and the budget holds. +type logFilePart struct { + path string + f *os.File + offset int64 + size int64 +} + +// logReader composes the upload of the current logging mode and leads it with +// the header of this request. full lifts the bound on the debug bytes. +// stripColor removes the color codes of the internal debug stream, which an +// upload must not carry. The size counts the bytes before that removal. +func (p *prog) logReader(full, stripColor bool) (*logReader, error) { + lr, err := p.modeLogReader(full) + if err != nil { + return nil, err + } + // An empty upload must stay empty, because the handlers answer 301 on a + // size of 0. + if lr.size == 0 { + return lr, nil + } + header := p.sendLogHeader() + var body io.Reader = io.MultiReader(bytes.NewReader(header), lr.r) + if stripColor { + body = newANSIStripReader(body) + } + return &logReader{ + r: &multiCloser{Reader: body, closers: []io.Closer{lr.r}}, + size: lr.size + int64(len(header)), + }, nil +} + +// ansiStripChunkSize is the size of one read from the source of an +// ansiStripReader. +const ansiStripChunkSize = 32 * 1024 + +// ansiStripReader removes the ANSI color codes from what r yields. The level +// encoder of the internal debug stream colors each level, and an upload must +// hold plain text. +type ansiStripReader struct { + r io.Reader + // pending holds the bytes that are read and not yet stripped. It ends in + // an escape sequence that the next chunk completes, or it is empty. + pending []byte + // out holds the stripped bytes that no Read took yet. + out []byte + err error +} + +func newANSIStripReader(r io.Reader) io.Reader { + return &ansiStripReader{r: r} +} + +func (s *ansiStripReader) Read(p []byte) (int, error) { + for len(s.out) == 0 { + if s.err != nil { + return 0, s.err + } + s.fill() + } + n := copy(p, s.out) + s.out = s.out[n:] + return n, nil +} + +// fill reads one chunk and strips the bytes that hold no unfinished escape +// sequence. An unfinished sequence waits for the next chunk. The end of the +// source flushes everything, because nothing can complete the sequence. +func (s *ansiStripReader) fill() { + chunk := make([]byte, ansiStripChunkSize) + n, err := s.r.Read(chunk) + s.pending = append(s.pending, chunk[:n]...) + keep := 0 + if err == nil { + keep = partialEscapeLen(s.pending) + } + ready := len(s.pending) - keep + s.out = ansiRegex.ReplaceAll(s.pending[:ready], nil) + s.pending = append([]byte(nil), s.pending[ready:]...) + s.err = err +} + +// partialEscapeLen returns the length of the escape sequence that starts in b +// and does not end in it, or 0 when b ends outside a sequence. +func partialEscapeLen(b []byte) int { + start := bytes.LastIndexByte(b, 0x1b) + if start < 0 { + return 0 + } + tail := b[start:] + if len(tail) == 1 { + return 1 + } + if tail[1] != '[' { + return 0 + } + for _, c := range tail[2:] { + if (c < '0' || c > '9') && c != ';' { + return 0 + } + } + return len(tail) +} + +// modeLogReader composes the body of the current logging mode. +func (p *prog) modeLogReader(full bool) (*logReader, error) { + if p.needInternalLogging() { + return p.internalLogReader(full) + } + return p.logPathReader(full) +} + +// internalLogReader puts the newest debug bytes first, then the marker, then +// the journal, so a support reader finds the retained stream after the marker. +func (p *prog) internalLogReader(full bool) (*logReader, error) { + lw, jlw := p.internalWriters() + if lw == nil { + return nil, errors.New("nil internal log writer") + } + if jlw == nil { + return nil, errors.New("nil internal journal writer") + } + + var upload logParts + debugBytes := upload.addStream(lw, logSendBudget(full)) + upload.addBytes([]byte(logWriterLogEndMarker)) + journalBytes := upload.addStream(jlw, 0) + // The marker alone is not a log, and the handlers answer 301 on an empty + // upload. + if debugBytes+journalBytes == 0 { + upload.close() + return nil, errInternalLogEmpty + } + return upload.reader(errInternalLogEmpty) +} + +// logPathReader composes the upload of the log_path stream. The journal needs +// internal logging, so this upload holds the debug files alone. A run that +// opened no log_path file has nothing to send. +func (p *prog) logPathReader(full bool) (*logReader, error) { + rf := logPathFile.Load() + if p.cfg.Service.LogPath == "" || rf == nil { + return emptyLogReader(), nil + } + var upload logParts + upload.addBoundedFiles(rf.paths(), logSendBudget(full)) + return upload.reader(errLogFileEmpty) +} + +// emptyLogReader yields no bytes, so the handlers answer 301 instead of +// sending an upload without a line. +func emptyLogReader() *logReader { + return &logReader{r: io.NopCloser(strings.NewReader(""))} +} + +// logSendBudget reports the debug bytes that one upload may carry. Zero means +// every file. +func logSendBudget(full bool) int64 { + if full { + return 0 + } + return logSendDebugBudget +} + +// boundedLogFiles opens the files, reads each size once, and picks the newest +// bytes that fit in budget. files come oldest first, and so do the parts. The +// oldest part starts at a line boundary, so the upload holds whole lines only. +// A budget of 0 or less takes every file. The files that fall out of the +// budget close here; the parts own the files they hold. +func boundedLogFiles(files []string, budget int64) []logFilePart { + parts := openLogParts(files) + remaining := budget + first := len(parts) + for first > 0 { + part := &parts[first-1] + if budget <= 0 || part.size <= remaining { + remaining -= part.size + first-- + continue + } + if offset, ok := lineStartAt(part.f, part.size-remaining); ok { + part.offset = offset + first-- + } + // Without a line boundary in the tail, the upload would start inside a + // line, so the file stays out. + break + } + for _, part := range parts[:first] { + part.f.Close() + } + return parts[first:] +} + +// openLogParts opens each file through the platform opener, whose share flags +// let a rotation move the file away while Windows reads it. A rotation can +// also move a file away between the listing and the open, so a file that does +// not open must not stop the rest. +func openLogParts(files []string) []logFilePart { + parts := make([]logFilePart, 0, len(files)) + for _, path := range files { + f, err := openLogFile(path, os.O_RDONLY) + if err != nil { + continue + } + st, err := f.Stat() + if err != nil { + f.Close() + continue + } + parts = append(parts, logFilePart{path: path, f: f, size: st.Size()}) + } + return parts +} + +// lineStartAt returns the offset where a whole line starts at or after from. +// A cut that already lands on a line start keeps the line that follows it. It +// reports false when no newline follows. +func lineStartAt(f *os.File, from int64) (int64, bool) { + if startsLine(f, from) { + return from, true + } + if _, err := f.Seek(from, io.SeekStart); err != nil { + return 0, false + } + buf := make([]byte, lineScanBufferSize) + offset := from + for { + n, readErr := f.Read(buf) + if index := bytes.IndexByte(buf[:n], '\n'); index >= 0 { + return offset + int64(index) + 1, true + } + offset += int64(n) + if readErr != nil { + return 0, false + } + } +} + +// startsLine reports whether from is the first byte of a line. +func startsLine(f *os.File, from int64) bool { + if from == 0 { + return true + } + var previous [1]byte + if _, err := f.ReadAt(previous[:], from-1); err != nil { + return false + } + return previous[0] == '\n' +} + +// logParts collects the readers of one upload and the files it must close. +type logParts struct { + readers []io.Reader + closers []io.Closer + size int64 +} + +func (lp *logParts) addBytes(b []byte) { + if len(b) == 0 { + return + } + lp.readers = append(lp.readers, bytes.NewReader(b)) + lp.size += int64(len(b)) +} + +// addFiles takes the open parts. Each reader ends at the size of the open, so +// the bytes that a writer adds later stay out of this upload. +func (lp *logParts) addFiles(parts []logFilePart) { + for _, part := range parts { + length := part.size - part.offset + if length <= 0 { + part.f.Close() + continue + } + lp.readers = append(lp.readers, io.NewSectionReader(part.f, part.offset, length)) + lp.closers = append(lp.closers, part.f) + lp.size += length + } +} + +// addBoundedFiles adds the newest bytes of files that fit in budget. A budget +// of 0 or less takes every file. +func (lp *logParts) addBoundedFiles(files []string, budget int64) { + lp.addFiles(boundedLogFiles(files, budget)) +} + +// addStream adds the bytes of one stream and reports how many it added. A +// stream that has a file takes its files, and a stream that keeps its lines in +// memory takes its buffer, so one failed file never hides the other stream. +func (lp *logParts) addStream(lw *logWriter, budget int64) int64 { + before := lp.size + if rf := lw.rotating(); rf != nil { + lp.addBoundedFiles(rf.paths(), budget) + return lp.size - before + } + lp.addBytes(lw.bufferedBytes()) + return lp.size - before +} + +// reader joins the parts. The size is the number of bytes that the reader +// yields, so the caller reports what it uploads. +func (lp *logParts) reader(empty error) (*logReader, error) { + if lp.size == 0 { + lp.close() + return nil, empty + } + return &logReader{ + r: &multiCloser{Reader: io.MultiReader(lp.readers...), closers: lp.closers}, + size: lp.size, + }, nil +} + +func (lp *logParts) close() { + for _, c := range lp.closers { + c.Close() + } +} + +// multiCloser wraps an io.Reader and closes multiple underlying closers. +type multiCloser struct { + io.Reader + closers []io.Closer +} + +func (mc *multiCloser) Close() error { + var firstErr error + for _, c := range mc.closers { + if err := c.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} diff --git a/cmd/cli/log_reader_ansi_test.go b/cmd/cli/log_reader_ansi_test.go new file mode 100644 index 00000000..5ed0eddb --- /dev/null +++ b/cmd/cli/log_reader_ansi_test.go @@ -0,0 +1,69 @@ +package cli + +import ( + "io" + "strings" + "testing" +) + +// splitReader yields its text in the pieces that the test names, one piece +// for each Read, so a test puts a chunk boundary where it wants one. +type splitReader struct { + pieces []string +} + +func (r *splitReader) Read(p []byte) (int, error) { + if len(r.pieces) == 0 { + return 0, io.EOF + } + n := copy(p, r.pieces[0]) + if n < len(r.pieces[0]) { + r.pieces[0] = r.pieces[0][n:] + } else { + r.pieces = r.pieces[1:] + } + return n, nil +} + +func Test_ansiStripReader_stripsSequencesSplitAcrossReads(t *testing.T) { + const red, reset = "\x1b[31m", "\x1b[0m" + for _, tc := range []struct { + name string + pieces []string + want string + }{ + {"one read", []string{"a " + red + "b" + reset + " c"}, "a b c"}, + {"split after the escape byte", []string{"a \x1b", "[31mb" + reset}, "a b"}, + {"split after the bracket", []string{"a \x1b[", "31mb"}, "a b"}, + {"split inside the parameters", []string{"a \x1b[3", "1mb"}, "a b"}, + {"split before the final byte", []string{"a \x1b[31", "mb"}, "a b"}, + {"two sequences in a row across a read", []string{"a " + red + "\x1b[", "1mb"}, "a b"}, + {"sequence open at the end of the source", []string{"a \x1b[31"}, "a \x1b[31"}, + {"lone escape at the end of the source", []string{"a \x1b"}, "a \x1b"}, + {"escape that starts no color sequence", []string{"a \x1bXb"}, "a \x1bXb"}, + {"letters inside the parameters stay", []string{"a \x1b[inva", "lidm b"}, "a \x1b[invalidm b"}, + {"empty source", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := io.ReadAll(newANSIStripReader(&splitReader{pieces: tc.pieces})) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + }) + } +} + +func Test_ansiStripReader_holdsALongTailAcrossTheChunkSize(t *testing.T) { + // The escape byte lands as the last byte of one chunk. + head := strings.Repeat("x", ansiStripChunkSize-1) + got, err := io.ReadAll(newANSIStripReader(&splitReader{pieces: []string{head + "\x1b", "[32my"}})) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != head+"y" { + t.Fatalf("got %d bytes ending in %q, want the text without the color", len(got), got[len(got)-4:]) + } +} diff --git a/cmd/cli/log_reader_test.go b/cmd/cli/log_reader_test.go new file mode 100644 index 00000000..a45ebd69 --- /dev/null +++ b/cmd/cli/log_reader_test.go @@ -0,0 +1,646 @@ +package cli + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +const ( + boundedLogTestLineSize = 100 + boundedLogTestLineCount = 30 + boundedLogTestFileCount = 5 + boundedLogTestBudget = 10000 +) + +// boundedLogTestLine returns a line of a fixed size that names its file, so a +// test can tell one part from another. +func boundedLogTestLine(file, line int) []byte { + head := fmt.Sprintf("file=%d line=%d ", file, line) + return []byte(head + strings.Repeat("x", boundedLogTestLineSize-len(head)-1) + "\n") +} + +// writeBoundedLogTestFiles creates the log files of one stream, oldest first. +func writeBoundedLogTestFiles(t *testing.T) []string { + t.Helper() + dir := t.TempDir() + files := make([]string, 0, boundedLogTestFileCount) + for file := 0; file < boundedLogTestFileCount; file++ { + var content bytes.Buffer + for line := 0; line < boundedLogTestLineCount; line++ { + content.Write(boundedLogTestLine(file, line)) + } + path := filepath.Join(dir, fmt.Sprintf("ctrld.log.%d", boundedLogTestFileCount-file)) + if err := os.WriteFile(path, content.Bytes(), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + files = append(files, path) + } + return files +} + +func logFilesContent(t *testing.T, files []string) []byte { + t.Helper() + var joined []byte + for _, path := range files { + joined = append(joined, []byte(rotatingFileTestContent(t, path))...) + } + return joined +} + +// boundedLogTestFileSize is the size of each file that the bounded reader +// tests select from. +const boundedLogTestFileSize = int64(boundedLogTestLineCount * boundedLogTestLineSize) + +// wantTailLine reports the first line that fits in budget, for a budget that +// ends inside the fourth newest file. A cut that lands on a line start keeps +// that line. +func wantTailLine(budget int64) int { + cut := boundedLogTestFileSize - (budget - 3*boundedLogTestFileSize) + line := cut / boundedLogTestLineSize + if cut%boundedLogTestLineSize != 0 { + line++ + } + return int(line) +} + +// assertTailPart checks that a part starts at a line boundary and that it +// starts with the expected whole line. +func assertTailPart(t *testing.T, part logFilePart, file, line int) { + t.Helper() + content := []byte(rotatingFileTestContent(t, part.path)) + if part.offset <= 0 || part.offset >= int64(len(content)) { + t.Fatalf("offset = %d, want a position inside %s", part.offset, part.path) + } + if content[part.offset-1] != '\n' { + t.Fatalf("offset %d in %s does not follow a newline", part.offset, part.path) + } + if want := boundedLogTestLine(file, line); !bytes.HasPrefix(content[part.offset:], want) { + t.Fatalf("tail of %s starts with %.24q, want %.24q", part.path, content[part.offset:], want) + } +} + +// closeLogParts closes the files of parts that no upload took. +func closeLogParts(parts []logFilePart) { + for _, part := range parts { + part.f.Close() + } +} + +// partsSize reports the bytes that the parts hold on disk. +func partsSize(t *testing.T, parts []logFilePart) int64 { + t.Helper() + var size int64 + for _, part := range parts { + content := rotatingFileTestContent(t, part.path) + size += int64(len(content)) - part.offset + } + return size +} + +// assertBoundedParts checks the shape that every bounded selection shares: +// the newest three files whole, oldest first, after the tail of the fourth. +func assertBoundedParts(t *testing.T, files []string, parts []logFilePart, tailLine int) { + t.Helper() + wantPaths := files[1:] + if len(parts) != len(wantPaths) { + t.Fatalf("parts = %d, want %d", len(parts), len(wantPaths)) + } + for i, part := range parts { + if part.path != wantPaths[i] { + t.Fatalf("parts[%d].path = %s, want %s", i, part.path, wantPaths[i]) + } + } + assertTailPart(t, parts[0], 1, tailLine) + for i, part := range parts[1:] { + if part.offset != 0 { + t.Fatalf("parts[%d].offset = %d, want the whole file", i+1, part.offset) + } + } +} + +func Test_boundedLogFiles(t *testing.T) { + files := writeBoundedLogTestFiles(t) + + t.Run("budget ends at a line boundary", func(t *testing.T) { + parts := boundedLogFiles(files, boundedLogTestBudget) + t.Cleanup(func() { closeLogParts(parts) }) + + assertBoundedParts(t, files, parts, wantTailLine(boundedLogTestBudget)) + // The cut lands on a line start, so the tail begins at the cut and the + // parts fill the budget exactly. + if size := partsSize(t, parts); size != boundedLogTestBudget { + t.Fatalf("size = %d, want the whole budget of %d", size, boundedLogTestBudget) + } + }) + + t.Run("budget ends inside a line", func(t *testing.T) { + const budget = boundedLogTestBudget + boundedLogTestLineSize/2 + parts := boundedLogFiles(files, budget) + t.Cleanup(func() { closeLogParts(parts) }) + + assertBoundedParts(t, files, parts, wantTailLine(budget)) + if size := partsSize(t, parts); size > budget { + t.Fatalf("size = %d, want at most %d", size, budget) + } + }) + + t.Run("no budget keeps every file", func(t *testing.T) { + parts := boundedLogFiles(files, 0) + t.Cleanup(func() { closeLogParts(parts) }) + + if len(parts) != len(files) { + t.Fatalf("parts = %d, want %d", len(parts), len(files)) + } + for i, part := range parts { + if part.path != files[i] || part.offset != 0 { + t.Fatalf("parts[%d] = %+v, want %s whole", i, part, files[i]) + } + } + want := int64(boundedLogTestFileCount) * boundedLogTestFileSize + if size := partsSize(t, parts); size != want { + t.Fatalf("size = %d, want %d", size, want) + } + }) +} + +// Test_logReaderDoesNotFollowASymlink covers the one open of the reader. A +// symlink in place of a log file points at a file of another owner, and the +// writer refuses it in the same way. +func Test_logReaderDoesNotFollowASymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the Windows opener takes no symlink flag") + } + dir := t.TempDir() + planted := filepath.Join(dir, "planted.log") + if err := os.WriteFile(planted, []byte("planted line\n"), 0600); err != nil { + t.Fatalf("write the planted file: %v", err) + } + path := filepath.Join(dir, "ctrld.log") + if err := os.Symlink(planted, path); err != nil { + t.Fatalf("symlink: %v", err) + } + + parts := boundedLogFiles([]string{path}, 4) + if len(parts) != 0 { + t.Fatalf("the selection opened a symlink: %+v", parts) + } + var upload logParts + upload.addFiles(parts) + if upload.size != 0 { + t.Fatalf("the upload holds %d bytes of a symlink, want none", upload.size) + } +} + +// internalLogReaderTestBudget keeps the files small: four lines fill one file, +// so a few writes build the whole backup chain. +var internalLogReaderTestBudget = logBudget{maxSize: 4 * rotatingFileTestLineSize, backups: 2} + +const internalLogReaderTestLines = 12 + +// newTestStreamWriter opens one persisted stream in the temporary home +// directory. +func newTestStreamWriter(t *testing.T, name string) *logWriter { + t.Helper() + lw := newLogWriterWithSize(logWriterSize) + if err := lw.setLogFile(absHomeDir(name), internalLogReaderTestBudget); err != nil { + t.Fatalf("setLogFile %s: %v", name, err) + } + t.Cleanup(lw.closeLogFile) + return lw +} + +// internalLogReaderTestSendBudget holds one debug file and a half, so the +// default upload cuts inside the oldest of the three files. +var internalLogReaderTestSendBudget = internalLogReaderTestBudget.maxSize * 3 / 2 + +// useTestSendBudget lowers the bound on the debug bytes of one upload, so a +// few small files cross it. +func useTestSendBudget(t *testing.T, budget int64) { + t.Helper() + origBudget := logSendDebugBudget + t.Cleanup(func() { logSendDebugBudget = origBudget }) + logSendDebugBudget = budget +} + +// setupInternalLogReaderTest returns a cd mode prog whose debug stream and +// journal stream both hold a full backup chain on disk. +func setupInternalLogReaderTest(t *testing.T) *prog { + t.Helper() + origSilent, origCdUID, origHomedir := silent, cdUID, homedir + t.Cleanup(func() { silent, cdUID, homedir = origSilent, origCdUID, origHomedir }) + captureMainLog(t) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID, homedir = false, "test-uid", t.TempDir() + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.internalLogWriter = newTestStreamWriter(t, logFileName) + p.internalJournalWriter = newTestStreamWriter(t, journalLogFileName) + for seq := 1; seq <= internalLogReaderTestLines; seq++ { + if _, err := p.internalLogWriter.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write debug line %d: %v", seq, err) + } + if _, err := p.internalJournalWriter.Write(rotatingFileTestLine(2, seq)); err != nil { + t.Fatalf("write journal line %d: %v", seq, err) + } + } + return p +} + +// readLogReader reads one upload and checks that the reported size is the +// number of bytes it yields. +func readLogReader(t *testing.T, p *prog, full bool) []byte { + t.Helper() + lr, err := p.logReader(full, false) + if err != nil { + t.Fatalf("logReader(%v): %v", full, err) + } + defer lr.r.Close() + data, err := io.ReadAll(lr.r) + if err != nil { + t.Fatalf("read upload: %v", err) + } + if lr.size != int64(len(data)) { + t.Fatalf("size = %d, want %d", lr.size, len(data)) + } + return data +} + +// splitSendHeader takes the header line off the front of an upload and checks +// that it names this request. It returns the rest of the upload. +func splitSendHeader(t *testing.T, upload []byte, since time.Time) []byte { + t.Helper() + end := bytes.IndexByte(upload, '\n') + if end < 0 { + t.Fatalf("upload holds no header line: %.64q", upload) + } + parsed, _ := parseLogHeader(t, upload[:end+1]) + if parsed.Message != logHeaderMessage { + t.Fatalf("line 1 of the upload = %q, want %q", parsed.Message, logHeaderMessage) + } + if parsed.Trigger != "send" { + t.Fatalf("trigger = %q, want %q", parsed.Trigger, "send") + } + rendered, err := time.Parse(time.RFC3339, parsed.Time) + if err != nil { + t.Fatalf("parse header time %q: %v", parsed.Time, err) + } + // The header format drops the fraction of a second, so the window starts + // at the second that holds since. + if rendered.Before(since.Truncate(time.Second)) || rendered.After(time.Now().Add(time.Second)) { + t.Fatalf("header time = %s, want a time between %s and now", rendered, since) + } + return upload[end+1:] +} + +// splitUpload takes the header off an upload and cuts the rest at the log end +// marker, which parts the debug bytes from the journal bytes. +func splitUpload(t *testing.T, upload []byte, since time.Time) (debugPart, journalPart []byte) { + t.Helper() + body := splitSendHeader(t, upload, since) + debugPart, journalPart, found := bytes.Cut(body, []byte(logWriterLogEndMarker)) + if !found { + t.Fatalf("upload holds no log end marker: %.64q", body) + } + return debugPart, journalPart +} + +func Test_logRequestPath(t *testing.T) { + if got := logRequestPath(sendLogsPath, false); got != sendLogsPath { + t.Fatalf("logRequestPath(%q, false) = %q, want the path itself", sendLogsPath, got) + } + want := viewLogsPath + "?full=1" + if got := logRequestPath(viewLogsPath, true); got != want { + t.Fatalf("logRequestPath(%q, true) = %q, want %q", viewLogsPath, got, want) + } +} + +func Test_logReader_internalComposition(t *testing.T) { + p := setupInternalLogReaderTest(t) + useTestSendBudget(t, internalLogReaderTestSendBudget) + + debugFiles := p.internalLogWriter.rotating().paths() + journalFiles := p.internalJournalWriter.rotating().paths() + wantFiles := internalLogReaderTestBudget.backups + 1 + if len(debugFiles) != wantFiles || len(journalFiles) != wantFiles { + t.Fatalf("files: debug %v, journal %v, want %d of each", debugFiles, journalFiles, wantFiles) + } + wholeDebug := logFilesContent(t, debugFiles) + wholeJournal := logFilesContent(t, journalFiles) + + start := time.Now() + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start) + + if int64(len(debugPart)) > logSendDebugBudget { + t.Fatalf("debug part = %d bytes, want at most the budget of %d", len(debugPart), logSendDebugBudget) + } + if len(debugPart) >= len(wholeDebug) { + t.Fatalf("debug part = %d bytes, want fewer than the %d bytes of %v", len(debugPart), len(wholeDebug), debugFiles) + } + if !bytes.HasSuffix(wholeDebug, debugPart) { + t.Fatalf("debug part = %.24q, want the newest %d bytes of %v", debugPart, len(debugPart), debugFiles) + } + cut := len(wholeDebug) - len(debugPart) + if wholeDebug[cut-1] != '\n' { + t.Fatalf("the debug part starts at %d, which does not follow a newline", cut) + } + if !bytes.Equal(journalPart, wholeJournal) { + t.Fatalf("journal part = %d bytes, want the %d bytes of %v", len(journalPart), len(wholeJournal), journalFiles) + } + + // The full upload lifts the bound, so it holds every debug file. + fullDebug, fullJournal := splitUpload(t, readLogReader(t, p, true), start) + if !bytes.Equal(fullDebug, wholeDebug) { + t.Fatalf("full debug part = %d bytes, want the %d bytes of %v", len(fullDebug), len(wholeDebug), debugFiles) + } + if !bytes.Equal(fullJournal, wholeJournal) { + t.Fatalf("full journal part = %d bytes, want the %d bytes of %v", len(fullJournal), len(wholeJournal), journalFiles) + } +} + +func Test_logReader_logPathComposition(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en9") + initLoggingWithBackup(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + files := logPathFile.Load().paths() + if want := []string{path + ".1", path}; !slices.Equal(files, want) { + t.Fatalf("paths() = %v, want %v", files, want) + } + + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + if want := logFilesContent(t, files); !bytes.Equal(got, want) { + t.Fatalf("upload = %d bytes, want the %d bytes of %v", len(got), len(want), files) + } + if bytes.Contains(got, []byte(logWriterLogEndMarker)) { + t.Fatalf("log_path upload holds the log end marker") + } +} + +// setupMemoryLogReaderTest returns a cd mode prog whose streams keep their +// lines in memory, because no file is open. +func setupMemoryLogReaderTest(t *testing.T) *prog { + t.Helper() + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + captureMainLog(t) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID = false, "test-uid" + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.internalLogWriter = newLogWriterWithSize(logWriterSize) + p.internalJournalWriter = newLogWriterWithSize(logWriterSize) + return p +} + +func Test_logReader_memoryFallbackStartsWithHeader(t *testing.T) { + p := setupMemoryLogReaderTest(t) + debugLine := []byte("debug line\n") + journalLine := []byte("journal line\n") + if _, err := p.internalLogWriter.Write(debugLine); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + if _, err := p.internalJournalWriter.Write(journalLine); err != nil { + t.Fatalf("write the journal buffer: %v", err) + } + + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + + var want []byte + want = append(want, debugLine...) + want = append(want, []byte(logWriterLogEndMarker)...) + want = append(want, journalLine...) + if !bytes.Equal(got, want) { + t.Fatalf("upload after the header = %q, want %q", got, want) + } +} + +// useTestHomeDir points the ctrld home directory at a temporary directory, so +// the internal log files of a test never reach the real one. +func useTestHomeDir(t *testing.T) { + t.Helper() + origHomedir := homedir + t.Cleanup(func() { homedir = origHomedir }) + homedir = t.TempDir() +} + +// newSeededJournalWriter writes the journal files of an earlier run and +// returns a new writer over them. Its memory buffer is empty, which is what a +// restart finds, so a rule that reads the buffer loses these lines. +func newSeededJournalWriter(t *testing.T, lines int) *logWriter { + t.Helper() + previous := newTestStreamWriter(t, journalLogFileName) + for seq := 1; seq <= lines; seq++ { + if _, err := previous.Write(rotatingFileTestLine(2, seq)); err != nil { + t.Fatalf("write journal line %d: %v", seq, err) + } + } + previous.closeLogFile() + return newTestStreamWriter(t, journalLogFileName) +} + +// mixedStreamJournalLines fill two journal files, because four lines fill one. +const mixedStreamJournalLines = 6 + +func Test_logReader_mixedStreams(t *testing.T) { + p := setupMemoryLogReaderTest(t) + useTestHomeDir(t) + p.internalJournalWriter = newSeededJournalWriter(t, mixedStreamJournalLines) + debugLine := []byte("debug line\n") + if _, err := p.internalLogWriter.Write(debugLine); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + journalFiles := p.internalJournalWriter.rotating().paths() + if len(journalFiles) != 2 { + t.Fatalf("journal files = %v, want two", journalFiles) + } + + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + + var want []byte + want = append(want, debugLine...) + want = append(want, []byte(logWriterLogEndMarker)...) + want = append(want, logFilesContent(t, journalFiles)...) + if !bytes.Equal(got, want) { + t.Fatalf("upload after the header = %d bytes, want the %d bytes of the debug buffer, the marker and %v", len(got), len(want), journalFiles) + } +} + +func Test_logReader_emptyStaysEmpty(t *testing.T) { + t.Run("internal mode without a line", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + + lr, err := p.logReader(false, false) + + if !errors.Is(err, errInternalLogEmpty) { + t.Fatalf("logReader error = %v, want %v", err, errInternalLogEmpty) + } + if lr != nil { + t.Fatalf("logReader = %+v, want no reader", lr) + } + }) + + t.Run("internal mode with empty files", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + useTestHomeDir(t) + p.internalLogWriter = newTestStreamWriter(t, logFileName) + p.internalJournalWriter = newTestStreamWriter(t, journalLogFileName) + + lr, err := p.logReader(false, false) + + if !errors.Is(err, errInternalLogEmpty) { + t.Fatalf("logReader error = %v, want %v", err, errInternalLogEmpty) + } + if lr != nil { + t.Fatalf("logReader = %+v, want no reader", lr) + } + }) + + t.Run("no log_path to read", func(t *testing.T) { + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID = false, "" + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + lr, err := p.logReader(false, false) + if err != nil { + t.Fatalf("logReader: %v", err) + } + defer lr.r.Close() + + if lr.size != 0 { + t.Fatalf("size = %d, want 0 so the handler answers 301", lr.size) + } + data, err := io.ReadAll(lr.r) + if err != nil { + t.Fatalf("read the upload: %v", err) + } + if len(data) != 0 { + t.Fatalf("upload = %q, want no bytes", data) + } + }) +} + +// Test_boundedLogFilesFixTheSizeAtSelection appends through the real writer +// between the selection and the reader creation. The upload must hold the +// bytes that the selection counted, so the budget holds under a live writer. +func Test_boundedLogFilesFixTheSizeAtSelection(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 1 << 20, backups: 1}) + if _, err := rf.Write([]byte("one\n")); err != nil { + t.Fatal(err) + } + parts := boundedLogFiles(rf.paths(), 4) + if _, err := rf.Write([]byte("two\n")); err != nil { + t.Fatal(err) + } + + var upload logParts + upload.addFiles(parts) + t.Cleanup(upload.close) + got, err := io.ReadAll(io.MultiReader(upload.readers...)) + if err != nil { + t.Fatal(err) + } + if upload.size != 4 || string(got) != "one\n" { + t.Fatalf("upload = %q (%d bytes), want %q within the budget of 4", got, upload.size, "one\n") + } +} + +// seedFullLogChain writes a full chain of files at the desktop budget: the +// current file and every backup, each file at its size limit. Every line names +// its file, so a reader can tell which file a part comes from. +func seedFullLogChain(t *testing.T, name string, budget logBudget) { + t.Helper() + path := absHomeDir(name) + for index := budget.backups; index >= 0; index-- { + filePath := path + if index > 0 { + filePath = fmt.Sprintf("%s.%d", path, index) + } + line := fmt.Sprintf("%s file=%d ", name, index) + line += strings.Repeat("x", rotatingFileTestLineSize-len(line)-1) + "\n" + content := bytes.Repeat([]byte(line), int(budget.maxSize)/len(line)) + if err := os.WriteFile(filePath, content, 0o600); err != nil { + t.Fatal(err) + } + } +} + +// Test_logReader_desktopBudgetUpload uses the desktop budgets, not the small +// test budgets: 50 MB of debug files and a full journal chain. The normal +// upload must stay under 17 MB, and the full upload must hold every file. +func Test_logReader_desktopBudgetUpload(t *testing.T) { + if testing.Short() { + t.Skip("writes 56 MB of log files") + } + origSilent, origCdUID, origHomedir := silent, cdUID, homedir + t.Cleanup(func() { silent, cdUID, homedir = origSilent, origCdUID, origHomedir }) + captureMainLog(t) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID, homedir = false, "test-uid", t.TempDir() + debugBudget, journalBudget := logBudgets() + seedFullLogChain(t, logFileName, debugBudget) + seedFullLogChain(t, journalLogFileName, journalBudget) + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.internalLogWriter = newLogWriterWithSize(logWriterSize) + p.internalJournalWriter = newLogWriterWithSize(logWriterSmallSize) + for _, stream := range []struct { + lw *logWriter + name string + budget logBudget + }{{p.internalLogWriter, logFileName, debugBudget}, {p.internalJournalWriter, journalLogFileName, journalBudget}} { + if err := stream.lw.setLogFile(absHomeDir(stream.name), stream.budget); err != nil { + t.Fatalf("setLogFile %s: %v", stream.name, err) + } + t.Cleanup(stream.lw.closeLogFile) + } + wholeDebug := logFilesContent(t, p.internalLogWriter.rotating().paths()) + wholeJournal := logFilesContent(t, p.internalJournalWriter.rotating().paths()) + if len(wholeDebug) < 50_000_000 { + t.Fatalf("the debug files hold %d bytes, want at least 50 MB", len(wholeDebug)) + } + + start := time.Now() + upload := readLogReader(t, p, false) + if len(upload) >= 17_000_000 { + t.Fatalf("normal upload = %d bytes, want less than 17 MB", len(upload)) + } + debugPart, journalPart := splitUpload(t, upload, start) + if int64(len(debugPart)) > logSendDebugBudget || !bytes.HasSuffix(wholeDebug, debugPart) { + t.Fatalf("debug part = %d bytes, want the newest bytes within %d", len(debugPart), logSendDebugBudget) + } + if !bytes.Equal(journalPart, wholeJournal) { + t.Fatalf("journal part = %d bytes, want the %d bytes of the journal chain", len(journalPart), len(wholeJournal)) + } + + fullDebug, fullJournal := splitUpload(t, readLogReader(t, p, true), start) + if !bytes.Equal(fullDebug, wholeDebug) || !bytes.Equal(fullJournal, wholeJournal) { + t.Fatalf("full upload = %d debug and %d journal bytes, want %d and %d", len(fullDebug), len(fullJournal), len(wholeDebug), len(wholeJournal)) + } +} diff --git a/cmd/cli/log_send.go b/cmd/cli/log_send.go new file mode 100644 index 00000000..d7492dc2 --- /dev/null +++ b/cmd/cli/log_send.go @@ -0,0 +1,134 @@ +package cli + +import ( + "context" + "io" + "net/http" + "sync" + "time" +) + +// logUploadGate is shared by both sockets. Reserve before opening or collecting +// anything, and cool down after completion, including failures. Logging reloads +// must never reset this state. +type logUploadGate struct { + mu sync.Mutex + busy bool + next time.Time +} + +func (g *logUploadGate) reserve() bool { + g.mu.Lock() + defer g.mu.Unlock() + if g.busy || time.Now().Before(g.next) { + return false + } + g.busy = true + return true +} + +func (g *logUploadGate) release() { + g.mu.Lock() + defer g.mu.Unlock() + g.busy = false + g.next = time.Now().Add(logWriterSentInterval) +} + +const delegatedSendTimeout = 5 * time.Minute + +type logSendPeerKey struct{} + +// logSendServer has no administrative mux. Its dependencies are immutable +// snapshots of daemon state, never request arguments or reloaded config pointers. +type logSendServer struct { + gate *logUploadGate + collect func(context.Context) (io.ReadCloser, error) + upload func(context.Context, io.ReadCloser) error + audit func(int64, int) + server *http.Server + cancel context.CancelFunc + mu sync.Mutex + stopped bool + active sync.WaitGroup +} + +func (s *logSendServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + uid, known := r.Context().Value(logSendPeerKey{}).(uint32) + caller := int64(-1) + if known { + caller = int64(uid) + } + code := http.StatusForbidden + accepted := false + admitted := false + defer func() { + if admitted { + defer s.active.Done() + } + if accepted && s.audit != nil { + s.audit(caller, code) + } + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Connection", "close") + w.WriteHeader(code) // status only: no logs, credentials, paths or API errors + }() + if !known { + return + } + if r.Method != http.MethodPost || r.RequestURI != sendLogsPath || r.URL.RawQuery != "" || r.ContentLength != 0 || len(r.TransferEncoding) != 0 || r.Header.Get("Expect") != "" { + code = http.StatusBadRequest + return + } + s.mu.Lock() + if s.stopped { + s.mu.Unlock() + code = http.StatusServiceUnavailable + return + } + s.active.Add(1) + admitted = true + s.mu.Unlock() + if !s.gate.reserve() { + code = http.StatusServiceUnavailable + return + } + accepted = true + defer s.gate.release() + ctx, cancel := context.WithTimeout(r.Context(), delegatedSendTimeout) + defer cancel() + body, err := s.collect(ctx) + if err != nil { + code = http.StatusPreconditionFailed + return + } + defer body.Close() + if err := s.upload(ctx, body); err != nil { + code = http.StatusBadGateway + return + } + code = http.StatusOK +} + +func (s *logSendServer) stop() { + s.mu.Lock() + s.stopped = true + s.mu.Unlock() + if s.cancel != nil { + s.cancel() + } + if s.server != nil { + _ = s.server.Close() + } + s.active.Wait() +} + +// The lifecycle lock also covers startup racing with shutdown. Stop fences new +// requests, cancels active uploads, and joins them before log writers close. +func (p *prog) stopLogSendServer() { + p.logSendMu.Lock() + defer p.logSendMu.Unlock() + p.logSendStopped = true + if p.logSend != nil { + p.logSend.stop() + } +} diff --git a/cmd/cli/log_send_audit_test.go b/cmd/cli/log_send_audit_test.go new file mode 100644 index 00000000..d2efbda4 --- /dev/null +++ b/cmd/cli/log_send_audit_test.go @@ -0,0 +1,37 @@ +package cli + +import ( + "context" + "errors" + "io" + "net/http/httptest" + "testing" +) + +func TestDelegatedLogSendAuditDoesNotAmplifyRejections(t *testing.T) { + var audits int + s := &logSendServer{gate: &logUploadGate{}, + collect: func(context.Context) (io.ReadCloser, error) { return nil, errors.New("synthetic failure") }, + audit: func(uid int64, status int) { + audits++ + if uid != 501 || status != 412 { + t.Errorf("audit uid=%d status=%d", uid, status) + } + }, + } + for i := 0; i < 100; i++ { + s.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", sendLogsPath, nil)) + req := httptest.NewRequest("GET", sendLogsPath+"?full=1", nil).WithContext(context.WithValue(context.Background(), logSendPeerKey{}, uint32(501))) + s.ServeHTTP(httptest.NewRecorder(), req) + } + if audits != 0 { + t.Fatalf("rejected request audits=%d", audits) + } + req := httptest.NewRequest("POST", sendLogsPath, nil).WithContext(context.WithValue(context.Background(), logSendPeerKey{}, uint32(501))) + for i := 0; i < 100; i++ { + s.ServeHTTP(httptest.NewRecorder(), req) + } + if audits != 1 { + t.Fatalf("admitted plus cooldown audits=%d, want 1", audits) + } +} diff --git a/cmd/cli/log_send_darwin.go b/cmd/cli/log_send_darwin.go new file mode 100644 index 00000000..7a85b31d --- /dev/null +++ b/cmd/cli/log_send_darwin.go @@ -0,0 +1,193 @@ +package cli + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "syscall" + "time" + "unsafe" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" + "golang.org/x/sys/unix" +) + +// /private/var/run is group-writable on macOS and fails the trust walk. +const logSendSocketDir = "/private/var/ctrld-log-send" +const logSendSocketPath = logSendSocketDir + "/send.sock" +const delegatedJournalBytes int64 = 6 << 20 + +func logSendPeerUID(fd int) (uint32, error) { + cred, err := unix.GetsockoptXucred(fd, unix.SOL_LOCAL, unix.LOCAL_PEERCRED) + if err != nil { + return 0, err + } + // XNU bsd/sys/ucred.h defines XUCRED_VERSION as 0; this version of + // x/sys/unix exposes Xucred but does not export that constant. + if cred.Version != 0 { + return 0, errLogSendTrust + } + return cred.Uid, nil +} + +// A POSIX mode alone cannot exclude a macOS ACL grant. Conservatively refuse +// extended ACLs (even deny-only ACLs), and fail closed if the kernel cannot report +// them. fgetattrlist returns a length and an attrreference; absent ACL => length 0. +// See XNU vfs_attrlist.c, ATTR_CMN_EXTENDED_SECURITY packing. Using the open fd +// avoids a pathname race and does not need cgo or a privileged helper process. +func logSendCheckACL(fd int) error { + attrs := unix.Attrlist{Bitmapcount: 5, Commonattr: unix.ATTR_CMN_EXTENDED_SECURITY} + var buf [12]byte + //lint:ignore SA1019 x/sys has no fgetattrlist libSystem wrapper; preserve descriptor-relative ACL checks in CGO_ENABLED=0 builds. + _, _, errno := syscall.Syscall6(unix.SYS_FGETATTRLIST, uintptr(fd), uintptr(unsafe.Pointer(&attrs)), uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf)), 0, 0) + if errno != 0 { + return errno + } + if binary.LittleEndian.Uint32(buf[:4]) != 12 || binary.LittleEndian.Uint32(buf[8:]) != 0 { + return errLogSendTrust + } + return nil +} + +func delegatedLogSendCLI() bool { return os.Geteuid() != 0 } +func runDelegatedLogSend(ctx context.Context, full bool) error { + err := requestDelegatedLogSend(ctx, full, func(ctx context.Context) (net.Conn, error) { + return dialLogSend(ctx, trustedLogTree{root: "/", uid: 0}, logSendSocketPath) + }) + if err != nil { + return fmt.Errorf("failed to send logs: %w", err) + } + mainLog.Load().Notice().Msg("Runtime logs sent successfully") + return nil +} + +func (p *prog) startLogSendServer() { + p.logSendMu.Lock() + defer p.logSendMu.Unlock() + if p.logSendStopped || p.logSend != nil { + return + } + p.mu.Lock() + enabled := p.cfg.Service.AllowUnprivilegedLogSend + p.mu.Unlock() + if !enabled { + return + } + s, err := p.newDelegatedLogSendServer() + if err != nil { + p.Error().Err(err).Msg("Unprivileged log send disabled: unsafe or unavailable configuration, logs or socket; correct as administrator and restart") + return + } + p.logSend = s + ctrld.Journal(p.Info()).Msg("Unprivileged log send enabled for all local users until service stop") +} + +func (p *prog) newDelegatedLogSendServer() (*logSendServer, error) { + if os.Geteuid() != 0 || cdUID == "" || configBase64 != "" { + return nil, errLogSendTrust + } + tree := trustedLogTree{root: "/", uid: 0} + collector := delegatedLogCollector{tree: tree, config: canonicalLogSendPath(configFileForWrite())} + // Capture source paths and budgets once. A reload cannot redirect this socket + // to new files, identities, or destinations, nor enable an initially off socket. + var files []*rotatingFile + if p.needInternalLogging() { + lw, jw := p.internalWriters() + if lw == nil || jw == nil { + return nil, errLogSendTrust + } + files = []*rotatingFile{lw.rotating(), jw.rotating()} + } else { + files = []*rotatingFile{logPathFile.Load()} + } + for i, rf := range files { + if rf == nil { + return nil, errLogSendTrust + } + rf.mu.Lock() + source := delegatedLogSource{path: canonicalLogSendPath(rf.path), backups: rf.budget.backups, budget: delegatedDebugBytes} + rf.mu.Unlock() + if i == 1 { + source.budget = delegatedJournalBytes + } + collector.sources = append(collector.sources, source) + } + listener, err := collector.listen(logSendSocketDir) + if err != nil { + return nil, err + } + uid, dev := cdUID, cdDev + s := &logSendServer{gate: &p.logUpload, collect: collector.collect, + upload: func(ctx context.Context, body io.ReadCloser) error { + return controld.SendLogs(ctrld.LoggerCtx(ctx, p.logger.Load()), &controld.LogsRequest{UID: uid, Data: body}, dev) + }, + audit: func(uid int64, code int) { + ctrld.Journal(p.Info()).Int64("caller_uid", uid).Int("status", code).Msg("Unprivileged log send outcome") + }, + } + s.serve(listener, logSendPeer) + return s, nil +} + +func (c delegatedLogCollector) listen(dir string) (net.Listener, error) { + // Validate before exposing any capability; repeat the same checks per upload. + if err := c.validate(context.Background()); err != nil { + return nil, err + } + return listenDelegatedLogSend(c.tree, dir) +} + +func listenDelegatedLogSend(tree trustedLogTree, dir string) (net.Listener, error) { + parent, err := tree.open(filepath.Dir(dir), true) + if err != nil { + return nil, err + } + defer parent.Close() + if err := unix.Mkdirat(int(parent.Fd()), filepath.Base(dir), 0755); err != nil && !errors.Is(err, unix.EEXIST) { + return nil, err + } + protected, err := tree.open(dir, true) + if err != nil { + return nil, err + } + protected.Close() + path := filepath.Join(dir, "send.sock") + if st, err := os.Lstat(path); err == nil { + stat, ok := st.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != tree.uid || st.Mode()&os.ModeSocket == 0 { + return nil, errLogSendTrust + } + c, err := net.DialTimeout("unix", path, time.Second) + if err == nil { + c.Close() + return nil, errors.New("log-send socket already active") + } + // Only an unequivocally stale endpoint may be removed, not timeout/permission errors. + if !errors.Is(err, syscall.ECONNREFUSED) { + return nil, err + } + if err := os.Remove(path); err != nil { + return nil, err + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + l, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"}) + if err != nil { + return nil, err + } + // No handler is serving until chmod succeeds. Parent ownership prevents + // replacement. The administrative control socket remains 0600. + if err := os.Chmod(path, 0666); err != nil { + l.Close() + return nil, err + } + l.SetUnlinkOnClose(true) + return l, nil +} diff --git a/cmd/cli/log_send_darwin_test.go b/cmd/cli/log_send_darwin_test.go new file mode 100644 index 00000000..4f3c2386 --- /dev/null +++ b/cmd/cli/log_send_darwin_test.go @@ -0,0 +1,188 @@ +package cli + +import ( + "context" + "encoding/base64" + "errors" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/viper" +) + +// These tests touch only owned temporary files and sockets: no service, PF, +// DNS, elevation, or Control D network calls. Execute on native macOS. +func TestDelegatedLogSendDarwinACL(t *testing.T) { + tree, c := logSendFixture(t) + if got := readDelegated(t, c); got != "89abcdef" { + t.Fatal(got) + } + for _, target := range []string{c.config, c.sources[0].path, tree.root} { + t.Run(filepath.Base(target), func(t *testing.T) { + cmd := exec.Command("/bin/chmod", "+a", "everyone allow read", target) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("set temp ACL: %v: %s", err, out) + } + defer func() { + if out, err := exec.Command("/bin/chmod", "-N", target).CombinedOutput(); err != nil { + t.Errorf("remove temp ACL: %v: %s", err, out) + } + }() + if r, err := c.collect(context.Background()); err == nil { + r.Close() + t.Fatal("extended ACL accepted") + } + }) + } +} + +func TestDelegatedLogSendDarwinProductionParent(t *testing.T) { + // Read only: exercise the actual ancestor chain, not a 0700 temp substitute. + tree := trustedLogTree{root: "/", uid: 0} + parent, err := tree.open(filepath.Dir(logSendSocketDir), true) + if err != nil { + t.Fatalf("production socket parent is not trusted: %v", err) + } + parent.Close() +} + +func TestDelegatedLogSendDarwinWritableParent(t *testing.T) { + tree, _ := logSendFixture(t) + parent := filepath.Join(tree.root, "run") + if err := os.Mkdir(parent, 0755); err != nil { + t.Fatal(err) + } + // Chmod after mkdir makes the fixture independent of the runner's umask. + if err := os.Chmod(parent, 0775); err != nil { + t.Fatal(err) + } + dir := filepath.Join(parent, "socket") + if l, err := listenDelegatedLogSend(tree, dir); !errors.Is(err, errLogSendTrust) { + if l != nil { + l.Close() + } + t.Fatalf("group-writable parent listener error=%v", err) + } + if err := os.Mkdir(dir, 0755); err != nil { + t.Fatal(err) + } + if c, err := dialLogSend(context.Background(), tree, filepath.Join(dir, "send.sock")); !errors.Is(err, errLogSendTrust) { + if c != nil { + c.Close() + } + t.Fatalf("group-writable parent dial error=%v", err) + } + st, err := os.Stat(parent) + if err != nil || st.Mode().Perm() != 0775 { + t.Fatalf("parent permissions were changed: %v, %v", st, err) + } +} + +func TestDelegatedLogSendDarwinEmptyStartup(t *testing.T) { + tree, collector := logSendFixture(t) + writeLogSendFixture(t, collector.sources[0].path, "") + dir := filepath.Join(tree.root, "socket") + l, err := collector.listen(dir) + if err != nil { + t.Fatalf("empty trusted logs prevent listener startup: %v", err) + } + s := &logSendServer{gate: &logUploadGate{}, collect: collector.collect, + upload: func(context.Context, io.ReadCloser) error { t.Error("uploaded empty logs"); return nil }, + } + s.serve(l, logSendPeer) + t.Cleanup(s.stop) + err = requestDelegatedLogSend(context.Background(), false, func(ctx context.Context) (net.Conn, error) { + return dialLogSend(ctx, tree, filepath.Join(dir, "send.sock")) + }) + if err == nil || !strings.Contains(err.Error(), "administrator must check") { + t.Fatalf("empty upload should return a precondition error: %v", err) + } + s.stop() + writeLogSendFixture(t, collector.config, "[service]\nallow_unprivileged_log_send = false\n") + if l, err := collector.listen(dir); err == nil { + l.Close() + t.Fatal("disabled config exposed a listener") + } +} + +func TestDelegatedLogSendDarwinListenerLifecycle(t *testing.T) { + tree, _ := logSendFixture(t) + dir := filepath.Join(tree.root, "socket") + l, err := listenDelegatedLogSend(tree, dir) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "send.sock") + st, err := os.Stat(path) + if err != nil { + l.Close() + t.Fatal(err) + } + if st.Mode().Perm() != 0666 { + t.Errorf("socket mode=%o", st.Mode().Perm()) + } + if other, err := listenDelegatedLogSend(tree, dir); err == nil { + other.Close() + t.Error("replaced live listener") + } + if err := l.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("socket retained: %v", err) + } + if err := os.Symlink(tree.root, dir+"-alias"); err != nil { + t.Fatal(err) + } + if other, err := listenDelegatedLogSend(tree, dir+"-alias"); err == nil { + other.Close() + t.Fatal("symlink parent accepted") + } +} + +func TestDelegatedLogSendConfigPathAfterCustomConfig(t *testing.T) { + oldV, oldFile, oldPath := v, defaultConfigFile, configPath + t.Cleanup(func() { v, defaultConfigFile, configPath = oldV, oldFile, oldPath }) + + dir := t.TempDir() + onDisk := filepath.Join(dir, "ctrld.toml") + if err := os.WriteFile(onDisk, []byte("[service]\nallow_unprivileged_log_send = true\n"), 0644); err != nil { + t.Fatal(err) + } + // Service start: --config names the file and readConfigFile reads it. + v = viper.NewWithOptions(viper.KeyDelimiter("::")) + configPath, defaultConfigFile = onDisk, onDisk + v.SetConfigFile(onDisk) + if err := v.ReadInConfig(); err != nil { + t.Fatal(err) + } + if got := configFileForWrite(); got != onDisk { + t.Fatalf("file config: got %q, want %q", got, onDisk) + } + + // --cd with a custom config: the production loader replaces viper with an + // in-memory config, so viper no longer names any file. + custom := base64.StdEncoding.EncodeToString([]byte("[service]\nallow_unprivileged_log_send = true\n")) + if err := readBase64Config(custom); err != nil { + t.Fatal(err) + } + if v.ConfigFileUsed() != "" { + t.Fatalf("precondition: viper still names %q", v.ConfigFileUsed()) + } + for _, tc := range []struct{ name, configPath, defaultFile string }{ + {"service --config", onDisk, "ctrld.toml"}, + {"default config file", "", onDisk}, + } { + t.Run(tc.name, func(t *testing.T) { + configPath, defaultConfigFile = tc.configPath, tc.defaultFile + if got := configFileForWrite(); got != onDisk { + t.Fatalf("custom config: got %q, want %q", got, onDisk) + } + }) + } +} diff --git a/cmd/cli/log_send_linux.go b/cmd/cli/log_send_linux.go new file mode 100644 index 00000000..b9246740 --- /dev/null +++ b/cmd/cli/log_send_linux.go @@ -0,0 +1,26 @@ +package cli + +import ( + "errors" + "golang.org/x/sys/unix" +) + +// Linux supplies real peer credentials for the portable boundary tests; the +// production capability and CLI remain disabled on this platform. +func logSendPeerUID(fd int) (uint32, error) { + cred, err := unix.GetsockoptUcred(fd, unix.SOL_SOCKET, unix.SO_PEERCRED) + if err != nil { + return 0, err + } + return cred.Uid, nil +} +func logSendCheckACL(fd int) error { + _, err := unix.Fgetxattr(fd, "system.posix_acl_access", nil) + if errors.Is(err, unix.ENODATA) || errors.Is(err, unix.ENOTSUP) { + return nil + } + if err != nil { + return err + } + return errLogSendTrust +} diff --git a/cmd/cli/log_send_listener_test.go b/cmd/cli/log_send_listener_test.go new file mode 100644 index 00000000..c8d7518d --- /dev/null +++ b/cmd/cli/log_send_listener_test.go @@ -0,0 +1,82 @@ +//go:build darwin || linux + +package cli + +import ( + "net" + "path/filepath" + "testing" + "time" +) + +func TestDelegatedLogSendListenerLimitAndClose(t *testing.T) { + tree, _ := logSendFixture(t) + path := filepath.Join(tree.root, "limit.sock") + base, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + l := &logSendListener{Listener: base, slots: make(chan struct{}, 1), done: make(chan struct{})} + defer l.Close() + client, err := net.Dial("unix", path) + if err != nil { + t.Fatal(err) + } + defer client.Close() + first, err := l.Accept() + if err != nil { + t.Fatal(err) + } + defer first.Close() + if uid, err := logSendPeer(first); err != nil || uid != tree.uid { + t.Fatalf("wrapped peer uid=%d err=%v", uid, err) + } + accepted := make(chan net.Conn, 1) + failed := make(chan error, 1) + go func() { + c, err := l.Accept() + if err != nil { + failed <- err + return + } + accepted <- c + }() + client2, err := net.Dial("unix", path) + if err != nil { + t.Fatal(err) + } + defer client2.Close() + select { + case c := <-accepted: + c.Close() + t.Fatal("connection limit bypassed") + case <-time.After(20 * time.Millisecond): + } + first.Close() + first.Close() // repeated closes must not release another connection's slot + var second net.Conn + select { + case second = <-accepted: + case err := <-failed: + t.Fatal(err) + case <-time.After(time.Second): + t.Fatal("slot not released") + } + defer second.Close() + go func() { + c, err := l.Accept() + if err == nil { + c.Close() + } + failed <- err + }() + l.Close() // blocked acquisition must terminate, even while second is still open + select { + case err := <-failed: + if err == nil { + t.Fatal("closed listener accepted") + } + case <-time.After(time.Second): + t.Fatal("close did not unblock accept") + } +} diff --git a/cmd/cli/log_send_other.go b/cmd/cli/log_send_other.go new file mode 100644 index 00000000..7e60fb80 --- /dev/null +++ b/cmd/cli/log_send_other.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package cli + +import "context" + +func delegatedLogSendCLI() bool { return false } +func runDelegatedLogSend(context.Context, bool) error { return nil } +func (p *prog) startLogSendServer() {} diff --git a/cmd/cli/log_send_startup_test.go b/cmd/cli/log_send_startup_test.go new file mode 100644 index 00000000..3e223758 --- /dev/null +++ b/cmd/cli/log_send_startup_test.go @@ -0,0 +1,51 @@ +//go:build darwin || linux + +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "testing" +) + +func TestDelegatedLogValidateEmptySources(t *testing.T) { + for _, journal := range []bool{false, true} { + for _, missing := range []bool{false, true} { + t.Run(fmt.Sprintf("journal=%t/missing=%t", journal, missing), func(t *testing.T) { + _, c := logSendFixture(t) + if journal { + c.sources = append(c.sources, delegatedLogSource{path: filepath.Join(c.tree.root, "journal.log"), budget: 4, backups: 2}) + } + for _, source := range c.sources { + writeLogSendFixture(t, source.path, "") + if missing { + if err := os.Remove(source.path); err != nil { + t.Fatal(err) + } + } + } + if err := c.validate(context.Background()); err != nil { + t.Fatalf("trusted empty sources prevent startup: %v", err) + } + if r, err := c.collect(context.Background()); !errors.Is(err, errLogFileEmpty) { + if r != nil { + r.Close() + } + t.Fatalf("empty upload error=%v, want %v", err, errLogFileEmpty) + } + // Files can gain data after startup without a service restart. + writeLogSendFixture(t, c.sources[0].path, "new-data") + want := "new-data" + if journal { + want += logWriterLogEndMarker + } + if got := readDelegated(t, c); got != want { + t.Fatalf("upload after append=%q, want %q", got, want) + } + }) + } + } +} diff --git a/cmd/cli/log_send_transport_unix.go b/cmd/cli/log_send_transport_unix.go new file mode 100644 index 00000000..b5e59672 --- /dev/null +++ b/cmd/cli/log_send_transport_unix.go @@ -0,0 +1,80 @@ +//go:build darwin || linux + +package cli + +import ( + "context" + "net" + "net/http" + "sync" + "syscall" + "time" +) + +// Unlike netutil.LimitListener, this wrapper preserves syscall.Conn so the +// HTTP server can authenticate the actual Unix peer after accepting it. +type logSendListener struct { + net.Listener + slots chan struct{} + done chan struct{} + once sync.Once +} + +type logSendConn struct { + net.Conn + release func() + once sync.Once +} + +func (c *logSendConn) SyscallConn() (syscall.RawConn, error) { + sc, ok := c.Conn.(syscall.Conn) + if !ok { + return nil, net.ErrClosed + } + return sc.SyscallConn() +} + +func (c *logSendConn) Close() error { + err := c.Conn.Close() + c.once.Do(c.release) + return err +} + +func (l *logSendListener) Accept() (net.Conn, error) { + select { + case l.slots <- struct{}{}: + case <-l.done: + return nil, net.ErrClosed + } + c, err := l.Listener.Accept() + if err != nil { + <-l.slots + return nil, err + } + return &logSendConn{Conn: c, release: func() { <-l.slots }}, nil +} + +func (l *logSendListener) Close() error { + l.once.Do(func() { close(l.done) }) + return l.Listener.Close() +} + +func (s *logSendServer) serve(l net.Listener, peer func(net.Conn) (uint32, error)) { + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + s.server = &http.Server{ + Handler: s, ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 3 * time.Second, + WriteTimeout: delegatedSendTimeout + 5*time.Second, MaxHeaderBytes: 1024, + BaseContext: func(net.Listener) context.Context { return ctx }, + ConnContext: func(ctx context.Context, c net.Conn) context.Context { + uid, err := peer(c) + if err != nil { + _ = c.Close() + return ctx + } + return context.WithValue(ctx, logSendPeerKey{}, uid) + }, + } + s.server.SetKeepAlivesEnabled(false) + go s.server.Serve(&logSendListener{Listener: l, slots: make(chan struct{}, 8), done: make(chan struct{})}) +} diff --git a/cmd/cli/log_send_unix.go b/cmd/cli/log_send_unix.go new file mode 100644 index 00000000..0d17f2c9 --- /dev/null +++ b/cmd/cli/log_send_unix.go @@ -0,0 +1,300 @@ +//go:build darwin || linux + +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/pelletier/go-toml/v2" + "golang.org/x/sys/unix" +) + +const delegatedDebugBytes int64 = 10 << 20 + +var errLogSendTrust = errors.New("unsafe log-send location") + +// trustedLogTree walks from a trusted anchor using descriptor-relative opens. +// Production always uses / and uid 0; tests anchor at their own private temp +// directory and uid, without needing root or touching system locations. +type trustedLogTree struct { + root string + uid uint32 +} + +func canonicalLogSendPath(path string) string { + path = filepath.Clean(path) + // macOS's system aliases. Do not resolve arbitrary symlinks in user-selected + // config/log paths: that could turn an attacker-controlled path into a trusted one. + for _, alias := range []string{"/var", "/etc", "/tmp"} { + if path == alias || strings.HasPrefix(path, alias+"/") { + return "/private" + path + } + } + return path +} + +func (t trustedLogTree) check(fd int, dir bool) error { + var st unix.Stat_t + if err := unix.Fstat(fd, &st); err != nil { + return err + } + kind := uint32(unix.S_IFREG) + if dir { + kind = unix.S_IFDIR + } + if st.Uid != t.uid || uint32(st.Mode)&unix.S_IFMT != kind || st.Mode&0022 != 0 || (!dir && st.Nlink != 1) { + return errLogSendTrust + } + return logSendCheckACL(fd) +} + +func (t trustedLogTree) open(path string, dir bool) (*os.File, error) { + if !filepath.IsAbs(path) { + return nil, errLogSendTrust + } + rel, err := filepath.Rel(t.root, filepath.Clean(path)) + if err != nil || rel == ".." || strings.HasPrefix(rel, "../") { + return nil, errLogSendTrust + } + fd, err := unix.Open(t.root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, err + } + if err = t.check(fd, true); err != nil { + unix.Close(fd) + return nil, err + } + if rel != "." { + parts := strings.Split(rel, string(os.PathSeparator)) + for i, part := range parts { + isDir := i < len(parts)-1 || dir + flags := unix.O_RDONLY | unix.O_CLOEXEC | unix.O_NOFOLLOW | unix.O_NONBLOCK + if isDir { + flags |= unix.O_DIRECTORY + } + next, openErr := unix.Openat(fd, part, flags, 0) + unix.Close(fd) + if openErr != nil { + return nil, openErr + } + fd = next + if err = t.check(fd, isDir); err != nil { + unix.Close(fd) + return nil, err + } + } + } else if !dir { + unix.Close(fd) + return nil, errLogSendTrust + } + return os.NewFile(uintptr(fd), path), nil +} + +type delegatedLogSource struct { + path string + backups int + budget int64 +} +type delegatedLogCollector struct { + tree trustedLogTree + config string + sources []delegatedLogSource +} + +// collect opens only immutable daemon-selected paths. It never calls the legacy +// reader (which tolerates untrusted homes and skips errors), never shells out for +// a network snapshot, and never reads an unbounded journal or memory fallback. +// Like the administrative upload, the body is stripped of the color codes of the +// internal debug stream; the source budgets count the bytes before that removal. +func (c delegatedLogCollector) collect(ctx context.Context) (io.ReadCloser, error) { + upload, err := c.open(ctx) + if err != nil { + return nil, err + } + lr, err := upload.reader(errLogFileEmpty) + if err != nil { + return nil, err + } + return &multiCloser{Reader: newANSIStripReader(lr.r), closers: []io.Closer{lr.r}}, nil +} + +// validate checks the same sources as an upload, but empty logs do not prevent +// startup. No log bytes are read, and every checked descriptor is closed. +func (c delegatedLogCollector) validate(ctx context.Context) error { + upload, err := c.open(ctx) + if err != nil { + return err + } + upload.close() + return nil +} + +func (c delegatedLogCollector) open(ctx context.Context) (*logParts, error) { + config, err := c.tree.open(c.config, false) + if err != nil { + return nil, err + } + data, err := io.ReadAll(io.LimitReader(config, (1<<20)+1)) + config.Close() + if err != nil || len(data) > 1<<20 { + return nil, errLogSendTrust + } + var permission struct { + Service struct { + Allow bool `toml:"allow_unprivileged_log_send"` + } `toml:"service"` + } + if err := toml.Unmarshal(data, &permission); err != nil || !permission.Service.Allow { + return nil, errLogSendTrust + } + var upload logParts + fail := func(err error) (*logParts, error) { upload.close(); return nil, err } + if len(c.sources) == 0 || len(c.sources) > 2 { + return fail(errLogSendTrust) + } + for index, source := range c.sources { + if source.backups < 0 || source.backups > 64 || source.budget <= 0 || source.budget > delegatedDebugBytes { + return fail(errLogSendTrust) + } + // Check the directory even when every file is missing. + dir, err := c.tree.open(filepath.Dir(source.path), true) + if err != nil { + return fail(err) + } + dir.Close() + remaining := source.budget + var parts []logFilePart + // Open newest first; hold at most 65 descriptors per stream. Missing rotated + // files are normal, but unsafe files are errors, not silently skipped. + for i := 0; i <= source.backups; i++ { + if err := ctx.Err(); err != nil { + for _, p := range parts { + p.f.Close() + } + return fail(err) + } + path := source.path + if i > 0 { + path = fmt.Sprintf("%s.%d", path, i) + } + f, err := c.tree.open(path, false) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + for _, p := range parts { + p.f.Close() + } + return fail(err) + } + st, err := f.Stat() + if err != nil { + f.Close() + for _, p := range parts { + p.f.Close() + } + return fail(err) + } + size := st.Size() + offset := int64(0) + if size > remaining { + offset = size - remaining + } + remaining -= size - offset + parts = append(parts, logFilePart{f: f, size: size, offset: offset}) + } + // Tail sections are bounded even for a single enormous line. Unlike the + // admin reader, delegated collection deliberately does no line-boundary scan. + for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 { + parts[i], parts[j] = parts[j], parts[i] + } + if index != 0 && (upload.size > 0 || remaining < source.budget) { + upload.addBytes([]byte(logWriterLogEndMarker)) + } + upload.addFiles(parts) + } + return &upload, nil +} + +func logSendPeer(c net.Conn) (uint32, error) { + u, ok := c.(syscall.Conn) + if !ok { + return 0, errLogSendTrust + } + raw, err := u.SyscallConn() + if err != nil { + return 0, err + } + var uid uint32 + var credErr error + if err = raw.Control(func(fd uintptr) { uid, credErr = logSendPeerUID(int(fd)) }); err != nil { + return 0, err + } + return uid, credErr +} + +// dialLogSend verifies the protected namespace and the actual connected peer. +// Socket mode is deliberately world-connectable; its parent must not be writable. +func dialLogSend(ctx context.Context, tree trustedLogTree, path string) (net.Conn, error) { + dir, err := tree.open(filepath.Dir(path), true) + if err != nil { + return nil, err + } + defer dir.Close() + var st unix.Stat_t + if err := unix.Fstatat(int(dir.Fd()), filepath.Base(path), &st, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return nil, err + } + if st.Uid != tree.uid || uint32(st.Mode)&unix.S_IFMT != unix.S_IFSOCK { + return nil, errLogSendTrust + } + d := net.Dialer{Timeout: 2 * time.Second} + conn, err := d.DialContext(ctx, "unix", path) + if err != nil { + return nil, err + } + uid, err := logSendPeer(conn) + if err != nil || uid != tree.uid { + conn.Close() + return nil, errLogSendTrust + } + return conn, nil +} + +func requestDelegatedLogSend(ctx context.Context, full bool, dial func(context.Context) (net.Conn, error)) error { + if full { + return errors.New("log send --full requires administrator privileges") + } + transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return dial(ctx) }, DisableKeepAlives: true} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: delegatedSendTimeout + 10*time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://unix"+sendLogsPath, nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return errors.New("log send unavailable: ask an administrator to enable allow_unprivileged_log_send and restart ctrld, or use sudo") + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusServiceUnavailable: + return errors.New("log send busy or cooling down; retry after one minute") + case http.StatusPreconditionFailed: + return errors.New("log send unavailable: administrator must check configuration and log permissions") + default: + return errors.New("log send failed; ask an administrator to check the daemon log") + } +} diff --git a/cmd/cli/log_send_unix_test.go b/cmd/cli/log_send_unix_test.go new file mode 100644 index 00000000..a28858a2 --- /dev/null +++ b/cmd/cli/log_send_unix_test.go @@ -0,0 +1,482 @@ +//go:build darwin || linux + +package cli + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + ctrld "github.com/Control-D-Inc/ctrld" + "golang.org/x/sys/unix" +) + +func logSendFixture(t *testing.T) (trustedLogTree, delegatedLogCollector) { + t.Helper() + // Short names also fit Darwin's 104-byte unix socket path limit. + dir, err := os.MkdirTemp("", "ls-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + tree := trustedLogTree{root: dir, uid: uint32(os.Geteuid())} + c := delegatedLogCollector{tree: tree, config: filepath.Join(dir, "config.toml"), sources: []delegatedLogSource{{path: filepath.Join(dir, "debug.log"), backups: 2, budget: 8}}} + writeLogSendFixture(t, c.config, "[service]\nallow_unprivileged_log_send = true\n") + writeLogSendFixture(t, c.sources[0].path, "0123456789abcdef") + return tree, c +} +func writeLogSendFixture(t *testing.T, path, data string) { + t.Helper() + if err := os.WriteFile(path, []byte(data), 0600); err != nil { + t.Fatal(err) + } +} +func readDelegated(t *testing.T, c delegatedLogCollector) string { + t.Helper() + r, err := c.collect(context.Background()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + b, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestDelegatedLogCollectionStripsColor(t *testing.T) { + _, c := logSendFixture(t) + colored := "\x1b[34mINFO\x1b[0m ok\n\x1b[31mERROR\x1b[0m bad\n" + writeLogSendFixture(t, c.sources[0].path, colored) + // The budget counts raw bytes, so it must cover the escapes too. + c.sources[0].budget = int64(len(colored)) + got := readDelegated(t, c) + if strings.Contains(got, "\x1b") { + t.Fatalf("delegated upload kept color codes: %q", got) + } + if want := "INFO ok\nERROR bad\n"; got != want { + t.Fatalf("body=%q, want %q", got, want) + } +} + +func TestDelegatedLogCollectionBoundedAndPinned(t *testing.T) { + _, c := logSendFixture(t) + writeLogSendFixture(t, c.sources[0].path+".1", strings.Repeat("older", 100)) + if got := readDelegated(t, c); got != "89abcdef" { + t.Fatalf("tail=%q", got) + } + c.sources = append(c.sources, delegatedLogSource{path: filepath.Join(c.tree.root, "journal.log"), budget: 4, backups: 2}) + writeLogSendFixture(t, c.sources[1].path, "journal-data") + if got := readDelegated(t, c); got != "89abcdef"+logWriterLogEndMarker+"data" { + t.Fatalf("bundle=%q", got) + } + r, err := c.collect(context.Background()) + if err != nil { + t.Fatal(err) + } + defer r.Close() + // An opened upload keeps the checked inode even if a privileged rotation + // replaces its name; it cannot switch to the replacement's bytes. + if err := os.Rename(c.sources[0].path, c.sources[0].path+".old"); err != nil { + t.Fatal(err) + } + writeLogSendFixture(t, c.sources[0].path, "replacement-secret") + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "secret") || !bytes.HasPrefix(data, []byte("89abcdef")) { + t.Fatalf("unpinned data: %q", data) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if r, err := c.collect(ctx); err == nil { + r.Close() + t.Fatal("canceled collection succeeded") + } +} + +func TestDelegatedLogCollectionTrustFailures(t *testing.T) { + cases := map[string]func(*testing.T, *delegatedLogCollector){ + "config writable": func(t *testing.T, c *delegatedLogCollector) { os.Chmod(c.config, 0666) }, + "log writable": func(t *testing.T, c *delegatedLogCollector) { os.Chmod(c.sources[0].path, 0666) }, + "directory writable": func(t *testing.T, c *delegatedLogCollector) { os.Chmod(c.tree.root, 0777) }, + "wrong owner": func(t *testing.T, c *delegatedLogCollector) { c.tree.uid++ }, + "config symlink": func(t *testing.T, c *delegatedLogCollector) { + os.Rename(c.config, c.config+".real") + os.Symlink(c.config+".real", c.config) + }, + "log symlink": func(t *testing.T, c *delegatedLogCollector) { + os.Remove(c.sources[0].path) + os.Symlink(c.config, c.sources[0].path) + }, + "backup symlink": func(t *testing.T, c *delegatedLogCollector) { os.Symlink(c.config, c.sources[0].path+".2") }, + "hardlink": func(t *testing.T, c *delegatedLogCollector) { + if err := os.Link(c.sources[0].path, c.sources[0].path+".link"); err != nil { + t.Fatal(err) + } + }, + "fifo": func(t *testing.T, c *delegatedLogCollector) { + os.Remove(c.sources[0].path) + if err := unix.Mkfifo(c.sources[0].path, 0600); err != nil { + t.Fatal(err) + } + }, + "directory as log": func(t *testing.T, c *delegatedLogCollector) { + os.Remove(c.sources[0].path) + os.Mkdir(c.sources[0].path, 0700) + }, + "disabled": func(t *testing.T, c *delegatedLogCollector) { + writeLogSendFixture(t, c.config, "[service]\nallow_unprivileged_log_send = false\n") + }, + "absent optin": func(t *testing.T, c *delegatedLogCollector) { writeLogSendFixture(t, c.config, "[service]\n") }, + "invalid config": func(t *testing.T, c *delegatedLogCollector) { writeLogSendFixture(t, c.config, "[service") }, + "huge config": func(t *testing.T, c *delegatedLogCollector) { + writeLogSendFixture(t, c.config, strings.Repeat("a", (1<<20)+1)) + }, + "missing config": func(t *testing.T, c *delegatedLogCollector) { os.Remove(c.config) }, + "relative log": func(t *testing.T, c *delegatedLogCollector) { c.sources[0].path = "debug.log" }, + "escape anchor": func(t *testing.T, c *delegatedLogCollector) { + c.sources[0].path = filepath.Join(c.tree.root, "..", "debug.log") + }, + "too many backups": func(t *testing.T, c *delegatedLogCollector) { c.sources[0].backups = 65 }, + "unbounded budget": func(t *testing.T, c *delegatedLogCollector) { c.sources[0].budget = 0 }, + "intermediate symlink": func(t *testing.T, c *delegatedLogCollector) { + os.Symlink(c.tree.root, filepath.Join(c.tree.root, "alias")) + c.sources[0].path = filepath.Join(c.tree.root, "alias", "debug.log") + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + _, c := logSendFixture(t) + mutate(t, &c) + if err := c.validate(context.Background()); err == nil { + t.Fatal("unsafe startup validation succeeded") + } + if r, err := c.collect(context.Background()); err == nil { + r.Close() + t.Fatal("unsafe collection succeeded") + } + }) + } +} + +func startLogSendFixture(t *testing.T, s *logSendServer, peer func(net.Conn) (uint32, error)) (*http.Client, string) { + t.Helper() + tree, _ := logSendFixture(t) + path := filepath.Join(tree.root, "send.sock") + l, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + s.serve(l, peer) + t.Cleanup(s.stop) + tr := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return dialLogSend(ctx, tree, path) }, DisableKeepAlives: true} + t.Cleanup(tr.CloseIdleConnections) + return &http.Client{Transport: tr, Timeout: 5 * time.Second}, path +} +func logSendStatus(t *testing.T, c *http.Client, method, path, body string) int { + t.Helper() + req, err := http.NewRequest(method, "http://unix"+path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if len(b) != 0 { + t.Fatalf("response leaked data: %q", b) + } + return resp.StatusCode +} + +func TestDelegatedLogSendRealTransportAndIsolation(t *testing.T) { + _, collector := logSendFixture(t) + var sent string + var caller int64 = -2 + var auditCode int + var mu sync.Mutex + s := &logSendServer{gate: &logUploadGate{}, collect: collector.collect, + upload: func(ctx context.Context, r io.ReadCloser) error { + b, err := io.ReadAll(r) + sent = string(b) + return err + }, + audit: func(uid int64, code int) { mu.Lock(); defer mu.Unlock(); caller, auditCode = uid, code }, + } + client, _ := startLogSendFixture(t, s, logSendPeer) + for _, path := range []string{"/clients", "/started", "/reload", "/deactivation", "/cd", "/iface", "/log/view", "/log/tail", "/log/send?full=1", "/log/send?url=https://attacker", "/log/send?path=/etc/passwd", "/log/send/", "/log/%73end", "//log/send"} { + if got := logSendStatus(t, client, "POST", path, ""); got != 400 { + t.Fatalf("%s status=%d", path, got) + } + } + for _, method := range []string{"GET", "PUT", "DELETE", "CONNECT"} { + if got := logSendStatus(t, client, method, sendLogsPath, ""); got != 400 { + t.Fatalf("%s status=%d", method, got) + } + } + if got := logSendStatus(t, client, "POST", sendLogsPath, `{"uid":"attacker","full":true}`); got != 400 { + t.Fatal(got) + } + if got := logSendStatus(t, client, "POST", sendLogsPath, ""); got != 200 { + t.Fatal(got) + } + if sent != "89abcdef" { + t.Fatalf("uploaded=%q", sent) + } + mu.Lock() + uid, code := caller, auditCode + mu.Unlock() + if uid != int64(os.Geteuid()) || code != 200 { + t.Fatalf("audit uid=%d status=%d", uid, code) + } + if got := logSendStatus(t, client, "POST", sendLogsPath, ""); got != 503 { + t.Fatal(got) + } +} + +func TestDelegatedLogSendGateSharedWithAdmin(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}, cs: &controlServer{mux: http.NewServeMux()}} + p.registerControlServerHandler() + entered, release := make(chan struct{}), make(chan struct{}) + var collections atomic.Int32 + s := &logSendServer{gate: &p.logUpload, collect: func(context.Context) (io.ReadCloser, error) { + collections.Add(1) + close(entered) + <-release + return io.NopCloser(strings.NewReader("data")), nil + }, upload: func(context.Context, io.ReadCloser) error { return errors.New("secret upstream error") }} + client, _ := startLogSendFixture(t, s, logSendPeer) + done := make(chan int, 1) + go func() { + resp, err := client.Post("http://unix"+sendLogsPath, "", nil) + if err != nil { + done <- 0 + return + } + defer resp.Body.Close() + done <- resp.StatusCode + }() + <-entered + var wg sync.WaitGroup + for i := 0; i < 24; i++ { + wg.Add(1) + go func() { + defer wg.Done() + r := httptest.NewRecorder() + p.cs.mux.ServeHTTP(r, httptest.NewRequest("POST", sendLogsPath+"?full=1", nil)) + if r.Code != 503 { + t.Errorf("admin concurrent=%d", r.Code) + } + }() + } + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + r := httptest.NewRecorder() + req := httptest.NewRequest("POST", sendLogsPath, nil).WithContext(context.WithValue(context.Background(), logSendPeerKey{}, uint32(42))) + s.ServeHTTP(r, req) + if r.Code != 503 { + t.Errorf("delegated concurrent=%d", r.Code) + } + }() + } + wg.Wait() + close(release) + if got := <-done; got != 502 { + t.Fatal(got) + } + if collections.Load() != 1 { + t.Fatal(collections.Load()) + } + r := httptest.NewRecorder() + p.cs.mux.ServeHTTP(r, httptest.NewRequest("POST", sendLogsPath, nil)) + if r.Code != 503 { + t.Fatalf("admin failure cooldown=%d", r.Code) + } + // Expire the cooldown explicitly, without sleeping a minute. The actual admin + // handler fails before an upload on an empty config, then reserves cooldown too. + p.logUpload.mu.Lock() + p.logUpload.next = time.Time{} + p.logUpload.mu.Unlock() + oldUID := cdUID + cdUID = "" + defer func() { cdUID = oldUID }() + r = httptest.NewRecorder() + p.cs.mux.ServeHTTP(r, httptest.NewRequest("POST", sendLogsPath, nil)) + if r.Code != 301 { + t.Fatalf("empty admin=%d", r.Code) + } + if got := logSendStatus(t, client, "POST", sendLogsPath, ""); got != 503 { + t.Fatalf("delegated after admin=%d", got) + } +} + +func TestDelegatedLogSendFailureCooldownAndMissingIdentity(t *testing.T) { + var calls int + s := &logSendServer{gate: &logUploadGate{}, collect: func(context.Context) (io.ReadCloser, error) { calls++; return nil, errors.New("secret path") }} + req := httptest.NewRequest("POST", sendLogsPath, nil) + r := httptest.NewRecorder() + s.ServeHTTP(r, req) + if r.Code != 403 || calls != 0 { + t.Fatalf("missing identity: code=%d calls=%d", r.Code, calls) + } + req = req.WithContext(context.WithValue(context.Background(), logSendPeerKey{}, uint32(501))) + r = httptest.NewRecorder() + s.ServeHTTP(r, req) + if r.Code != 412 || r.Body.Len() != 0 || calls != 1 { + t.Fatalf("collection failure: %+v calls=%d", r, calls) + } + r = httptest.NewRecorder() + s.ServeHTTP(r, req) + if r.Code != 503 || calls != 1 { + t.Fatal("failure did not reserve cooldown") + } + client, _ := startLogSendFixture(t, &logSendServer{}, func(net.Conn) (uint32, error) { return 0, errors.New("no credentials") }) + if resp, err := client.Post("http://unix"+sendLogsPath, "", nil); err == nil { + resp.Body.Close() + t.Fatal("unverified peer accepted") + } +} + +func TestDelegatedLogSendShutdownCancelsAndJoins(t *testing.T) { + entered, exited := make(chan struct{}), make(chan struct{}) + s := &logSendServer{gate: &logUploadGate{}, collect: func(context.Context) (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("data")), nil }, upload: func(ctx context.Context, _ io.ReadCloser) error { + close(entered) + <-ctx.Done() + close(exited) + return ctx.Err() + }} + client, path := startLogSendFixture(t, s, logSendPeer) + done := make(chan struct{}) + go func() { + defer close(done) + resp, err := client.Post("http://unix"+sendLogsPath, "", nil) + if err == nil { + resp.Body.Close() + } + }() + <-entered + s.stop() + select { + case <-exited: + default: + t.Fatal("stop did not join upload") + } + <-done + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("socket not removed: %v", err) + } + p := &prog{} + p.stopLogSendServer() + if !p.logSendStopped { + t.Fatal("startup not fenced") + } +} + +func TestDelegatedLogSendClientAndImpersonation(t *testing.T) { + tree, _ := logSendFixture(t) + path := filepath.Join(tree.root, "send.sock") + var dials int + dial := func(context.Context) (net.Conn, error) { dials++; return nil, errors.New("not running") } + if err := requestDelegatedLogSend(context.Background(), true, dial); err == nil || dials != 0 { + t.Fatal("full dialed") + } + if err := requestDelegatedLogSend(context.Background(), false, dial); err == nil || !strings.Contains(err.Error(), "administrator") { + t.Fatal(err) + } + writeLogSendFixture(t, path, "not a socket") + if c, err := dialLogSend(context.Background(), tree, path); err == nil { + c.Close() + t.Fatal("regular file accepted") + } + os.Remove(path) + s := &logSendServer{gate: &logUploadGate{}, collect: func(context.Context) (io.ReadCloser, error) { return io.NopCloser(strings.NewReader("x")), nil }, upload: func(context.Context, io.ReadCloser) error { return nil }} + l, err := net.Listen("unix", path) + if err != nil { + t.Fatal(err) + } + s.serve(l, logSendPeer) + defer s.stop() + if err := requestDelegatedLogSend(context.Background(), false, func(ctx context.Context) (net.Conn, error) { return dialLogSend(ctx, tree, path) }); err != nil { + t.Fatal(err) + } + os.Chmod(tree.root, 0777) + if c, err := dialLogSend(context.Background(), tree, path); err == nil { + c.Close() + t.Fatal("writable location accepted") + } + os.Chmod(tree.root, 0700) + // A nonroot peer cannot impersonate root, even though it can create a socket + // in its own directory. Production validation starts at /, not this test anchor. + bad := tree + bad.uid++ + if c, err := dialLogSend(context.Background(), bad, path); err == nil { + c.Close() + t.Fatal("wrong owner accepted") + } + os.Symlink(path, path+".alias") + if c, err := dialLogSend(context.Background(), tree, path+".alias"); err == nil { + c.Close() + t.Fatal("symlink socket accepted") + } +} + +func TestDelegatedLogSendRawRequestsBounded(t *testing.T) { + s := &logSendServer{gate: &logUploadGate{}, collect: func(context.Context) (io.ReadCloser, error) { + t.Error("invalid request collected logs") + return nil, errors.New("unexpected") + }} + _, path := startLogSendFixture(t, s, logSendPeer) + for _, raw := range []string{ + "POST /log/send HTTP/1.1\r\nHost: unix\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", + "POST /log/send HTTP/1.1\r\nHost: unix\r\nContent-Length: 999999999\r\n\r\n", + "POST /log/send HTTP/1.1\r\nHost: unix\r\nExpect: 100-continue\r\n\r\n", + "POST /log/send HTTP/1.1\r\nHost: unix\r\nX-Large: " + strings.Repeat("x", 8192) + "\r\n\r\n", + } { + conn, err := net.Dial("unix", path) + if err != nil { + t.Fatal(err) + } + conn.SetDeadline(time.Now().Add(5 * time.Second)) + io.WriteString(conn, raw) + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + conn.Close() + t.Fatal(err) + } + if resp.StatusCode != 400 && resp.StatusCode != 431 { + t.Errorf("unexpected status %d", resp.StatusCode) + } + resp.Body.Close() + conn.Close() + } +} + +func TestDelegatedLogSendCanonicalAliases(t *testing.T) { + for from, to := range map[string]string{"/var/log/ctrld.log": "/private/var/log/ctrld.log", "/etc/ctrld.toml": "/private/etc/ctrld.toml", "/private/var/log/a": "/private/var/log/a", "/Users/u/alias/log": "/Users/u/alias/log"} { + if got := canonicalLogSendPath(from); got != to { + t.Fatalf("%q => %q", from, got) + } + } +} diff --git a/cmd/cli/log_setup_test.go b/cmd/cli/log_setup_test.go new file mode 100644 index 00000000..a372a095 --- /dev/null +++ b/cmd/cli/log_setup_test.go @@ -0,0 +1,173 @@ +package cli + +import ( + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +// setupLogSetupTest puts ctrld in cd mode with no log_path and a temporary +// home directory, and puts back every global that the logging setup touches. +// The globals are the reason why no test of this package calls t.Parallel. +func setupLogSetupTest(t *testing.T) string { + t.Helper() + dir := t.TempDir() + origLogPath, origLogLevel := cfg.Service.LogPath, cfg.Service.LogLevel + origSilent, origVerbose, origCdUID, origHomedir := silent, verbose, cdUID, homedir + origConsoleWriter := consoleWriter + origMainLog := mainLog.Load() + origLogPathFile, origPendingRotation := logPathFile.Load(), pendingLogPathRotation.Load() + t.Cleanup(func() { + if rf := logPathFile.Load(); rf != nil && rf != origLogPathFile { + _ = rf.close() + } + logPathFile.Store(origLogPathFile) + pendingLogPathRotation.Store(origPendingRotation) + cfg.Service.LogPath, cfg.Service.LogLevel = origLogPath, origLogLevel + silent, verbose, cdUID, homedir = origSilent, origVerbose, origCdUID, origHomedir + consoleWriter = origConsoleWriter + mainLog.Store(origMainLog) + }) + cfg.Service.LogPath, cfg.Service.LogLevel = "", "info" + silent, verbose, cdUID, homedir = false, 0, "", dir + consoleWriter = newHumanReadableZapCore(io.Discard, ctrld.NoticeLevel) + mainLog.Store(ctrld.NopLogger) + logPathFile.Store(nil) + pendingLogPathRotation.Store(nil) + stubHeaderSnapshotSources(t) + stubLogHeaderNetworkRead(t, "en0") + return dir +} + +// Test_initLoggingAfterProvisioning_opensTheJournal drives a --cd-org start: +// the first logging setup runs before the resolver UID is known, and the +// provisioning step then opens the internal streams. +func Test_initLoggingAfterProvisioning_opensTheJournal(t *testing.T) { + dir := setupLogSetupTest(t) + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + journalPath := filepath.Join(dir, journalLogFileName) + if _, err := os.Stat(journalPath); !os.IsNotExist(err) { + t.Fatalf("journal exists before the UID is known (stat err = %v)", err) + } + + cdUID = "provisioned-uid" + p.initLoggingAfterProvisioning() + + firstLine, headers := logHeaderFileLines(t, journalPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("journal starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + mainLog.Load().Warn().Msg("retained after provisioning") + if content := rotatingFileTestContent(t, journalPath); !strings.Contains(content, "retained after provisioning") { + t.Fatalf("journal misses the line after provisioning: %s", content) + } + + p.initLoggingAfterProvisioning() + if _, headers := logHeaderFileLines(t, journalPath); headers != 1 { + t.Fatalf("a second call added a header: %d headers", headers) + } +} + +// Test_switchLogPath_leadsTheNewFileAndClosesTheInternalFiles drives the +// log_path that the API config sets after the first logging setup. +func Test_switchLogPath_leadsTheNewFileAndClosesTheInternalFiles(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + mainLog.Load().Info().Msg("line before the switch") + + newPath := filepath.Join(dir, "logs", "ctrld.log") + cfg.Service.LogPath = newPath + p.switchLogPath("", newPath) + + firstLine, headers := logHeaderFileLines(t, newPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("log_path file starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + lw, jlw := p.internalWriters() + if lw.rotating() != nil || jlw.rotating() != nil { + t.Fatal("the internal files stayed open after the switch") + } + if files := p.logHeaderFiles(); !slices.Equal(files, []string{newPath}) { + t.Fatalf("log_files = %v, want the log_path only", files) + } + // cd mode has no local log_path, so the lines of the run stay in the + // internal debug stream that this call closes. + if content := rotatingFileTestContent(t, newPath); strings.Contains(content, "line before the switch") { + t.Fatalf("a line moved although the run had no local log_path: %s", content) + } +} + +// Test_switchLogPath_carriesTheOldFileBelowOneHeader drives a run that already +// wrote a local log_path file. The new file must hold one header, and the +// header must lead the lines that the old file holds. +func Test_switchLogPath_carriesTheOldFileBelowOneHeader(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + oldPath := filepath.Join(dir, "old", "ctrld.log") + cfg.Service.LogPath = oldPath + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + mainLog.Load().Info().Msg("line before the switch") + + newPath := filepath.Join(dir, "logs", "ctrld.log") + cfg.Service.LogPath = newPath + p.switchLogPath(oldPath, newPath) + + firstLine, headers := logHeaderFileLines(t, newPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("log_path file starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + if !slices.Equal(firstLine.LogFiles, []string{newPath}) { + t.Fatalf("header log_files = %v, want the new log_path", firstLine.LogFiles) + } + if content := rotatingFileTestContent(t, newPath); !strings.Contains(content, "line before the switch") { + t.Fatalf("the new file misses the lines of the old file: %s", content) + } +} + +// Test_initLoggingAfterProvisioning_skipsTheGuardedRuns covers the two runs +// that keep no internal files: a run with a local log_path, and a silent run. +// Neither may gain a journal when the provisioning step learns the UID. +func Test_initLoggingAfterProvisioning_skipsTheGuardedRuns(t *testing.T) { + for _, tc := range []struct { + name string + guard func(t *testing.T, dir string) + }{ + {"log_path set", func(t *testing.T, dir string) { + cfg.Service.LogPath = filepath.Join(dir, "operator.log") + }}, + {"silent set", func(t *testing.T, _ string) { silent = true }}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := setupLogSetupTest(t) + tc.guard(t, dir) + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + cdUID = "provisioned-uid" + p.initLoggingAfterProvisioning() + + journalPath := filepath.Join(dir, journalLogFileName) + if _, err := os.Stat(journalPath); !os.IsNotExist(err) { + t.Fatalf("the guarded run opened a journal (stat err = %v)", err) + } + }) + } +} diff --git a/cmd/cli/log_tail_follow.go b/cmd/cli/log_tail_follow.go new file mode 100644 index 00000000..045b4e18 --- /dev/null +++ b/cmd/cli/log_tail_follow.go @@ -0,0 +1,80 @@ +package cli + +import ( + "context" + "io" + "net/http" + "os" + "time" +) + +// logTailPollInterval is the time between two reads of a tailed log file. +const logTailPollInterval = 200 * time.Millisecond + +// followLogFile streams the lines of path until ctx ends. It first sends the +// last numLines lines, or starts at the end when numLines is 0. A rotation +// moves the file away or truncates it, so the poll reopens the path when the +// file behind it changes or shrinks. +func followLogFile(ctx context.Context, w io.Writer, flusher http.Flusher, path string, numLines int) { + f, err := os.Open(path) + if err != nil { + return + } + defer func() { f.Close() }() + + if numLines > 0 { + if tail := tailFileLastLines(f, numLines); len(tail) > 0 { + w.Write(tail) + flusher.Flush() + } + } else { + f.Seek(0, io.SeekEnd) + } + + buf := make([]byte, 4096) + ticker := time.NewTicker(logTailPollInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + n, err := f.Read(buf) + if n > 0 { + if _, werr := w.Write(buf[:n]); werr != nil { + return + } + flusher.Flush() + } + if err != nil && err != io.EOF { + return + } + if n == 0 && logFileReplaced(f, path) { + reopened, err := os.Open(path) + if err != nil { + continue + } + f.Close() + f = reopened + } + case <-ctx.Done(): + return + } + } +} + +// logFileReplaced reports whether path names another file than f, or a file +// shorter than the read position. Both mean that a rotation happened. +func logFileReplaced(f *os.File, path string) bool { + current, err := os.Stat(path) + if err != nil { + return false + } + open, err := f.Stat() + if err != nil { + return true + } + if !os.SameFile(current, open) { + return true + } + position, err := f.Seek(0, io.SeekCurrent) + return err == nil && current.Size() < position +} diff --git a/cmd/cli/log_tail_test.go b/cmd/cli/log_tail_test.go new file mode 100644 index 00000000..37ad4110 --- /dev/null +++ b/cmd/cli/log_tail_test.go @@ -0,0 +1,339 @@ +package cli + +import ( + "io" + "os" + "strings" + "sync" + "testing" + "time" +) + +// ============================================================================= +// logWriter.tailLastLines tests +// ============================================================================= + +func Test_logWriter_tailLastLines_Empty(t *testing.T) { + lw := newLogWriterWithSize(4096) + if got := lw.tailLastLines(10); got != nil { + t.Fatalf("expected nil for empty buffer, got %q", got) + } +} + +func Test_logWriter_tailLastLines_ZeroLines(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("line1\nline2\n")) + if got := lw.tailLastLines(0); got != nil { + t.Fatalf("expected nil for n=0, got %q", got) + } +} + +func Test_logWriter_tailLastLines_NegativeLines(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("line1\nline2\n")) + if got := lw.tailLastLines(-1); got != nil { + t.Fatalf("expected nil for n=-1, got %q", got) + } +} + +func Test_logWriter_tailLastLines_FewerThanN(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("line1\nline2\n")) + got := string(lw.tailLastLines(10)) + want := "line1\nline2\n" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_logWriter_tailLastLines_ExactN(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("line1\nline2\nline3\n")) + got := string(lw.tailLastLines(3)) + want := "line1\nline2\nline3\n" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_logWriter_tailLastLines_MoreThanN(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("line1\nline2\nline3\nline4\nline5\n")) + got := string(lw.tailLastLines(2)) + want := "line4\nline5\n" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_logWriter_tailLastLines_NoTrailingNewline(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("line1\nline2\nline3")) + // Without trailing newline, "line3" is a partial line. + // Asking for 1 line returns the last newline-terminated line plus the partial. + got := string(lw.tailLastLines(1)) + want := "line2\nline3" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_logWriter_tailLastLines_SingleLineNoNewline(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("only line")) + got := string(lw.tailLastLines(5)) + want := "only line" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_logWriter_tailLastLines_SingleLineWithNewline(t *testing.T) { + lw := newLogWriterWithSize(4096) + lw.Write([]byte("only line\n")) + got := string(lw.tailLastLines(1)) + want := "only line\n" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +// ============================================================================= +// logWriter.Subscribe tests +// ============================================================================= + +func Test_logWriter_Subscribe_Basic(t *testing.T) { + lw := newLogWriterWithSize(4096) + ch, unsub := lw.Subscribe() + defer unsub() + + msg := []byte("hello world\n") + lw.Write(msg) + + select { + case got := <-ch: + if string(got) != string(msg) { + t.Fatalf("got %q, want %q", got, msg) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for subscriber data") + } +} + +func Test_logWriter_Subscribe_MultipleSubscribers(t *testing.T) { + lw := newLogWriterWithSize(4096) + ch1, unsub1 := lw.Subscribe() + defer unsub1() + ch2, unsub2 := lw.Subscribe() + defer unsub2() + + msg := []byte("broadcast\n") + lw.Write(msg) + + for i, ch := range []<-chan []byte{ch1, ch2} { + select { + case got := <-ch: + if string(got) != string(msg) { + t.Fatalf("subscriber %d: got %q, want %q", i, got, msg) + } + case <-time.After(time.Second): + t.Fatalf("subscriber %d: timed out", i) + } + } +} + +func Test_logWriter_Subscribe_Unsubscribe(t *testing.T) { + lw := newLogWriterWithSize(4096) + ch, unsub := lw.Subscribe() + + // Verify subscribed. + lw.Write([]byte("before unsub\n")) + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("timed out before unsub") + } + + unsub() + + // Channel should be closed after unsub. + if _, ok := <-ch; ok { + t.Fatal("channel should be closed after unsubscribe") + } + + // Verify subscriber list is empty. + lw.mu.Lock() + count := len(lw.subscribers) + lw.mu.Unlock() + if count != 0 { + t.Fatalf("expected 0 subscribers after unsub, got %d", count) + } +} + +func Test_logWriter_Subscribe_UnsubscribeIdempotent(t *testing.T) { + lw := newLogWriterWithSize(4096) + _, unsub := lw.Subscribe() + unsub() + // Second unsub should not panic. + unsub() +} + +func Test_logWriter_Subscribe_SlowSubscriberDropped(t *testing.T) { + lw := newLogWriterWithSize(4096) + ch, unsub := lw.Subscribe() + defer unsub() + + // Fill the subscriber channel (buffer size is 256). + for i := 0; i < 300; i++ { + lw.Write([]byte("msg\n")) + } + + // Should have 256 buffered messages, rest dropped. + count := 0 + for { + select { + case <-ch: + count++ + default: + goto done + } + } +done: + if count != 256 { + t.Fatalf("expected 256 buffered messages, got %d", count) + } +} + +func Test_logWriter_Subscribe_ConcurrentWriteAndRead(t *testing.T) { + lw := newLogWriterWithSize(64 * 1024) + ch, unsub := lw.Subscribe() + defer unsub() + + const numWrites = 100 + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < numWrites; i++ { + lw.Write([]byte("concurrent write\n")) + } + }() + + received := 0 + timeout := time.After(5 * time.Second) + for received < numWrites { + select { + case <-ch: + received++ + case <-timeout: + t.Fatalf("timed out after receiving %d/%d messages", received, numWrites) + } + } + wg.Wait() +} + +// ============================================================================= +// tailFileLastLines tests +// ============================================================================= + +func writeTempFile(t *testing.T, content string) *os.File { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "tail-test-*") + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteString(content); err != nil { + t.Fatal(err) + } + return f +} + +func Test_tailFileLastLines_Empty(t *testing.T) { + f := writeTempFile(t, "") + defer f.Close() + if got := tailFileLastLines(f, 10); got != nil { + t.Fatalf("expected nil for empty file, got %q", got) + } +} + +func Test_tailFileLastLines_FewerThanN(t *testing.T) { + f := writeTempFile(t, "line1\nline2\n") + defer f.Close() + got := string(tailFileLastLines(f, 10)) + want := "line1\nline2\n" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_tailFileLastLines_ExactN(t *testing.T) { + f := writeTempFile(t, "a\nb\nc\n") + defer f.Close() + got := string(tailFileLastLines(f, 3)) + want := "a\nb\nc\n" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_tailFileLastLines_MoreThanN(t *testing.T) { + f := writeTempFile(t, "line1\nline2\nline3\nline4\nline5\n") + defer f.Close() + got := string(tailFileLastLines(f, 2)) + want := "line4\nline5\n" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_tailFileLastLines_NoTrailingNewline(t *testing.T) { + f := writeTempFile(t, "line1\nline2\nline3") + defer f.Close() + // Without trailing newline, partial last line comes with the previous line. + got := string(tailFileLastLines(f, 1)) + want := "line2\nline3" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func Test_tailFileLastLines_LargerThanChunk(t *testing.T) { + // Build content larger than the 4096 chunk size to exercise multi-chunk reads. + var sb strings.Builder + for i := 0; i < 200; i++ { + sb.WriteString(strings.Repeat("x", 50)) + sb.WriteByte('\n') + } + f := writeTempFile(t, sb.String()) + defer f.Close() + got := string(tailFileLastLines(f, 3)) + lines := strings.Split(strings.TrimRight(got, "\n"), "\n") + if len(lines) != 3 { + t.Fatalf("expected 3 lines, got %d: %q", len(lines), got) + } + expectedLine := strings.Repeat("x", 50) + for _, line := range lines { + if line != expectedLine { + t.Fatalf("unexpected line content: %q", line) + } + } +} + +func Test_tailFileLastLines_SeeksToEnd(t *testing.T) { + f := writeTempFile(t, "line1\nline2\nline3\n") + defer f.Close() + tailFileLastLines(f, 1) + + // After tailFileLastLines, file position should be at the end. + pos, err := f.Seek(0, io.SeekCurrent) + if err != nil { + t.Fatal(err) + } + stat, err := f.Stat() + if err != nil { + t.Fatal(err) + } + if pos != stat.Size() { + t.Fatalf("expected file position at end (%d), got %d", stat.Size(), pos) + } +} diff --git a/cmd/cli/log_test_helpers_test.go b/cmd/cli/log_test_helpers_test.go new file mode 100644 index 00000000..68c7ffb5 --- /dev/null +++ b/cmd/cli/log_test_helpers_test.go @@ -0,0 +1,67 @@ +package cli + +import ( + "encoding/json" + "io" + "strings" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" +) + +// journalMarker is the serialized form of the journal field in a JSON line. +var journalMarker = []byte(`"` + journalField + `":true`) + +// hasJournalMarker reports whether a log line carries the journal field. A +// JSON line writes the field without a space, a console line with one. +func hasJournalMarker(line string) bool { + return strings.Contains(line, string(journalMarker)) || strings.Contains(line, `"`+journalField+`": true`) +} + +// newTestJSONLogger writes one JSON object for each line to every writer, at +// debug level, with the field names of the journal. A test parses the lines +// back, so the encoder must be the JSON one. +func newTestJSONLogger(writers ...io.Writer) *ctrld.Logger { + cores := make([]zapcore.Core, 0, len(writers)) + for _, w := range writers { + cores = append(cores, zapcore.NewCore(newJournalEncoder(), zapcore.AddSync(w), zapcore.DebugLevel)) + } + return &ctrld.Logger{Logger: zap.New(zapcore.NewTee(cores...))} +} + +// captureJSONMainLog swaps mainLog for a JSON logger that writes into a +// buffer at debug level and puts the old logger back when the test ends. The +// JSON form lets a test parse the lines it captured. It swaps a global, so +// no test of this package runs in parallel. +func captureJSONMainLog(t *testing.T) *syncBuffer { + t.Helper() + buf := &syncBuffer{} + old := mainLog.Load() + mainLog.Store(newTestJSONLogger(buf)) + t.Cleanup(func() { mainLog.Store(old) }) + return buf +} + +// parseLogLineFields returns the fields of one log line. A JSON line is one +// object. A console line of the debug stream ends with its fields as one +// object after the last tab. +func parseLogLineFields(t *testing.T, line string) map[string]json.RawMessage { + t.Helper() + line = strings.TrimSpace(line) + object := line + if !strings.HasPrefix(line, "{") { + index := strings.LastIndex(line, "\t{") + if index < 0 { + t.Fatalf("log line holds no fields: %q", line) + } + object = line[index+1:] + } + fields := map[string]json.RawMessage{} + if err := json.Unmarshal([]byte(object), &fields); err != nil { + t.Fatalf("parse fields of %q: %v", line, err) + } + return fields +} diff --git a/cmd/cli/log_writer.go b/cmd/cli/log_writer.go index ab6b855f..b8c47bc3 100644 --- a/cmd/cli/log_writer.go +++ b/cmd/cli/log_writer.go @@ -2,74 +2,253 @@ package cli import ( "bytes" - "errors" - "fmt" + "encoding/json" "io" "os" + "regexp" "strings" "sync" "time" - "github.com/rs/zerolog" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" "github.com/Control-D-Inc/ctrld" ) +// Log writer constants for buffer management and log formatting const ( - logWriterSize = 1024 * 1024 * 5 // 5 MB - logWriterSmallSize = 1024 * 1024 * 1 // 1 MB - logWriterInitialSize = 32 * 1024 // 32 KB - logWriterSentInterval = time.Minute + // logWriterSize is the default buffer size for log writers + // This provides sufficient space for runtime logs without excessive memory usage + logWriterSize = 1024 * 1024 * 5 // 5 MB + + // logWriterSmallSize is used for memory-constrained environments + // This reduces memory footprint while still maintaining log functionality + logWriterSmallSize = 1024 * 1024 * 1 // 1 MB + + // logWriterInitialSize is the initial buffer allocation + // This provides immediate space for early log entries + logWriterInitialSize = 32 * 1024 // 32 KB + + // logWriterSentInterval controls how often logs are sent to external systems + // This balances real-time logging with system performance + logWriterSentInterval = time.Minute + + // logWriterInitEndMarker marks the end of initialization logs + // This helps separate startup logs from runtime logs logWriterInitEndMarker = "\n\n=== INIT_END ===\n\n" - logWriterLogEndMarker = "\n\n=== LOG_END ===\n\n" + + // logWriterLogEndMarker marks the end of log sections + // This provides clear boundaries for log parsing and analysis + logWriterLogEndMarker = "\n\n=== LOG_END ===\n\n" + + logFileName = "ctrld.log" ) +// Custom level encoders that handle NOTICE level +// Since NOTICE and WARN share the same numeric value (1), we handle them specially +// in the encoder to display NOTICE messages with the "NOTICE" prefix. +// Note: WARN messages will also display as "NOTICE" because they share the same level value. +// This is the intended behavior for visual distinction. + +// noticeLevelEncoder provides custom level encoding for NOTICE level +// This ensures NOTICE messages are clearly distinguished from other log levels +func noticeLevelEncoder(l zapcore.Level, enc zapcore.PrimitiveArrayEncoder) { + switch l { + case ctrld.NoticeLevel: + enc.AppendString("NOTICE") + default: + zapcore.CapitalLevelEncoder(l, enc) + } +} + +// noticeColorLevelEncoder provides colored level encoding for NOTICE level +// This uses cyan color to make NOTICE messages visually distinct in terminal output +func noticeColorLevelEncoder(l zapcore.Level, enc zapcore.PrimitiveArrayEncoder) { + switch l { + case ctrld.NoticeLevel: + enc.AppendString("\x1b[36mNOTICE\x1b[0m") // Cyan color for NOTICE + default: + zapcore.CapitalColorLevelEncoder(l, enc) + } +} + +// logViewResponse represents the response structure for log viewing requests +// This provides a consistent JSON format for log data retrieval type logViewResponse struct { Data string `json:"data"` } +// logSentResponse represents the response structure for log sending operations +// This includes size information and error details for debugging type logSentResponse struct { Size int64 `json:"size"` Error string `json:"error"` } -type logReader struct { - r io.ReadCloser - size int64 +// logSubscriber represents a subscriber to live log output. +type logSubscriber struct { + ch chan []byte } // logWriter is an internal buffer to keep track of runtime log when no logging is enabled. +// When a file is configured via setLogFile, writes also go to that file, which +// rotates within its budget, so logs survive restarts. type logWriter struct { - mu sync.Mutex - buf bytes.Buffer - size int + mu sync.Mutex + buf bytes.Buffer + size int + subscribers []*logSubscriber + + file *rotatingFile } // newLogWriter creates an internal log writer. +// This provides the default log writer with standard buffer size func newLogWriter() *logWriter { return newLogWriterWithSize(logWriterSize) } // newSmallLogWriter creates an internal log writer with small buffer size. +// This is used in memory-constrained environments or for temporary logging func newSmallLogWriter() *logWriter { return newLogWriterWithSize(logWriterSmallSize) } // newLogWriterWithSize creates an internal log writer with a given buffer size. +// This allows customization of log buffer size based on specific requirements func newLogWriterWithSize(size int) *logWriter { - lw := &logWriter{size: size} - return lw + return &logWriter{size: size} +} + +// setLogFile persists the stream to a file that rotates within the budget. +func (lw *logWriter) setLogFile(path string, budget logBudget) error { + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + return err + } + rf.onRotate = emitLogRotated + lw.mu.Lock() + defer lw.mu.Unlock() + lw.file = rf + return nil +} + +// closeLogFile closes the backing file if open. +func (lw *logWriter) closeLogFile() { + lw.mu.Lock() + defer lw.mu.Unlock() + if lw.file == nil { + return + } + lw.file.close() + lw.file = nil +} + +// rotating returns the backing file, so a caller works with the file itself +// instead of one pass-through method for each of its calls. A stream that +// keeps its lines in memory only has no file. +func (lw *logWriter) rotating() *rotatingFile { + lw.mu.Lock() + defer lw.mu.Unlock() + return lw.file } +// bufferedBytes copies the memory buffer, so a reader of the copy holds no +// writer lock while it works. +func (lw *logWriter) bufferedBytes() []byte { + lw.mu.Lock() + defer lw.mu.Unlock() + buffered := make([]byte, lw.buf.Len()) + copy(buffered, lw.buf.Bytes()) + return buffered +} + +// Subscribe returns a channel that receives new log data as it's written, +// and an unsubscribe function to clean up when done. +func (lw *logWriter) Subscribe() (<-chan []byte, func()) { + lw.mu.Lock() + defer lw.mu.Unlock() + sub := &logSubscriber{ch: make(chan []byte, 256)} + lw.subscribers = append(lw.subscribers, sub) + unsub := func() { + lw.mu.Lock() + defer lw.mu.Unlock() + for i, s := range lw.subscribers { + if s == sub { + lw.subscribers = append(lw.subscribers[:i], lw.subscribers[i+1:]...) + close(sub.ch) + break + } + } + } + return sub.ch, unsub +} + +// tailLastLines returns the last n lines from the current buffer. +func (lw *logWriter) tailLastLines(n int) []byte { + lw.mu.Lock() + defer lw.mu.Unlock() + data := lw.buf.Bytes() + if n <= 0 || len(data) == 0 { + return nil + } + // Find the last n newlines from the end. + count := 0 + pos := len(data) + for pos > 0 { + pos-- + if data[pos] == '\n' { + count++ + if count == n+1 { + pos++ // move past this newline + break + } + } + } + result := make([]byte, len(data)-pos) + copy(result, data[pos:]) + return result +} + +// Write implements io.Writer interface for logWriter func (lw *logWriter) Write(p []byte) (int, error) { + // The file takes the line outside lw.mu: it holds its own lock, and it + // reports its rotation with an event that comes back through this writer. + // A file error must not stop the memory buffer, which is the fallback for + // the log readers. + if rf := lw.rotating(); rf != nil { + _, _ = rf.Write(p) + } + return lw.writeStreams(p) +} + +// writeStreams feeds the subscribers and the memory buffer. +// This manages buffer overflow by discarding old data while preserving important markers +func (lw *logWriter) writeStreams(p []byte) (int, error) { lw.mu.Lock() defer lw.mu.Unlock() + // Fan-out to subscribers (non-blocking). + if len(lw.subscribers) > 0 { + cp := make([]byte, len(p)) + copy(cp, p) + for _, sub := range lw.subscribers { + select { + case sub.ch <- cp: + default: + // Drop if subscriber is slow to avoid blocking the logger. + } + } + } + // If writing p causes overflows, discard old data. + // This prevents unbounded memory growth while maintaining recent logs if lw.buf.Len()+len(p) > lw.size { buf := lw.buf.Bytes() haveEndMarker := false // If there's init end marker already, preserve the data til the marker. + // This ensures initialization logs are always available for debugging if idx := bytes.LastIndex(buf, []byte(logWriterInitEndMarker)); idx >= 0 { buf = buf[:idx+len(logWriterInitEndMarker)] haveEndMarker = true @@ -93,56 +272,265 @@ func (lw *logWriter) Write(p []byte) (int, error) { return lw.buf.Write(p) } +// emitLogRotated logs a rotation. The file that rotated guards this call, +// because the event is a line that can rotate that file again. A rotation +// that could not move the file keeps the lines it holds, so it earns a +// warning instead of the normal report. +func emitLogRotated(r logRotation) { + event := journal(mainLog.Load().Info()) + message := "Log rotated" + if r.err != nil { + event = journal(mainLog.Load().Warn()).Err(r.err) + message = "Log rotation failed" + } + event = event.Str("file", r.file).Int64("bytes_written", r.bytesWritten) + // A file that holds a header only carries no event, and the year 1 of the + // zero time reads as a wrong date. + if !r.firstWrite.IsZero() { + event = event.Time("first_event_at", r.firstWrite) + } + if !r.lastWrite.IsZero() { + event = event.Time("last_event_at", r.lastWrite) + } + event.Int("backups", r.backups).Msg(message) +} + // initLogging initializes global logging setup. func (p *prog) initLogging(backup bool) { - zerolog.TimeFieldFormat = time.RFC3339 + ".000" - logWriters := initLoggingWithBackup(backup) + logCores := initLoggingWithBackup(backup) // Initializing internal logging after global logging. - p.initInternalLogging(logWriters) + p.initInternalLogging(logCores) + p.logger.Store(mainLog.Load()) + + if rf := logPathFile.Load(); rf != nil { + p.refreshLogHeader() + if err := rf.writeHeader(); err != nil { + p.Warn().Err(err).Msg("Could not write log header") + } + } + // A start-time backup happens before the logger reaches the new file, so + // its event waits here for a logger that can report it. + if record := pendingLogPathRotation.Swap(nil); record != nil { + emitLogRotated(*record) + } +} + +// initLoggingAfterProvisioning opens the internal streams of a run whose +// resolver UID came from the provisioning token. The first logging setup ran +// before that UID was known, so it opened no debug file and no journal. +func (p *prog) initLoggingAfterProvisioning() { + if !p.needInternalLogging() { + return + } + if lw, _ := p.internalWriters(); lw != nil { + return + } + p.initLogging(false) +} + +// switchLogPath moves the logging of this run to the log_path that the API +// config set. The header of the new file leads it, the lines of an earlier +// local log_path file follow that header, and the internal streams close, +// because a run with a log_path keeps no internal files. A run without a local +// log_path carries no lines: its history stays in the internal debug stream, +// which this call closes. +func (p *prog) switchLogPath(oldLogPath, newLogPath string) { + // After processCDFlags, log config may change, so reset mainLog and re-init logging. + mainLog.Store(&ctrld.Logger{Logger: zap.NewNop()}) + + carried := carriedLogLines(oldLogPath, newLogPath) + p.initLogging(false) + p.carryLogLines(carried) +} + +// carriedLogLines reads the lines of the old log_path file and empties the new +// file, so the header of the new file leads it. It drops the header lines of +// the old file, because each of them names the old file. +func carriedLogLines(oldLogPath, newLogPath string) []byte { + buf, err := os.ReadFile(oldLogPath) + if err != nil { + return nil + } + if err := os.Remove(newLogPath); err != nil && !os.IsNotExist(err) { + mainLog.Load().Warn().Err(err).Msg("could not clear the new log file") + return nil + } + return withoutLogHeaderLines(buf) +} + +// carryLogLines appends the lines of the old log_path file below the header of +// the new one. +func (p *prog) carryLogLines(lines []byte) { + if len(lines) == 0 { + return + } + rf := logPathFile.Load() + if rf == nil { + return + } + if _, err := rf.Write(lines); err != nil { + p.Warn().Err(err).Msg("Could not copy old log file") + } +} + +// withoutLogHeaderLines removes every header line of a log file. +func withoutLogHeaderLines(buf []byte) []byte { + var kept bytes.Buffer + for _, line := range bytes.SplitAfter(buf, []byte("\n")) { + if isLogHeaderLine(line) { + continue + } + kept.Write(line) + } + return kept.Bytes() +} + +// isLogHeaderLine reports whether one line of a log file is a header line. +func isLogHeaderLine(line []byte) bool { + var parsed struct { + Message string `json:"message"` + } + if err := json.Unmarshal(bytes.TrimSpace(line), &parsed); err != nil { + return false + } + return parsed.Message == logHeaderMessage } // initInternalLogging performs internal logging if there's no log enabled. -func (p *prog) initInternalLogging(writers []io.Writer) { +func (p *prog) initInternalLogging(externalCores []zapcore.Core) { if !p.needInternalLogging() { + // A log_path that the API set takes over from the internal files. + p.closeInternalLogs() return } + headersWritten := false + var headerRotations []logRotation p.initInternalLogWriterOnce.Do(func() { - mainLog.Load().Notice().Msg("internal logging enabled") + // mainLog, not p.logger: a test drives this setup on a bare prog. + mainLog.Load().Notice().Msg("Internal logging enabled") p.internalLogWriter = newLogWriter() - p.internalLogSent = time.Now().Add(-logWriterSentInterval) - p.internalWarnLogWriter = newSmallLogWriter() + p.internalJournalWriter = newSmallLogWriter() + p.openInternalLogFiles() + // A restart appends to the file it finds, so the header marks the + // point where this run starts. + p.refreshLogHeader() + headerRotations = p.writeLogHeaders() + headersWritten = true }) + lw, jlw := p.internalWriters() + + // Create zap cores for different writers + cores := make([]zapcore.Core, 0, len(externalCores)+2) + cores = append(cores, externalCores...) + + // Run the internal logging at debug level, so we could + // have enough information for troubleshooting. The journal core keeps + // the lines that survive a restart. + cores = append(cores, newHumanReadableZapCore(lw, zapcore.DebugLevel), newJournalCore(jlw)) + + // Create a multi-core logger + mainLog.Store(&ctrld.Logger{Logger: zap.New(zapcore.NewTee(cores...))}) + // A header that crossed the budget rotated its file before the logger + // reached the files, so its event waits for this logger. + for _, rotated := range headerRotations { + emitLogRotated(rotated) + } + if headersWritten { + return + } + // A later call reaches a changed config, so the stored bytes need the new + // values. The files keep the header they already hold. + p.applyDebugLogBudget(debugLogBudget(&p.cfg.Service)) + p.refreshLogHeader() +} + +// applyDebugLogBudget gives the open debug files the limits of the config that +// runs now. The files open before the API config arrives, and a managed config +// refresh can change the limits with no restart. Only the internal file gets a +// prune: ctrld owns its name, while log_path can name a file in a directory +// that holds the files of other programs. +func (p *prog) applyDebugLogBudget(budget logBudget) { + if debugWriter, _ := p.internalWriters(); debugWriter != nil { + if rf := debugWriter.rotating(); rf != nil { + rf.setBudget(budget) + pruneNumberedBackups(rf.currentPath(), budget.backups) + } + } + if rf := logPathFile.Load(); rf != nil { + rf.setBudget(budget) + } +} + +// openInternalLogFiles persists both internal streams, so they survive a +// restart. +func (p *prog) openInternalLogFiles() { + _, journalBudget := logBudgets() + openInternalLogFile(p.internalLogWriter, absHomeDir(logFileName), debugLogBudget(&p.cfg.Service), "internal") + openInternalLogFile(p.internalJournalWriter, absHomeDir(journalLogFileName), journalBudget, "journal") +} + +// openInternalLogFile persists one internal stream. A stream whose file cannot +// open keeps its lines in memory. Only these files get a prune: ctrld owns +// their names, while log_path can name a file in a directory that holds the +// files of other programs. +func openInternalLogFile(lw *logWriter, path string, budget logBudget, name string) { + pruneNumberedBackups(path, budget.backups) + if err := lw.setLogFile(path, budget); err != nil { + mainLog.Load().Warn().Err(err).Msgf("Could not enable persistent %s logging", name) + return + } + mainLog.Load().Notice().Msgf("%s log file: %s", name, path) +} + +// internalWriters returns the debug stream and the journal stream. Both stay +// nil until initInternalLogging opens them, and outside cd mode. +func (p *prog) internalWriters() (debugWriter, journalWriter *logWriter) { p.mu.Lock() - lw := p.internalLogWriter - wlw := p.internalWarnLogWriter - p.mu.Unlock() - // If ctrld was run without explicit verbose level, - // run the internal logging at debug level, so we could - // have enough information for troubleshooting. - if verbose == 0 { - for i := range writers { - w := &zerolog.FilteredLevelWriter{ - Writer: zerolog.LevelWriterAdapter{Writer: writers[i]}, - Level: zerolog.NoticeLevel, - } - writers[i] = w + defer p.mu.Unlock() + return p.internalLogWriter, p.internalJournalWriter +} + +// openLogFiles returns the files that this run writes, debug first. A stream +// that keeps its lines in memory only has no file, and log_path belongs to +// this list although no internal writer owns it. +func (p *prog) openLogFiles() []*rotatingFile { + debugWriter, journalWriter := p.internalWriters() + files := make([]*rotatingFile, 0, 3) + for _, writer := range []*logWriter{debugWriter, journalWriter} { + if writer == nil { + continue + } + if rf := writer.rotating(); rf != nil { + files = append(files, rf) } - zerolog.SetGlobalLevel(zerolog.DebugLevel) } - writers = append(writers, lw) - writers = append(writers, &zerolog.FilteredLevelWriter{ - Writer: zerolog.LevelWriterAdapter{Writer: wlw}, - Level: zerolog.WarnLevel, - }) - multi := zerolog.MultiLevelWriter(writers...) - l := mainLog.Load().Output(multi).With().Logger() - mainLog.Store(&l) - ctrld.ProxyLogger.Store(&l) + if rf := logPathFile.Load(); rf != nil { + files = append(files, rf) + } + return files +} + +// closeInternalLogs closes both internal files, so a stop leaves no open +// handle behind. +func (p *prog) closeInternalLogs() { + debugWriter, journalWriter := p.internalWriters() + for _, writer := range []*logWriter{debugWriter, journalWriter} { + if writer == nil { + continue + } + writer.closeLogFile() + } } // needInternalLogging reports whether prog needs to run internal logging. func (p *prog) needInternalLogging() bool { + // Do not run in silent mode: the user explicitly asked for no logging, so + // ctrld must not create or write the persisted internal log file (nor reset + // the global level back to debug). See https://github.com/Control-D-Inc/ctrld/issues/320. + if silent { + return false + } // Do not run in non-cd mode. if cdUID == "" { return false @@ -154,51 +542,71 @@ func (p *prog) needInternalLogging() bool { return true } -func (p *prog) logReader() (*logReader, error) { - if p.needInternalLogging() { - p.mu.Lock() - lw := p.internalLogWriter - wlw := p.internalWarnLogWriter - p.mu.Unlock() - if lw == nil { - return nil, errors.New("nil internal log writer") - } - if wlw == nil { - return nil, errors.New("nil internal warn log writer") - } - // Normal log content. - lw.mu.Lock() - lwReader := bytes.NewReader(lw.buf.Bytes()) - lwSize := lw.buf.Len() - lw.mu.Unlock() - // Warn log content. - wlw.mu.Lock() - wlwReader := bytes.NewReader(wlw.buf.Bytes()) - wlwSize := wlw.buf.Len() - wlw.mu.Unlock() - reader := io.MultiReader(lwReader, bytes.NewReader([]byte(logWriterLogEndMarker)), wlwReader) - lr := &logReader{r: io.NopCloser(reader)} - lr.size = int64(lwSize + wlwSize) - if lr.size == 0 { - return nil, errors.New("internal log is empty") - } - return lr, nil - } - if p.cfg.Service.LogPath == "" { - return &logReader{r: io.NopCloser(strings.NewReader(""))}, nil - } - f, err := os.Open(normalizeLogFilePath(p.cfg.Service.LogPath)) - if err != nil { - return nil, err - } - lr := &logReader{r: f} - if st, err := f.Stat(); err == nil { - lr.size = st.Size() - } else { - return nil, fmt.Errorf("f.Stat: %w", err) - } - if lr.size == 0 { - return nil, errors.New("log file is empty") +// newHumanReadableZapCore creates a zap core optimized for human-readable log output. +// +// Features: +// - Uses development encoder configuration for enhanced readability +// - Console encoding with colored log levels for easy visual scanning +// - Millisecond precision timestamps in human-friendly format +// - Structured field output with clear key-value pairs +// - Ideal for development, debugging, and interactive terminal sessions +// +// Parameters: +// - w: The output writer (e.g., os.Stdout, file, buffer) +// - level: Minimum log level to capture (e.g., Debug, Info, Warn, Error) +// +// Returns a zapcore.Core configured for human consumption. +func newHumanReadableZapCore(w io.Writer, level zapcore.Level) zapcore.Core { + encoderConfig := zap.NewDevelopmentEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout(time.StampMilli) + encoderConfig.EncodeLevel = noticeColorLevelEncoder + encoder := zapcore.NewConsoleEncoder(encoderConfig) + return zapcore.NewCore(encoder, zapcore.AddSync(w), level) +} + +// newMachineFriendlyZapCore creates a zap core optimized for machine processing and log aggregation. +// +// Features: +// - Uses production encoder configuration for consistent, parseable output +// - Console encoding with non-colored log levels for log parsing tools +// - Millisecond precision timestamps in ISO-like format +// - Structured field output optimized for log aggregation systems +// - Ideal for production environments, log shipping, and automated analysis +// +// Parameters: +// - w: The output writer (e.g., os.Stdout, file, buffer) +// - level: Minimum log level to capture (e.g., Debug, Info, Warn, Error) +// +// Returns a zapcore.Core configured for machine consumption and log aggregation. +func newMachineFriendlyZapCore(w io.Writer, level zapcore.Level) zapcore.Core { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout(time.StampMilli) + encoderConfig.EncodeLevel = noticeLevelEncoder + encoder := zapcore.NewConsoleEncoder(encoderConfig) + return zapcore.NewCore(encoder, zapcore.AddSync(w), level) +} + +// ansiRegex is a regular expression to match ANSI color codes. +var ansiRegex = regexp.MustCompile(`\x1b\[[0-9;]*m`) + +// newLogReader creates a reader for log buffer content with optional ANSI color stripping. +// +// This function provides flexible log content access by allowing consumers to choose +// between raw log data (with ANSI color codes) or stripped content (without color codes). +// The color stripping is useful when logs need to be processed by tools that don't +// handle ANSI escape sequences properly, or when storing logs in plain text format. +// +// Parameters: +// - buf: The log buffer containing the log data to read +// - stripColor: If true, strips ANSI color codes from the log content; +// if false, returns raw log content with color codes preserved +// +// Returns an io.Reader that provides access to the processed log content. +func newLogReader(buf *bytes.Buffer, stripColor bool) io.Reader { + if stripColor { + return newANSIStripReader(strings.NewReader(buf.String())) } - return lr, nil + return strings.NewReader(buf.String()) } diff --git a/cmd/cli/log_writer_silent_test.go b/cmd/cli/log_writer_silent_test.go new file mode 100644 index 00000000..75b52f2a --- /dev/null +++ b/cmd/cli/log_writer_silent_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +// Test_needInternalLogging_silent is a regression test for +// https://github.com/Control-D-Inc/ctrld/issues/320: running with --silent must +// not enable internal logging, otherwise ctrld creates and writes +// /ctrld.log (and, when verbose==0, resets the global level back to +// debug) despite the user asking for silence. +func Test_needInternalLogging_silent(t *testing.T) { + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + + tests := []struct { + name string + silent bool + cdUID string + logPath string + want bool + }{ + {"silent suppresses internal logging in cd mode", true, "test-uid", "", false}, + {"cd mode enables internal logging", false, "test-uid", "", true}, + {"non-cd mode disabled", false, "", "", false}, + {"explicit log path disables internal logging", false, "test-uid", "/var/log/ctrld.log", false}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + silent = tt.silent + cdUID = tt.cdUID + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.cfg.Service.LogPath = tt.logPath + if got := p.needInternalLogging(); got != tt.want { + t.Fatalf("needInternalLogging() = %v, want %v", got, tt.want) + } + }) + } +} + +// Test_initInternalLogging_silentCreatesNoFile drives the real initInternalLogging +// path and asserts that a --silent --cd run does not create /ctrld.log, +// which is the observable failure reported in +// https://github.com/Control-D-Inc/ctrld/issues/320. +func Test_initInternalLogging_silentCreatesNoFile(t *testing.T) { + origSilent, origCdUID, origHomedir := silent, cdUID, homedir + t.Cleanup(func() { silent, cdUID, homedir = origSilent, origCdUID, origHomedir }) + + dir := t.TempDir() + homedir = dir + cdUID = "test-uid" // cd mode, which would otherwise enable internal logging + silent = true + stubHeaderSnapshotSources(t) + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initInternalLogging(nil) + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("silent mode must not create %s (stat err = %v)", path, err) + } + } +} + +// Test_initInternalLogging_createsJournalFile drives a cd mode run and asserts +// that the journal file holds the retained lines only, while the debug file +// holds every line. +func Test_initInternalLogging_createsJournalFile(t *testing.T) { + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog := mainLog.Load() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + }) + if origMainLog == nil { + mainLog.Store(ctrld.NopLogger) + } + + dir := t.TempDir() + homedir = dir + cdUID = "test-uid" + silent = false + verbose = 0 + stubHeaderSnapshotSources(t) + + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + + debugPath := filepath.Join(dir, logFileName) + journalPath := filepath.Join(dir, journalLogFileName) + for _, path := range []string{debugPath, journalPath} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("stat %s: %v", path, err) + } + } + + const ( + warnMsg = "retained warn line" + markedMsg = "retained marked info line" + plainMsg = "dropped plain info line" + ) + logger := mainLog.Load() + logger.Warn().Msg(warnMsg) + journal(logger.Info()).Msg(markedMsg) + logger.Info().Msg(plainMsg) + + journalText := rotatingFileTestContent(t, journalPath) + for _, msg := range []string{warnMsg, markedMsg} { + if !strings.Contains(journalText, msg) { + t.Fatalf("journal file misses %q: %s", msg, journalText) + } + } + if strings.Contains(journalText, plainMsg) { + t.Fatalf("journal file kept %q: %s", plainMsg, journalText) + } + + debugText := rotatingFileTestContent(t, debugPath) + for _, msg := range []string{warnMsg, markedMsg, plainMsg} { + if !strings.Contains(debugText, msg) { + t.Fatalf("debug file misses %q: %s", msg, debugText) + } + } +} diff --git a/cmd/cli/log_writer_test.go b/cmd/cli/log_writer_test.go index 5336d4eb..6aaada26 100644 --- a/cmd/cli/log_writer_test.go +++ b/cmd/cli/log_writer_test.go @@ -1,9 +1,21 @@ package cli import ( + "bytes" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "slices" "strings" "sync" "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) func Test_logWriter_Write(t *testing.T) { @@ -83,3 +95,602 @@ func Test_logWriter_MarkerInitEnd(t *testing.T) { t.Fatalf("unexpected log content: %s", lw.buf.String()) } } + +// testLogWriterBudget keeps the rotation tests small: four lines fill the +// current file. +var testLogWriterBudget = logBudget{maxSize: 4 * rotatingFileTestLineSize, backups: 2} + +func newTestFileLogWriter(t *testing.T, budget logBudget) (*logWriter, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "test.log") + lw := newLogWriterWithSize(logWriterSize) + if err := lw.setLogFile(path, budget); err != nil { + t.Fatalf("setLogFile: %v", err) + } + t.Cleanup(lw.closeLogFile) + return lw, path +} + +func Test_logWriter_SetLogFile(t *testing.T) { + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + + msg := "hello file\n" + lw.Write([]byte(msg)) + + // Verify data in memory buffer. + if lw.buf.String() != msg { + t.Fatalf("buffer: got %q, want %q", lw.buf.String(), msg) + } + // Verify data on disk. + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if string(data) != msg { + t.Fatalf("file: got %q, want %q", data, msg) + } +} + +func Test_logWriter_FileRotation(t *testing.T) { + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + + // Three rotations: each file takes four lines. + for seq := 1; seq <= 16; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) + } + + for _, suffix := range []string{".1", ".2"} { + if _, err := os.Stat(path + suffix); err != nil { + t.Fatalf("stat %s: %v", path+suffix, err) + } + } + if _, err := os.Stat(path + ".3"); !os.IsNotExist(err) { + t.Fatalf("stat %s.3 = %v, want a not exist error", path, err) + } + + st, err := os.Stat(path) + if err != nil { + t.Fatalf("stat current: %v", err) + } + if want := testLogWriterBudget.maxSize + rotatingFileTestLineSize; st.Size() > want { + t.Fatalf("current file holds %d bytes after rotation, want at most %d", st.Size(), want) + } +} + +func Test_logWriter_RotatingPaths(t *testing.T) { + lw := newLogWriterWithSize(logWriterSize) + if lw.rotating() != nil { + t.Fatal("a writer without a file has a rotating file, want none") + } + + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + if got, want := lw.rotating().paths(), []string{path}; !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } + + // Two rotations, so both backups exist. + for seq := 1; seq <= 9; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) + } + want := []string{path + ".2", path + ".1", path} + if got := lw.rotating().paths(); !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func Test_logWriter_FileAppendOnRestart(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.log") + + // Simulate first run. + lw1 := newLogWriterWithSize(logWriterSize) + if err := lw1.setLogFile(path, testLogWriterBudget); err != nil { + t.Fatalf("setLogFile: %v", err) + } + lw1.Write([]byte("run1\n")) + lw1.closeLogFile() + + // Simulate second run (restart) — file should be appended. + lw2 := newLogWriterWithSize(logWriterSize) + if err := lw2.setLogFile(path, testLogWriterBudget); err != nil { + t.Fatalf("setLogFile: %v", err) + } + lw2.Write([]byte("run2\n")) + lw2.closeLogFile() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + want := "run1\nrun2\n" + if string(data) != want { + t.Fatalf("file: got %q, want %q", data, want) + } +} + +func Test_logWriter_EmitsLogRotated(t *testing.T) { + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + + // The logger writes back into the writer that rotates, which is the case + // that deadlocks when the writer still holds its lock. + buf := &syncBuffer{} + logger := newTestJSONLogger(lw, buf) + old := mainLog.Load() + mainLog.Store(logger) + t.Cleanup(func() { mainLog.Store(old) }) + + done := make(chan struct{}) + go func() { + defer close(done) + for seq := 1; seq <= 5; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) + } + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("the writer did not return: the rotation event blocked on the writer lock") + } + + st, err := os.Stat(path + ".1") + if err != nil { + t.Fatalf("stat %s.1: %v", path, err) + } + + var rotatedLines []string + for _, line := range strings.Split(buf.String(), "\n") { + if strings.Contains(line, "Log rotated") { + rotatedLines = append(rotatedLines, line) + } + } + if len(rotatedLines) != 1 { + t.Fatalf("got %d Log rotated events, want 1: %q", len(rotatedLines), buf.String()) + } + if !strings.Contains(rotatedLines[0], string(journalMarker)) { + t.Fatalf("the event is not marked for the journal: %s", rotatedLines[0]) + } + var event struct { + File string `json:"file"` + BytesWritten int64 `json:"bytes_written"` + } + if err := json.Unmarshal([]byte(rotatedLines[0]), &event); err != nil { + t.Fatalf("unmarshal %s: %v", rotatedLines[0], err) + } + if event.File != path+".1" { + t.Fatalf("file = %q, want %q", event.File, path+".1") + } + if event.BytesWritten != st.Size() { + t.Fatalf("bytes_written = %d, want %d", event.BytesWritten, st.Size()) + } +} + +func Test_logWriter_NestedRotationStillEmitsEvent(t *testing.T) { + debugLog, debugPath := newTestFileLogWriter(t, testLogWriterBudget) + journalLog, journalPath := newTestFileLogWriter(t, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}) + + buf := &syncBuffer{} + logger := newTestJSONLogger(debugLog, journalLog, buf) + old := mainLog.Load() + mainLog.Store(logger) + t.Cleanup(func() { mainLog.Store(old) }) + + // Fill the journal, so the rotation event of the debug file is the line + // that rotates the journal. That nested rotation needs its own event. + journalLog.Write(rotatingFileTestLine(1, 1)) + journalLog.Write(rotatingFileTestLine(1, 2)) + for seq := 1; seq <= 5; seq++ { + debugLog.Write(rotatingFileTestLine(1, seq)) + } + + rotatedFiles := map[string]bool{} + for _, line := range strings.Split(buf.String(), "\n") { + if !strings.Contains(line, "Log rotated") { + continue + } + var event struct { + File string `json:"file"` + } + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("parse %q: %v", line, err) + } + rotatedFiles[event.File] = true + } + for _, want := range []string{debugPath + ".1", journalPath + ".1"} { + if !rotatedFiles[want] { + t.Fatalf("no Log rotated event for %s; events: %v", want, rotatedFiles) + } + } + if len(rotatedFiles) != 2 { + t.Fatalf("got %d rotated files, want 2: %v", len(rotatedFiles), rotatedFiles) + } +} + +// rotationEventFields parses one log line into its raw fields, so a test can +// ask whether a field is there at all. +func rotationEventFields(t *testing.T, line string) map[string]json.RawMessage { + t.Helper() + fields := map[string]json.RawMessage{} + if err := json.Unmarshal([]byte(strings.TrimSpace(line)), &fields); err != nil { + t.Fatalf("unmarshal %q: %v", line, err) + } + return fields +} + +func Test_emitLogRotated_eventTimes(t *testing.T) { + + t.Run("a file with no event names no time", func(t *testing.T) { + buf := captureJSONMainLog(t) + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", backups: 1}) + + fields := rotationEventFields(t, buf.String()) + for _, name := range []string{"first_event_at", "last_event_at"} { + if raw, ok := fields[name]; ok { + t.Errorf("%s = %s, want no field for a file with no event", name, raw) + } + } + }) + + t.Run("a file with events names both times", func(t *testing.T) { + buf := captureJSONMainLog(t) + now := time.Now() + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", backups: 1, firstWrite: now, lastWrite: now}) + + fields := rotationEventFields(t, buf.String()) + for _, name := range []string{"first_event_at", "last_event_at"} { + if _, ok := fields[name]; !ok { + t.Errorf("%s is missing: %v", name, fields) + } + } + }) +} + +func Test_emitLogRotated_reportsAFailedRotation(t *testing.T) { + buf := captureJSONMainLog(t) + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", err: errors.New("rename refused")}) + + var event struct { + Level string `json:"level"` + Message string `json:"message"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(buf.String())), &event); err != nil { + t.Fatalf("unmarshal %q: %v", buf.String(), err) + } + if event.Message != "Log rotation failed" { + t.Errorf("message = %q, want %q", event.Message, "Log rotation failed") + } + if event.Level != "warn" { + t.Errorf("level = %q, want %q", event.Level, "warn") + } + if event.Error != "rename refused" { + t.Errorf("error = %q, want %q", event.Error, "rename refused") + } +} + +// TestNoticeLevel tests that the custom NOTICE level works correctly +func TestNoticeLevel(t *testing.T) { + // Create a buffer to capture log output + var buf bytes.Buffer + + // Create encoder config with custom NOTICE level support + encoderConfig := zap.NewDevelopmentEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout("15:04:05.000") + encoderConfig.EncodeLevel = noticeLevelEncoder + + // Test with NOTICE level + encoder := zapcore.NewConsoleEncoder(encoderConfig) + core := zapcore.NewCore(encoder, zapcore.AddSync(&buf), ctrld.NoticeLevel) + logger := zap.New(core) + ctrldLogger := &ctrld.Logger{Logger: logger} + + // Log messages at different levels + ctrldLogger.Debug().Msg("This is a DEBUG message") + ctrldLogger.Info().Msg("This is an INFO message") + ctrldLogger.Notice().Msg("This is a NOTICE message") + ctrldLogger.Warn().Msg("This is a WARN message") + ctrldLogger.Error().Msg("This is an ERROR message") + + output := buf.String() + + // Verify that DEBUG and INFO messages are NOT logged (filtered out) + if strings.Contains(output, "DEBUG") { + t.Error("DEBUG message should not be logged when level is NOTICE") + } + if strings.Contains(output, "INFO") { + t.Error("INFO message should not be logged when level is NOTICE") + } + + // Verify that NOTICE, WARN, and ERROR messages ARE logged + if !strings.Contains(output, "NOTICE") { + t.Error("NOTICE message should be logged when level is NOTICE") + } + if !strings.Contains(output, "WARN") { + t.Error("WARN message should be logged when level is NOTICE") + } + if !strings.Contains(output, "ERROR") { + t.Error("ERROR message should be logged when level is NOTICE") + } + + // Verify the NOTICE message content + if !strings.Contains(output, "This is a NOTICE message") { + t.Error("NOTICE message content should be present") + } + + t.Logf("Log output with NOTICE level:\n%s", output) +} + +func TestNewLogReader(t *testing.T) { + tests := []struct { + name string + bufContent string + stripColor bool + expected string + description string + }{ + { + name: "empty_buffer_no_color_strip", + bufContent: "", + stripColor: false, + expected: "", + description: "Empty buffer should return empty reader", + }, + { + name: "empty_buffer_with_color_strip", + bufContent: "", + stripColor: true, + expected: "", + description: "Empty buffer with color strip should return empty reader", + }, + { + name: "plain_text_no_color_strip", + bufContent: "This is plain text without any color codes", + stripColor: false, + expected: "This is plain text without any color codes", + description: "Plain text should be returned as-is when not stripping colors", + }, + { + name: "plain_text_with_color_strip", + bufContent: "This is plain text without any color codes", + stripColor: true, + expected: "This is plain text without any color codes", + description: "Plain text should be returned as-is when stripping colors", + }, + { + name: "text_with_ansi_codes_no_strip", + bufContent: "Normal text \x1b[31mred text\x1b[0m normal again", + stripColor: false, + expected: "Normal text \x1b[31mred text\x1b[0m normal again", + description: "ANSI color codes should be preserved when not stripping", + }, + { + name: "text_with_ansi_codes_with_strip", + bufContent: "Normal text \x1b[31mred text\x1b[0m normal again", + stripColor: true, + expected: "Normal text red text normal again", + description: "ANSI color codes should be removed when stripping colors", + }, + { + name: "multiple_ansi_codes_no_strip", + bufContent: "\x1b[1mBold\x1b[0m \x1b[32mGreen\x1b[0m \x1b[34mBlue\x1b[0m text", + stripColor: false, + expected: "\x1b[1mBold\x1b[0m \x1b[32mGreen\x1b[0m \x1b[34mBlue\x1b[0m text", + description: "Multiple ANSI codes should be preserved when not stripping", + }, + { + name: "multiple_ansi_codes_with_strip", + bufContent: "\x1b[1mBold\x1b[0m \x1b[32mGreen\x1b[0m \x1b[34mBlue\x1b[0m text", + stripColor: true, + expected: "Bold Green Blue text", + description: "Multiple ANSI codes should be removed when stripping colors", + }, + { + name: "complex_ansi_sequences_no_strip", + bufContent: "\x1b[1;31;42mBold red on green\x1b[0m \x1b[38;5;208mOrange\x1b[0m", + stripColor: false, + expected: "\x1b[1;31;42mBold red on green\x1b[0m \x1b[38;5;208mOrange\x1b[0m", + description: "Complex ANSI sequences should be preserved when not stripping", + }, + { + name: "complex_ansi_sequences_with_strip", + bufContent: "\x1b[1;31;42mBold red on green\x1b[0m \x1b[38;5;208mOrange\x1b[0m", + stripColor: true, + expected: "Bold red on green Orange", + description: "Complex ANSI sequences should be removed when stripping colors", + }, + { + name: "ansi_codes_with_newlines_no_strip", + bufContent: "Line 1\n\x1b[31mRed line\x1b[0m\nLine 3", + stripColor: false, + expected: "Line 1\n\x1b[31mRed line\x1b[0m\nLine 3", + description: "ANSI codes with newlines should be preserved when not stripping", + }, + { + name: "ansi_codes_with_newlines_with_strip", + bufContent: "Line 1\n\x1b[31mRed line\x1b[0m\nLine 3", + stripColor: true, + expected: "Line 1\nRed line\nLine 3", + description: "ANSI codes with newlines should be removed when stripping colors", + }, + { + name: "malformed_ansi_codes_no_strip", + bufContent: "Text \x1b[invalidm \x1b[0m normal", + stripColor: false, + expected: "Text \x1b[invalidm \x1b[0m normal", + description: "Malformed ANSI codes should be preserved when not stripping", + }, + { + name: "malformed_ansi_codes_with_strip", + bufContent: "Text \x1b[invalidm \x1b[0m normal", + stripColor: true, + expected: "Text \x1b[invalidm normal", + description: "Non-matching ANSI sequences should be preserved when stripping colors", + }, + { + name: "large_buffer_no_strip", + bufContent: strings.Repeat("A", 10000) + "\x1b[31m" + strings.Repeat("B", 1000) + "\x1b[0m", + stripColor: false, + expected: strings.Repeat("A", 10000) + "\x1b[31m" + strings.Repeat("B", 1000) + "\x1b[0m", + description: "Large buffer should handle ANSI codes correctly when not stripping", + }, + { + name: "large_buffer_with_strip", + bufContent: strings.Repeat("A", 10000) + "\x1b[31m" + strings.Repeat("B", 1000) + "\x1b[0m", + stripColor: true, + expected: strings.Repeat("A", 10000) + strings.Repeat("B", 1000), + description: "Large buffer should remove ANSI codes correctly when stripping", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a buffer with the test content + buf := &bytes.Buffer{} + buf.WriteString(tt.bufContent) + + // Create the log reader + reader := newLogReader(buf, tt.stripColor) + + // Read all content from the reader + content, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("Failed to read from log reader: %v", err) + } + + // Verify the content matches expected + actual := string(content) + if actual != tt.expected { + t.Errorf("Expected content: %q, got: %q", tt.expected, actual) + t.Logf("Description: %s", tt.description) + } + }) + } +} + +func TestNewLogReader_ReaderBehavior(t *testing.T) { + // Test that the returned reader behaves correctly + buf := &bytes.Buffer{} + buf.WriteString("Test content with \x1b[31mred\x1b[0m text") + + // Test with color stripping + reader := newLogReader(buf, true) + + // Test reading in chunks + chunk1 := make([]byte, 10) + n1, err := reader.Read(chunk1) + if err != nil && err != io.EOF { + t.Fatalf("Unexpected error reading first chunk: %v", err) + } + if n1 != 10 { + t.Errorf("Expected to read 10 bytes, got %d", n1) + } + + // Test reading remaining content + remaining, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("Failed to read remaining content: %v", err) + } + + // Verify total content + totalContent := string(chunk1[:n1]) + string(remaining) + expected := "Test content with red text" + if totalContent != expected { + t.Errorf("Expected total content: %q, got: %q", expected, totalContent) + } +} + +func TestNewLogReader_ConcurrentAccess(t *testing.T) { + // Test concurrent access to the same buffer + buf := &bytes.Buffer{} + buf.WriteString("Concurrent test with \x1b[32mgreen\x1b[0m text") + + var wg sync.WaitGroup + numGoroutines := 10 + results := make(chan string, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + reader := newLogReader(buf, true) + content, err := io.ReadAll(reader) + if err != nil { + t.Errorf("Failed to read content: %v", err) + return + } + results <- string(content) + }() + } + + wg.Wait() + close(results) + + // Verify all goroutines got the same result + expected := "Concurrent test with green text" + for result := range results { + if result != expected { + t.Errorf("Expected: %q, got: %q", expected, result) + } + } +} + +func TestNewLogReader_ANSIRegexEdgeCases(t *testing.T) { + // Test edge cases for ANSI regex matching + tests := []struct { + name string + input string + expected string + }{ + { + name: "empty_escape_sequence", + input: "Text \x1b[m normal", + expected: "Text normal", + }, + { + name: "multiple_semicolons", + input: "Text \x1b[1;2;3;4m normal", + expected: "Text normal", + }, + { + name: "numeric_only", + input: "Text \x1b[123m normal", + expected: "Text normal", + }, + { + name: "mixed_numeric_semicolon", + input: "Text \x1b[1;23;456m normal", + expected: "Text normal", + }, + { + name: "no_closing_bracket", + input: "Text \x1b[31 normal", + expected: "Text \x1b[31 normal", + }, + { + name: "no_opening_bracket", + input: "Text 31m normal", + expected: "Text 31m normal", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := &bytes.Buffer{} + buf.WriteString(tt.input) + + reader := newLogReader(buf, true) + content, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("Failed to read content: %v", err) + } + + actual := string(content) + if actual != tt.expected { + t.Errorf("Expected: %q, got: %q", tt.expected, actual) + } + }) + } +} diff --git a/cmd/cli/loop.go b/cmd/cli/loop.go index 3504bc34..94e42181 100644 --- a/cmd/cli/loop.go +++ b/cmd/cli/loop.go @@ -14,6 +14,11 @@ import ( const ( loopTestDomain = ".test" loopTestQtype = dns.TypeTXT + + // defaultLoopCheckTimeout bounds one loop probe when the upstream sets no + // timeout. It matches the DNS client's own default, so the unconfigured + // case keeps its previous behavior. + defaultLoopCheckTimeout = 2 * time.Second ) // newLoopGuard returns new loopGuard. @@ -84,8 +89,11 @@ func (p *prog) detectLoop(msg *dns.Msg) { // // See: https://thekelleys.org.uk/dnsmasq/docs/dnsmasq-man.html func (p *prog) checkDnsLoop() { - mainLog.Load().Debug().Msg("start checking DNS loop") + p.Debug().Msg("Start checking DNS loop") upstream := make(map[string]*ctrld.UpstreamConfig) + // The probe loop below runs on the upstream UID, and the failure report + // needs the config key. Record the reference while the key is in hand. + reference := make(map[string]string) p.loopMu.Lock() for n, uc := range p.cfg.Upstream { if p.um.isDown("upstream." + n) { @@ -93,15 +101,17 @@ func (p *prog) checkDnsLoop() { } // Do not send test query to external upstream. if !canBeLocalUpstream(uc.Domain) { - mainLog.Load().Debug().Msgf("skipping external: upstream.%s", n) + p.Debug().Msgf("Skipping external: upstream.%s", n) continue } uid := uc.UID() p.loop[uid] = false upstream[uid] = uc + reference[uid] = upstreamPrefix + n } p.loopMu.Unlock() + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) for uid := range p.loop { msg := loopTestMsg(uid) uc := upstream[uid] @@ -109,16 +119,37 @@ func (p *prog) checkDnsLoop() { if uc == nil { continue } - resolver, err := ctrld.NewResolver(uc) + resolver, err := ctrld.NewResolver(loggerCtx, uc) if err != nil { - mainLog.Load().Warn().Err(err).Msgf("could not perform loop check for upstream: %q, endpoint: %q", uc.Name, uc.Endpoint) + p.logUpstreamProbeFailure(reference[uid], uc, err, p.Warn, "Could not perform loop check") continue } - if _, err := resolver.Resolve(context.Background(), msg); err != nil { - mainLog.Load().Warn().Err(err).Msgf("could not send DNS loop check query for upstream: %q, endpoint: %q", uc.Name, uc.Endpoint) + // Bound the probe, like checkUpstreamOnce does. Without a deadline the + // DNS client falls back to its own default, so one unreachable local + // upstream stalls this serial loop far past the configured timeout. + timeout := defaultLoopCheckTimeout + if uc.Timeout > 0 { + timeout = time.Millisecond * time.Duration(uc.Timeout) + } + if err := resolveLoopTestMsg(resolver, msg, timeout); err != nil { + p.logUpstreamProbeFailure(reference[uid], uc, err, p.Warn, "Could not send DNS loop check query") } } - mainLog.Load().Debug().Msg("end checking DNS loop") + p.Debug().Msg("End checking DNS loop") +} + +// resolveLoopTestMsg sends one loop test query under its own deadline. The +// cancel runs before the next upstream, so a long loop leaks no contexts. +// +// The context deliberately carries no logger. A resolver logs its own failure +// at error level with the endpoint in the message, which would put an Internal +// Domain resolver address in the retained journal; logUpstreamProbeFailure is +// what reports this failure, and it bounds what the line may hold. +func resolveLoopTestMsg(resolver ctrld.Resolver, msg *dns.Msg, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + _, err := resolver.Resolve(ctx, msg) + return err } // checkDnsLoopTicker performs p.checkDnsLoop every minute. @@ -137,7 +168,7 @@ func (p *prog) checkDnsLoopTicker(ctx context.Context) { } } -// loopTestMsg generates DNS message for checking loop. +// loopTestMsg creates a DNS test message for loop detection func loopTestMsg(uid string) *dns.Msg { msg := new(dns.Msg) msg.SetQuestion(dns.Fqdn(uid+loopTestDomain), loopTestQtype) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 07839756..2df009f3 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -2,71 +2,97 @@ package cli import ( "io" + "net" "os" "path/filepath" "sync/atomic" - "time" "github.com/kardianos/service" - "github.com/rs/zerolog" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" "github.com/Control-D-Inc/ctrld" ) +// Global variables for CLI configuration and state management +// These are used across multiple commands and need to persist throughout the application lifecycle var ( - configPath string - configBase64 string - daemon bool - listenAddress string - primaryUpstream string - secondaryUpstream string - domains []string - logPath string - homedir string - cacheSize int - cfg ctrld.Config - verbose int - silent bool - cdUID string - cdOrg string - customHostname string - cdDev bool - iface string - ifaceStartStop string - nextdns string - cdUpstreamProto string - deactivationPin int64 - skipSelfChecks bool - cleanup bool - startOnly bool - rfc1918 bool - - mainLog atomic.Pointer[zerolog.Logger] - consoleWriter zerolog.ConsoleWriter - noConfigStart bool + configPath string + configBase64 string + daemon bool + listenAddress string + primaryUpstream string + secondaryUpstream string + domains []string + logPath string + homedir string + cacheSize int + cfg ctrld.Config + verbose int + silent bool + cdUID string + cdOrg string + customHostname string + cdDev bool + iface string + ifaceStartStop string + nextdns string + cdUpstreamProto string + deactivationPin int64 + skipSelfChecks bool + cleanup bool + startOnly bool + rfc1918 bool + interceptMode string // "", "off", "dns", or "hard" — set via --intercept-mode flag or config + dnsIntercept bool // derived: interceptMode == "dns" || interceptMode == "hard" + hardIntercept bool // derived: interceptMode == "hard" + firewallMode string // "off" or "on" — set via --firewall-mode flag or config + firewallModeFlagChanged bool // true when --firewall-mode was explicitly provided + + mainLog atomic.Pointer[ctrld.Logger] + consoleWriter zapcore.Core + consoleWriterLevel zapcore.Level + noConfigStart bool ) +// Flag name constants for consistent reference across the codebase +// Using constants prevents typos and makes refactoring easier const ( cdUidFlagName = "cd" cdOrgFlagName = "cd-org" customHostnameFlagName = "custom-hostname" nextdnsFlagName = "nextdns" + + // autoIface is the sentinel --iface value meaning "use the default gateway interface". + autoIface = "auto" ) +// init initializes the default logger before any CLI commands are executed +// This ensures logging is available even during early initialization phases func init() { - l := zerolog.New(io.Discard) - mainLog.Store(&l) + l := zap.NewNop() + mainLog.Store(&ctrld.Logger{Logger: l}) } +// Main is the entry point for the CLI application +// It initializes configuration, sets up the CLI structure, and executes the root command func Main() { + // Fast path for pf interception probe subprocess. + if len(os.Args) >= 4 && os.Args[1] == "pf-probe-send" { + pfProbeSend(os.Args[2], os.Args[3]) + return + } + ctrld.InitConfig(v, "ctrld") - initCLI() + rootCmd := initCLI() if err := rootCmd.Execute(); err != nil { mainLog.Load().Error().Msg(err.Error()) os.Exit(1) } } +// normalizeLogFilePath converts relative log file paths to absolute paths +// This ensures log files are created in predictable locations regardless of working directory func normalizeLogFilePath(logFilePath string) string { if logFilePath == "" || filepath.IsAbs(logFilePath) || service.Interactive() { return logFilePath @@ -82,40 +108,36 @@ func normalizeLogFilePath(logFilePath string) string { } // initConsoleLogging initializes console logging, then storing to mainLog. +// This sets up human-readable logging output for interactive use func initConsoleLogging() { - consoleWriter = zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) { - w.TimeFormat = time.StampMilli - }) - multi := zerolog.MultiLevelWriter(consoleWriter) - l := mainLog.Load().Output(multi).With().Timestamp().Logger() - mainLog.Store(&l) - + consoleWriterLevel = ctrld.NoticeLevel switch { case silent: - zerolog.SetGlobalLevel(zerolog.NoLevel) + // For silent mode, use a no-op logger to suppress all output + l := zap.NewNop() + mainLog.Store(&ctrld.Logger{Logger: l}) case verbose == 1: - ctrld.ProxyLogger.Store(&l) - zerolog.SetGlobalLevel(zerolog.InfoLevel) + // Info level provides basic operational information + consoleWriterLevel = zapcore.InfoLevel case verbose > 1: - ctrld.ProxyLogger.Store(&l) - zerolog.SetGlobalLevel(zerolog.DebugLevel) - default: - zerolog.SetGlobalLevel(zerolog.NoticeLevel) + // Debug level provides detailed diagnostic information + consoleWriterLevel = zapcore.DebugLevel } + consoleWriter = newHumanReadableZapCore(os.Stdout, consoleWriterLevel) + l := zap.New(consoleWriter) + mainLog.Store(&ctrld.Logger{Logger: l}) } // initInteractiveLogging is like initLogging, but the ProxyLogger is discarded // to be used for all interactive commands. // // Current log file config will also be ignored. +// This prevents log file conflicts during interactive command execution func initInteractiveLogging() { old := cfg.Service.LogPath cfg.Service.LogPath = "" - zerolog.TimeFieldFormat = time.RFC3339 + ".000" initLoggingWithBackup(false) cfg.Service.LogPath = old - l := zerolog.New(io.Discard) - ctrld.ProxyLogger.Store(&l) } // initLoggingWithBackup initializes log setup base on current config. @@ -124,68 +146,83 @@ func initInteractiveLogging() { // This is only used in runCmd for special handling in case of logging config // change in cd mode. Without special reason, the caller should use initLogging // wrapper instead of calling this function directly. -func initLoggingWithBackup(doBackup bool) []io.Writer { +func initLoggingWithBackup(doBackup bool) []zapcore.Core { var writers []io.Writer - if logFilePath := normalizeLogFilePath(cfg.Service.LogPath); logFilePath != "" { - // Create parent directory if necessary. - if err := os.MkdirAll(filepath.Dir(logFilePath), 0750); err != nil { - mainLog.Load().Error().Msgf("failed to create log path: %v", err) - os.Exit(1) - } - - // Default open log file in append mode. - flags := os.O_CREATE | os.O_RDWR | os.O_APPEND - if doBackup { - // Backup old log file with .1 suffix. - if err := os.Rename(logFilePath, logFilePath+oldLogSuffix); err != nil && !os.IsNotExist(err) { - mainLog.Load().Error().Msgf("could not backup old log file: %v", err) - } else { - // Backup was created, set flags for truncating old log file. - flags = os.O_CREATE | os.O_RDWR - } - } - logFile, err := openLogFile(logFilePath, flags) - if err != nil { - mainLog.Load().Error().Msgf("failed to create log file: %v", err) - os.Exit(1) - } - writers = append(writers, logFile) + if rf := openLogPathWriter(doBackup); rf != nil { + writers = append(writers, rf) } - writers = append(writers, consoleWriter) - multi := zerolog.MultiLevelWriter(writers...) - l := mainLog.Load().Output(multi).With().Logger() - mainLog.Store(&l) - // TODO: find a better way. - ctrld.ProxyLogger.Store(&l) - - zerolog.SetGlobalLevel(zerolog.NoticeLevel) + + // Create zap cores for different writers + // Multiple cores allow logging to both console and file simultaneously + var cores []zapcore.Core + cores = append(cores, consoleWriter) + + // Determine log level based on verbosity and configuration + // This provides flexible logging control for different use cases logLevel := cfg.Service.LogLevel switch { case silent: - zerolog.SetGlobalLevel(zerolog.NoLevel) - return writers + // For silent mode, use a no-op logger to suppress all output + l := zap.NewNop() + mainLog.Store(&ctrld.Logger{Logger: l}) + return cores case verbose == 1: logLevel = "info" case verbose > 1: logLevel = "debug" } - if logLevel == "" { - return writers + + // Parse log level string to zapcore.Level + // This provides human-readable log level configuration + var level zapcore.Level + switch logLevel { + case "debug": + level = zapcore.DebugLevel + case "info": + level = zapcore.InfoLevel + case "notice": + level = ctrld.NoticeLevel + case "warn": + level = zapcore.WarnLevel + case "error": + level = zapcore.ErrorLevel + default: + level = zapcore.InfoLevel // default level } - level, err := zerolog.ParseLevel(logLevel) - if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not set log level") - return writers + + consoleWriter.Enabled(level) + // Add cores for all writers + // This enables multi-destination logging (console + file) + for _, writer := range writers { + core := newMachineFriendlyZapCore(writer, level) + cores = append(cores, core) } - zerolog.SetGlobalLevel(level) - return writers + + // Create a multi-core logger + // This allows simultaneous logging to multiple destinations + multiCore := zapcore.NewTee(cores...) + logger := zap.New(multiCore) + mainLog.Store(&ctrld.Logger{Logger: logger}) + + return cores } +// initCache initializes DNS cache configuration +// This improves performance by caching frequently requested DNS responses func initCache() { if !cfg.Service.CacheEnable { return } if cfg.Service.CacheSize == 0 { + // Default cache size provides good balance between memory usage and performance cfg.Service.CacheSize = 4096 } } + +// pfProbeSend is a minimal subprocess that sends a pre-built DNS query packet +// to the specified host on port 53. +func pfProbeSend(host, hexPacket string) { + if err := sendPFProbe(host, hexPacket, net.DialTimeout, os.Stdout); err != nil { + os.Exit(1) + } +} diff --git a/cmd/cli/main_test.go b/cmd/cli/main_test.go index 6ed26c73..d9b7cef6 100644 --- a/cmd/cli/main_test.go +++ b/cmd/cli/main_test.go @@ -1,17 +1,91 @@ package cli import ( + "fmt" "os" + "os/exec" "strings" + "sync" "testing" - "github.com/rs/zerolog" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" ) -var logOutput strings.Builder +// logOutput is the log sink for the whole test binary. Tests share it with any +// background goroutine the code under test starts (watchdogs, timers), so it +// must tolerate concurrent writes. +var logOutput syncBuffer + +// syncBuffer is a strings.Builder guarded by a mutex. +type syncBuffer struct { + mu sync.Mutex + sb strings.Builder +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.String() +} + +// envFakeVersionOutput makes this test binary impersonate a ctrld executable: when +// set, the process writes the value to stdout and exits without running any test, so +// binaryVersion() can be exercised on every platform without building or shipping a +// fixture binary. The value envFakeVersionSilent produces no output at all, which +// reproduces the unusable ctrld.exe_previous seen in the Firewall Mode incident. +// +// This must be handled before m.Run(), which is what parses the test flags: the child +// is invoked as " --version" and would otherwise die on an unknown flag. +const ( + envFakeVersionOutput = "CTRLD_TEST_FAKE_VERSION_OUTPUT" + envFakeVersionSilent = "" +) func TestMain(m *testing.M) { - l := zerolog.New(&logOutput) - mainLog.Store(&l) + if out := os.Getenv(envFakeVersionOutput); out != "" { + if out != envFakeVersionSilent { + fmt.Println(out) + } + os.Exit(0) + } + + // Create a custom writer that writes to logOutput + writer := zapcore.AddSync(&logOutput) + + // Create zap encoder + encoderConfig := zap.NewDevelopmentEncoderConfig() + encoder := zapcore.NewConsoleEncoder(encoderConfig) + + // Create core that writes to our string builder + core := zapcore.NewCore(encoder, writer, zap.DebugLevel) + + // Create logger + l := zap.New(core) + + mainLog.Store(&ctrld.Logger{Logger: l}) + + // Stub the self-upgrade command builder for the whole test binary. The real + // builder execs os.Executable() — which under `go test` IS this test binary + // — with positional args ("upgrade", ...). `go test` stops flag parsing at + // the first positional arg and ignores the rest, so the child just re-runs + // the entire suite, hits the upgrade tests again, and spawns more children: + // a fork bomb of detached processes that stalls the host and (on Windows) + // holds the test binary's image locked, breaking CI artifact cleanup. + // Point it at the test binary with a no-match -test.run so any test that + // reaches performUpgrade still exercises the cmd.Start() success path while + // the child exits immediately without recursing. + newUpgradeCmd = func(exe string) *exec.Cmd { + return exec.Command(exe, "-test.run=^$") + } + os.Exit(m.Run()) } diff --git a/cmd/cli/metadata_scope_test.go b/cmd/cli/metadata_scope_test.go new file mode 100644 index 00000000..bb499e35 --- /dev/null +++ b/cmd/cli/metadata_scope_test.go @@ -0,0 +1,82 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// requireRuntimeMetadata asserts the runtime half of the startup-versus-runtime +// wire contract in docs/username-detection.md: no username discovery and no +// hostname hints. +func requireRuntimeMetadata(t *testing.T, caller string, req *controld.ResolverConfigRequest) { + t.Helper() + if req == nil { + t.Fatalf("%s did not send a resolver config request", caller) + } + if _, ok := req.Metadata["username"]; ok { + t.Errorf("%s request included username", caller) + } + if req.IncludeHostnameHints { + t.Errorf("%s request asked for hostname hints", caller) + } +} + +func TestDeactivationPinRefreshUsesRuntimeMetadata(t *testing.T) { + oldFetch, oldUID := fetchResolverConfig, cdUID + oldPin, oldKnown := cdDeactivationPin.Load(), deactivationPinKnown.Load() + t.Cleanup(func() { + fetchResolverConfig, cdUID = oldFetch, oldUID + cdDeactivationPin.Store(oldPin) + deactivationPinKnown.Store(oldKnown) + deactivationLockedUntil.Store(0) + }) + cdUID = "test-uid" + deactivationLockedUntil.Store(0) + + var captured *controld.ResolverConfigRequest + fetchResolverConfig = func(_ context.Context, req *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + captured = req + return &controld.ResolverConfig{}, nil + } + + p := &prog{cs: &controlServer{mux: http.NewServeMux()}, pinCodeValidCh: make(chan struct{}, 1)} + p.logger.Store(discardLogger()) + p.registerControlServerHandler() + body, _ := json.Marshal(&deactivationRequest{Pin: defaultDeactivationPin}) + req := httptest.NewRequest(http.MethodPost, deactivationPath, strings.NewReader(string(body))) + p.cs.mux.ServeHTTP(httptest.NewRecorder(), req) + + requireRuntimeMetadata(t, "deactivation-PIN refresh", captured) +} + +func TestSelfUninstallCheckUsesRuntimeMetadata(t *testing.T) { + oldFetch, oldUID := fetchResolverConfig, cdUID + t.Cleanup(func() { fetchResolverConfig, cdUID = oldFetch, oldUID }) + cdUID = "test-uid" + + var captured *controld.ResolverConfigRequest + fetchResolverConfig = func(_ context.Context, req *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + captured = req + // A generic failure is not a device-deleted answer, so no uninstall runs. + return nil, errors.New("stop after capturing metadata") + } + + p := &prog{} + p.logger.Store(discardLogger()) + p.canSelfUninstall.Store(true) + p.refusedQueryCount = selfUninstallMaxQueries + 1 + refused := new(dns.Msg) + refused.SetRcode(newDnsMsgWithHostname("example.com.", dns.TypeA), dns.RcodeRefused) + p.doSelfUninstall(&proxyResponse{answer: refused}) + + requireRuntimeMetadata(t, "self-uninstall check", captured) +} diff --git a/cmd/cli/metrics.go b/cmd/cli/metrics.go index 565cdcc5..7eb394fd 100644 --- a/cmd/cli/metrics.go +++ b/cmd/cli/metrics.go @@ -15,6 +15,7 @@ import ( ) // metricsServer represents a server to expose Prometheus metrics via HTTP. +// This provides monitoring and observability for the DNS proxy service type metricsServer struct { server *http.Server mux *http.ServeMux @@ -24,6 +25,7 @@ type metricsServer struct { } // newMetricsServer returns new metrics server. +// This initializes the HTTP server for exposing Prometheus metrics func newMetricsServer(addr string, reg *prometheus.Registry) (*metricsServer, error) { mux := http.NewServeMux() ms := &metricsServer{ @@ -37,11 +39,13 @@ func newMetricsServer(addr string, reg *prometheus.Registry) (*metricsServer, er } // register adds handlers for given pattern. +// This provides a clean interface for adding HTTP endpoints to the metrics server func (ms *metricsServer) register(pattern string, handler http.Handler) { ms.mux.Handle(pattern, handler) } // registerMetricsServerHandler adds handlers for metrics server. +// This sets up both Prometheus format and JSON format endpoints for metrics func (ms *metricsServer) registerMetricsServerHandler() { ms.register("/metrics", promhttp.HandlerFor( ms.reg, @@ -74,6 +78,7 @@ func (ms *metricsServer) registerMetricsServerHandler() { } // start runs the metricsServer. +// This starts the HTTP server for metrics exposure func (ms *metricsServer) start() error { listener, err := net.Listen("tcp", ms.addr) if err != nil { @@ -85,6 +90,7 @@ func (ms *metricsServer) start() error { } // stop shutdowns the metricsServer within 2 seconds timeout. +// This ensures graceful shutdown of the metrics server func (ms *metricsServer) stop() error { if !ms.started { return nil @@ -95,6 +101,7 @@ func (ms *metricsServer) stop() error { } // runMetricsServer initializes metrics stats and runs the metrics server if enabled. +// This sets up the complete metrics infrastructure including Prometheus collectors func (p *prog) runMetricsServer(ctx context.Context, reloadCh chan struct{}) { if !p.metricsEnabled() { return @@ -113,9 +120,25 @@ func (p *prog) runMetricsServer(ctx context.Context, reloadCh chan struct{}) { } addr := p.cfg.Service.MetricsListener + if addr != "" { + host, port, err := net.SplitHostPort(addr) + if err != nil { + mainLog.Load().Warn().Err(err).Msgf("Invalid metrics listener address (%s); expected host:port", addr) + } else { + if host == "" { + host = "127.0.0.1" + addr = net.JoinHostPort(host, port) + } + ip := net.ParseIP(host) + if (ip != nil && !ip.IsLoopback()) || (ip == nil && host != "localhost") { + mainLog.Load().Warn().Msgf("Metrics server is bound to a non-loopback address (%s). This exposes sensitive data without authentication.", addr) + } + } + } + ms, err := newMetricsServer(addr, reg) if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not create new metrics server") + mainLog.Load().Warn().Err(err).Msg("Could not create new metrics server") return } // Only start listener address if defined. @@ -130,9 +153,9 @@ func (p *prog) runMetricsServer(ctx context.Context, reloadCh chan struct{}) { statsVersion.WithLabelValues(commit, runtime.Version(), curVersion()).Inc() reg.MustRegister(statsTimeStart) statsTimeStart.Set(float64(time.Now().Unix())) - mainLog.Load().Debug().Msgf("starting metrics server on: %s", addr) + mainLog.Load().Debug().Msgf("Starting metrics server on: %s", addr) if err := ms.start(); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not start metrics server") + mainLog.Load().Warn().Err(err).Msg("Could not start metrics server") return } } @@ -144,7 +167,7 @@ func (p *prog) runMetricsServer(ctx context.Context, reloadCh chan struct{}) { } if err := ms.stop(); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not stop metrics server") + mainLog.Load().Warn().Err(err).Msg("Could not stop metrics server") return } } diff --git a/cmd/cli/net_darwin.go b/cmd/cli/net_darwin.go index 62331610..c9c59135 100644 --- a/cmd/cli/net_darwin.go +++ b/cmd/cli/net_darwin.go @@ -1,16 +1,50 @@ package cli import ( - "bufio" "bytes" + "context" + "errors" "io" "net" "os/exec" "strings" + "sync" + "time" +) + +// hardwarePortInfo holds the two names that macOS gives a network device: the +// hardware port of the adapter and the network service that uses it. +type hardwarePortInfo struct { + Port string + Service string +} + +// A network event reads the ports of every interface, so the two networksetup +// calls run at most once per window. +const hardwarePortsTTL = time.Minute + +// networksetupTimeout bounds one command. A stuck read must not hold the +// network callback that waits for the port names. +const networksetupTimeout = 5 * time.Second + +// maxNetworksetupOutput bounds one read. A host with many ports stays far +// below it, so more than this is a broken command. +const maxNetworksetupOutput = 1 << 20 + +var errNetworksetupOutputTooLarge = errors.New("networksetup output above the bound") + +var ( + hardwarePortsMu sync.Mutex + hardwarePortsCache map[string]hardwarePortInfo + hardwarePortsReadAt time.Time + + // Tests replace these seams to avoid a networksetup call. + readHardwarePortsFn = readHardwarePorts + hardwarePortsNowFn = time.Now ) func patchNetIfaceName(iface *net.Interface) (bool, error) { - b, err := exec.Command("networksetup", "-listnetworkserviceorder").Output() + b, err := networksetupOutput("-listnetworkserviceorder") if err != nil { return false, err } @@ -24,53 +58,162 @@ func patchNetIfaceName(iface *net.Interface) (bool, error) { } func networkServiceName(ifaceName string, r io.Reader) string { - scanner := bufio.NewScanner(r) - prevLine := "" + return serviceNamesByDevice(r)[ifaceName] +} + +// parseHardwarePorts maps each device of "networksetup -listallhardwareports" +// to its hardware port. Each block names the port before the device. +func parseHardwarePorts(r io.Reader) map[string]hardwarePortInfo { + ports := make(map[string]hardwarePortInfo) + scanner := scanNetworksetup(r) + port := "" for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "*") { - // Network services is disabled. + line := strings.TrimSpace(scanner.Text()) + if after, ok := strings.CutPrefix(line, "Hardware Port:"); ok { + port = strings.TrimSpace(after) continue } - if !strings.Contains(line, "Device: "+ifaceName) { - prevLine = line + after, ok := strings.CutPrefix(line, "Device:") + if !ok { continue } - parts := strings.SplitN(prevLine, " ", 2) - if len(parts) == 2 { - return strings.TrimSpace(parts[1]) + device := strings.TrimSpace(after) + if device == "" { + continue + } + ports[device] = hardwarePortInfo{Port: port} + port = "" + } + if scanner.Err() != nil { + // A truncated read names fewer ports than the host has. + return nil + } + return ports +} + +// addServiceNames fills the service of every known device. A device that no +// hardware port lists is not a port of this host. +func addServiceNames(m map[string]hardwarePortInfo, r io.Reader) { + for device, service := range serviceNamesByDevice(r) { + info, known := m[device] + if !known { + continue } + info.Service = service + m[device] = info } - return "" } -// validInterface reports whether the *net.Interface is a valid one. -func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bool { - _, ok := validIfacesMap[iface.Name] - return ok +// hardwarePortsByDevice returns the port and service of every device. The +// commands run outside the mutex, so a slow read blocks no other caller. +func hardwarePortsByDevice() map[string]hardwarePortInfo { + if ports, fresh := cachedHardwarePorts(); fresh { + return ports + } + return storeHardwarePorts(readHardwarePortsFn()) +} + +// cachedHardwarePorts reports whether the read window still holds. +func cachedHardwarePorts() (map[string]hardwarePortInfo, bool) { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + if hardwarePortsReadAt.IsZero() { + return nil, false + } + return hardwarePortsCache, hardwarePortsNowFn().Sub(hardwarePortsReadAt) < hardwarePortsTTL +} + +// storeHardwarePorts starts the window again and keeps the last good map. A +// failing command must run once per window, not once per lookup. +func storeHardwarePorts(ports map[string]hardwarePortInfo) map[string]hardwarePortInfo { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = hardwarePortsNowFn() + if ports != nil { + hardwarePortsCache = ports + } + return hardwarePortsCache } -// validInterfacesMap returns a set of all valid hardware ports. -func validInterfacesMap() map[string]struct{} { - b, err := exec.Command("networksetup", "-listallhardwareports").Output() +// readHardwarePorts runs the two networksetup commands. +func readHardwarePorts() map[string]hardwarePortInfo { + b, err := networksetupOutput("-listallhardwareports") if err != nil { return nil } - return parseListAllHardwarePorts(bytes.NewReader(b)) + ports := parseHardwarePorts(bytes.NewReader(b)) + if ports == nil { + // A truncated list must keep the last good map. + return nil + } + b, err = networksetupOutput("-listnetworkserviceorder") + if err != nil { + return ports + } + addServiceNames(ports, bytes.NewReader(b)) + return ports } -// parseListAllHardwarePorts parses output of "networksetup -listallhardwareports" -// and returns map presents all hardware ports. -func parseListAllHardwarePorts(r io.Reader) map[string]struct{} { - m := make(map[string]struct{}) - scanner := bufio.NewScanner(r) - for scanner.Scan() { - line := scanner.Text() - after, ok := strings.CutPrefix(line, "Device: ") - if !ok { - continue - } - m[after] = struct{}{} +// networksetupOutput runs one networksetup list under a time bound and a size +// bound. +func networksetupOutput(arg string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), networksetupTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "networksetup", arg) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + out, readErr := readBoundedOutput(stdout) + if readErr != nil { + // A command ends only when something reads the rest of its output. + _, _ = io.Copy(io.Discard, stdout) + } + if err := cmd.Wait(); err != nil { + return nil, err + } + return out, readErr +} + +// readBoundedOutput reads at most maxNetworksetupOutput bytes and reports an +// error above that. +func readBoundedOutput(r io.Reader) ([]byte, error) { + out, err := io.ReadAll(io.LimitReader(r, maxNetworksetupOutput+1)) + if err != nil { + return nil, err } - return m + if len(out) > maxNetworksetupOutput { + return nil, errNetworksetupOutputTooLarge + } + return out, nil +} + +// platformInterfaceMeta returns the macOS names of an interface. The class +// comes from the interface name, so it stays empty here. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := hardwarePortsByDevice()[name] + return "", info.Port, info.Service +} + +// platformVirtualInterfaces returns no set. The name rules already class the +// AirDrop and the link-local interfaces of macOS. +func platformVirtualInterfaces() map[string]struct{} { + return nil +} + +// refreshInterfaceMeta drops the read window. The cache has no port name for +// an adapter that appeared, so the next lookup reads the ports again. +func refreshInterfaceMeta() { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = time.Time{} +} + +// validInterface reports whether the *net.Interface is a valid one. +func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bool { + _, ok := validIfacesMap[iface.Name] + return ok } diff --git a/cmd/cli/net_darwin_test.go b/cmd/cli/net_darwin_test.go new file mode 100644 index 00000000..4bd9d437 --- /dev/null +++ b/cmd/cli/net_darwin_test.go @@ -0,0 +1,284 @@ +package cli + +import ( + "strings" + "testing" + "time" +) + +const listAllHardwarePortsFixture = ` +Hardware Port: Wi-Fi +Device: en0 +Ethernet Address: a0:78:17:68:56:3f + +Hardware Port: iPhone USB +Device: en7 +Ethernet Address: 6e:5a:6b:12:34:56 + +Hardware Port: Thunderbolt Bridge +Device: bridge0 +Ethernet Address: 36:21:bb:3a:7a:40 + +Hardware Port: Ethernet Adapter (en6) +Device: en6 +Ethernet Address: 3a:3e:fc:1e:ab:41 + +Hardware Port: com.wireguard.macos +Device: + +VLAN Configurations +=================== +` + +const listNetworkServiceOrderFixture = `An asterisk (*) denotes that a network service is disabled. +(1) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) + +(2) iPhone USB +(Hardware Port: iPhone USB, Device: en7) + +(3) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) + +(*) Ethernet Adapter (en6) +(Hardware Port: Ethernet Adapter (en6), Device: en6) + +(4) ca-001-stg +(Hardware Port: com.wireguard.macos, Device: ) + +` + +func Test_parseHardwarePorts(t *testing.T) { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + + tests := []struct { + device string + wantPort string + wantService string + }{ + {"en0", "Wi-Fi", "Wi-Fi"}, + {"en7", "iPhone USB", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge", "Thunderbolt Bridge"}, + {"en6", "Ethernet Adapter (en6)", ""}, + } + if len(ports) != len(tests) { + t.Fatalf("parseHardwarePorts returned %d devices, want %d: %v", len(ports), len(tests), ports) + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + info := ports[tc.device] + if info.Port != tc.wantPort { + t.Errorf("port of %s = %q, want %q", tc.device, info.Port, tc.wantPort) + } + if info.Service != tc.wantService { + t.Errorf("service of %s = %q, want %q", tc.device, info.Service, tc.wantService) + } + }) + } +} + +func Test_networkServiceName(t *testing.T) { + tests := []struct { + device string + want string + }{ + {"en0", "Wi-Fi"}, + {"en7", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge"}, + {"en6", ""}, + {"en9", ""}, + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + got := networkServiceName(tc.device, strings.NewReader(listNetworkServiceOrderFixture)) + if got != tc.want { + t.Errorf("networkServiceName(%q) = %q, want %q", tc.device, got, tc.want) + } + }) + } +} + +func Test_parseHardwarePorts_dropsATruncatedRead(t *testing.T) { + longLine := strings.Repeat("x", maxNetworksetupLine+1) + + if got := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture + longLine)); got != nil { + t.Errorf("parseHardwarePorts of a truncated read = %v, want nil", got) + } + if got := serviceNamesByDevice(strings.NewReader(listNetworkServiceOrderFixture + longLine)); got != nil { + t.Errorf("serviceNamesByDevice of a truncated read = %v, want nil", got) + } +} + +func Test_readBoundedOutput(t *testing.T) { + out, err := readBoundedOutput(strings.NewReader(listAllHardwarePortsFixture)) + if err != nil || string(out) != listAllHardwarePortsFixture { + t.Fatalf("readBoundedOutput of the fixture = %q, %v, want the fixture and no error", out, err) + } + if _, err := readBoundedOutput(strings.NewReader(strings.Repeat("x", maxNetworksetupOutput+1))); err == nil { + t.Fatal("readBoundedOutput of an output above the bound gave no error") + } +} + +// stubHardwarePorts replaces the command and clock seams of the hardware port +// cache, and restores them with the cache itself. +func stubHardwarePorts(t *testing.T, read func() map[string]hardwarePortInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readHardwarePortsFn, hardwarePortsNowFn + hardwarePortsMu.Lock() + cacheBefore, readAtBefore := hardwarePortsCache, hardwarePortsReadAt + hardwarePortsCache, hardwarePortsReadAt = nil, time.Time{} + hardwarePortsMu.Unlock() + t.Cleanup(func() { + readHardwarePortsFn, hardwarePortsNowFn = readBefore, nowBefore + hardwarePortsMu.Lock() + hardwarePortsCache, hardwarePortsReadAt = cacheBefore, readAtBefore + hardwarePortsMu.Unlock() + }) + readHardwarePortsFn, hardwarePortsNowFn = read, now +} + +func Test_hardwarePortsByDevice_caches(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi", Service: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("first read port = %q, want Wi-Fi", got) + } + now = now.Add(59 * time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_keepsLastGoodRead(t *testing.T) { + now := time.Unix(1700000000, 0) + fail := false + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + if fail { + return nil + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + fail = true + now = now.Add(2 * time.Minute) + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a failed read = %q, want Wi-Fi", got) + } +} + +func Test_hardwarePortsByDevice_keepsTheWindowAfterAFailedRead(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return nil + }, func() time.Time { return now }) + + hardwarePortsByDevice() + now = now.Add(time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads of a failing command within the window = %d, want 1", reads) + } + now = now.Add(2 * time.Minute) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the window = %d, want 2", reads) + } +} + +func Test_refreshInterfaceMeta_dropsTheReadWindow(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the window = %d, want 1", reads) + } + refreshInterfaceMeta() + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a refresh = %q, want Wi-Fi", got) + } + if reads != 2 { + t.Fatalf("reads after a refresh = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_readsOutsideTheLock(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + // A command runs outside the mutex, so this read can take it. + if !hardwarePortsMu.TryLock() { + t.Error("the command ran under the hardware ports mutex") + } else { + hardwarePortsMu.Unlock() + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port = %q, want Wi-Fi", got) + } +} + +// fixtureHardwarePorts returns the port map of the two command fixtures. +func fixtureHardwarePorts() map[string]hardwarePortInfo { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + return ports +} + +func Test_transitionTestGlobalsReadNoPorts(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return fixtureHardwarePorts() + }, func() time.Time { return now }) + + TestNetworkChangeDoesNotRestoreDownDefaultRouteSources(t) + TestNetworkChangeSourceMovedToUpInterface(t) + + if reads != 0 { + t.Fatalf("port reads of two transition tests = %d, want 0", reads) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + return map[string]hardwarePortInfo{"en7": {Port: "iPhone USB", Service: "iPhone USB"}} + }, func() time.Time { return now }) + + class, hardwarePort, service := platformInterfaceMeta("en7") + if class != "" { + t.Errorf("class = %q, want empty", class) + } + if hardwarePort != "iPhone USB" || service != "iPhone USB" { + t.Errorf("port, service = %q, %q, want iPhone USB twice", hardwarePort, service) + } + if class, hardwarePort, service = platformInterfaceMeta("utun3"); class != "" || hardwarePort != "" || service != "" { + t.Errorf("unknown device gave %q, %q, %q, want empty strings", class, hardwarePort, service) + } + if platformVirtualInterfaces() != nil { + t.Error("platformVirtualInterfaces is not nil on darwin") + } +} diff --git a/cmd/cli/net_linux.go b/cmd/cli/net_linux.go index ea17d3d8..4f098a48 100644 --- a/cmd/cli/net_linux.go +++ b/cmd/cli/net_linux.go @@ -2,47 +2,56 @@ package cli import ( "net" - "net/netip" "os" + "path/filepath" + "slices" + "strconv" "strings" +) + +// The kernel gives an Ethernet-like link the type ARPHRD_ETHER. +const arphrdEther = 1 - "tailscale.com/net/netmon" +// Tests point the readers at a temporary directory. +var ( + sysfsNetRoot = "/sys/class/net" + sysfsVirtualNetRoot = "/sys/devices/virtual/net" ) +// noiseInterfacePrefixes name the interfaces of a container host. They change +// every few seconds, and no traffic of the host goes through them. +var noiseInterfacePrefixes = []string{"docker", "veth", "virbr", "br-"} + +// usbTetherDrivers name the kernel modules of a phone that shares its data +// connection over the cable. +var usbTetherDrivers = []string{"ipheth", "rndis_host"} + +// sysfsInfo holds what sysfs knows about one interface. +type sysfsInfo struct { + Type int + DevType string + Driver string + Wireless bool +} + +// patchNetIfaceName patches network interface names on Linux +// This is a no-op on Linux as interface names don't need special handling func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } // validInterface reports whether the *net.Interface is a valid one. // Only non-virtual interfaces are considered valid. +// This prevents DNS configuration on virtual interfaces like docker, veth, etc. func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bool { _, ok := validIfacesMap[iface.Name] return ok } -// validInterfacesMap returns a set containing non virtual interfaces. -func validInterfacesMap() map[string]struct{} { - m := make(map[string]struct{}) - vis := virtualInterfaces() - netmon.ForeachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { - if _, existed := vis[i.Name]; existed { - return - } - m[i.Name] = struct{}{} - }) - // Fallback to default route interface if found nothing. - if len(m) == 0 { - defaultRoute, err := netmon.DefaultRoute() - if err != nil { - return m - } - m[defaultRoute.InterfaceName] = struct{}{} - } - return m -} - -// virtualInterfaces returns a map of virtual interfaces on current machine. -func virtualInterfaces() map[string]struct{} { +// sysfsVirtualInterfaces returns the virtual interfaces of the host. The +// nameserver code has its own reader of the fixed path; this one reads the +// root that a test can point elsewhere. +func sysfsVirtualInterfaces() map[string]struct{} { s := make(map[string]struct{}) - entries, _ := os.ReadDir("/sys/devices/virtual/net") + entries, _ := os.ReadDir(sysfsVirtualNetRoot) for _, entry := range entries { if entry.IsDir() { s[strings.TrimSpace(entry.Name())] = struct{}{} @@ -50,3 +59,104 @@ func virtualInterfaces() map[string]struct{} { } return s } + +// sysfsInterfaceInfo reads the sysfs entry of an interface. A missing file +// leaves its field empty. +func sysfsInterfaceInfo(name string) sysfsInfo { + dir := filepath.Join(sysfsNetRoot, name) + info := sysfsInfo{DevType: sysfsUeventValue(dir, "DEVTYPE"), Driver: sysfsDriver(dir)} + info.Type, _ = strconv.Atoi(readSysfsAttribute(dir, "type")) + if entry, err := os.Stat(filepath.Join(dir, "wireless")); err == nil && entry.IsDir() { + info.Wireless = true + } + return info +} + +// readSysfsAttribute reads one attribute without its trailing newline. +func readSysfsAttribute(dir, file string) string { + b, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +// sysfsUeventValue reads one entry of the uevent file. +func sysfsUeventValue(dir, key string) string { + for _, line := range strings.Split(readSysfsAttribute(dir, "uevent"), "\n") { + after, ok := strings.CutPrefix(strings.TrimSpace(line), key+"=") + if !ok { + continue + } + return strings.TrimSpace(after) + } + return "" +} + +// sysfsDriver names the kernel module behind an interface. The driver link of +// the device points at the module, and the uevent of the device names it too. +func sysfsDriver(dir string) string { + device := filepath.Join(dir, "device") + if target, err := os.Readlink(filepath.Join(device, "driver")); err == nil { + return filepath.Base(target) + } + if driver := sysfsUeventValue(device, "DRIVER"); driver != "" { + return driver + } + return sysfsUeventValue(dir, "DRIVER") +} + +// sysfsHardwarePort names the link in the words that the link type mapping +// reads. The port names of macOS use the same words. +func sysfsHardwarePort(info sysfsInfo) string { + switch { + case info.Wireless: + return "Wi-Fi" + case slices.Contains(usbTetherDrivers, info.Driver): + return hardwarePortIPhoneUSB + case info.DevType != "": + return info.DevType + case info.Type == arphrdEther: + return "Ethernet" + } + return "" +} + +// sysfsClass names the kind of an interface. The name rule comes before the +// device type, so an interface that went away keeps its class. A bridge, a +// bond, and a VLAN carry the traffic of the host, so they stay hardware. +func sysfsClass(name string, info sysfsInfo) string { + switch { + case info.DevType == "tun" || info.DevType == "tap": + return "tunnel" + case hasInterfacePrefix(name, noiseInterfacePrefixes...): + return "virtual" + case info.Wireless || info.Type == arphrdEther: + return "hardware" + } + return "" +} + +// platformInterfaceMeta returns the Linux class and link name of an interface. +// Linux has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := sysfsInterfaceInfo(name) + return sysfsClass(name, info), sysfsHardwarePort(info), "" +} + +// platformVirtualInterfaces returns the container interfaces of the host. The +// kernel lists the uplink bridges and the bonds beside them, and those carry +// the traffic of the host. +func platformVirtualInterfaces() map[string]struct{} { + noise := make(map[string]struct{}) + for name := range sysfsVirtualInterfaces() { + if hasInterfacePrefix(name, noiseInterfacePrefixes...) { + noise[name] = struct{}{} + } + } + return noise +} + +// refreshInterfaceMeta does no work. Each sysfs read answers from the kernel +// of this moment, so there is no cache to refresh. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_linux_test.go b/cmd/cli/net_linux_test.go new file mode 100644 index 00000000..cbf61c5c --- /dev/null +++ b/cmd/cli/net_linux_test.go @@ -0,0 +1,201 @@ +package cli + +import ( + "maps" + "os" + "path/filepath" + "testing" +) + +// writeSysfsInterface builds one interface entry under a fake sysfs root. +func writeSysfsInterface(t *testing.T, root, name string, files map[string]string, wireless bool) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + for file, content := range files { + if err := os.WriteFile(filepath.Join(dir, file), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", file, err) + } + } + if !wireless { + return + } + if err := os.MkdirAll(filepath.Join(dir, "wireless"), 0o755); err != nil { + t.Fatalf("create the wireless directory: %v", err) + } +} + +// writeSysfsDriverLink points the device entry of an interface at a driver. +func writeSysfsDriverLink(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.Symlink(filepath.Join(root, "drivers", driver), filepath.Join(device, "driver")); err != nil { + t.Fatalf("link the driver of %s: %v", name, err) + } +} + +// writeSysfsDriverUevent names the driver in the uevent of the device. +func writeSysfsDriverUevent(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.WriteFile(filepath.Join(device, "uevent"), []byte("DRIVER="+driver+"\n"), 0o644); err != nil { + t.Fatalf("write the uevent of %s: %v", name, err) + } +} + +// stubSysfsNetRoot points the sysfs reader at a temporary directory. +func stubSysfsNetRoot(t *testing.T) string { + t.Helper() + before := sysfsNetRoot + t.Cleanup(func() { sysfsNetRoot = before }) + sysfsNetRoot = t.TempDir() + return sysfsNetRoot +} + +func Test_sysfsInterfaceInfo(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=wlan0\nDEVTYPE=wlan\nIFINDEX=3\n", + }, true) + writeSysfsInterface(t, root, "eth0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=eth0\nIFINDEX=2\n", + }, false) + writeSysfsInterface(t, root, "tun0", map[string]string{ + "type": "65534\n", + "uevent": "DEVTYPE=tun\n", + }, false) + + tests := []struct { + name string + want sysfsInfo + }{ + {"wlan0", sysfsInfo{Type: 1, DevType: "wlan", Wireless: true}}, + {"eth0", sysfsInfo{Type: 1}}, + {"tun0", sysfsInfo{Type: 65534, DevType: "tun"}}, + {"absent0", sysfsInfo{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsInterfaceInfo(tc.name); got != tc.want { + t.Errorf("sysfsInterfaceInfo(%q) = %+v, want %+v", tc.name, got, tc.want) + } + }) + } +} + +func Test_sysfsClass(t *testing.T) { + tests := []struct { + name string + iface string + info sysfsInfo + want string + }{ + {"wireless", "wlan0", sysfsInfo{Wireless: true}, "hardware"}, + {"ethernet", "eth0", sysfsInfo{Type: 1}, "hardware"}, + {"tun devtype", "tun0", sysfsInfo{Type: 65534, DevType: "tun"}, "tunnel"}, + {"tap devtype", "tap0", sysfsInfo{Type: 1, DevType: "tap"}, "tunnel"}, + {"unknown", "sit0", sysfsInfo{Type: 65534}, ""}, + {"docker bridge", "docker0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"container link", "veth1a2b3c", sysfsInfo{Type: 1}, "virtual"}, + {"libvirt bridge", "virbr0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"compose bridge", "br-1a2b3c", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"uplink bridge", "br0", sysfsInfo{Type: 1, DevType: "bridge"}, "hardware"}, + {"bond", "bond0", sysfsInfo{Type: 1, DevType: "bond"}, "hardware"}, + {"vlan uplink", "eth0.100", sysfsInfo{Type: 1, DevType: "vlan"}, "hardware"}, + {"removed container link", "veth9f8e7d", sysfsInfo{}, "virtual"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsClass(tc.iface, tc.info); got != tc.want { + t.Errorf("sysfsClass(%q, %+v) = %q, want %q", tc.iface, tc.info, got, tc.want) + } + }) + } +} + +func Test_platformVirtualInterfaces_namesTheContainerInterfaces(t *testing.T) { + root := t.TempDir() + before := sysfsVirtualNetRoot + t.Cleanup(func() { sysfsVirtualNetRoot = before }) + sysfsVirtualNetRoot = root + for _, name := range []string{"lo", "docker0", "veth1a2b3c", "virbr0", "br-1a2b3c", "br0", "bond0"} { + if err := os.MkdirAll(filepath.Join(root, name), 0o755); err != nil { + t.Fatalf("create %s: %v", name, err) + } + } + + want := map[string]struct{}{"docker0": {}, "veth1a2b3c": {}, "virbr0": {}, "br-1a2b3c": {}} + if got := platformVirtualInterfaces(); !maps.Equal(got, want) { + t.Errorf("platformVirtualInterfaces = %v, want %v", got, want) + } + if got := sysfsVirtualInterfaces(); len(got) != 7 { + t.Errorf("virtualInterfaces = %v, want every entry of the kernel directory", got) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "ctrld-test-eth0", map[string]string{ + "type": "1\n", + "uevent": "DEVTYPE=bridge\n", + }, false) + + class, hardwarePort, service := platformInterfaceMeta("ctrld-test-eth0") + if class != "hardware" { + t.Errorf("class = %q, want hardware", class) + } + if hardwarePort != "bridge" { + t.Errorf("hardware port = %q, want bridge", hardwarePort) + } + if service != "" { + t.Errorf("service = %q, want empty", service) + } +} + +func Test_platformInterfaceMeta_namesTheLink(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=wlan\n"}, true) + writeSysfsInterface(t, root, "eth0", map[string]string{"type": "1\n", "uevent": "INTERFACE=eth0\n"}, false) + writeSysfsInterface(t, root, "br0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=bridge\n"}, false) + writeSysfsInterface(t, root, "tun0", map[string]string{"type": "65534\n", "uevent": "DEVTYPE=tun\n"}, false) + writeSysfsInterface(t, root, "usb0", map[string]string{"type": "1\n"}, false) + writeSysfsDriverLink(t, root, "usb0", "ipheth") + writeSysfsInterface(t, root, "usb1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "usb1", "rndis_host") + writeSysfsInterface(t, root, "eth1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "eth1", "e1000e") + + tests := []struct { + iface, class, hardwarePort, linkType string + }{ + {"wlan0", "hardware", "Wi-Fi", "wifi"}, + {"eth0", "hardware", "Ethernet", "ethernet"}, + {"eth1", "hardware", "Ethernet", "ethernet"}, + {"br0", "hardware", "bridge", "unknown"}, + {"tun0", "tunnel", "tun", "tunnel"}, + {"usb0", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + {"usb1", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + } + for _, tc := range tests { + t.Run(tc.iface, func(t *testing.T) { + class, hardwarePort, service := platformInterfaceMeta(tc.iface) + if class != tc.class || hardwarePort != tc.hardwarePort || service != "" { + t.Errorf("platformInterfaceMeta(%q) = %q, %q, %q, want %q, %q, empty", + tc.iface, class, hardwarePort, service, tc.class, tc.hardwarePort) + } + if got := linkTypeFor(hardwarePort, class); got != tc.linkType { + t.Errorf("linkTypeFor(%q, %q) = %q, want %q", hardwarePort, class, got, tc.linkType) + } + }) + } +} diff --git a/cmd/cli/net_others.go b/cmd/cli/net_others.go index f3472781..03c80dd0 100644 --- a/cmd/cli/net_others.go +++ b/cmd/cli/net_others.go @@ -4,19 +4,25 @@ package cli import ( "net" - - "tailscale.com/net/netmon" ) +// patchNetIfaceName patches network interface names on non-Linux/Darwin platforms func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } +// validInterface checks if an interface is valid on non-Linux/Darwin platforms func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bool { return true } -// validInterfacesMap returns a set containing only default route interfaces. -func validInterfacesMap() map[string]struct{} { - defaultRoute, err := netmon.DefaultRoute() - if err != nil { - return nil - } - return map[string]struct{}{defaultRoute.InterfaceName: {}} +// platformInterfaceMeta returns no names. These platforms have no source for +// them, so the interface name decides the class. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + return "", "", "" +} + +// platformVirtualInterfaces returns no set. +func platformVirtualInterfaces() map[string]struct{} { + return nil } + +// refreshInterfaceMeta does no work. These platforms keep no interface names, +// so an adapter that appears leaves nothing stale. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_windows.go b/cmd/cli/net_windows.go index bed06b57..d77858ed 100644 --- a/cmd/cli/net_windows.go +++ b/cmd/cli/net_windows.go @@ -5,6 +5,9 @@ import ( "log" "net" "os" + "strings" + "sync" + "time" "github.com/microsoft/wmi/pkg/base/host" "github.com/microsoft/wmi/pkg/base/instance" @@ -13,6 +16,31 @@ import ( "github.com/microsoft/wmi/pkg/hardware/network/netadapter" ) +// adapterInfo holds what one WMI query knows about a network adapter. +type adapterInfo struct { + Name string + Description string + Virtual bool + ConnectorPresent bool + Hardware bool +} + +// Every network change reads the adapter set, so one query serves a window. +const adapterCacheTTL = time.Minute + +var ( + adapterCacheMu sync.Mutex + adapterCache []adapterInfo + // The set before the last read still names the adapter that went away, + // so a removed adapter keeps its class in the journal. + adapterCachePrevious []adapterInfo + adapterCacheReadAt time.Time + + // Tests replace these seams to avoid a WMI query. + readAdaptersFn = readAdapters + adapterCacheNowFn = time.Now +) + func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } @@ -24,17 +52,108 @@ func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bo return ok } -// validInterfacesMap returns a set of all physical interfaces. -func validInterfacesMap() map[string]struct{} { - m := make(map[string]struct{}) - for _, ifaceName := range validInterfaces() { - m[ifaceName] = struct{}{} +// adapters returns the adapter set of the host, from the cache while it is +// younger than the window. The query runs under the mutex, so two callers +// start one query. +func adapters() []adapterInfo { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + now := adapterCacheNowFn() + if !adapterCacheReadAt.IsZero() && now.Sub(adapterCacheReadAt) < adapterCacheTTL { + return adapterCache + } + read := readAdaptersFn() + // A failing query waits for the window too, so it starts no query storm. + adapterCacheReadAt = now + if read == nil { + // Keep the last good read, so a failed query loses no adapter. + return adapterCache + } + adapterCachePrevious, adapterCache = adapterCache, read + return read +} + +// refreshInterfaceMeta drops the read window. The cache still describes the +// adapter that appeared or went away, and the next lookup reads the set. +func refreshInterfaceMeta() { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + adapterCacheReadAt = time.Time{} +} + +// tunnelAdapterWords name the VPN drivers of Windows. The description is the +// only signal, because WMI describes a tunnel adapter as an Ethernet adapter. +var tunnelAdapterWords = []string{"tap", "wintun", "wireguard", "vpn"} + +// adapterClass names the kind of an adapter. WMI marks a software adapter +// virtual and gives it no connector. +func adapterClass(adapter adapterInfo) string { + switch { + case tunnelAdapter(adapter.Description): + return "tunnel" + case adapter.Virtual && !adapter.ConnectorPresent: + return "virtual" + } + return "hardware" +} + +// tunnelAdapter reports a description that names a VPN driver. +func tunnelAdapter(description string) bool { + lower := strings.ToLower(description) + for _, word := range tunnelAdapterWords { + if strings.Contains(lower, word) { + return true + } + } + return false +} + +// platformInterfaceMeta returns the Windows class and adapter description of +// an interface. Windows has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + adapter, known := adapterByName(name) + if !known { + return "", "", "" + } + return adapterClass(adapter), adapter.Description, "" +} + +// adapterByName finds an adapter in the current set, or in the set before the +// last read. An adapter that went away keeps its names that way. +func adapterByName(name string) (adapterInfo, bool) { + if adapter, known := findAdapter(adapters(), name); known { + return adapter, true + } + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + return findAdapter(adapterCachePrevious, name) +} + +func findAdapter(set []adapterInfo, name string) (adapterInfo, bool) { + for _, adapter := range set { + if adapter.Name == name { + return adapter, true + } } - return m + return adapterInfo{}, false } -// validInterfaces returns a list of all physical interfaces. -func validInterfaces() []string { +// platformVirtualInterfaces returns the adapters that carry no traffic of the +// host network. +func platformVirtualInterfaces() map[string]struct{} { + virtual := make(map[string]struct{}) + for _, adapter := range adapters() { + if adapterClass(adapter) != "virtual" { + continue + } + virtual[adapter.Name] = struct{}{} + } + return virtual +} + +// readAdapters runs one WMI query for every network adapter. It returns nil +// when the query fails. +func readAdapters() []adapterInfo { log.SetOutput(io.Discard) defer log.SetOutput(os.Stderr) whost := host.NewWmiLocalHost() @@ -47,47 +166,53 @@ func validInterfaces() []string { mainLog.Load().Warn().Err(err).Msg("failed to get wmi network adapter") return nil } - var adapters []string + var read []adapterInfo for _, i := range instances { adapter, err := netadapter.NewNetworkAdapter(i) if err != nil { mainLog.Load().Warn().Err(err).Msg("failed to get network adapter") continue } - - name, err := adapter.GetPropertyName() - if err != nil { - mainLog.Load().Warn().Err(err).Msg("failed to get interface name") + info, ok := readAdapterInfo(adapter) + if !ok { continue } + read = append(read, info) + } + return read +} - // From: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh968170(v=vs.85) - // - // "Indicates if a connector is present on the network adapter. This value is set to TRUE - // if this is a physical adapter or FALSE if this is not a physical adapter." - physical, err := adapter.GetPropertyConnectorPresent() - if err != nil { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("failed to get network adapter connector present property") - continue - } - if !physical { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("skipping non-physical adapter") - continue - } +// readAdapterInfo reads the properties of one adapter. It reports false when a +// property that decides validity is unreadable. +func readAdapterInfo(adapter *netadapter.NetworkAdapter) (adapterInfo, bool) { + name, err := adapter.GetPropertyName() + if err != nil { + mainLog.Load().Warn().Err(err).Msg("failed to get interface name") + return adapterInfo{}, false + } - // Check if it's a hardware interface. Checking only for connector present is not enough - // because some interfaces are not physical but have a connector. - hardware, err := adapter.GetPropertyHardwareInterface() - if err != nil { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("failed to get network adapter hardware interface property") - continue - } - if !hardware { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("skipping non-hardware interface") - continue - } + // From: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh968170(v=vs.85) + // + // "Indicates if a connector is present on the network adapter. This value is set to TRUE + // if this is a physical adapter or FALSE if this is not a physical adapter." + connectorPresent, err := adapter.GetPropertyConnectorPresent() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter connector present property") + return adapterInfo{}, false + } - adapters = append(adapters, name) + // Check if it's a hardware interface. Checking only for connector present is not enough + // because some interfaces are not physical but have a connector. + hardware, err := adapter.GetPropertyHardwareInterface() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter hardware interface property") + return adapterInfo{}, false } - return adapters + + info := adapterInfo{Name: name, ConnectorPresent: connectorPresent, Hardware: hardware} + // The class and the description describe the adapter only, so an + // unreadable one costs no validity. + info.Description, _ = adapter.GetPropertyInterfaceDescription() + info.Virtual, _ = adapter.GetPropertyVirtual() + return info, true } diff --git a/cmd/cli/net_windows_test.go b/cmd/cli/net_windows_test.go index a15f119b..e52b52be 100644 --- a/cmd/cli/net_windows_test.go +++ b/cmd/cli/net_windows_test.go @@ -3,18 +3,24 @@ package cli import ( "bufio" "bytes" + "context" + "maps" "slices" "strings" + "sync" "testing" "time" + + "github.com/Control-D-Inc/ctrld" ) func Test_validInterfaces(t *testing.T) { verbose = 3 initConsoleLogging() start := time.Now() - ifaces := validInterfaces() + im := ctrld.ValidInterfaces(ctrld.LoggerCtx(context.Background(), mainLog.Load())) t.Logf("Using Windows API takes: %d", time.Since(start).Milliseconds()) + ifaces := slices.Collect(maps.Keys(im)) start = time.Now() ifacesPowershell := validInterfacesPowershell() @@ -40,3 +46,143 @@ func validInterfacesPowershell() []string { } return res } + +func Test_adapterClass(t *testing.T) { + tests := []struct { + name string + adapter adapterInfo + want string + }{ + {"virtual without a connector", adapterInfo{Virtual: true}, "virtual"}, + {"virtual with a connector", adapterInfo{Virtual: true, ConnectorPresent: true}, "hardware"}, + {"physical adapter", adapterInfo{ConnectorPresent: true, Hardware: true}, "hardware"}, + {"unknown adapter", adapterInfo{}, "hardware"}, + {"tap driver", adapterInfo{Description: "TAP-Windows Adapter V9", Virtual: true}, "tunnel"}, + {"wintun driver", adapterInfo{Description: "Wintun Userspace Tunnel", Virtual: true}, "tunnel"}, + {"wireguard driver", adapterInfo{Description: "WireGuard Tunnel #1", Virtual: true}, "tunnel"}, + {"vpn description", adapterInfo{Description: "Cisco AnyConnect VPN Adapter", ConnectorPresent: true}, "tunnel"}, + {"lower case description", adapterInfo{Description: "openvpn tap-windows6"}, "tunnel"}, + {"ethernet adapter", adapterInfo{Description: "Intel(R) Ethernet Connection I219-V", ConnectorPresent: true, Hardware: true}, "hardware"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := adapterClass(tc.adapter); got != tc.want { + t.Errorf("adapterClass(%+v) = %q, want %q", tc.adapter, got, tc.want) + } + }) + } +} + +// stubAdapters replaces the WMI and clock seams of the adapter cache, and +// restores them with the cache itself. +func stubAdapters(t *testing.T, read func() []adapterInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readAdaptersFn, adapterCacheNowFn + adapterCacheMu.Lock() + cacheBefore, readAtBefore := adapterCache, adapterCacheReadAt + adapterCache, adapterCacheReadAt = nil, time.Time{} + adapterCacheMu.Unlock() + t.Cleanup(func() { + readAdaptersFn, adapterCacheNowFn = readBefore, nowBefore + adapterCacheMu.Lock() + adapterCache, adapterCacheReadAt = cacheBefore, readAtBefore + adapterCacheMu.Unlock() + }) + readAdaptersFn, adapterCacheNowFn = read, now +} + +func Test_adapterCache(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubAdapters(t, func() []adapterInfo { + reads++ + return []adapterInfo{ + {Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}, + {Name: "Hyper-V Switch", Description: "Hyper-V Virtual Switch", Virtual: true}, + } + }, func() time.Time { return now }) + + if got := adapters(); len(got) != 2 || got[0].Name != "Ethernet" { + t.Fatalf("adapters = %v, want the two fixture adapters", got) + } + now = now.Add(59 * time.Second) + adapters() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + adapters() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } + refreshInterfaceMeta() + if reads != 2 { + t.Fatalf("reads for a refresh alone = %d, want 2", reads) + } + adapters() + if reads != 3 { + t.Fatalf("reads after a refresh = %d, want 3", reads) + } + + virtual := platformVirtualInterfaces() + if _, ok := virtual["Hyper-V Switch"]; !ok || len(virtual) != 1 { + t.Fatalf("platformVirtualInterfaces = %v, want the virtual switch only", virtual) + } + class, hardwarePort, service := platformInterfaceMeta("Ethernet") + if class != "hardware" || hardwarePort != "Intel I219-V" || service != "" { + t.Fatalf("platformInterfaceMeta = %q, %q, %q, want hardware, Intel I219-V, empty", class, hardwarePort, service) + } +} + +func Test_adapterCacheReadsOnceForTwoCallers(t *testing.T) { + now := time.Unix(1700000000, 0) + var mu sync.Mutex + reads := 0 + stubAdapters(t, func() []adapterInfo { + mu.Lock() + reads++ + mu.Unlock() + time.Sleep(10 * time.Millisecond) + return []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + }, func() time.Time { return now }) + + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + adapters() + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + if reads != 1 { + t.Fatalf("queries for four callers = %d, want 1", reads) + } +} + +func Test_adapterCacheKeepsTheRemovedAdapter(t *testing.T) { + now := time.Unix(1700000000, 0) + removed := false + stubAdapters(t, func() []adapterInfo { + read := []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + if removed { + return read + } + return append(read, adapterInfo{Name: "WireGuard", Description: "WireGuard Tunnel #1", Virtual: true}) + }, func() time.Time { return now }) + + adapters() + removed = true + refreshInterfaceMeta() + + class, hardwarePort, _ := platformInterfaceMeta("WireGuard") + if class != "tunnel" || hardwarePort != "WireGuard Tunnel #1" { + t.Fatalf("meta of the removed adapter = %q, %q, want tunnel, WireGuard Tunnel #1", class, hardwarePort) + } + if class, _, _ := platformInterfaceMeta("Absent"); class != "" { + t.Fatalf("meta of an adapter that no read named = %q, want empty", class) + } +} diff --git a/cmd/cli/netlink_linux.go b/cmd/cli/netlink_linux.go deleted file mode 100644 index d757f8b7..00000000 --- a/cmd/cli/netlink_linux.go +++ /dev/null @@ -1,34 +0,0 @@ -package cli - -import ( - "context" - - "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" -) - -func (p *prog) watchLinkState(ctx context.Context) { - ch := make(chan netlink.LinkUpdate) - done := make(chan struct{}) - defer close(done) - if err := netlink.LinkSubscribe(ch, done); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not subscribe link") - return - } - for { - select { - case <-ctx.Done(): - return - case lu := <-ch: - if lu.Change == 0xFFFFFFFF { - continue - } - if lu.Change&unix.IFF_UP != 0 { - mainLog.Load().Debug().Msgf("link state changed, re-bootstrapping") - for _, uc := range p.cfg.Upstream { - uc.ReBootstrap() - } - } - } - } -} diff --git a/cmd/cli/netlink_others.go b/cmd/cli/netlink_others.go deleted file mode 100644 index 5a298b99..00000000 --- a/cmd/cli/netlink_others.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !linux - -package cli - -import "context" - -func (p *prog) watchLinkState(ctx context.Context) {} diff --git a/cmd/cli/network_delta.go b/cmd/cli/network_delta.go new file mode 100644 index 00000000..18ea6bd9 --- /dev/null +++ b/cmd/cli/network_delta.go @@ -0,0 +1,301 @@ +package cli + +import ( + "net" + "net/netip" + "slices" + "strings" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// interfaceChange describes one interface that a network delta changed. +type interfaceChange struct { + Name string + Action string + Class string + HardwarePort string + Service string + IPsBefore []string + IPsAfter []string + Flags string + MTU int + IsDefaultRoute bool +} + +// interfaceMeta holds the platform description of one interface. An empty +// Class asks the caller to read the class from the interface name. +type interfaceMeta struct { + Class string + HardwarePort string + Service string + LinkType string +} + +// interfaceMetaFunc reads the platform description of one interface. +type interfaceMetaFunc func(name string) interfaceMeta + +// virtualInterfaceSetFn names the virtual adapters of the host. Each platform +// file fills it, because a name alone does not tell a container bridge from a +// physical port. +var virtualInterfaceSetFn = func() map[string]struct{} { return nil } + +// interfaceSnapshot holds the part of one interface state that the diff +// compares. +type interfaceSnapshot struct { + present bool + up bool + mac string + mtu int + flags string + ips []string +} + +// interfaceClass groups an interface by the traffic it carries. The class +// decides which changes are noise. +func interfaceClass(name string, virtual bool) string { + switch { + case strings.HasPrefix(name, "lo"): + return "loopback" + case hasInterfacePrefix(name, "awdl", "llw"): + return "airdrop" + case hasInterfacePrefix(name, "utun", "tun", "tap", "wg", "ipsec", "ppp"): + return "tunnel" + case virtual: + return "virtual" + default: + return "hardware" + } +} + +// deltaBeforeState picks the state that the diff of one callback compares +// against, and makes a current callback the baseline of the next one. netmon +// caches major snapshots only, so every minor callback of an epoch carries the +// same Old, and a difference that lasts would look new in each of them. A +// callback that a newer snapshot replaced reads the baseline and leaves it. +// The caller holds networkSourceMu. +// +// netmon runs its callbacks in their own goroutines, so a minor callback of a +// new epoch can run before the major callback that opened the epoch. That +// major then reports its own changes against its Old and leaves the newer +// baseline in place. A major whose Old is the cache of the epoch before +// continues from the last reported state of that epoch, so a change that a +// minor callback reported does not appear again. +func (p *prog) deltaBeforeState(delta *netmon.ChangeDelta, epoch *netmon.State, current, major bool) *netmon.State { + before := p.networkDeltaState + switch { + case before == nil: + before = delta.Old + case current && major && p.networkDeltaEpoch == epoch: + return delta.Old + case p.networkDeltaEpoch != epoch && delta.Old != p.networkDeltaEpoch: + before = delta.Old + } + if current { + p.networkDeltaState, p.networkDeltaEpoch = delta.New, epoch + } + return before +} + +// diffNetworkDelta describes every interface that changed between two states. +// It walks every interface, not only the valid hardware ports, because the +// journal must show the port that went away too. +func diffNetworkDelta(beforeState, afterState *netmon.State) []interfaceChange { + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + virtual := virtualInterfaceSetFn() + var changes []interfaceChange + for _, name := range interfaceNameUnion(before, after) { + snapBefore, snapAfter := readInterfaceSnapshot(before, name), readInterfaceSnapshot(after, name) + action := interfaceAction(snapBefore, snapAfter) + if action == "" { + continue + } + known := snapAfter + if !snapAfter.present { + known = snapBefore + } + _, isVirtual := virtual[name] + changes = append(changes, interfaceChange{ + Name: name, + Action: action, + Class: interfaceClass(name, isVirtual), + IPsBefore: snapBefore.ips, + IPsAfter: snapAfter.ips, + Flags: known.flags, + MTU: known.mtu, + IsDefaultRoute: name == after.DefaultRouteInterface, + }) + } + return changes +} + +// noiseDelta reports a delta that no part of the daemon must act on. AirDrop +// and the virtual adapters of a container host change every few seconds, and +// each change would otherwise start the whole reconcile chain. A time jump +// means the host woke, so the daemon acts on the delta whatever it holds. +func noiseDelta(beforeState, afterState *netmon.State, timeJumped bool, changes []interfaceChange) bool { + if len(changes) == 0 || timeJumped { + return false + } + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + if before.DefaultRouteInterface != after.DefaultRouteInterface { + return false + } + if before.HaveV4 != after.HaveV4 || before.HaveV6 != after.HaveV6 { + return false + } + source4, source6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + for _, change := range changes { + if !noiseClass(change.Class) || change.IsDefaultRoute { + return false + } + if carriesHostTraffic(change.IPsBefore, source4, source6) { + return false + } + if carriesHostTraffic(change.IPsAfter, source4, source6) { + return false + } + } + return true +} + +// noiseClass names the classes that change on their own. The kernel moves the +// link-local address of an AirDrop or a container adapter every few seconds. +func noiseClass(class string) bool { + return class == "airdrop" || class == "virtual" +} + +// carriesHostTraffic reports an address list that reaches another host. A +// bridge, a bond, or a tunnel adapter can own the traffic of the host, so a +// routable address or the resolver source address makes the interface real +// whatever its name says. +func carriesHostTraffic(ips []string, source4, source6 net.IP) bool { + for _, ip := range ips { + prefix, err := netip.ParsePrefix(ip) + if err != nil { + continue + } + addr := prefix.Addr() + if addr.IsGlobalUnicast() || sameAddress(addr, source4) || sameAddress(addr, source6) { + return true + } + } + return false +} + +// sameAddress compares one interface address with a resolver source address. +func sameAddress(addr netip.Addr, ip net.IP) bool { + source, ok := netip.AddrFromSlice(ip) + return ok && source.Unmap() == addr +} + +// describeInterfaceChanges fills the platform fields of the changes that reach +// the journal. macOS reads them with two subprocesses, so only a delta that +// the daemon acts on pays for them. The platform class wins where it knows +// one, because a name rule cannot see a virtual adapter on Windows. +func describeInterfaceChanges(changes []interfaceChange, meta interfaceMetaFunc) { + if meta == nil { + return + } + for i := range changes { + info := meta(changes[i].Name) + changes[i].HardwarePort, changes[i].Service = info.HardwarePort, info.Service + if info.Class != "" { + changes[i].Class = info.Class + } + } +} + +// interfaceAction names the first difference that the journal reports. The +// order runs from the coarse change to the fine one, so one interface gives +// one action. +func interfaceAction(before, after interfaceSnapshot) string { + switch { + case before.present && !after.present: + return "removed" + case !before.present && after.present: + return "added" + case before.up && !after.up: + return "down" + case !before.up && after.up: + return "up" + case hasExtraEntry(before.ips, after.ips): + return "ip_removed" + case hasExtraEntry(after.ips, before.ips): + return "ip_added" + case before.mac != after.mac: + return "mac_changed" + case before.mtu != after.mtu: + return "mtu_changed" + default: + return "" + } +} + +func readInterfaceSnapshot(state *netmon.State, name string) interfaceSnapshot { + snapshot := interfaceSnapshot{ips: sortedPrefixStrings(state.InterfaceIPs[name])} + iface, ok := state.Interface[name] + if !ok { + return snapshot + } + snapshot.present = true + if iface.Interface == nil { + return snapshot + } + snapshot.up = iface.IsUp() + snapshot.mtu = iface.MTU + snapshot.flags = iface.Flags.String() + if len(iface.HardwareAddr) > 0 { + snapshot.mac = iface.HardwareAddr.String() + } + return snapshot +} + +func interfaceNameUnion(before, after *netmon.State) []string { + names := make([]string, 0, len(before.Interface)+len(after.Interface)) + for name := range before.Interface { + names = append(names, name) + } + for name := range after.Interface { + if _, both := before.Interface[name]; !both { + names = append(names, name) + } + } + slices.Sort(names) + return names +} + +func sortedPrefixStrings(prefixes []netip.Prefix) []string { + ips := prefixStrings(prefixes) + slices.Sort(ips) + return ips +} + +// networkStateOrEmpty keeps the diff free of nil checks. The first delta after +// start carries no old state. +func networkStateOrEmpty(state *netmon.State) *netmon.State { + if state != nil { + return state + } + return &netmon.State{} +} + +func hasInterfacePrefix(name string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// hasExtraEntry reports an entry of list that other does not hold. +func hasExtraEntry(list, other []string) bool { + for _, entry := range list { + if !slices.Contains(other, entry) { + return true + } + } + return false +} diff --git a/cmd/cli/network_delta_test.go b/cmd/cli/network_delta_test.go new file mode 100644 index 00000000..4b847eb6 --- /dev/null +++ b/cmd/cli/network_delta_test.go @@ -0,0 +1,489 @@ +package cli + +import ( + "context" + "net" + "net/netip" + "slices" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// deltaTestInterface describes one interface of a test network state. +type deltaTestInterface struct { + name string + up bool + mac string + mtu int + ips []string +} + +var ( + deltaTestUpFlags = net.FlagUp | net.FlagBroadcast | net.FlagMulticast + deltaTestDownFlags = net.Flags(0) +) + +func deltaTestState(t *testing.T, route string, ifaces ...deltaTestInterface) *netmon.State { + t.Helper() + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range ifaces { + flags := deltaTestDownFlags + if iface.up { + flags = deltaTestUpFlags + } + var mac net.HardwareAddr + if iface.mac != "" { + parsed, err := net.ParseMAC(iface.mac) + if err != nil { + t.Fatalf("ParseMAC(%q) error: %v", iface.mac, err) + } + mac = parsed + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{ + Name: iface.name, + Flags: flags, + MTU: iface.mtu, + HardwareAddr: mac, + }} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func sameInterfaceChange(got, want interfaceChange) bool { + return got.Name == want.Name && + got.Action == want.Action && + got.Class == want.Class && + got.HardwarePort == want.HardwarePort && + got.Service == want.Service && + slices.Equal(got.IPsBefore, want.IPsBefore) && + slices.Equal(got.IPsAfter, want.IPsAfter) && + got.Flags == want.Flags && + got.MTU == want.MTU && + got.IsDefaultRoute == want.IsDefaultRoute +} + +func assertInterfaceChanges(t *testing.T, got, want []interfaceChange) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("changes = %+v, want %+v", got, want) + } + for i := range want { + if !sameInterfaceChange(got[i], want[i]) { + t.Errorf("change %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func Test_diffNetworkDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + nilBefore bool + before []deltaTestInterface + after []deltaTestInterface + meta interfaceMetaFunc + want []interfaceChange + }{ + { + name: "airdrop_mac_change_with_link_local_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: []interfaceChange{{ + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "airdrop_mac_and_link_local_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, { + name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::2/64"}, + }}, + want: []interfaceChange{{ + Name: "awdl0", Action: "ip_removed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::2/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "hardware_address_added_with_airdrop_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }, awdl0NewMAC}, + want: []interfaceChange{ + { + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }, + { + Name: "en0", Action: "ip_added", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24", "192.0.2.11/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "interface_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, en5}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en5", Action: "removed", Class: "hardware", + IPsBefore: []string{"192.0.2.50/24"}, IPsAfter: nil, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "interface_added", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{en0, en5}, + want: []interfaceChange{{ + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "address_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "ip_removed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24", "192.0.2.11/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_down", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_up", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "up", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "mtu_changed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }}, + }, + { + name: "default_route_moves_to_the_new_interface", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }, en5}, + want: []interfaceChange{ + { + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, + }, + { + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "nil_old_state_adds_every_interface", + routeNew: "en0", + nilBefore: true, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: nil, + }, + { + name: "meta_fills_the_platform_fields", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "utun3", up: true, mtu: 1400}}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }, {name: "utun3", up: true, mtu: 1380}}, + meta: func(name string) interfaceMeta { + if name == "utun3" { + return interfaceMeta{Class: "virtual"} + } + return interfaceMeta{HardwarePort: "Wi-Fi", Service: "Wi-Fi"} + }, + want: []interfaceChange{ + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + HardwarePort: "Wi-Fi", Service: "Wi-Fi", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + { + Name: "utun3", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1380, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + var before *netmon.State + if !tt.nilBefore { + before = deltaTestState(t, tt.routeOld, tt.before...) + } + changes := diffNetworkDelta(before, deltaTestState(t, tt.routeNew, tt.after...)) + describeInterfaceChanges(changes, tt.meta) + assertInterfaceChanges(t, changes, tt.want) + }) + } +} + +func Test_diffNetworkDeltaVirtualSet(t *testing.T) { + virtual := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = virtual }) + virtualInterfaceSetFn = func() map[string]struct{} { return map[string]struct{}{"docker0": {}} } + + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1500}, + ) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1400, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1400}, + ) + assertInterfaceChanges(t, diffNetworkDelta(before, after), []interfaceChange{ + { + Name: "docker0", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1400, + }, + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + }) +} + +func Test_interfaceClass(t *testing.T) { + for _, tt := range []struct { + name string + virtual bool + want string + }{ + {name: "lo0", want: "loopback"}, + {name: "awdl0", want: "airdrop"}, + {name: "llw0", want: "airdrop"}, + {name: "utun3", want: "tunnel"}, + {name: "tun0", want: "tunnel"}, + {name: "tap0", want: "tunnel"}, + {name: "wg0", want: "tunnel"}, + {name: "ipsec0", want: "tunnel"}, + {name: "ppp0", want: "tunnel"}, + {name: "docker0", virtual: true, want: "virtual"}, + {name: "docker0", want: "hardware"}, + {name: "en0", want: "hardware"}, + } { + t.Run(tt.name+"_"+tt.want, func(t *testing.T) { + if got := interfaceClass(tt.name, tt.virtual); got != tt.want { + t.Errorf("interfaceClass(%q, %v) = %q, want %q", tt.name, tt.virtual, got, tt.want) + } + }) + } +} + +func Test_noiseDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en0Extra := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + } + + docker0 := deltaTestInterface{name: "docker0", up: true, mtu: 1500} + docker0NewMTU := deltaTestInterface{name: "docker0", up: true, mtu: 1400} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + before []deltaTestInterface + after []deltaTestInterface + virtual []string + timeJumped bool + beforeV4, beforeV6 bool + afterV4, afterV6 bool + source4 string + want bool + }{ + { + name: "airdrop_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: true, + }, + { + name: "hardware_and_airdrop", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0Extra, awdl0NewMAC}, + want: false, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: false, + }, + { + name: "default_route_moved", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0, en5, awdl0}, + after: []deltaTestInterface{en0, en5, awdl0NewMAC}, + want: false, + }, + { + name: "virtual_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: true, + }, + { + name: "a_time_jump_is_never_noise", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + timeJumped: true, + want: false, + }, + { + name: "the_default_route_runs_over_a_virtual_interface", + routeOld: "docker0", routeNew: "docker0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "a_virtual_interface_holds_a_routable_address", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1500, ips: []string{"172.17.0.1/16"}}}, + after: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1400, ips: []string{"172.17.0.1/16"}}}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "an_airdrop_interface_holds_the_resolver_source", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1484, ips: []string{"169.254.10.5/16"}}}, + after: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1400, ips: []string{"169.254.10.5/16"}}}, + source4: "169.254.10.5", + want: false, + }, + { + name: "have_v4_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + beforeV4: true, + want: false, + }, + { + name: "have_v6_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + afterV6: true, + want: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + stubSnapshotVirtualSet(t, tt.virtual...) + stubResolverSource(t, tt.source4) + before := deltaTestState(t, tt.routeOld, tt.before...) + after := deltaTestState(t, tt.routeNew, tt.after...) + before.HaveV4, before.HaveV6 = tt.beforeV4, tt.beforeV6 + after.HaveV4, after.HaveV6 = tt.afterV4, tt.afterV6 + changes := diffNetworkDelta(before, after) + if got := noiseDelta(before, after, tt.timeJumped, changes); got != tt.want { + t.Errorf("noiseDelta = %v, want %v, changes %+v", got, tt.want, changes) + } + }) + } +} + +// stubResolverSource sets the address that the resolver sends its queries +// from. The tests own that global, so each case starts from a known source. +func stubResolverSource(t *testing.T, source4 string) { + t.Helper() + original4, original6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(context.Background(), original4) + ctrld.SetDefaultLocalIPv6(context.Background(), original6) + }) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP(source4)) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) +} diff --git a/cmd/cli/network_events.go b/cmd/cli/network_events.go new file mode 100644 index 00000000..f73cc99a --- /dev/null +++ b/cmd/cli/network_events.go @@ -0,0 +1,167 @@ +package cli + +import ( + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// networkEventsNowFn is the clock of the network events. Tests replace it. +var networkEventsNowFn = time.Now + +const ( + hostWokeMessage = "Host woke" + + // noiseDeltaMessage names the deltas that no part of the daemon acts on. + noiseDeltaMessage = "Network delta noise" + + // interfaceChangedMessage names one interface that a delta changed. + interfaceChangedMessage = "Network interface changed" + + // networkTransitionMessage names the outcome of one delta. + networkTransitionMessage = "Network transition" + + // transitionOutcomeIgnored names a delta that left every usable interface + // as it was. + transitionOutcomeIgnored = "ignored" + + // transitionOutcomeSnapshotSuperseded names a delta whose snapshot netmon + // already replaced. A newer callback reports the same interfaces. + transitionOutcomeSnapshotSuperseded = "snapshot_superseded" +) + +// noteHostWoke reports one wake to the wake reporter. The caller passes the +// network of the wake, because the state that ctrld holds still describes the +// network before the sleep. netmon reports a time jump and measures no gap, so +// its report waits for the detector, and the journal gets one event per wake. +func (p *prog) noteHostWoke(source string, gap time.Duration, state *netmon.State) { + now := networkEventsNowFn() + // A resume moves the resolvers of the host, so every wake source starts the + // fast poll. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(now) + } + p.wake.note(now, wakeReport{source: source, gap: gap, state: state}, logHostWoke) +} + +// logHostWoke writes the Host woke journal event of one report. +func logHostWoke(report wakeReport) { + gapKnown := report.gap > 0 + event := journal(mainLog.Load().Info()).Str("source", report.source).Bool("gap_known", gapKnown) + if gapKnown { + event.Int64("gap_ms", report.gap.Milliseconds()) + } + event.Str("default_route", defaultRouteOf(report.state)). + Int("interfaces_up", upInterfaceCount(report.state)). + Msg(hostWokeMessage) +} + +// noteNoiseDelta records one delta of a noise storm. AirDrop and the virtual +// adapters of a container host change every few seconds, so the journal keeps +// one line per window while the debug stream keeps them all. +func (p *prog) noteNoiseDelta(changes []interfaceChange) { + names := changedInterfaceNames(changes) + mainLog.Load().Debug().Strs("interfaces", names).Msg(noiseDeltaMessage) + if summary, emit := p.networkNoise.add(names, networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// flushNoiseSummary closes the open noise window, because the delta that the +// daemon acts on ends the storm. +func (p *prog) flushNoiseSummary() { + if summary, emit := p.networkNoise.flush(networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// logNoiseSummary writes one journal line for a window of noise deltas. +func logNoiseSummary(summary noiseSummary) { + journal(mainLog.Load().Info()). + Int("count", summary.Count). + Strs("interfaces", summary.Interfaces). + Str("first_at", summary.First.Format(time.RFC3339)). + Str("last_at", summary.Last.Format(time.RFC3339)). + Msg(noiseDeltaMessage) +} + +// logInterfaceChanges puts the interfaces of one transition in the journal. +// Support reads back the port that lost its address at the time of an outage. +// An AirDrop or a container adapter changes every few seconds, so the +// transition names it and the journal keeps the interfaces that carry the +// traffic of the host. +func logInterfaceChanges(transitionID uint64, changes []interfaceChange) { + for _, change := range changes { + if noiseClass(change.Class) { + continue + } + journal(mainLog.Load().Info()). + Uint64("transition_id", transitionID). + Str("interface", change.Name). + Str("action", change.Action). + Str("class", change.Class). + Str("hardware_port", change.HardwarePort). + Str("service", change.Service). + Strs("ips_before", change.IPsBefore). + Strs("ips_after", change.IPsAfter). + Str("flags", change.Flags). + Int("mtu", change.MTU). + Bool("is_default_route", change.IsDefaultRoute). + Msg(interfaceChangedMessage) + } +} + +// networkTransitionEvent picks the level of one transition line. An ignored +// delta and a superseded snapshot arrive many times per minute, so the journal +// keeps the outcomes that can change the resolver state only. +func networkTransitionEvent(outcome string) *ctrld.LogEvent { + if outcome == transitionOutcomeIgnored || outcome == transitionOutcomeSnapshotSuperseded { + return mainLog.Load().Debug() + } + return journal(mainLog.Load().Info()) +} + +// changedInterfaceNames names the interfaces of one delta, in the order of the +// diff. +func changedInterfaceNames(changes []interfaceChange) []string { + names := make([]string, 0, len(changes)) + for _, change := range changes { + names = append(names, change.Name) + } + return names +} + +// hasAddOrRemove reports a delta that changed the adapter set of the host. The +// platform names of an adapter come from a cache, and that cache holds nothing +// for an adapter that just appeared. +func hasAddOrRemove(changes []interfaceChange) bool { + for _, change := range changes { + if change.Action == "added" || change.Action == "removed" { + return true + } + } + return false +} + +func defaultRouteOf(state *netmon.State) string { + if state == nil { + return "" + } + return state.DefaultRouteInterface +} + +// upInterfaceCount counts the interfaces with the up flag. A netmon entry +// without an embedded interface counts as down. +func upInterfaceCount(state *netmon.State) int { + if state == nil { + return 0 + } + up := 0 + for _, iface := range state.Interface { + if iface.Interface != nil && iface.IsUp() { + up++ + } + } + return up +} diff --git a/cmd/cli/network_events_test.go b/cmd/cli/network_events_test.go new file mode 100644 index 00000000..458a292f --- /dev/null +++ b/cmd/cli/network_events_test.go @@ -0,0 +1,827 @@ +package cli + +import ( + "context" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// hostWokeTestState names the network that a wake event reports. +func hostWokeTestState() *netmon.State { + return &netmon.State{ + DefaultRouteInterface: "en0", + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "en5": {Interface: &net.Interface{Name: "en5"}}, + "utun": {Interface: &net.Interface{Name: "utun", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{}, + } +} + +// hostWokeTestClock hands the wake events a clock that the test moves. +func hostWokeTestClock(t *testing.T, clock *time.Time) { + t.Helper() + original := networkEventsNowFn + t.Cleanup(func() { networkEventsNowFn = original }) + networkEventsNowFn = func() time.Time { return *clock } +} + +// hostWokeTestHold captures the hold timer of the wake reporter, so a test +// decides when the hold ends. +func hostWokeTestHold(p *prog) *[]func() { + var armed []func() + p.wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + return &armed +} + +// Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap covers the order of the +// 2026-09-14 capture. netmon sees the time jump first and measures no gap, so +// its report waits for the detector, and the journal gets one event with the +// gap. +func Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil })} + p.logger.Store(mainLog.Load()) + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events := jsonLogEvents(t, logs, hostWokeMessage); len(events) != 0 { + t.Fatalf("got %d Host woke events before the hold ended, want 0", len(events)) + } + if got := p.dnsConfig.nextDelay(clock); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } + clock = clock.Add(2 * time.Second) + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events, want one report with the gap of the detector", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_known", true) + wantField(t, events[0], "gap_ms", float64(90000)) + wantField(t, events[0], "default_route", "en0") + wantField(t, events[0], "interfaces_up", float64(2)) + wantField(t, events[0], "journal", true) + + clock = clock.Add(5 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 { + t.Fatalf("got %d Host woke events, want no second report inside the window", len(events)) + } +} + +// Test_noteHostWokeNetmonAloneReportsAfterTheHold covers a host without the +// detector. The netmon report goes out without a gap when the hold ends. +func Test_noteHostWokeNetmonAloneReportsAfterTheHold(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{} + p.logger.Store(mainLog.Load()) + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if len(*armed) != 1 { + t.Fatalf("holds armed = %d, want 1", len(*armed)) + } + (*armed)[0]() + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events after the hold, want 1", len(events)) + } + wantField(t, events[0], "source", "netmon") + wantField(t, events[0], "gap_known", false) + if _, measured := events[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", events[0]) + } +} + +// Test_noteHostWokeDetectorFirstReportsOneEvent covers the other order. The +// gap is known with the first report, so netmon adds nothing. +func Test_noteHostWokeDetectorFirstReportsOneEvent(t *testing.T) { + logs := captureDebugMainLog(t) + resume := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + clock := resume + hostWokeTestClock(t, &clock) + p := &prog{} + p.logger.Store(mainLog.Load()) + armed := hostWokeTestHold(p) + + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + clock = resume.Add(2 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds inside the window, want 1 and 0", len(events), len(*armed)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(90000)) + + // netmon polls every 15 s, so the window has to outlast one poll. + clock = resume.Add(29 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds, want one report for a wake that netmon reports late", len(events), len(*armed)) + } + + clock = resume.Add(31 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 2 { + t.Fatalf("got %d Host woke events after the window, want 2", len(events)) + } +} + +// networkEventsHarness drives the network callback on a clock that the test +// owns. Every host read stays behind a stub, so no test starts networksetup or +// reads the route table of the machine. +type networkEventsHarness struct { + prog *prog + logs *syncBuffer + now time.Time + reconciled int + ignored int +} + +func newNetworkEventsHarness(t *testing.T, valid ...string) *networkEventsHarness { + t.Helper() + sourceTestGlobals(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) + h := &networkEventsHarness{ + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + logs: captureDebugMainLog(t), + now: time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC), + } + h.prog.logger.Store(mainLog.Load()) + oldNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return h.now } + t.Cleanup(func() { networkEventsNowFn = oldNow }) + validSet := make(map[string]struct{}, len(valid)) + for _, name := range valid { + validSet[name] = struct{}{} + } + networkChangeValidInterfacesFn = func(context.Context) map[string]struct{} { return validSet } + networkChangeReconcileFn = func(_ *prog, _ context.Context, _ uint64) { h.reconciled++ } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { h.ignored++ } + return h +} + +func (h *networkEventsHarness) handle(before, after *netmon.State, timeJumped bool) { + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: before, New: after, TimeJumped: timeJumped}, true) +} + +// stubNetmonCache gives every callback of the harness one cache snapshot. +// netmon caches major snapshots only, so the epoch of a minor callback stays +// at that snapshot. +func (h *networkEventsHarness) stubNetmonCache(t *testing.T, cache *netmon.State) { + t.Helper() + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } +} + +// handleMinor drives one minor callback. netmon puts the cached major snapshot +// in Old, whatever the callback before it reported. +func (h *networkEventsHarness) handleMinor(cache, after *netmon.State) { + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: cache, New: after}, false) +} + +// interfaceEventsFor counts the journal events of one interface. +func (h *networkEventsHarness) interfaceEventsFor(t *testing.T, name string) int { + t.Helper() + count := 0 + for _, event := range jsonLogEvents(t, h.logs, interfaceChangedMessage) { + if event["interface"] == name { + count++ + } + } + return count +} + +// eventsAtLevel selects the captured events of one message and one level. +func (h *networkEventsHarness) eventsAtLevel(t *testing.T, message, level string) []map[string]any { + t.Helper() + var selected []map[string]any + for _, event := range jsonLogEvents(t, h.logs, message) { + if event["level"] == level { + selected = append(selected, event) + } + } + return selected +} + +var networkEventsHardwareIface = deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, +} + +// networkEventsAddressChange returns a delta that moves the address of en0. +func networkEventsAddressChange(t *testing.T) (before, after *netmon.State) { + t.Helper() + return deltaTestState(t, "en0", networkEventsHardwareIface), + deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) +} + +// transitionEvent returns the one transition event of a callback. +func (h *networkEventsHarness) transitionEvent(t *testing.T) map[string]any { + t.Helper() + events := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want 1", len(events), networkTransitionMessage) + } + return events[0] +} + +// wantNoField fails when an event carries a field that it must not. +func wantNoField(t *testing.T, event map[string]any, field string) { + t.Helper() + if value, carried := event[field]; carried { + t.Fatalf("field %q: got %v, want no field", field, value) + } +} + +// noiseDeltaStates returns a delta that moves the AirDrop link-local address. +func noiseDeltaStates(t *testing.T, step int) (before, after *netmon.State) { + t.Helper() + return noiseDeltaStatesFor(t, step, "awdl0") +} + +// noiseDeltaStatesFor returns a delta that moves the link-local address of +// each named noise interface. +func noiseDeltaStatesFor(t *testing.T, step int, names ...string) (before, after *netmon.State) { + t.Helper() + noise := func(address int) []deltaTestInterface { + interfaces := []deltaTestInterface{networkEventsHardwareIface} + for _, name := range names { + interfaces = append(interfaces, deltaTestInterface{name: name, up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}}) + } + return interfaces + } + return deltaTestState(t, "en0", noise(step)...), deltaTestState(t, "en0", noise(step+1)...) +} + +func wantStringsField(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, isText := value.(string) + if !isText { + t.Fatalf("field %q: got %v, want strings", field, event[field]) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("field %q: got %v, want %v", field, got, want) + } +} + +// Test_networkEventsNoiseDeltaSkipsTheHandler covers the AirDrop link, the +// low-latency Wi-Fi link, and both in one callback, as an AirDrop transfer +// gives them. +func Test_networkEventsNoiseDeltaSkipsTheHandler(t *testing.T) { + for _, names := range [][]string{{"awdl0"}, {"llw0"}, {"awdl0", "llw0"}} { + t.Run(strings.Join(names, "+"), func(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + + before, after := noiseDeltaStatesFor(t, 1, names...) + h.handle(before, after, false) + + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("noise delta ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + debugLines := h.eventsAtLevel(t, noiseDeltaMessage, "debug") + if len(debugLines) != 1 { + t.Fatalf("got %d debug %q lines, want 1", len(debugLines), noiseDeltaMessage) + } + wantStringsField(t, debugLines[0], "interfaces", names...) + if events := jsonLogEvents(t, h.logs, networkTransitionMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a noise delta, want 0", len(events), networkTransitionMessage) + } + }) + } +} + +func Test_networkEventsNoiseStormEmitsOneSummaryPerWindow(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step <= 10; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want 2", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[0], "journal", true) + wantField(t, summaries[0], "count", float64(1)) + wantField(t, summaries[1], "count", float64(10)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(10*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +func Test_networkEventsLogsEveryChangedInterface(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en5") + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24", "2001:db8::10/64"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}, + ) + before.HaveV4, before.HaveV6 = true, true + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}, + ) + after.HaveV4, after.HaveV6 = true, false + + h.handle(before, after, false) + + changes := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(changes) != 2 { + t.Fatalf("got %d %q events, want 2", len(changes), interfaceChangedMessage) + } + for index, name := range []string{"en0", "en5"} { + wantField(t, changes[index], "journal", true) + wantField(t, changes[index], "level", "info") + wantField(t, changes[index], "interface", name) + wantField(t, changes[index], "action", "ip_removed") + wantField(t, changes[index], "class", "hardware") + wantField(t, changes[index], "transition_id", float64(1)) + wantField(t, changes[index], "mtu", float64(1500)) + } + wantField(t, changes[0], "is_default_route", true) + wantField(t, changes[1], "is_default_route", false) + wantStringsField(t, changes[0], "ips_before", "192.0.2.10/24", "2001:db8::10/64") + wantStringsField(t, changes[0], "ips_after", "192.0.2.20/24") + + transitions := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(transitions) != 1 { + t.Fatalf("got %d %q events, want 1", len(transitions), networkTransitionMessage) + } + wantField(t, transitions[0], "journal", true) + wantField(t, transitions[0], "level", "info") + wantField(t, transitions[0], "outcome", "accepted") + wantField(t, transitions[0], "time_jumped", false) + wantField(t, transitions[0], "have_v4_before", true) + wantField(t, transitions[0], "have_v4_after", true) + wantField(t, transitions[0], "have_v6_before", true) + wantField(t, transitions[0], "have_v6_after", false) + wantStringsField(t, transitions[0], "changed_interfaces", "en0", "en5") + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1", h.reconciled) + } + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 1 { + t.Fatalf("got %d %q events, want 1", len(snapshots), networkSnapshotMessage) + } +} + +func Test_networkEventsTimeJumpReportsTheWake(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + armed := hostWokeTestHold(h.prog) + before, after := noiseDeltaStates(t, 1) + + h.handle(before, after, true) + for _, fire := range *armed { + fire() + } + + wakes := jsonLogEvents(t, h.logs, hostWokeMessage) + if len(wakes) != 1 { + t.Fatalf("got %d %q events, want 1", len(wakes), hostWokeMessage) + } + wantField(t, wakes[0], "journal", true) + wantField(t, wakes[0], "source", "netmon") + wantField(t, wakes[0], "gap_known", false) + wantField(t, wakes[0], "default_route", "en0") + if _, measured := wakes[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", wakes[0]) + } +} + +func Test_networkEventsStoreTheIPv6Flag(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + // The one-time probe wins over the first store, so consume it first. + available := ctrld.HasIPv6(context.Background()) + t.Cleanup(func() { ctrld.SetIPv6Available(available) }) + ctrld.SetIPv6Available(true) + + before := deltaTestState(t, "en0", networkEventsHardwareIface) + before.HaveV6 = true + after := deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) + after.HaveV6 = false + + h.handle(before, after, false) + + if ctrld.HasIPv6(context.Background()) { + t.Fatal("HasIPv6 = true after a callback with have_v6 false, want false") + } +} + +// Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt covers a difference +// that lasts. The cached major snapshot holds no link-local address of en0, so +// a diff against the cache reports the same change in every minor callback. +func Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + en0WithLinkLocal := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "fe80::a/64"}, + } + airdrop := func(address int) deltaTestInterface { + return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + } + cache := deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(1)) + h.stubNetmonCache(t, cache) + + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(2))) + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(3))) + + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("got %d %q events for en0, want 1", got, interfaceChangedMessage) + } + if noise := h.eventsAtLevel(t, noiseDeltaMessage, "debug"); len(noise) != 1 { + t.Fatalf("got %d debug %q lines, want 1 for the second delta", len(noise), noiseDeltaMessage) + } +} + +// Test_networkEventsNoiseDeltaReadsNoPlatformMeta pins the class-first order. +// The macOS meta read starts two subprocesses per call, so a storm of noise +// deltas must not reach it. +func Test_networkEventsNoiseDeltaReadsNoPlatformMeta(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + reads := countPlatformMetaReads(t) + + before, after := noiseDeltaStates(t, 1) + h.handle(before, after, false) + + if *reads != 0 { + t.Fatalf("a noise delta read the platform meta %d times, want 0", *reads) + } +} + +// countPlatformMetaReads counts the platform meta reads of a test. One read +// starts two subprocesses on macOS, so the count is the cost of a delta. +func countPlatformMetaReads(t *testing.T) *int { + t.Helper() + reads := 0 + original := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = original }) + platformInterfaceMetaFn = func(string) (class, hardwarePort, service string) { + reads++ + return "", "", "" + } + return &reads +} + +// Test_networkEventsNoiseStormEndsWithTheNextTransition covers the last window +// of a storm. The count of the deltas that a real change follows must not wait +// for the next storm. +func Test_networkEventsNoiseStormEndsWithTheNextTransition(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step < 4; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + h.now = start.Add(5 * time.Minute) + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want the first delta and the end of the storm", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[1], "count", float64(3)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(3*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +// Test_networkEventsStoreTheNetworkBeforeTheHeaderRender proves that the +// callback stores its network under the source lock and renders the header +// after the unlock, so the header read of the hardware ports starts no +// process under the lock. +func Test_networkEventsStoreTheNetworkBeforeTheHeaderRender(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + lockHeld, renders := false, 0 + original := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = original }) + // The header render reads the routes, so this seam runs where the + // callback renders the header. + defaultRoutesFn = func() (defaultRoute, defaultRoute) { + renders++ + if h.prog.networkSourceMu.TryLock() { + h.prog.networkSourceMu.Unlock() + } else { + lockHeld = true + } + return defaultRoute{Gateway: "192.0.2.1"}, defaultRoute{} + } + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + if renders == 0 { + t.Fatal("the callback rendered no header") + } + if lockHeld { + t.Fatal("the header render ran under the source lock") + } + if h.prog.lastNetworkState.Load() != after { + t.Fatal("the callback did not store its network") + } +} + +// Test_networkEventsIgnoredTransitionStaysOutOfTheJournal pins the level of an +// ignored delta. Such a delta arrives many times per minute, so it belongs in +// the debug stream and not in the journal. +func Test_networkEventsIgnoredTransitionStaysOutOfTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}) + after := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}) + + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", transitionOutcomeIgnored) + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 0 { + t.Fatalf("got %d %q events for an ignored delta, want 0", len(snapshots), networkSnapshotMessage) + } + if h.ignored != 1 || h.reconciled != 0 { + t.Fatalf("ignored delta: ignored=%d reconciled=%d, want 1/0", h.ignored, h.reconciled) + } +} + +// Test_networkEventsSupersededTransitionEntersTheJournal pins the level of a +// delta that a newer accepted delta replaced. Support reads it to tell a lost +// reconcile from one that never started. +func Test_networkEventsSupersededTransitionEntersTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + h.prog.networkAcceptedGen.Store(5) + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "superseded") + wantField(t, transition, "level", "info") + wantField(t, transition, "journal", true) + if h.reconciled != 0 { + t.Fatalf("a superseded delta reconciled %d times, want 0", h.reconciled) + } +} + +// Test_networkEventsSupersededSnapshotReportsTheOutcome covers the delta whose +// snapshot netmon already replaced. The callback logged nothing at all, so the +// journal showed a silent gap between two deltas. +func Test_networkEventsSupersededSnapshotReportsTheOutcome(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + cache := deltaTestState(t, "en0", networkEventsHardwareIface) + h.stubNetmonCache(t, cache) + reads := countPlatformMetaReads(t) + stale, after := networkEventsAddressChange(t) + + // netmon holds a newer snapshot than the one that this callback carries. + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: stale, New: after}, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "snapshot_superseded") + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + wantStringsField(t, transition, "changed_interfaces", "en0") + if events := jsonLogEvents(t, h.logs, interfaceChangedMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a superseded snapshot, want 0", len(events), interfaceChangedMessage) + } + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("superseded snapshot ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + if *reads != 0 { + t.Fatalf("a superseded snapshot read the platform meta %d times, want 0", *reads) + } +} + +// Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass covers a mixed +// delta. An AirDrop change beside a real one must not fill the journal, but +// the transition still names every interface that changed. +func Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::1/64"}}) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::2/64"}}) + + h.handle(before, after, false) + + events := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want one for the hardware port", len(events), interfaceChangedMessage) + } + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "class", "hardware") + wantStringsField(t, h.transitionEvent(t), "changed_interfaces", "awdl0", "en0") +} + +// reorderTestState builds a state with en0 on one address and awdl0 on one +// link-local address, so a test can replay callbacks out of order. +func reorderTestState(en0 string, awdl string) *netmon.State { + return &netmon.State{ + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "awdl0": {Interface: &net.Interface{Name: "awdl0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en0": {netip.MustParsePrefix(en0)}, + "awdl0": {netip.MustParsePrefix(awdl)}, + }, + DefaultRouteInterface: "en0", + HaveV4: true, + } +} + +// firstV4Of returns the first IPv4 address of one interface of a state. +func firstV4Of(state *netmon.State, name string) net.IP { + for _, prefix := range state.InterfaceIPs[name] { + if prefix.Addr().Is4() { + return net.IP(prefix.Addr().AsSlice()) + } + } + return nil +} + +// Test_networkEventsSupersededCallbackKeepsTheBaseline replays the callbacks +// out of order: B runs, a late A arrives while the monitor already caches C, +// then C runs. The late A must not become the baseline of C, or the address +// change from B to C looks like AirDrop noise and the stale source stays. +func Test_networkEventsSupersededCallbackKeepsTheBaseline(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + stateA := reorderTestState("192.0.2.20/24", "fe80::1/64") + stateB := reorderTestState("192.0.2.10/24", "fe80::1/64") + stateC := reorderTestState("192.0.2.20/24", "fe80::2/64") + cache := stateB + oldCurrent := networkChangeCurrentStateFn + t.Cleanup(func() { networkChangeCurrentStateFn = oldCurrent }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + networkChangeReconcileFn = func(p *prog, _ context.Context, _ uint64) { + h.reconciled++ + ctrld.SetDefaultLocalIPv4(context.Background(), firstV4Of(p.networkSourceState, "en0")) + } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: stateA, New: stateB}, true) + cache = stateC + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: &netmon.State{}, New: stateA}, true) + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: stateB, New: stateC}, true) + + if h.reconciled != 2 { + t.Fatalf("reconciles = %d, want 2: the late callback must not hide the change from B to C", h.reconciled) + } + if got := ctrld.GetDefaultLocalIPv4().String(); got != "192.0.2.20" { + t.Fatalf("resolver source = %s, want 192.0.2.20", got) + } +} + +// orderTestState builds a state for the callback order tests: en0 on one +// address, plus extra interfaces with their addresses and their up flag. +func orderTestState(route, en0 string, extra map[string][]string, down map[string]bool) *netmon.State { + state := &netmon.State{ + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + DefaultRouteInterface: route, + HaveV4: true, + } + add := func(name string, up bool, ips []string) { + flags := net.Flags(0) + if up { + flags = net.FlagUp + } + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: flags}} + for _, ip := range ips { + state.InterfaceIPs[name] = append(state.InterfaceIPs[name], netip.MustParsePrefix(ip)) + } + } + if en0 != "" { + add("en0", true, []string{en0}) + } + for name, ips := range extra { + add(name, !down[name], ips) + } + return state +} + +// Test_networkEventsMajorAfterItsMinorKeepsItsChanges runs a minor callback +// of the new epoch before the major callback that opened the epoch. netmon +// starts one goroutine per callback, so this order happens. The major must +// still report the address change and reconcile. +func Test_networkEventsMajorAfterItsMinorKeepsItsChanges(t *testing.T) { + for _, order := range []string{"in_order", "minor_first", "minor_first_time_jumped"} { + t.Run(order, func(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + hostWokeTestHold(h.prog) + s0 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s3 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::2/64"}}, nil) + h.stubNetmonCache(t, s2) + monitor := netmon.NewStatic() + major := &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2, TimeJumped: order == "minor_first_time_jumped"} + minor := &netmon.ChangeDelta{Monitor: monitor, Old: s2, New: s3} + if order == "in_order" { + h.prog.handleNetworkChange(context.Background(), major, true) + h.prog.handleNetworkChange(context.Background(), minor, false) + } else { + h.prog.handleNetworkChange(context.Background(), minor, false) + h.prog.handleNetworkChange(context.Background(), major, true) + } + + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1 for the move of en0 from .10 to .20", h.reconciled) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } + if h.prog.networkDeltaState != s3 { + t.Fatal("the baseline must stay at the newest state of the epoch") + } + }) + } +} + +// Test_networkEventsEpochChangeReportsAMinorChangeOnce closes an epoch with a +// major callback whose Old is the cache of the epoch. The interface that a +// minor callback of that epoch reported must not appear again. +func Test_networkEventsEpochChangeReportsAMinorChangeOnce(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + s0 := orderTestState("en0", "192.0.2.10/24", nil, nil) + s1 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + cache := s0 + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s1}, false) + cache = s2 + h.prog.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2}, true) + + if got := h.interfaceEventsFor(t, "ipsec0"); got != 1 { + t.Fatalf("ipsec0 interface events = %d, want 1", got) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } +} + +// Test_networkEventsNewValidInterfaceReconciles plugs a valid interface in +// with its address in one delta while the other valid interface is down. The +// new interface is active, so the change must reconcile. +func Test_networkEventsNewValidInterfaceReconciles(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en7") + before := orderTestState("", "", map[string][]string{"en0": nil}, map[string]bool{"en0": true}) + before.HaveV4 = false + after := orderTestState("en7", "", map[string][]string{"en0": nil, "en7": {"192.0.2.30/24"}}, map[string]bool{"en0": true}) + + h.handle(before, after, false) + + if h.reconciled != 1 { + event := h.transitionEvent(t) + t.Fatalf("reconciled = %d, want 1; outcome %v", h.reconciled, event["outcome"]) + } +} diff --git a/cmd/cli/network_manager_linux.go b/cmd/cli/network_manager_linux.go index 1a8c22b9..dc847e3a 100644 --- a/cmd/cli/network_manager_linux.go +++ b/cmd/cli/network_manager_linux.go @@ -23,66 +23,67 @@ systemd-resolved=false var networkManagerCtrldConfFile = filepath.Join(nmConfDir, nmCtrldConfFilename) // hasNetworkManager reports whether NetworkManager executable found. +// hasNetworkManager checks if NetworkManager is available on the system func hasNetworkManager() bool { exe, _ := exec.LookPath("NetworkManager") return exe != "" } -func setupNetworkManager() error { +func (p *prog) setupNetworkManager() error { if !hasNetworkManager() { return nil } if content, _ := os.ReadFile(nmCtrldConfContent); string(content) == nmCtrldConfContent { - mainLog.Load().Debug().Msg("NetworkManager already setup, nothing to do") + p.Debug().Msg("NetworkManager already setup, nothing to do") return nil } err := os.WriteFile(networkManagerCtrldConfFile, []byte(nmCtrldConfContent), os.FileMode(0644)) if os.IsNotExist(err) { - mainLog.Load().Debug().Msg("NetworkManager is not available") + p.Debug().Msg("NetworkManager is not available") return nil } if err != nil { - mainLog.Load().Debug().Err(err).Msg("could not write NetworkManager ctrld config file") + p.Debug().Err(err).Msg("Could not write NetworkManager ctrld config file") return err } - reloadNetworkManager() - mainLog.Load().Debug().Msg("setup NetworkManager done") + p.reloadNetworkManager() + p.Debug().Msg("Setup NetworkManager done") return nil } -func restoreNetworkManager() error { +func (p *prog) restoreNetworkManager() error { if !hasNetworkManager() { return nil } err := os.Remove(networkManagerCtrldConfFile) if os.IsNotExist(err) { - mainLog.Load().Debug().Msg("NetworkManager is not available") + p.Debug().Msg("NetworkManager is not available") return nil } if err != nil { - mainLog.Load().Debug().Err(err).Msg("could not remove NetworkManager ctrld config file") + p.Debug().Err(err).Msg("Could not remove NetworkManager ctrld config file") return err } - reloadNetworkManager() - mainLog.Load().Debug().Msg("restore NetworkManager done") + p.reloadNetworkManager() + p.Debug().Msg("Restore NetworkManager done") return nil } -func reloadNetworkManager() { +func (p *prog) reloadNetworkManager() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() conn, err := dbus.NewSystemConnectionContext(ctx) if err != nil { - mainLog.Load().Error().Err(err).Msg("could not create new system connection") + p.Error().Err(err).Msg("Could not create new system connection") return } defer conn.Close() waitCh := make(chan string) if _, err := conn.ReloadUnitContext(ctx, nmSystemdUnitName, "ignore-dependencies", waitCh); err != nil { - mainLog.Load().Debug().Err(err).Msg("could not reload NetworkManager") + p.Debug().Err(err).Msg("Could not reload NetworkManager") return } <-waitCh diff --git a/cmd/cli/network_manager_others.go b/cmd/cli/network_manager_others.go index 323d2f2e..e6e5f687 100644 --- a/cmd/cli/network_manager_others.go +++ b/cmd/cli/network_manager_others.go @@ -2,14 +2,14 @@ package cli -func setupNetworkManager() error { - reloadNetworkManager() +func (p *prog) setupNetworkManager() error { + p.reloadNetworkManager() return nil } -func restoreNetworkManager() error { - reloadNetworkManager() +func (p *prog) restoreNetworkManager() error { + p.reloadNetworkManager() return nil } -func reloadNetworkManager() {} +func (p *prog) reloadNetworkManager() {} diff --git a/cmd/cli/network_monitor_test.go b/cmd/cli/network_monitor_test.go new file mode 100644 index 00000000..bf888806 --- /dev/null +++ b/cmd/cli/network_monitor_test.go @@ -0,0 +1,56 @@ +package cli + +import ( + "context" + "net" + "testing" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +type sourceTestMonitor struct { + callback netmon.ChangeFunc + started bool +} + +func (*sourceTestMonitor) Close() error { return nil } + +func (m *sourceTestMonitor) RegisterChangeCallback(fn netmon.ChangeFunc) func() { + m.callback = fn + return func() {} +} +func (*sourceTestMonitor) IsMajorChangeFrom(*netmon.State, *netmon.State) bool { return true } +func (m *sourceTestMonitor) Start() { m.started = true } + +func TestNetworkMonitorRegistersSourceCleanup(t *testing.T) { + sourceTestGlobals(t) + old := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = old }) + mon := &sourceTestMonitor{} + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { return mon, nil } + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(context.Background(), net.ParseIP("2001:db8::10")) + calls := 0 + networkChangeReconcileFn = func(*prog, context.Context, uint64) { + calls++ + if ctrld.GetDefaultLocalIPv4() != nil { + t.Fatal("departed IPv4 survived to reconciliation") + } + } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}) + if err := p.monitorNetworkChanges(context.Background()); err != nil { + t.Fatal(err) + } + if !mon.started || mon.callback == nil { + t.Fatal("monitor did not register/start") + } + mon.callback(&netmon.ChangeDelta{ + Old: sourceTestState("en0", true, "192.0.2.10/24", "2001:db8::10/64"), + New: sourceTestState("en0", true, "2001:db8::10/64"), + }) + if calls != 1 { + t.Fatalf("registered callback never reached source reconciliation: %d", calls) + } +} diff --git a/cmd/cli/network_service_order.go b/cmd/cli/network_service_order.go new file mode 100644 index 00000000..b025fee5 --- /dev/null +++ b/cmd/cli/network_service_order.go @@ -0,0 +1,101 @@ +package cli + +import ( + "bufio" + "errors" + "io" + "regexp" + "strings" +) + +// maxNetworksetupLine bounds one line, so one token fills no memory. +const maxNetworksetupLine = 64 * 1024 + +var serviceOrderEntry = regexp.MustCompile(`^\(([1-9][0-9]*|\*)\) (.+)$`) +var serviceOrderDeviceName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]*$`) + +type networkServiceEntry struct { + Name, Device string + Disabled bool +} + +// scanNetworksetup reads the output line by line under a bounded buffer. +func scanNetworksetup(r io.Reader) *bufio.Scanner { + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxNetworksetupLine) + return scanner +} + +// parseNetworkServiceOrder decodes networksetup -listnetworkserviceorder. +// Selection policy belongs to the caller: retain disabled and duplicate-device +// entries so a DNS writer can reject mappings a first-match reader accepts. +func parseNetworkServiceOrder(r io.Reader) ([]networkServiceEntry, error) { + const header = "An asterisk (*) denotes that a network service is disabled." + scanner := scanNetworksetup(r) + var entries []networkServiceEntry + var pending *networkServiceEntry + headerSeen := false + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + continue + } + if !headerSeen { + if strings.TrimSpace(line) != header { + return nil, errors.New("invalid network service listing") + } + headerSeen = true + continue + } + if pending == nil { + m := serviceOrderEntry.FindStringSubmatch(line) + if m == nil || strings.ContainsAny(m[2], "\r\n\x00") { + return nil, errors.New("invalid network service entry") + } + // Only the index marks a disabled service. An enabled name may + // itself start with an asterisk; preserve it and its whitespace. + pending = &networkServiceEntry{Name: m[2], Disabled: m[1] == "*"} + continue + } + if !strings.HasPrefix(line, "(Hardware Port: ") || !strings.HasSuffix(line, ")") { + return nil, errors.New("invalid network service device") + } + at := strings.LastIndex(line, ", Device: ") + if at < 0 { + return nil, errors.New("missing network service device") + } + device := line[at+len(", Device: ") : len(line)-1] + if device != "" && !serviceOrderDeviceName.MatchString(device) { + return nil, errors.New("invalid network device name") + } + pending.Device = device + entries = append(entries, *pending) + pending = nil + } + if err := scanner.Err(); err != nil { + return nil, err + } + if !headerSeen || pending != nil { + return nil, errors.New("incomplete network service listing") + } + return entries, nil +} + +// serviceNamesByDevice keeps the first enabled service on each device for +// interface-name lookup and hardware-port metadata, not DNS-target admission. +func serviceNamesByDevice(r io.Reader) map[string]string { + entries, err := parseNetworkServiceOrder(r) + if err != nil { + return nil + } + names := make(map[string]string) + for _, entry := range entries { + if entry.Disabled || entry.Device == "" { + continue + } + if _, seen := names[entry.Device]; !seen { + names[entry.Device] = entry.Name + } + } + return names +} diff --git a/cmd/cli/network_service_order_test.go b/cmd/cli/network_service_order_test.go new file mode 100644 index 00000000..f82fed82 --- /dev/null +++ b/cmd/cli/network_service_order_test.go @@ -0,0 +1,102 @@ +package cli + +import ( + "errors" + "io" + "reflect" + "strings" + "testing" +) + +func TestNetworkServiceOrderEntries(t *testing.T) { + const listing = `An asterisk (*) denotes that a network service is disabled. +(1) *Téléphone personnel +(Hardware Port: USB, Device: en8) +(*) Disabled USB +(Hardware Port: USB, Device: en8) +(2) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) +(3) Tunnel +(Hardware Port: VPN, Device: ) +` + want := []networkServiceEntry{ + {Name: "*Téléphone personnel", Device: "en8"}, + {Name: "Disabled USB", Device: "en8", Disabled: true}, + {Name: " Wi-Fi", Device: "en0"}, + {Name: "Tunnel"}, + } + got, err := parseNetworkServiceOrder(strings.NewReader(listing)) + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("entries=%+v err=%v", got, err) + } +} + +func TestNetworkServiceOrderSelectionPolicies(t *testing.T) { + base := nativeTargetTestServiceOrder("*iPhone USB", "en8") + for _, tc := range []struct { + name, listing, lookup, target string + }{ + {"enabled asterisk name", base, "*iPhone USB", "*iPhone USB"}, + {"embedded asterisk", nativeTargetTestServiceOrder("My * USB", "en8"), "My * USB", "My * USB"}, + {"name whitespace", nativeTargetTestServiceOrder(" USB ", "en8"), " USB ", " USB "}, + {"disabled", strings.Replace(base, "(1)", "(*)", 1), "", ""}, + {"disabled neighbor", base + "(*) Ethernet\n(Hardware Port: Ethernet, Device: en9)\n", "*iPhone USB", "*iPhone USB"}, + {"enabled then disabled same device", base + "(*) Old USB\n(Hardware Port: USB, Device: en8)\n", "*iPhone USB", ""}, + {"disabled then enabled same device", strings.Replace(base, "(1)", "(*)", 1) + "(2) New USB\n(Hardware Port: USB, Device: en8)\n", "New USB", ""}, + {"two enabled same device", base + "(2) Other USB\n(Hardware Port: USB, Device: en8)\n", "*iPhone USB", ""}, + {"duplicate name on another device", base + "(2) *iPhone USB\n(Hardware Port: USB, Device: en9)\n", "*iPhone USB", ""}, + {"case-insensitive duplicate", base + "(*) *IPHONE USB\n(Hardware Port: USB, Device: en9)\n", "*iPhone USB", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + names := serviceNamesByDevice(strings.NewReader(tc.listing)) + if names["en8"] != tc.lookup { + t.Fatalf("lookup=%v, want %q", names, tc.lookup) + } + if tc.name == "disabled neighbor" { + if _, exists := names["en9"]; exists { + t.Fatal("disabled neighbor inherited an enabled service name") + } + } + got, err := targetNetworkServiceName(tc.listing, "en8") + if got != tc.target || (err != nil) != (tc.target == "") { + t.Fatalf("target=%q err=%v, want %q", got, err, tc.target) + } + }) + } +} + +func TestNetworkServiceOrderRejectsIncompleteReads(t *testing.T) { + base := nativeTargetTestServiceOrder("USB", "en8") + for _, listing := range []string{ + "", "unexpected output", base + "(2) Incomplete\n", + base + "(2) \n(Hardware Port: USB, Device: en9)\n", + base + "(2) Broken\nmissing device line\n", + base + strings.Repeat("x", maxNetworksetupLine+1), + strings.Replace(base, "Device: en8", "Device: en8 extra", 1), + strings.Replace(base, "(1) USB", "(0) USB", 1), + strings.Replace(base, "(1) USB", "(1) USB\x00", 1), + } { + entries, err := parseNetworkServiceOrder(strings.NewReader(listing)) + if err == nil || entries != nil { + t.Fatalf("invalid listing returned entries=%+v err=%v", entries, err) + } + if got := serviceNamesByDevice(strings.NewReader(listing)); got != nil { + t.Fatalf("invalid listing returned partial names: %v", got) + } + if got, err := targetNetworkServiceName(listing, "en8"); err == nil || got != "" { + t.Fatalf("invalid listing admitted target=%q err=%v", got, err) + } + } + readErr := errors.New("interrupted service-order read") + reader := func() io.Reader { return io.MultiReader(strings.NewReader(base), serviceOrderErrorReader{readErr}) } + if entries, err := parseNetworkServiceOrder(reader()); !errors.Is(err, readErr) || entries != nil { + t.Fatalf("partial read returned entries=%+v err=%v", entries, err) + } + if names := serviceNamesByDevice(reader()); names != nil { + t.Fatalf("failed read returned partial names: %v", names) + } +} + +type serviceOrderErrorReader struct{ err error } + +func (r serviceOrderErrorReader) Read([]byte) (int, error) { return 0, r.err } diff --git a/cmd/cli/network_shutdown_test.go b/cmd/cli/network_shutdown_test.go new file mode 100644 index 00000000..2e2296a3 --- /dev/null +++ b/cmd/cli/network_shutdown_test.go @@ -0,0 +1,342 @@ +package cli + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func guardNetworkShutdownHostMutations(t *testing.T) { + t.Helper() + oldReset, oldRemove := recoveryResetDNSFn, removeInterceptDNSTargetFn + recoveryResetDNSFn = func(*prog, bool, bool) { panic("unexpected recovery DNS reset") } + // Also fence the first platform boundary inside real resetDNS, so mutation + // testing a bypassed recoveryResetDNSFn can never touch the test runner's DNS/PF. + removeInterceptDNSTargetFn = func(*prog, string) { panic("real resetDNS crossed host safety boundary") } + t.Cleanup(func() { recoveryResetDNSFn, removeInterceptDNSTargetFn = oldReset, oldRemove }) +} + +func waitNetworkShutdownSignal(t *testing.T, ch <-chan struct{}, what string) { + t.Helper() + select { + case <-ch: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for %s", what) + } +} + +func waitNetworkShutdownGate(t *testing.T, p *prog) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for !p.networkActivityClosed() { + if time.Now().After(deadline) { + t.Fatal("shutdown did not close admission") + } + time.Sleep(time.Millisecond) + } +} + +func assertNetworkShutdownBlocked(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + t.Fatal("shutdown returned before admitted work finished") + case <-time.After(20 * time.Millisecond): + } +} + +func TestNetworkShutdownRejectsLateMonitorPublication(t *testing.T) { + p := &prog{} + p.logger.Store(mainLog.Load()) + p.closeNetMonitor() + // A zero monitor is never started or closed: on rejected publication the + // initializer retains ownership. No OS monitor or DNS policy is installed. + if p.setNetMonitor(new(netmon.Monitor)) { + t.Fatal("monitor published after shutdown") + } + if p.netMonitor != nil { + t.Fatal("shutdown retained a monitor") + } + if p.beginNetworkActivity() { + p.netMonitorWG.Done() + t.Fatal("work admitted after shutdown") + } +} + +func TestNetworkShutdownDrainsCallbacksAndConcurrentClosers(t *testing.T) { + guardNetworkShutdownHostMutations(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + var calls atomic.Int32 + // Stub the entire callback body before any platform or DNS boundary. This + // is the same wrapper used by RegisterChangeCallback in production. + callback := p.networkChangeCallback(func(*netmon.ChangeDelta) { + calls.Add(1) + close(entered) + <-release + // An admitted callback may try to schedule recovery after Close fenced + // admission. Neither synchronous nor deferred work may start then. + p.debounceRecovery(p.networkAcceptedGen.Load()) + p.handleRecovery(RecoveryReasonNetworkChange) + }) + go callback(nil) + waitNetworkShutdownSignal(t, entered, "callback admission") + + closed1, closed2 := make(chan struct{}), make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed1) }() + waitNetworkShutdownGate(t, p) + go func() { p.closeNetMonitor(); close(closed2) }() + // Model a callback dispatched by netmon before Close but scheduled later. + callback(nil) + assertNetworkShutdownBlocked(t, closed1) + assertNetworkShutdownBlocked(t, closed2) + unblock() + waitNetworkShutdownSignal(t, closed1, "first close") + waitNetworkShutdownSignal(t, closed2, "second close") + callback(nil) + if calls.Load() != 1 || p.recoveryGen.Load() != 0 || p.recoveryDebounceTimer != nil { + t.Fatalf("late work ran: callbacks=%d recovery=%d timer=%v", calls.Load(), p.recoveryGen.Load(), p.recoveryDebounceTimer) + } +} + +func TestNetworkShutdownCancelsAndDrainsRecovery(t *testing.T) { + testNetworkShutdownCancelsAndDrainsRecovery(t, func(p *prog) { + p.handleRecovery(RecoveryReasonRegularFailure) + }) +} + +func TestNetworkShutdownDrainsDeferredRecovery(t *testing.T) { + testNetworkShutdownCancelsAndDrainsRecovery(t, func(p *prog) { + p.networkChangeCallback(func(*netmon.ChangeDelta) { p.debounceRecovery(p.networkAcceptedGen.Load()) })(nil) + }) +} + +func testNetworkShutdownCancelsAndDrainsRecovery(t *testing.T, start func(*prog)) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + guardNetworkShutdownHostMutations(t) + oldReset, oldIntercept := recoveryResetDNSFn, dnsIntercept + dnsIntercept = false + defer func() { recoveryResetDNSFn, dnsIntercept = oldReset, oldIntercept }() + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + var resets atomic.Int32 + // resetDNS is the FIRST host-mutating boundary in traditional recovery. + // Block there; never invoke real resetDNS, setDNS, PF, or NRPT operations. + recoveryResetDNSFn = func(p *prog, isStart, restoreStatic bool) { + resets.Add(1) + p.recoveryCancelMu.Lock() + cancel := p.recoveryCancel + p.recoveryCancel = func() { cancel(); close(canceled) } + p.recoveryCancelMu.Unlock() + close(entered) + <-release + } + recovered := make(chan struct{}) + go func() { start(p); close(recovered) }() + waitNetworkShutdownSignal(t, entered, "recovery reset seam") + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownSignal(t, canceled, "recovery cancellation") + assertNetworkShutdownBlocked(t, closed) + if p.recoveryOwnsState(p.recoveryGen.Load()) { + t.Fatal("shutdown recovery still owns permission to reapply DNS") + } + unblock() + waitNetworkShutdownSignal(t, closed, "recovery drain") + waitNetworkShutdownSignal(t, recovered, "recovery return") + p.handleRecovery(RecoveryReasonRegularFailure) + p.handleRecovery(RecoveryReasonNetworkChange) + if resets.Load() != 1 || p.recoveryRunning.Load() || p.recoveryBypass.Load() || p.recoveryCancel != nil { + t.Fatalf("recovery survived close: resets=%d running=%v bypass=%v", resets.Load(), p.recoveryRunning.Load(), p.recoveryBypass.Load()) + } + ends := jsonLogEvents(t, logs, "Recovery end") + if len(ends) != 1 { + t.Fatalf("recovery end events: got %d, want 1", len(ends)) + } + wantField(t, ends[0], "outcome", recoveryOutcomeCanceled) + wantField(t, ends[0], "bypass_active", false) +} + +func TestNetworkShutdownCancelsIndefiniteRecoveryWait(t *testing.T) { + guardNetworkShutdownHostMutations(t) + oldReset, oldIntercept := recoveryResetDNSFn, dnsIntercept + dnsIntercept = false + defer func() { recoveryResetDNSFn, dnsIntercept = oldReset, oldIntercept }() + // The only upstream is a loopback port nothing listens on, so recovery + // can never succeed. An empty upstream map no longer does that: recovery + // then checks upstream.os, and on a host with working DNS that answers, + // completes the recovery and turns the test into a probe of the host. + dead := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, Endpoint: deadUDPAddr(t), Timeout: 300} + dead.Init(context.Background()) + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{"0": dead}} + p := &prog{cfg: cfg} + p.logger.Store(mainLog.Load()) + // Recovery reads the upstream monitor, which run creates before network + // monitoring can start any recovery. + p.um = newUpstreamMonitor(cfg, p.logger.Load()) + entered := make(chan struct{}) + recoveryResetDNSFn = func(*prog, bool, bool) { close(entered) } + recovered := make(chan struct{}) + // Shutdown must cancel that wait without waiting on a mutex held for the + // recovery's entire lifetime. + go func() { p.handleRecovery(RecoveryReasonRegularFailure); close(recovered) }() + waitNetworkShutdownSignal(t, entered, "recovery reset seam") + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownSignal(t, closed, "canceled indefinite recovery") + waitNetworkShutdownSignal(t, recovered, "recovery return") +} + +func TestNetworkShutdownStopsDeferredRecovery(t *testing.T) { + guardNetworkShutdownHostMutations(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + p.debounceRecovery(p.networkAcceptedGen.Load()) + p.recoveryDebounceMu.Lock() + timer := p.recoveryDebounceTimer + p.recoveryDebounceMu.Unlock() + if timer == nil { + t.Fatal("debounce did not schedule recovery") + } + p.closeNetMonitor() + if timer.Stop() { + t.Fatal("shutdown did not stop pending debounce timer") + } + // Model the already-fired timer's call to handleRecovery, which Stop + // cannot retract, then a callback trying to create another timer. + p.handleRecovery(RecoveryReasonNetworkChange) + p.debounceRecovery(p.networkAcceptedGen.Load()) + if p.recoveryDebounceTimer != nil || p.recoveryGen.Load() != 0 { + t.Fatal("deferred recovery restarted after shutdown") + } +} + +func TestNetworkShutdownRejectsRecoveryAdmittedBeforeClose(t *testing.T) { + p := &prog{} + p.logger.Store(mainLog.Load()) + // Pause between handleRecovery's activity admission and beginRecovery. + if !p.beginNetworkActivity() { + t.Fatal("initial admission failed") + } + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownGate(t, p) + _, _, _, ok := p.beginRecovery(RecoveryReasonNetworkChange) + p.netMonitorWG.Done() + waitNetworkShutdownSignal(t, closed, "admitted recovery drain") + if ok || p.recoveryCancel != nil || p.recoveryRunning.Load() { + t.Fatal("recovery published a new owner after shutdown cancellation") + } +} + +type shutdownTestMonitor struct { + callback netmon.ChangeFunc + starts, closes int +} + +func (m *shutdownTestMonitor) Start() { m.starts++ } +func (m *shutdownTestMonitor) Close() error { m.closes++; return nil } +func (m *shutdownTestMonitor) RegisterChangeCallback(fn netmon.ChangeFunc) func() { + m.callback = fn + return func() {} +} +func (*shutdownTestMonitor) IsMajorChangeFrom(*netmon.State, *netmon.State) bool { + panic("closed callback reached network discovery") +} + +func TestNetworkMonitorProductionPublicationAndCallbackFence(t *testing.T) { + old := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = old }) + for _, late := range []bool{false, true} { + p := &prog{} + p.logger.Store(mainLog.Load()) + m := &shutdownTestMonitor{} + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { + if late { + p.closeNetMonitor() + } + return m, nil + } + if err := p.monitorNetworkChanges(context.Background()); err != nil { + t.Fatal(err) + } + p.closeNetMonitor() + if m.closes != 1 { + t.Fatalf("monitor closed %d times", m.closes) + } + wantStarts := 1 + if late { + wantStarts = 0 + } + if m.starts != wantStarts { + t.Fatalf("monitor starts=%d want=%d", m.starts, wantStarts) + } + // Exercise the callback captured from the actual production registration. + // Removing its wrapper reaches the panic above, before any OS mutation. + m.callback(&netmon.ChangeDelta{}) + } +} + +func TestFinishRunDrainsNetworkJournalOnAbort(t *testing.T) { + captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + p := &prog{ + cfg: &ctrld.Config{}, stopCh: make(chan struct{}), + runAbortCh: make(chan struct{}), runDone: make(chan struct{}), + dnsWatcherStopCh: make(chan struct{}), + } + p.logger.Store(mainLog.Load()) + p.newNetworkJournal() + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + defer unblock() + // The poll is a synthetic first platform boundary, even on Darwin. + p.dnsConfig = newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + close(entered) + <-release + return nil, nil + }) + p.waitNetworkJournal = p.startNetworkJournal() + waitNetworkShutdownSignal(t, entered, "journal poll") + close(p.runDone) + done := make(chan struct{}) + go func() { p.finishRun(); close(done) }() + waitNetworkShutdownSignal(t, p.runAbortCh, "run abort") + assertNetworkShutdownBlocked(t, done) + unblock() + waitNetworkShutdownSignal(t, done, "journal drain on abort") + if stopRequested(p.stopCh) { + t.Fatal("finishRun closed the caller-owned stop channel") + } +} + +func TestNetworkShutdownDrainsBeforeOSRestoration(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + entered, release, restored := make(chan struct{}), make(chan struct{}), make(chan struct{}) + p.onStopped = []func(){func() { close(restored) }} + go p.networkChangeCallback(func(*netmon.ChangeDelta) { close(entered); <-release })(nil) + waitNetworkShutdownSignal(t, entered, "callback entry") + done := make(chan struct{}) + go func() { _ = p.restoreOSState(); close(done) }() + waitNetworkShutdownGate(t, p) + assertNetworkShutdownBlocked(t, restored) + close(release) + waitNetworkShutdownSignal(t, done, "restoration") + waitNetworkShutdownSignal(t, restored, "restore hook") +} diff --git a/cmd/cli/network_snapshot.go b/cmd/cli/network_snapshot.go new file mode 100644 index 00000000..8690341c --- /dev/null +++ b/cmd/cli/network_snapshot.go @@ -0,0 +1,318 @@ +package cli + +import ( + "cmp" + "net/netip" + "slices" + "strings" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" + "tailscale.com/net/tsaddr" +) + +// hardwarePortIPhoneUSB is the macOS port name of a phone that shares its +// connection over the cable. +const hardwarePortIPhoneUSB = "iPhone USB" + +// snapshotInterfaceLimit bounds the interface list of one snapshot. A +// container host holds hundreds of interfaces, and each entry costs bytes in +// every journal line and in every log file header. +const snapshotInterfaceLimit = 32 + +// v6GlobalPrefix holds the v6 addresses that reach the Internet. +var v6GlobalPrefix = netip.MustParsePrefix("2000::/3") + +// hotspotGatewayPrefix is the range that an iPhone hotspot hands out. A +// gateway inside it names a tethered host on every platform. +var hotspotGatewayPrefix = netip.MustParsePrefix("172.20.10.0/28") + +// ethernetPortWords name a wired port. macOS spells the same link in several +// ways, one per adapter family. +var ethernetPortWords = []string{"Ethernet", "LAN", "Thunderbolt"} + +// defaultRoute names the next hop and the interface of one default route. Both +// values come from the same entry of the route table, so a reader can trust +// the pair. +type defaultRoute struct { + Gateway string + Interface string +} + +// snapshotInterface is one interface of a network snapshot. +type snapshotInterface struct { + Name string + Class string + Up bool + IPs []string + HardwarePort string + Service string +} + +// networkSnapshot describes the host network at one moment. The journal event +// and the header of a log file render the same value. +type networkSnapshot struct { + DefaultRouteV4 string + DefaultRouteV6 string + GatewayV4 string + GatewayV6 string + HaveV4 bool + HaveV6 bool + Interfaces []snapshotInterface + InterfacesOmitted int + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + LinkType string + Tethered bool + CLATPresent bool + NAT64Prefix string + DNSLessTarget bool +} + +// snapshotInputs holds every value a snapshot needs. The caller reads them, so +// the builder stays free of locks and of system calls. +type snapshotInputs struct { + State *netmon.State + RouteV4 defaultRoute + RouteV6 defaultRoute + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + NAT64Prefix string + CLATPresent bool + Meta interfaceMetaFunc +} + +// buildNetworkSnapshot makes the snapshot of one moment. It is pure, so a +// caller can take the inputs under a lock and build outside it. +func buildNetworkSnapshot(in snapshotInputs) networkSnapshot { + snapshot := networkSnapshot{ + GatewayV4: in.RouteV4.Gateway, + GatewayV6: in.RouteV6.Gateway, + DefaultRouteV4: in.RouteV4.Interface, + DefaultRouteV6: in.RouteV6.Interface, + Resolvers: in.Resolvers, + SourceIPv4: in.SourceIPv4, + SourceIPv6: in.SourceIPv6, + InterceptTarget: in.InterceptTarget, + BypassActive: in.BypassActive, + RecoveryRunning: in.RecoveryRunning, + PFStabilizing: in.PFStabilizing, + NAT64Prefix: in.NAT64Prefix, + CLATPresent: in.CLATPresent, + DNSLessTarget: in.InterceptTarget != "", + } + + routeMeta := interfaceMeta{} + var routeAddresses []netip.Prefix + if in.State != nil { + meta := interfaceMetaWithClass(in.Meta) + snapshot.HaveV4, snapshot.HaveV6 = in.State.HaveV4, in.State.HaveV6 + snapshot.Interfaces, snapshot.InterfacesOmitted = snapshotInterfaces(in.State, meta) + // Only a platform that reads no route table takes the route of the + // network monitor. On macOS an empty v4 route is a fact, and the + // monitor can still hold the interface of the last network. + if in.RouteV4.Interface == "" && in.RouteV6.Interface == "" { + snapshot.DefaultRouteV4 = defaultRouteV4Interface(in.State) + } + linkInterface := snapshotLinkInterface(in) + routeMeta = meta(linkInterface) + routeAddresses = in.State.InterfaceIPs[linkInterface] + } + snapshot.LinkType = linkTypeFor(routeMeta.HardwarePort, routeMeta.Class) + snapshot.Tethered = tetheredNetwork(in.RouteV4.Gateway, routeMeta.HardwarePort, routeAddresses) + return snapshot +} + +// snapshotLinkInterface names the interface that the link fields describe. +// The route table wins over the network monitor: during a switch the monitor +// can still hold the old interface, and one record must not mix the routes of +// one moment with the link type of another. It must run with a state. +func snapshotLinkInterface(in snapshotInputs) string { + for _, name := range []string{in.RouteV4.Interface, in.RouteV6.Interface} { + if name != "" { + return name + } + } + return in.State.DefaultRouteInterface +} + +// interfaceMetaWithClass fills the class that the platform metadata leaves +// empty. A host that names no port still tells a tunnel from a hardware port +// by the interface name. +func interfaceMetaWithClass(meta interfaceMetaFunc) interfaceMetaFunc { + return func(name string) interfaceMeta { + info := interfaceMeta{} + if meta != nil { + info = meta(name) + } + if info.Class == "" { + info.Class = interfaceClass(name, false) + } + return info + } +} + +// snapshotInterfaces lists the interfaces of the state, sorted by name, and +// counts the ones it leaves out. A reader compares two snapshots line by line, +// so the order must not move. An interface that is down and holds no address +// carries no traffic, and a container host holds hundreds of them. +func snapshotInterfaces(state *netmon.State, meta interfaceMetaFunc) (list []snapshotInterface, omitted int) { + list = make([]snapshotInterface, 0, len(state.Interface)) + for name, iface := range state.Interface { + up := iface.Interface != nil && iface.IsUp() + ips := prefixStrings(state.InterfaceIPs[name]) + if !up && len(ips) == 0 { + omitted++ + continue + } + info := meta(name) + list = append(list, snapshotInterface{ + Name: name, + Class: info.Class, + Up: up, + IPs: ips, + HardwarePort: info.HardwarePort, + Service: info.Service, + }) + } + slices.SortFunc(list, func(a, b snapshotInterface) int { + return cmp.Compare(a.Name, b.Name) + }) + if len(list) > snapshotInterfaceLimit { + omitted += len(list) - snapshotInterfaceLimit + list = list[:snapshotInterfaceLimit] + } + return list, omitted +} + +// defaultRouteV4Interface names the interface that the network monitor holds +// for the default route. It fills the v4 route of a platform that reads no +// route table. An interface without a usable v4 address carries no v4 traffic, +// whatever the monitor says. +func defaultRouteV4Interface(state *netmon.State) string { + route := state.DefaultRouteInterface + if route == "" { + return "" + } + for _, prefix := range state.InterfaceIPs[route] { + if usableV4(prefix.Addr().Unmap()) { + return route + } + } + return "" +} + +// globalAddress reports whether an address reaches another host. +func globalAddress(address netip.Addr) bool { + return address.IsGlobalUnicast() && !address.IsLoopback() && !address.IsLinkLocalUnicast() +} + +// usableV4 and usableV6 grade one address the way the network monitor grades +// it. A fresh read of the interfaces must report the same families as a +// monitor callback, or two snapshots of one moment disagree. The monitor +// counts a unique local address of the v6 family, because some networks reach +// the Internet from it. The range of the Tailscale mesh is the exception, +// because that mesh carries no Internet traffic. +func usableV4(address netip.Addr) bool { + return address.Is4() && globalAddress(address) +} + +func usableV6(address netip.Addr) bool { + if !address.Is6() { + return false + } + if v6GlobalPrefix.Contains(address) { + return true + } + return address.IsPrivate() && !tsaddr.TailscaleULARange().Contains(address) +} + +// tetheredNetwork reports whether the host runs on a phone connection. A +// tethered host pays for its traffic and loses the link on every phone move. +// A platform that reads no route table gives no gateway, so the address that +// the phone handed to the host tells the same story. +func tetheredNetwork(gatewayV4, hardwarePort string, routeAddresses []netip.Prefix) bool { + if hardwarePort == hardwarePortIPhoneUSB { + return true + } + if gateway, err := netip.ParseAddr(gatewayV4); err == nil && hotspotGatewayPrefix.Contains(gateway.Unmap()) { + return true + } + for _, prefix := range routeAddresses { + if hotspotGatewayPrefix.Contains(prefix.Addr().Unmap()) { + return true + } + } + return false +} + +// linkTypeFor names the link of the default route. The port name tells more +// than the interface name, which is only a number on macOS. +func linkTypeFor(hardwarePort, class string) string { + switch { + case hardwarePort == "Wi-Fi": + return "wifi" + case hardwarePort == hardwarePortIPhoneUSB: + return "usb_tether" + case ethernetPort(hardwarePort): + return "ethernet" + case class == "tunnel": + return "tunnel" + } + return "unknown" +} + +func ethernetPort(hardwarePort string) bool { + for _, word := range ethernetPortWords { + if strings.Contains(hardwarePort, word) { + return true + } + } + return false +} + +// snapshotDict renders the snapshot with the field names of the spec. The +// journal event and the header line share it, so both stay in one shape. +func snapshotDict(snapshot networkSnapshot) *ctrld.LogEvent { + interfaces := ctrld.Arr() + for _, iface := range snapshot.Interfaces { + interfaces = interfaces.Dict(ctrld.Dict(). + Str("name", iface.Name). + Str("class", iface.Class). + Bool("up", iface.Up). + Strs("ips", iface.IPs). + Str("hardware_port", iface.HardwarePort). + Str("service", iface.Service)) + } + return ctrld.Dict(). + Str("default_route_v4", snapshot.DefaultRouteV4). + Str("default_route_v6", snapshot.DefaultRouteV6). + Str("gateway_v4", snapshot.GatewayV4). + Str("gateway_v6", snapshot.GatewayV6). + Bool("have_v4", snapshot.HaveV4). + Bool("have_v6", snapshot.HaveV6). + Array("interfaces", interfaces). + Int("interfaces_omitted", snapshot.InterfacesOmitted). + Strs("resolvers", snapshot.Resolvers). + Str("source_ipv4", snapshot.SourceIPv4). + Str("source_ipv6", snapshot.SourceIPv6). + Str("intercept_target", snapshot.InterceptTarget). + Bool("bypass_active", snapshot.BypassActive). + Bool("recovery_running", snapshot.RecoveryRunning). + Bool("pf_stabilizing", snapshot.PFStabilizing). + Str("link_type", snapshot.LinkType). + Bool("tethered", snapshot.Tethered). + Bool("clat_present", snapshot.CLATPresent). + Str("nat64_prefix", snapshot.NAT64Prefix). + Bool("dns_less_target", snapshot.DNSLessTarget) +} diff --git a/cmd/cli/network_snapshot_prog.go b/cmd/cli/network_snapshot_prog.go new file mode 100644 index 00000000..22b1f435 --- /dev/null +++ b/cmd/cli/network_snapshot_prog.go @@ -0,0 +1,226 @@ +package cli + +import ( + "net" + "reflect" + "slices" + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// networkSnapshotMessage is the message of the snapshot event. A log tool +// selects every snapshot of a run by this message. +const networkSnapshotMessage = "Network snapshot" + +// snapshotTriggerStart names the snapshot of the start of a run. +const snapshotTriggerStart = "start" + +// snapshotMinInterval bounds the snapshots of a flapping network. +const snapshotMinInterval = 60 * time.Second + +// snapshotNowFn is the clock of the snapshot limit. A test replaces it. +var snapshotNowFn = time.Now + +// defaultRoutesFn reads the default route of each family. A test replaces it, +// because the route table of the host is not a fixture. +var defaultRoutesFn = defaultRoutes + +// platformInterfaceMetaFn reads the platform names of one interface. A test +// replaces it, because the ports of the host are not a fixture. +var platformInterfaceMetaFn = platformInterfaceMeta + +// The delta diff has to know the virtual adapters of the host, and only the +// platform file reads them. +func init() { + virtualInterfaceSetFn = platformVirtualInterfaces +} + +// interfaceMetaFor describes one interface for the snapshot and for the delta +// diff. +func interfaceMetaFor(name string) interfaceMeta { + platformClass, hardwarePort, service := platformInterfaceMetaFn(name) + meta := interfaceMeta{ + Class: interfaceClassFor(name, platformClass), + HardwarePort: hardwarePort, + Service: service, + } + meta.LinkType = linkTypeFor(hardwarePort, meta.Class) + return meta +} + +// interfaceClassFor picks the class of one interface. A name rule wins for the +// loopback, the AirDrop, and the tunnel interfaces, because Windows describes +// a tunnel adapter as an Ethernet adapter. +func interfaceClassFor(name, platformClass string) string { + if class := interfaceClass(name, false); class != "hardware" { + return class + } + if platformClass != "" { + return platformClass + } + if _, virtual := virtualInterfaceSetFn()[name]; virtual { + return "virtual" + } + return "hardware" +} + +// snapshotInputs reads the sources of one snapshot from the network that the +// last monitor callback reported. No callback ran before the first header of a +// run, so that header reads the network itself. +func (p *prog) snapshotInputs() snapshotInputs { + state := p.lastNetworkState.Load() + if state == nil { + state = freshNetworkState() + } + return p.snapshotInputsFor(state) +} + +// freshSnapshotInputs reads the sources of one snapshot from the network of +// this moment. Support reads the header of an upload to learn the network of +// the moment the user asked, which can differ from the last callback. +func (p *prog) freshSnapshotInputs() snapshotInputs { + return p.snapshotInputsFor(freshNetworkState()) +} + +// networkSnapshot describes the host network of this moment. +func (p *prog) networkSnapshot() networkSnapshot { + return buildNetworkSnapshot(p.snapshotInputs()) +} + +// logNetworkSnapshot puts the host network in the journal. Support reads the +// snapshot of each trigger to learn the network at the time of an outage. +func (p *prog) logNetworkSnapshot(trigger string) { + snapshot := p.networkSnapshot() + if !p.snapshotWrites.allow(trigger, snapshot, snapshotNowFn()) { + return + } + journal(mainLog.Load().Info()). + Str("trigger", trigger). + Dict("network", snapshotDict(snapshot)). + Msg(networkSnapshotMessage) +} + +// snapshotLimiter holds the snapshot that reached the journal last, so a burst +// of triggers writes one line. +type snapshotLimiter struct { + mu sync.Mutex + last networkSnapshot + written time.Time + seen bool +} + +// allow reports whether one snapshot reaches the journal, and records the +// snapshots that do. +func (s *snapshotLimiter) allow(trigger string, snapshot networkSnapshot, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + if !s.permits(trigger, snapshot, now) { + return false + } + s.last, s.written, s.seen = snapshot, now, true + return true +} + +// permits decides one snapshot. The first snapshot of a run tells a reader +// where the run started. A snapshot equal to the one before it carries +// nothing. A new default route and a new resolver set explain an outage, so +// they pass at once. Every other snapshot waits out the interval, because a +// flapping interface writes some kilobytes per flap. +func (s *snapshotLimiter) permits(trigger string, snapshot networkSnapshot, now time.Time) bool { + if trigger == snapshotTriggerStart || !s.seen { + return true + } + if sameNetworkSnapshot(s.last, snapshot) { + return false + } + if routeOrResolversChanged(s.last, snapshot) { + return true + } + return now.Sub(s.written) >= snapshotMinInterval +} + +// sameNetworkSnapshot compares two snapshots field by field. The trigger is no +// part of a snapshot, so two triggers of one network give one line. +func sameNetworkSnapshot(before, after networkSnapshot) bool { + return reflect.DeepEqual(before, after) +} + +// routeOrResolversChanged names the changes that a reader of an outage needs +// at once. +func routeOrResolversChanged(before, after networkSnapshot) bool { + return before.DefaultRouteV4 != after.DefaultRouteV4 || + before.DefaultRouteV6 != after.DefaultRouteV6 || + !slices.Equal(before.Resolvers, after.Resolvers) +} + +// snapshotInputsFor reads every source of one snapshot. It takes no writer +// lock, so any path of the daemon can log a snapshot. +func (p *prog) snapshotInputsFor(state *netmon.State) snapshotInputs { + routeV4, routeV6 := defaultRoutesFn() + inputs := snapshotInputs{ + State: state, + RouteV4: routeV4, + RouteV6: routeV6, + Resolvers: ctrld.OsResolverNameservers(), + SourceIPv4: addressString(ctrld.GetDefaultLocalIPv4()), + SourceIPv6: addressString(ctrld.GetDefaultLocalIPv6()), + InterceptTarget: p.interceptTargetSnapshot(), + BypassActive: p.recoveryBypass.Load(), + RecoveryRunning: p.recoveryRunning.Load(), + PFStabilizing: p.pfStabilizing.Load(), + Meta: interfaceMetaFor, + } + if prefix, ok := p.dns64PrefixSnapshot(); ok { + inputs.NAT64Prefix = prefix.String() + } + if state != nil { + inputs.CLATPresent = dns64StateHasCLAT(state) + } + return inputs +} + +// freshNetworkState reads the interfaces of the host now. netmon fills the +// route and the families of a callback state, so a read of this package has to +// fill them itself. +func freshNetworkState() *netmon.State { + state, err := readNetworkSourceStateFn() + if err != nil || state == nil { + return nil + } + fresh := *state + if route, err := netmon.DefaultRouteInterface(); err == nil { + fresh.DefaultRouteInterface = route + } + fresh.HaveV4, fresh.HaveV6 = reachableAddressFamilies(&fresh) + return &fresh +} + +// reachableAddressFamilies reports which families the host can reach. It +// grades each address with the rule of the network monitor, so the first +// header of a run reports the same families as the first callback. +func reachableAddressFamilies(state *netmon.State) (haveV4, haveV6 bool) { + for name, iface := range state.Interface { + if iface.Interface == nil || !iface.IsUp() { + continue + } + for _, prefix := range state.InterfaceIPs[name] { + address := prefix.Addr().Unmap() + haveV4 = haveV4 || usableV4(address) + haveV6 = haveV6 || usableV6(address) + } + } + return haveV4, haveV6 +} + +// addressString names one source address. An address that ctrld does not hold +// leaves an empty field, because net.IP prints an empty value as the word for +// a nil address. +func addressString(ip net.IP) string { + if len(ip) == 0 { + return "" + } + return ip.String() +} diff --git a/cmd/cli/network_snapshot_prog_test.go b/cmd/cli/network_snapshot_prog_test.go new file mode 100644 index 00000000..881e568c --- /dev/null +++ b/cmd/cli/network_snapshot_prog_test.go @@ -0,0 +1,351 @@ +package cli + +import ( + "context" + "net" + "net/netip" + "reflect" + "slices" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// stubSnapshotPlatformMeta makes the platform names of each interface a +// fixture, so no test depends on the ports of the host. +func stubSnapshotPlatformMeta(t *testing.T, meta map[string]interfaceMeta) { + t.Helper() + orig := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = orig }) + platformInterfaceMetaFn = func(name string) (class, hardwarePort, service string) { + info := meta[name] + return info.Class, info.HardwarePort, info.Service + } +} + +// stubSnapshotVirtualSet names the virtual adapters of the fixture host. +func stubSnapshotVirtualSet(t *testing.T, names ...string) { + t.Helper() + orig := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = orig }) + virtual := make(map[string]struct{}, len(names)) + for _, name := range names { + virtual[name] = struct{}{} + } + virtualInterfaceSetFn = func() map[string]struct{} { return virtual } +} + +// stubSnapshotGateways replaces the route table read, because the gateways of +// the host are not a fixture. +func stubSnapshotGateways(t *testing.T, v4, v6 string) { + t.Helper() + stubSnapshotRoutes(t, defaultRoute{Gateway: v4}, defaultRoute{Gateway: v6}) +} + +// stubSnapshotRoutes replaces the route table read with one default route per +// family. +func stubSnapshotRoutes(t *testing.T, v4, v6 defaultRoute) { + t.Helper() + orig := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = orig }) + defaultRoutesFn = func() (defaultRoute, defaultRoute) { return v4, v6 } +} + +// stubSnapshotDNS64 reports a dual-stack host, so the NAT64 check starts no +// discovery and touches no name server. +func stubSnapshotDNS64(t *testing.T) { + t.Helper() + orig := dns64NetworkClassFn + t.Cleanup(func() { dns64NetworkClassFn = orig }) + dns64NetworkClassFn = func() (hasUsableIPv4, hasCLAT bool, err error) { return true, false, nil } +} + +// stubSnapshotSources sets the local source addresses and puts the values of +// the host back when the test ends. +func stubSnapshotSources(t *testing.T, v4, v6 string) { + t.Helper() + origV4, origV6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(context.Background(), origV4) + ctrld.SetDefaultLocalIPv6(context.Background(), origV6) + }) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(context.Background(), net.ParseIP(v6)) +} + +// tetheredSnapshotProg returns a prog on a phone hotspot over the cable. +func tetheredSnapshotProg(t *testing.T) *prog { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "172.20.10.1", "") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "en7": {HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + }) + stubSnapshotVirtualSet(t) + stubSnapshotSources(t, "172.20.10.5", "") + + p := &prog{} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + HaveV4: true, + Interface: map[string]netmon.Interface{ + "en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}, + "lo0": {Interface: &net.Interface{Name: "lo0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en7": {netip.MustParsePrefix("172.20.10.5/28")}, + "lo0": {netip.MustParsePrefix("127.0.0.1/8")}, + }, + }) + p.recoveryBypass.Store(true) + p.pfStabilizing.Store(true) + p.interceptDNSTargetSetValue = "127.0.0.1" + p.publishInterceptTarget("127.0.0.1") + return p +} + +func Test_networkSnapshotFromTheStoredState(t *testing.T) { + p := tetheredSnapshotProg(t) + + snapshot := p.networkSnapshot() + + checks := []struct { + field string + got, want any + }{ + {"default_route_v4", snapshot.DefaultRouteV4, "en7"}, + {"default_route_v6", snapshot.DefaultRouteV6, ""}, + {"gateway_v4", snapshot.GatewayV4, "172.20.10.1"}, + {"have_v4", snapshot.HaveV4, true}, + {"source_ipv4", snapshot.SourceIPv4, "172.20.10.5"}, + {"source_ipv6", snapshot.SourceIPv6, ""}, + {"intercept_target", snapshot.InterceptTarget, "127.0.0.1"}, + {"dns_less_target", snapshot.DNSLessTarget, true}, + {"bypass_active", snapshot.BypassActive, true}, + {"recovery_running", snapshot.RecoveryRunning, false}, + {"pf_stabilizing", snapshot.PFStabilizing, true}, + {"link_type", snapshot.LinkType, "usb_tether"}, + {"tethered", snapshot.Tethered, true}, + {"nat64_prefix", snapshot.NAT64Prefix, ""}, + {"clat_present", snapshot.CLATPresent, false}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } + + want := []snapshotInterface{ + {Name: "en7", Class: "hardware", Up: true, IPs: []string{"172.20.10.5/28"}, HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + {Name: "lo0", Class: "loopback", Up: true, IPs: []string{"127.0.0.1/8"}}, + } + if !reflect.DeepEqual(snapshot.Interfaces, want) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_networkSnapshotReadsTheNetworkWithoutAStoredState covers the first +// header of a run. No network callback reached ctrld yet, so the snapshot must +// read the interfaces itself and name the families it finds. +func Test_networkSnapshotReadsTheNetworkWithoutAStoredState(t *testing.T) { + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotSources(t, "", "") + stubLogHeaderNetworkRead(t, "en9") + p := &prog{} + p.logger.Store(mainLog.Load()) + + snapshot := p.networkSnapshot() + + if len(snapshot.Interfaces) != 1 || snapshot.Interfaces[0].Name != "en9" || !snapshot.Interfaces[0].Up { + t.Fatalf("interfaces = %+v, want one up en9", snapshot.Interfaces) + } + if !snapshot.HaveV4 { + t.Error("have_v4 = false, want true for a global unicast v4 address") + } + if snapshot.HaveV6 { + t.Error("have_v6 = true, want false without a v6 address") + } +} + +// Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon covers a host on a +// unique local address. The network monitor counts such an address as usable, +// so a fresh read must count it too. +func Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV6 bool + }{ + {name: "unique_local", ips: []string{"fd00::2/64"}, wantV6: true}, + {name: "tailscale_range", ips: []string{"fd7a:115c:a1e0::2/128"}}, + {name: "global", ips: []string{"2001:db8::2/64"}, wantV6: true}, + {name: "link_local", ips: []string{"fe80::2/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, tt.ips}) + + _, haveV6 := reachableAddressFamilies(state) + + if haveV6 != tt.wantV6 { + t.Errorf("have_v6 = %v, want %v for %v", haveV6, tt.wantV6, tt.ips) + } + }) + } +} + +func Test_networkSnapshotLogsOneJournalLine(t *testing.T) { + logs := captureDebugMainLog(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("start") + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("%q lines = %d, want 1", networkSnapshotMessage, len(events)) + } + wantField(t, events[0], "trigger", "start") + wantField(t, events[0], journalField, true) + wantField(t, events[0], "level", "info") + network, ok := events[0]["network"].(map[string]any) + if !ok { + t.Fatalf("network = %v, want an object", events[0]["network"]) + } + wantField(t, network, "default_route_v4", "en7") + wantField(t, network, "tethered", true) + wantField(t, network, "link_type", "usb_tether") +} + +// snapshotTestClock hands out the time that a test sets, because the snapshot +// limit counts seconds between two lines. +type snapshotTestClock struct { + now time.Time +} + +// stubSnapshotClock puts the clock of the snapshot limit under the control of +// one test. +func stubSnapshotClock(t *testing.T) *snapshotTestClock { + t.Helper() + clock := &snapshotTestClock{now: time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)} + orig := snapshotNowFn + t.Cleanup(func() { snapshotNowFn = orig }) + snapshotNowFn = func() time.Time { return clock.now } + return clock +} + +func (c *snapshotTestClock) advance(d time.Duration) { c.now = c.now.Add(d) } + +// snapshotTriggerLines names the trigger of every snapshot line, in order. +func snapshotTriggerLines(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +// Test_logNetworkSnapshotSkipsAnEqualSnapshot covers a recovery on a network +// that did not change. The begin and the end of that recovery render the same +// object, so the second line carries nothing. +func Test_logNetworkSnapshotSkipsAnEqualSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("recovery_begin") + clock.advance(5 * time.Minute) + p.logNetworkSnapshot("recovery_end") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"recovery_begin"}) { + t.Errorf("snapshot triggers = %v, want the first line alone", got) + } +} + +// Test_logNetworkSnapshotCapsTheRate covers a flapping interface. Two +// snapshots that differ in no route and in no resolver give one line per +// minute. +func Test_logNetworkSnapshotCapsTheRate(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.logNetworkSnapshot("transition") + + if got := snapshotTriggerLines(t, logs); len(got) != 2 { + t.Errorf("snapshot lines = %d, want 2: %v", len(got), got) + } +} + +// Test_logNetworkSnapshotWritesARouteChangeAtOnce covers the change that +// explains an outage. The cap must not hold it back. +func Test_logNetworkSnapshotWritesARouteChangeAtOnce(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + stubSnapshotRoutes(t, defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}, defaultRoute{}) + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("recovery_begin") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "transition"}) { + t.Errorf("snapshot triggers = %v, want the two transitions", got) + } +} + +// Test_logNetworkSnapshotAlwaysWritesTheStartLine covers the first line of a +// run. A reader needs the network of the start, whatever came before it. +func Test_logNetworkSnapshotAlwaysWritesTheStartLine(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.logNetworkSnapshot(snapshotTriggerStart) + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "start"}) { + t.Errorf("snapshot triggers = %v, want the transition and the start", got) + } +} + +func Test_interfaceMetaFor(t *testing.T) { + stubSnapshotVirtualSet(t, "bridge100") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "lo0": {Class: "hardware"}, + "en0": {HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "eth0": {Class: "hardware", HardwarePort: "Intel I219-V"}, + }) + + for _, tt := range []struct { + name string + want interfaceMeta + }{ + {"lo0", interfaceMeta{Class: "loopback", LinkType: "unknown"}}, + {"awdl0", interfaceMeta{Class: "airdrop", LinkType: "unknown"}}, + {"utun3", interfaceMeta{Class: "tunnel", LinkType: "tunnel"}}, + {"en0", interfaceMeta{Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi", LinkType: "wifi"}}, + {"bridge100", interfaceMeta{Class: "virtual", LinkType: "unknown"}}, + {"eth0", interfaceMeta{Class: "hardware", HardwarePort: "Intel I219-V", LinkType: "unknown"}}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := interfaceMetaFor(tt.name); got != tt.want { + t.Errorf("interfaceMetaFor(%q) = %+v, want %+v", tt.name, got, tt.want) + } + }) + } +} diff --git a/cmd/cli/network_snapshot_test.go b/cmd/cli/network_snapshot_test.go new file mode 100644 index 00000000..f568bfbe --- /dev/null +++ b/cmd/cli/network_snapshot_test.go @@ -0,0 +1,595 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "testing" + + "tailscale.com/net/netmon" +) + +// snapshotTestInterface names one interface of a test network state. +type snapshotTestInterface struct { + name string + up bool + ips []string +} + +// snapshotTestState builds a netmon state with several interfaces, because the +// shared helper of the transition tests holds one interface only. +func snapshotTestState(route string, interfaces ...snapshotTestInterface) *netmon.State { + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range interfaces { + flags := net.Flags(0) + if iface.up { + flags = net.FlagUp + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{Name: iface.name, Flags: flags}} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func snapshotTestMeta(meta map[string]interfaceMeta) interfaceMetaFunc { + return func(name string) interfaceMeta { return meta[name] } +} + +func Test_tetheredNetwork(t *testing.T) { + for _, tt := range []struct { + name string + gatewayV4 string + hardwarePort string + hostIPs []string + want bool + }{ + {name: "hotspot_gateway", gatewayV4: "172.20.10.1", hardwarePort: "Wi-Fi", want: true}, + {name: "hotspot_range_end", gatewayV4: "172.20.10.15", hardwarePort: "Wi-Fi", want: true}, + {name: "above_hotspot_range", gatewayV4: "172.20.10.16", hardwarePort: "Wi-Fi"}, + {name: "below_hotspot_range", gatewayV4: "172.20.9.255", hardwarePort: "Wi-Fi"}, + {name: "home_gateway", gatewayV4: "192.168.1.1", hardwarePort: "Wi-Fi"}, + {name: "iphone_usb_port", gatewayV4: "192.168.1.1", hardwarePort: "iPhone USB", want: true}, + {name: "no_gateway_no_port"}, + {name: "unparsable_gateway", gatewayV4: "not-an-address"}, + {name: "v6_gateway", gatewayV4: "fe80::1", hardwarePort: "Ethernet"}, + {name: "hotspot_host_address_without_a_gateway", hostIPs: []string{"172.20.10.4/28"}, want: true}, + {name: "hotspot_host_address_of_a_second_address", hostIPs: []string{"fe80::1/64", "172.20.10.4/28"}, want: true}, + {name: "home_host_address_without_a_gateway", hostIPs: []string{"192.168.1.20/24"}}, + } { + t.Run(tt.name, func(t *testing.T) { + hostIPs := make([]netip.Prefix, 0, len(tt.hostIPs)) + for _, ip := range tt.hostIPs { + hostIPs = append(hostIPs, netip.MustParsePrefix(ip)) + } + if got := tetheredNetwork(tt.gatewayV4, tt.hardwarePort, hostIPs); got != tt.want { + t.Fatalf("tetheredNetwork(%q, %q, %v) = %v, want %v", tt.gatewayV4, tt.hardwarePort, tt.hostIPs, got, tt.want) + } + }) + } +} + +// Test_buildNetworkSnapshot_tetheredWithoutAGateway covers Linux and Windows. +// Both read no route table in this release, so the address of the host is the +// only sign of a phone hotspot. +func Test_buildNetworkSnapshot_tetheredWithoutAGateway(t *testing.T) { + state := snapshotTestState("wlan0", snapshotTestInterface{"wlan0", true, []string{"172.20.10.4/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for a host address of the hotspot range") + } +} + +// Test_usableV6 pins the grade of a v6 address to the rule of the network +// monitor. A fresh read and a monitor callback must report the same have_v6. +func Test_usableV6(t *testing.T) { + for _, tt := range []struct { + address string + want bool + }{ + {"2001:db8::1", true}, + {"2606:4700:4700::1111", true}, + {"fd00::1", true}, + {"fd7a:115c:a1e0::1", false}, + {"fe80::1", false}, + {"::1", false}, + {"192.0.2.10", false}, + } { + t.Run(tt.address, func(t *testing.T) { + if got := usableV6(netip.MustParseAddr(tt.address).Unmap()); got != tt.want { + t.Errorf("usableV6(%q) = %v, want %v", tt.address, got, tt.want) + } + }) + } +} + +func Test_linkTypeFor(t *testing.T) { + for _, tt := range []struct { + name string + hardwarePort string + class string + want string + }{ + {"wifi_port", "Wi-Fi", "hardware", "wifi"}, + {"ethernet_port", "Ethernet", "hardware", "ethernet"}, + {"thunderbolt_port", "Thunderbolt Ethernet Slot 1", "hardware", "ethernet"}, + {"usb_lan_port", "USB 10/100/1000 LAN", "hardware", "ethernet"}, + {"thunderbolt_bridge", "Thunderbolt Bridge", "hardware", "ethernet"}, + {"iphone_usb_port", "iPhone USB", "hardware", "usb_tether"}, + {"tunnel_class", "", "tunnel", "tunnel"}, + // The Windows adapter description reaches linkTypeFor as the port + // name, and a wired word in it wins over the tunnel class. + {"ethernet_word_wins_over_tunnel_class", "SSL VPN Virtual Ethernet Adapter", "tunnel", "ethernet"}, + {"no_port_no_class", "", "", "unknown"}, + {"unknown_port", "Bluetooth PAN", "hardware", "unknown"}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := linkTypeFor(tt.hardwarePort, tt.class); got != tt.want { + t.Fatalf("linkTypeFor(%q, %q) = %q, want %q", tt.hardwarePort, tt.class, got, tt.want) + } + }) + } +} + +func Test_buildNetworkSnapshot_tetheredWiFiHotspot(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, []string{"172.20.10.2/28"}}) + state.HaveV4 = true + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "172.20.10.1", Interface: "en0"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for the hotspot gateway") + } + if snapshot.LinkType != "wifi" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "wifi") + } + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "" { + t.Errorf("default routes = %q/%q, want en0 and empty", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if !snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("have_v4/have_v6 = %v/%v, want true/false", snapshot.HaveV4, snapshot.HaveV6) + } + want := []snapshotInterface{{Name: "en0", Class: "hardware", Up: true, IPs: []string{"172.20.10.2/28"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}} + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +func Test_buildNetworkSnapshot_usbTether(t *testing.T) { + state := snapshotTestState("en7", snapshotTestInterface{"en7", true, []string{"172.20.10.3/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en7": {Class: "hardware", HardwarePort: "iPhone USB", Service: "iPhone USB"}}), + }) + + if snapshot.LinkType != "usb_tether" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "usb_tether") + } + if !snapshot.Tethered { + t.Error("tethered = false, want true for the iPhone USB port") + } +} + +func Test_buildNetworkSnapshot_tunnelDefaultRoute(t *testing.T) { + state := snapshotTestState("utun3", snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {Class: "tunnel"}}), + }) + + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } + if snapshot.Tethered { + t.Error("tethered = true, want false without a hotspot gateway") + } +} + +// Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable covers the +// platforms that read no route table. The v4 route falls back to the interface +// of the network monitor, and that interface must hold a usable v4 address. +func Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV4 string + otherIPs []string + }{ + {name: "dual_stack", ips: []string{"192.0.2.10/24", "2001:db8::10/64"}, wantV4: "en0"}, + {name: "v4_only", ips: []string{"192.0.2.10/24", "fe80::1/64"}, wantV4: "en0"}, + {name: "v6_only", ips: []string{"2001:db8::10/64"}}, + {name: "link_local_only", ips: []string{"169.254.1.2/16", "fe80::1/64"}}, + {name: "loopback_only", ips: []string{"127.0.0.1/8"}}, + {name: "address_on_another_interface", ips: nil, otherIPs: []string{"192.0.2.20/24", "2001:db8::20/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, tt.ips}, + snapshotTestInterface{"en1", true, tt.otherIPs}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if snapshot.DefaultRouteV4 != tt.wantV4 { + t.Errorf("default_route_v4 = %q, want %q", snapshot.DefaultRouteV4, tt.wantV4) + } + if snapshot.DefaultRouteV6 != "" { + t.Errorf("default_route_v6 = %q, want an empty value without a route table", snapshot.DefaultRouteV6) + } + }) + } +} + +// Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable covers a host +// whose v6 default route leaves over a tunnel. The v6 route must name that +// tunnel, not the interface of the v4 route. +func Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24", "2001:db8::10/64"}}, + snapshotTestInterface{"utun4", true, []string{"2001:db8:1::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}, + }) + + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "utun4" { + t.Errorf("default routes = %q/%q, want en0 and utun4", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if snapshot.GatewayV4 != "192.0.2.1" || snapshot.GatewayV6 != "fe80::1%utun4" { + t.Errorf("gateways = %q/%q, want the gateways of the route table", snapshot.GatewayV4, snapshot.GatewayV6) + } +} + +func Test_buildNetworkSnapshot_sortsInterfacesAndTakesMeta(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", false, []string{"192.0.2.11/24"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{ + "en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "awdl0": {Class: "airdrop"}, + "en1": {Class: "hardware", HardwarePort: "Ethernet", Service: "Ethernet"}, + }), + }) + + want := []snapshotInterface{ + {Name: "awdl0", Class: "airdrop", Up: true, IPs: []string{"fe80::2/64"}}, + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + {Name: "en1", Class: "hardware", Up: false, IPs: []string{"192.0.2.11/24"}, HardwarePort: "Ethernet", Service: "Ethernet"}, + } + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Fatalf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_buildNetworkSnapshot_capsTheInterfaceList covers a container host. Such +// a host holds hundreds of interfaces, and a snapshot of all of them fills the +// journal. +func Test_buildNetworkSnapshot_capsTheInterfaceList(t *testing.T) { + interfaces := make([]snapshotTestInterface, 0, 40) + for i := range 36 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("veth%02d", i), true, nil}) + } + for i := range 4 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("down%d", i), false, nil}) + } + state := snapshotTestState("veth00", interfaces...) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if len(snapshot.Interfaces) != snapshotInterfaceLimit { + t.Fatalf("interfaces = %d entries, want %d", len(snapshot.Interfaces), snapshotInterfaceLimit) + } + if snapshot.InterfacesOmitted != 8 { + t.Errorf("interfaces_omitted = %d, want 8", snapshot.InterfacesOmitted) + } + if first, last := snapshot.Interfaces[0].Name, snapshot.Interfaces[snapshotInterfaceLimit-1].Name; first != "veth00" || last != "veth31" { + t.Errorf("interfaces run from %q to %q, want veth00 to veth31", first, last) + } +} + +// Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces covers the rule that +// selects the interfaces of the list. +func Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}, + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"en2", false, []string{"192.0.2.20/24"}}, + snapshotTestInterface{"en3", false, nil}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + var names []string + for _, iface := range snapshot.Interfaces { + names = append(names, iface.Name) + } + if !slices.Equal(names, []string{"en0", "en1", "en2"}) { + t.Errorf("interfaces = %v, want en0, en1, and en2", names) + } + if snapshot.InterfacesOmitted != 1 { + t.Errorf("interfaces_omitted = %d, want 1", snapshot.InterfacesOmitted) + } +} + +func Test_buildNetworkSnapshot_classesByNameWithoutMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["en0"] != "hardware" { + t.Fatalf("classes = %v, want utun3 tunnel and en0 hardware", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_fillsTheEmptyClassOfTheMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {HardwarePort: ""}}), + }) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["awdl0"] != "airdrop" { + t.Fatalf("classes = %v, want utun3 tunnel and awdl0 airdrop", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_nilStateKeepsFlags(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{ + RouteV4: defaultRoute{Gateway: "192.168.1.1"}, + RouteV6: defaultRoute{Gateway: "fe80::1"}, + Resolvers: []string{"192.168.1.1:53"}, + SourceIPv4: "192.168.1.20", + SourceIPv6: "2001:db8::20", + InterceptTarget: "127.0.0.1", + BypassActive: true, + RecoveryRunning: true, + PFStabilizing: true, + NAT64Prefix: "64:ff9b::/96", + CLATPresent: true, + }) + + if len(snapshot.Interfaces) != 0 { + t.Errorf("interfaces = %+v, want none", snapshot.Interfaces) + } + if snapshot.DefaultRouteV4 != "" || snapshot.DefaultRouteV6 != "" || snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("state fields are set without a state: %+v", snapshot) + } + if snapshot.LinkType != "unknown" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "unknown") + } + if !snapshot.BypassActive || !snapshot.RecoveryRunning || !snapshot.PFStabilizing || !snapshot.CLATPresent { + t.Errorf("flags dropped: %+v", snapshot) + } + if !snapshot.DNSLessTarget { + t.Error("dns_less_target = false, want true with an intercept target") + } + if snapshot.GatewayV4 != "192.168.1.1" || snapshot.GatewayV6 != "fe80::1" { + t.Errorf("gateways = %q/%q, want the input values", snapshot.GatewayV4, snapshot.GatewayV6) + } + if snapshot.SourceIPv4 != "192.168.1.20" || snapshot.SourceIPv6 != "2001:db8::20" { + t.Errorf("sources = %q/%q, want the input values", snapshot.SourceIPv4, snapshot.SourceIPv6) + } + if snapshot.NAT64Prefix != "64:ff9b::/96" || !slices.Equal(snapshot.Resolvers, []string{"192.168.1.1:53"}) { + t.Errorf("prefix and resolvers dropped: %+v", snapshot) + } +} + +func Test_buildNetworkSnapshot_noInterceptTargetIsNotDNSLess(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{State: snapshotTestState("en0", snapshotTestInterface{"en0", true, nil})}) + if snapshot.DNSLessTarget { + t.Error("dns_less_target = true, want false without an intercept target") + } +} + +// snapshotDictKeys are the field names that spec R2.2 names for the snapshot, +// plus the count of the interfaces that the list leaves out. +var snapshotDictKeys = []string{ + "default_route_v4", "default_route_v6", "gateway_v4", "gateway_v6", + "have_v4", "have_v6", "interfaces", "interfaces_omitted", "resolvers", + "source_ipv4", "source_ipv6", "intercept_target", "bypass_active", + "recovery_running", "pf_stabilizing", "link_type", "tethered", + "clat_present", "nat64_prefix", "dns_less_target", +} + +var snapshotInterfaceDictKeys = []string{"class", "hardware_port", "ips", "name", "service", "up"} + +// renderSnapshotDict writes the dict under one field and parses the line, so +// the test reads the keys that a reader of the journal sees. +func renderSnapshotDict(t *testing.T, snapshot networkSnapshot) (string, map[string]any) { + t.Helper() + buf := &syncBuffer{} + newTestJSONLogger(buf).Info().Dict("network", snapshotDict(snapshot)).Msg("Network snapshot") + + line := strings.TrimSpace(buf.String()) + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + network, ok := event["network"].(map[string]any) + if !ok { + t.Fatalf("line has no network object: %q", line) + } + return line, network +} + +func Test_snapshotDict_rendersEveryKeyOnce(t *testing.T) { + line, network := renderSnapshotDict(t, buildNetworkSnapshot(snapshotInputs{})) + + if got := sortedKeys(network); !slices.Equal(got, slices.Sorted(slices.Values(snapshotDictKeys))) { + t.Fatalf("keys = %v, want %v", got, snapshotDictKeys) + } + for _, key := range snapshotDictKeys { + if count := strings.Count(line, `"`+key+`":`); count != 1 { + t.Errorf("key %q appears %d times, want 1: %s", key, count, line) + } + } + if network["interfaces"] == nil || len(network["interfaces"].([]any)) != 0 { + t.Errorf("interfaces = %v, want an empty array", network["interfaces"]) + } + if network["resolvers"] == nil || len(network["resolvers"].([]any)) != 0 { + t.Errorf("resolvers = %v, want an empty array", network["resolvers"]) + } +} + +func Test_snapshotDict_rendersSortedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + Resolvers: []string{"192.0.2.1:53"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + _, network := renderSnapshotDict(t, snapshot) + + interfaces, ok := network["interfaces"].([]any) + if !ok || len(interfaces) != 3 { + t.Fatalf("interfaces = %v, want three entries", network["interfaces"]) + } + var names []string + for _, entry := range interfaces { + object, ok := entry.(map[string]any) + if !ok { + t.Fatalf("interface entry is not an object: %v", entry) + } + if got := sortedKeys(object); !slices.Equal(got, snapshotInterfaceDictKeys) { + t.Fatalf("interface keys = %v, want %v", got, snapshotInterfaceDictKeys) + } + names = append(names, object["name"].(string)) + } + if !slices.Equal(names, []string{"awdl0", "en0", "en1"}) { + t.Errorf("interface order = %v, want awdl0, en0, en1", names) + } + if ips, ok := interfaces[2].(map[string]any)["ips"].([]any); !ok || len(ips) != 0 { + t.Errorf("ips of en1 = %v, want an empty array", interfaces[2].(map[string]any)["ips"]) + } + wantValues := map[string]any{"gateway_v4": "192.0.2.1", "link_type": "wifi", "default_route_v4": "en0", "tethered": false} + for key, want := range wantValues { + wantField(t, network, key, want) + } + if resolvers, ok := network["resolvers"].([]any); !ok || len(resolvers) != 1 || resolvers[0] != "192.0.2.1:53" { + t.Errorf("resolvers = %v, want one entry", network["resolvers"]) + } +} + +func Test_defaultRoutes_returnAddressesOrNothing(t *testing.T) { + v4, v6 := defaultRoutes() + // A host without a route of a family gives an empty value, so the test + // logs the pair and holds every value it gets to the format. + t.Logf("default routes: v4 %+v, v6 %+v", v4, v6) + + if v4.Gateway != "" { + address, err := netip.ParseAddr(v4.Gateway) + if err != nil || !address.Is4() { + t.Errorf("gateway_v4 = %q, want an IPv4 address or nothing: %v", v4.Gateway, err) + } + } + if v6.Gateway != "" { + address, err := netip.ParseAddr(v6.Gateway) + if err != nil || !address.Is6() { + t.Errorf("gateway_v6 = %q, want an IPv6 address or nothing: %v", v6.Gateway, err) + } + } + // A tunnel default has no next hop, so an interface without a gateway is a + // valid pair. A gateway without an interface is not. + if v4.Gateway != "" && v4.Interface == "" { + t.Errorf("gateway_v4 = %q without an interface, want the interface of the route", v4.Gateway) + } + if v6.Gateway != "" && v6.Interface == "" { + t.Errorf("gateway_v6 = %q without an interface, want the interface of the route", v6.Gateway) + } +} + +func sameSnapshotInterface(a, b snapshotInterface) bool { + return a.Name == b.Name && a.Class == b.Class && a.Up == b.Up && + a.HardwarePort == b.HardwarePort && a.Service == b.Service && + slices.Equal(a.IPs, b.IPs) +} + +func sortedKeys(object map[string]any) []string { + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +// Test_buildNetworkSnapshot_linkFieldsFollowTheRouteTable covers a network +// switch. The network monitor still holds the old Ethernet interface, while the +// route table already selects Wi-Fi. The link fields must describe the +// interface of the routes in the same record. +func Test_buildNetworkSnapshot_linkFieldsFollowTheRouteTable(t *testing.T) { + meta := snapshotTestMeta(map[string]interfaceMeta{ + "en0": {Class: "hardware", HardwarePort: "Ethernet"}, + "en1": {Class: "hardware", HardwarePort: "Wi-Fi"}, + }) + for _, tt := range []struct { + name string + routeV4 defaultRoute + routeV6 defaultRoute + }{ + {name: "dual_stack", routeV4: defaultRoute{Interface: "en1"}, routeV6: defaultRoute{Interface: "en1"}}, + {name: "v6_only", routeV6: defaultRoute{Interface: "en1"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}, + snapshotTestInterface{"en1", true, []string{"172.20.10.2/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state, RouteV4: tt.routeV4, RouteV6: tt.routeV6, Meta: meta}) + + if snapshot.LinkType != "wifi" { + t.Errorf("link_type = %q, want %q for the interface of the route table", snapshot.LinkType, "wifi") + } + if !snapshot.Tethered { + t.Error("tethered = false, want true for the hotspot address of the route interface") + } + if snapshot.DefaultRouteV4 != tt.routeV4.Interface { + t.Errorf("default_route_v4 = %q, want %q from the route table", snapshot.DefaultRouteV4, tt.routeV4.Interface) + } + }) + } +} diff --git a/cmd/cli/network_source_error_test.go b/cmd/cli/network_source_error_test.go new file mode 100644 index 00000000..846bceac --- /dev/null +++ b/cmd/cli/network_source_error_test.go @@ -0,0 +1,34 @@ +package cli + +import ( + "context" + "errors" + "net" + "testing" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +func TestSourceReadFailureDefersAcceptedWrites(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + major := sourceTestState("en0", true, "192.0.2.10/24") + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return major } + readNetworkSourceStateFn = func() (*netmon.State, error) { return nil, errors.New("read failed") } + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.1")) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) + p := sourceTestProg(&prog{cfg: &ctrld.Config{}, networkSourceEpoch: major}) + calls := 0 + networkChangeReconcileFn = func(*prog, context.Context, uint64) { calls++ } + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{ + Monitor: netmon.NewStatic(), + Old: sourceTestState("en0", true, "192.0.2.1/24"), New: major, + }, true) + if calls != 1 || !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.1")) { + t.Fatal("unknown source validity changed a binding or skipped accepted recovery") + } +} diff --git a/cmd/cli/network_source_state_test.go b/cmd/cli/network_source_state_test.go new file mode 100644 index 00000000..dfb197e8 --- /dev/null +++ b/cmd/cli/network_source_state_test.go @@ -0,0 +1,147 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/netip" + "slices" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func Test_interfaceAddressPrefix(t *testing.T) { + for _, tc := range []struct { + name string + address net.Addr + want string + }{ + {"IPv4 with a 4-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10).To4(), Mask: net.CIDRMask(24, 32)}, "192.0.2.10/24"}, + {"IPv4 in the 16-byte form with a 4-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(24, 32)}, "192.0.2.10/24"}, + {"IPv4 in the 16-byte form with a 16-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(120, 128)}, "192.0.2.10/24"}, + {"IPv6 with a 64-bit mask", &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, "2001:db8::1/64"}, + {"IPv4 without a mask", &net.IPAddr{IP: net.IPv4(192, 0, 2, 10)}, "192.0.2.10/32"}, + {"IPv6 without a mask", &net.IPAddr{IP: net.ParseIP("2001:db8::1")}, "2001:db8::1/128"}, + {"IPv4 with a nil mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10)}, "192.0.2.10/32"}, + {"IPv4 with an irregular mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.IPMask{255, 0, 255, 0}}, "192.0.2.10/32"}, + } { + t.Run(tc.name, func(t *testing.T) { + prefix, err := interfaceAddressPrefix(tc.address) + if err != nil { + t.Fatalf("interfaceAddressPrefix: %v", err) + } + if prefix.String() != tc.want { + t.Fatalf("prefix = %s, want %s", prefix, tc.want) + } + }) + } + if _, err := interfaceAddressPrefix(&net.TCPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 53}); err == nil { + t.Fatal("a TCP address gave a prefix, want an error") + } +} + +// sourceTestInterfaces returns one up interface with a /24 and a /64 address, +// through the reader seam of networkSourceState. +func sourceTestInterfaces() ([]net.Interface, func(*net.Interface) ([]net.Addr, error)) { + interfaces := []net.Interface{{Index: 1, Name: "en0", Flags: net.FlagUp}} + addresses := func(iface *net.Interface) ([]net.Addr, error) { + if iface.Name != "en0" { + return nil, errors.New("unknown interface") + } + return []net.Addr{ + &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(24, 32)}, + &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, + }, nil + } + return interfaces, addresses +} + +func Test_networkSourceState_keepsTheMasks(t *testing.T) { + state, err := networkSourceState(sourceTestInterfaces()) + if err != nil { + t.Fatalf("networkSourceState: %v", err) + } + if !state.Interface["en0"].IsUp() { + t.Fatal("en0 is not up") + } + if got := prefixStrings(state.InterfaceIPs["en0"]); !slices.Equal(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { + t.Fatalf("prefixes = %v, want the /24 and the /64 with the host address", got) + } +} + +// stubSourceStateRead makes the fresh network read return the test interfaces +// through the real state builder. +func stubSourceStateRead(t *testing.T) { + t.Helper() + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { return networkSourceState(sourceTestInterfaces()) } +} + +// Test_freshSnapshotAndHeaderKeepTheMasks proves that the send-time header and +// the fresh snapshot show the /24 and the /64 of the host, as the network +// monitor does. +func Test_freshSnapshotAndHeaderKeepTheMasks(t *testing.T) { + stubHeaderSnapshotSources(t) + stubSourceStateRead(t) + p := &prog{cfg: &ctrld.Config{}} + p.logger.Store(mainLog.Load()) + want := []string{"192.0.2.10/24", "2001:db8::1/64"} + + snapshot := buildNetworkSnapshot(p.freshSnapshotInputs()) + index := slices.IndexFunc(snapshot.Interfaces, func(i snapshotInterface) bool { return i.Name == "en0" }) + if index < 0 { + t.Fatalf("snapshot holds no en0: %+v", snapshot.Interfaces) + } + if got := snapshot.Interfaces[index].IPs; !slices.Equal(got, want) { + t.Fatalf("snapshot ips = %v, want %v", got, want) + } + + parsed, _ := parseLogHeader(t, p.sendLogHeader()) + var headerIPs []string + for _, iface := range parsed.Network.Interfaces { + if iface.Name == "en0" { + headerIPs = iface.IPs + } + } + if !slices.Equal(headerIPs, want) { + t.Fatalf("header ips = %v, want %v", headerIPs, want) + } +} + +// Test_sourceValidationKeepsTheHostAddress proves that a state with masked +// prefixes still names the host address, not the subnet address, as the +// source of the process. +func Test_sourceValidationKeepsTheHostAddress(t *testing.T) { + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(context.Background(), before4) + ctrld.SetDefaultLocalIPv6(context.Background(), before6) + }) + state, err := networkSourceState(sourceTestInterfaces()) + if err != nil { + t.Fatalf("networkSourceState: %v", err) + } + + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(context.Background(), net.ParseIP("2001:db8::1")) + validateDefaultLocalIPsFromDelta(context.Background(), state, 1) + if got := ctrld.GetDefaultLocalIPv4(); !got.Equal(net.ParseIP("192.0.2.10")) { + t.Fatalf("source IPv4 = %v, want the host address kept", got) + } + if got := ctrld.GetDefaultLocalIPv6(); !got.Equal(net.ParseIP("2001:db8::1")) { + t.Fatalf("source IPv6 = %v, want the host address kept", got) + } + + // The subnet address is not an address of the host. + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.0")) + validateDefaultLocalIPsFromDelta(context.Background(), state, 2) + if got := ctrld.GetDefaultLocalIPv4(); got != nil { + t.Fatalf("source IPv4 = %v, want the subnet address cleared", got) + } + if reason := sourceInvalidReason(state, netip.MustParseAddr("192.0.2.10").AsSlice()); reason != "" { + t.Fatalf("host address reported %q, want a valid source", reason) + } +} diff --git a/cmd/cli/network_transition.go b/cmd/cli/network_transition.go new file mode 100644 index 00000000..182ed980 --- /dev/null +++ b/cmd/cli/network_transition.go @@ -0,0 +1,277 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/netip" + "sync" + "sync/atomic" + "syscall" + "time" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" +) + +type networkChangeMonitor interface { + Close() error + RegisterChangeCallback(netmon.ChangeFunc) func() + IsMajorChangeFrom(*netmon.State, *netmon.State) bool + Start() +} + +// Tests replace these functions to avoid changes to host DNS and PF state. +var ( + newNetworkChangeMonitorFn = func(logf func(string, ...any)) (networkChangeMonitor, error) { + return netmon.New(logf) + } + networkChangeCurrentStateFn = func(delta *netmon.ChangeDelta) *netmon.State { + if delta.Monitor != nil { + return delta.Monitor.InterfaceState() + } + return delta.New + } + readNetworkSourceStateFn = readNetworkSourceState + networkChangeValidInterfacesFn = ctrld.ValidInterfaces + networkChangeDefaultRouteIPFn = (*prog).defaultRouteIP + networkChangeReconcileFn = (*prog).reconcileNetworkChange + networkChangeIgnoredInterceptFn = (*prog).handleDNSInterceptIgnoredNetworkChange + handleRecoveryForTransitionFn = (*prog).handleRecoveryForTransition + recoveryResetDNSFn = (*prog).resetDNS +) + +// Minor callbacks have no observation sequence in netmon v1.74.0. Read only +// current interface addresses and flags rather than trusting their arrival order. +func readNetworkSourceState() (*netmon.State, error) { + interfaces, err := net.Interfaces() + if err != nil { + return nil, err + } + return networkSourceState(interfaces, (*net.Interface).Addrs) +} + +// networkSourceState builds the state of the given interfaces. The address +// reader is a parameter, so a test feeds addresses without a host network. +func networkSourceState(interfaces []net.Interface, addressesOf func(*net.Interface) ([]net.Addr, error)) (*netmon.State, error) { + state := &netmon.State{Interface: map[string]netmon.Interface{}, InterfaceIPs: map[string][]netip.Prefix{}} + for _, iface := range interfaces { + addresses, err := addressesOf(&iface) + if err != nil { + return nil, err + } + state.Interface[iface.Name] = netmon.Interface{Interface: &iface} + for _, address := range addresses { + prefix, err := interfaceAddressPrefix(address) + if err != nil { + return nil, err + } + state.InterfaceIPs[iface.Name] = append(state.InterfaceIPs[iface.Name], prefix) + } + } + return state, nil +} + +// interfaceAddressPrefix turns one interface address into a prefix. An address +// with a subnet mask keeps the mask and its host bits, so a fresh state shows +// the same /24 or /64 as the network monitor. An address without a mask is a +// host prefix. +func interfaceAddressPrefix(address net.Addr) (netip.Prefix, error) { + var ip net.IP + var mask net.IPMask + switch address := address.(type) { + case *net.IPNet: + ip, mask = address.IP, address.Mask + case *net.IPAddr: + ip = address.IP + default: + return netip.Prefix{}, errors.New("unsupported interface address type") + } + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Prefix{}, errors.New("invalid interface address") + } + addr = addr.Unmap() + return netip.PrefixFrom(addr, prefixBits(addr, mask)), nil +} + +// prefixBits reads the length of a mask for an unmapped address. The mask of +// an IPv4 address is 4 or 16 bytes long, and a 16-byte mask counts the 96 +// bits of the IPv6 mapping too. A missing or irregular mask means a host +// prefix. +func prefixBits(addr netip.Addr, mask net.IPMask) int { + bits, total := mask.Size() + if total == 0 { + return addr.BitLen() + } + if addr.Is4() && total == 128 { + bits -= 96 + } + if bits < 0 || bits > addr.BitLen() { + return addr.BitLen() + } + return bits +} + +// netmon v1.74.0 caches only major snapshots. Minor and time-jump events +// belong to their Old snapshot, not their distinct New snapshot. +func networkSnapshotCurrent(delta *netmon.ChangeDelta, major bool, current *netmon.State) bool { + if major || delta.Monitor == nil { + return current == delta.New + } + return current == delta.Old +} + +// Callers hold networkSourceMu. Preserve minor changes within the current +// major epoch, but use a new major snapshot even before its callback runs. +func (p *prog) sourceCommitState(delta *netmon.ChangeDelta) *netmon.State { + if delta.Monitor != nil { + if current := networkChangeCurrentStateFn(delta); current != p.networkSourceEpoch { + return current + } + } + return p.networkSourceState +} + +// sourceInvalidReason considers all up interfaces, not just hardware ports: a +// preferred source remains valid if it moved to another active interface. +func sourceInvalidReason(state *netmon.State, ip net.IP) string { + if ip == nil || state == nil { + return "" + } + reason := "address_removed" + for name, prefixes := range state.InterfaceIPs { + for _, prefix := range prefixes { + if !ip.Equal(net.ParseIP(prefix.Addr().String())) { + continue + } + iface, exists := state.Interface[name] + if exists && iface.IsUp() { + return "" + } + if exists { + reason = "interface_down" + } else if reason != "interface_down" { + reason = "interface_removed" + } + } + } + return reason +} + +func validateDefaultLocalIPsFromDelta(ctx context.Context, state *netmon.State, transitionID uint64) { + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + reason4, reason6 := sourceInvalidReason(state, before4), sourceInvalidReason(state, before6) + if reason4 != "" { + ctrld.SetDefaultLocalIPv4(ctx, nil) + } + if reason6 != "" { + ctrld.SetDefaultLocalIPv6(ctx, nil) + } + if reason4 == "" && reason6 == "" { + return + } + // One warning per actual invalidation, not per notification or query. The + // normal warning buffer retains this summary when debug traffic rotates. + ctrld.LoggerFromCtx(ctx).Warn().Uint64("transition_id", transitionID). + Str("default_route", state.DefaultRouteInterface). + Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). + Str("source_ipv4_after", ctrld.GetDefaultLocalIPv4().String()). + Str("source_ipv6_after", ctrld.GetDefaultLocalIPv6().String()). + Str("ipv4_clear_reason", reason4).Str("ipv6_clear_reason", reason6). + Msg("Removed stale resolver source") +} + +type recoveryDiagnostic struct { + logger *ctrld.Logger + transitionID uint64 + generation uint64 + reason RecoveryReason + started time.Time + firstFailure sync.Once + failed atomic.Bool + + // The flow fills these while it runs. The end event reports them, so one + // journal line tells what the recovery found and what it changed. + recoveredUpstream string + dhcpServers []string + interceptTargetAction string + bypassActive bool +} + +func recoveryReasonName(reason RecoveryReason) string { + switch reason { + case RecoveryReasonNetworkChange: + return "network_change" + case RecoveryReasonRegularFailure: + return "upstream_failure" + case RecoveryReasonOSFailure: + return "os_resolver_failure" + default: + return "unknown" + } +} + +func (d *recoveryDiagnostic) event(e *ctrld.LogEvent) *ctrld.LogEvent { + return e.Uint64("transition_id", d.transitionID).Uint64("recovery_generation", d.generation). + Str("recovery_reason", recoveryReasonName(d.reason)) +} + +func (d *recoveryDiagnostic) failure(err error) { + d.firstFailure.Do(func() { + d.failed.Store(true) + // Classify only; resolver errors can contain queried names or endpoints. + failure := "upstream_error" + var networkError net.Error + if errors.Is(err, syscall.EADDRNOTAVAIL) { + failure = "source_unavailable" + } else if ctrldnet.IsUnreachable(err) { + failure = "network_unreachable" + } else if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &networkError) && networkError.Timeout()) { + failure = "timeout" + } + d.event(d.log().Warn()).Str("failure", failure). + Msg("Recovery waiting for upstream; first failure") + }) +} + +// The outcomes that one recovery pass ends with. +const ( + recoveryOutcomeCompleted = "completed" + recoveryOutcomeCanceled = "canceled" + recoveryOutcomeSuperseded = "superseded" + recoveryOutcomeFailed = "failed" +) + +// journalRecoveredUpstream bounds the upstream name of the end event. An empty +// name means that no upstream recovered, so it stays empty. +func journalRecoveredUpstream(upstream string) string { + if upstream == "" { + return "" + } + return journalUpstreamName(upstream) +} + +func (d *recoveryDiagnostic) end(outcome string) { + e := d.log().Info() + if outcome == recoveryOutcomeCanceled || outcome == recoveryOutcomeFailed || d.failed.Load() { + e = d.log().Warn() + } + journal(d.event(e)).Str("outcome", outcome).Bool("had_failure", d.failed.Load()). + Int64("duration_ms", time.Since(d.started).Milliseconds()). + Str("recovered_upstream", journalRecoveredUpstream(d.recoveredUpstream)). + Strs("dhcp_servers", d.dhcpServers). + Int("dhcp_server_count", len(d.dhcpServers)). + Str("intercept_target_action", d.interceptTargetAction). + Bool("bypass_active", d.bypassActive). + Msg("Recovery end") +} + +func (d *recoveryDiagnostic) log() *ctrld.Logger { + if d.logger != nil { + return d.logger + } + return mainLog.Load() +} diff --git a/cmd/cli/network_transition_concurrency_test.go b/cmd/cli/network_transition_concurrency_test.go new file mode 100644 index 00000000..7ba8f876 --- /dev/null +++ b/cmd/cli/network_transition_concurrency_test.go @@ -0,0 +1,89 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" + "time" + + "tailscale.com/net/netmon" + + ctrld "github.com/Control-D-Inc/ctrld" +) + +func TestNetworkChangeOverlappingCallbackCannotRestoreRemovedSource(t *testing.T) { + sourceTestGlobals(t) + logs := captureTransitionLogs(t) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + networkChangeDefaultRouteIPFn = func(*prog) string { + close(entered) + <-release + return "192.0.2.10" + } + reconciliations := 0 + networkChangeReconcileFn = func(*prog, context.Context, uint64) { reconciliations++ } + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) {} + p := sourceTestProg(&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}) + stateA := sourceTestState("en0", true, "192.0.2.10/24") + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + <-done + }) + go func() { + defer close(done) + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.20/24"), New: stateA}, true) + }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("first callback did not reach route discovery") + } + // The newer callback completes while A is paused. Its ignored/address-loss + // path clears the source and deliberately schedules no replacement recovery. + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: stateA, New: sourceTestState("en0", true)}, true) + close(release) + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("superseded callback did not finish") + } + if ctrld.GetDefaultLocalIPv4() != nil { + t.Fatal("older callback restored the removed source") + } + if reconciliations != 1 || !strings.Contains(logs.String(), `"outcome":"accepted"`) { + t.Fatal("an ignored address-loss delta must preserve the accepted recovery") + } +} + +func TestNetworkChangeLateOldSnapshotIsRejected(t *testing.T) { + sourceTestGlobals(t) + old := networkChangeCurrentStateFn + t.Cleanup(func() { networkChangeCurrentStateFn = old }) + latest := sourceTestState("en0", true) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return latest } + ctrld.SetDefaultLocalIPv4(context.Background(), nil) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) + networkChangeReconcileFn = func(*prog, context.Context, uint64) { t.Fatal("old snapshot reconciled") } + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) { t.Fatal("old snapshot reconciled") } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: latest, New: sourceTestState("en0", true, "192.0.2.10/24")}, true) + if p.networkTransitionGen.Load() != 0 || ctrld.GetDefaultLocalIPv4() != nil { + t.Fatal("old snapshot became a new transition/source") + } +} + +func TestRecoveryRejectsSupersededTransition(t *testing.T) { + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + p.networkAcceptedGen.Store(2) + p.handleRecoveryForTransition(RecoveryReasonNetworkChange, 1) + if p.recoveryGen.Load() != 0 || p.recoveryRunning.Load() { + t.Fatal("superseded transition started recovery") + } +} diff --git a/cmd/cli/network_transition_reordered_test.go b/cmd/cli/network_transition_reordered_test.go new file mode 100644 index 00000000..af124f4c --- /dev/null +++ b/cmd/cli/network_transition_reordered_test.go @@ -0,0 +1,96 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/netip" + "testing" + "time" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +func TestLateSameEpochCallbacksCannotRestoreRemovedSource(t *testing.T) { + for _, initial := range []string{"192.0.2.1", "192.0.2.10"} { + t.Run(initial, func(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + mon := netmon.NewStatic() + before := sourceTestState("en0", true, "192.0.2.1/24") + major := sourceTestState("ipsec0", true, "192.0.2.1/24") + major.Interface["ipsec0"] = netmon.Interface{Interface: &net.Interface{Name: "ipsec0", Flags: net.FlagUp | net.FlagPointToPoint}} + major.InterfaceIPs["ipsec0"] = []netip.Prefix{netip.MustParsePrefix("192.0.2.10/24")} + latest := sourceTestState("ipsec0", true, "192.0.2.1/24") + latest.Interface["ipsec0"] = major.Interface["ipsec0"] + // netmon ignores ipsec address changes on Darwin. Its major cache + // stays at A while B removes the source. These are OS-read seams. + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return major } + reads := 0 + readNetworkSourceStateFn = func() (*netmon.State, error) { reads++; return latest, nil } + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP(initial)) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + reconciled := 0 + networkChangeReconcileFn = func(*prog, context.Context, uint64) { reconciled++ } + // B finishes before A's major callback even enters. A second, older + // minor callback also arrives late in that same cache epoch. + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: mon, Old: major, New: latest}, false) + olderMinor := *major + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: mon, Old: major, New: &olderMinor}, false) + if len(p.networkSourceState.InterfaceIPs["ipsec0"]) != 0 { + t.Fatal("late minor callback replaced current source validity") + } + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: mon, Old: before, New: major}, true) + want := net.IP(nil) + if initial == "192.0.2.1" { + want = net.ParseIP(initial) + } + if !ctrld.GetDefaultLocalIPv4().Equal(want) { + t.Fatalf("late major restored an obsolete source: got %v want %v", ctrld.GetDefaultLocalIPv4(), want) + } + if reads != 3 || reconciled != 1 { + t.Fatalf("fresh reads=%d reconciliation=%d", reads, reconciled) + } + }) + } +} + +func TestMinorSourceReadFailureDoesNotSkipReconciliation(t *testing.T) { + sourceTestGlobals(t) + oldRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = oldRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { return nil, errors.New("read failed") } + mon := netmon.NewStatic() + old := mon.InterfaceState() + next := *old + networkChangeValidInterfacesFn = func(context.Context) map[string]struct{} { return map[string]struct{}{} } + calls := 0 + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) { calls++ } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}) + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Monitor: mon, Old: old, New: &next}, false) + if calls != 1 || p.networkSourceState != nil || !p.networkSourceReadFailed { + t.Fatal("source-read failure skipped reconciliation or treated unknown data as current") + } +} + +func TestReadNetworkSourceState(t *testing.T) { + state, err := readNetworkSourceState() + if err != nil { + t.Fatal(err) + } + for name, iface := range state.Interface { + if iface.Interface == nil || iface.Name != name { + t.Fatal("interface snapshots share the wrong object") + } + for _, prefix := range state.InterfaceIPs[name] { + if !prefix.IsValid() { + t.Fatalf("invalid source prefix on %s", name) + } + } + } +} diff --git a/cmd/cli/network_transition_review_test.go b/cmd/cli/network_transition_review_test.go new file mode 100644 index 00000000..c3853f5e --- /dev/null +++ b/cmd/cli/network_transition_review_test.go @@ -0,0 +1,130 @@ +package cli + +import ( + "context" + "net" + "sync/atomic" + "testing" + "time" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +func TestNetmonMinorAndWakeSnapshotReconciliation(t *testing.T) { + for _, timeJump := range []bool{false, true} { + t.Run(map[bool]string{false: "minor", true: "time_jump"}[timeJump], func(t *testing.T) { + sourceTestGlobals(t) + mon := netmon.NewStatic() + // NewStatic has no monitoring resources to close. + old := mon.InterfaceState() + next := *old + // An identical snapshot is non-major; NewStatic has no OS monitor for IsMajorChangeFrom. + major := false + networkChangeValidInterfacesFn = func(context.Context) map[string]struct{} { return map[string]struct{}{} } + calls := 0 + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) { calls++ } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}) + delta := &netmon.ChangeDelta{Monitor: mon, Old: old, New: &next, TimeJumped: timeJump} + //lint:ignore SA1019 This fixture reproduces netmon v1.74.0's legacy wake flag. + delta.Major = timeJump + p.handleNetworkChange(context.Background(), delta, major) + if calls != 1 { + t.Fatalf("real netmon non-major delta discarded: ignored-path calls=%d transition ID=%d", calls, p.networkTransitionGen.Load()) + } + }) + } +} + +func TestIgnoredDeltaPreservesAcceptedRecovery(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + oldRecovery := handleRecoveryForTransitionFn + t.Cleanup(func() { handleRecoveryForTransitionFn = oldRecovery }) + calls := make(chan uint64, 2) + handleRecoveryForTransitionFn = func(_ *prog, _ RecoveryReason, id uint64) { calls <- id } + networkChangeReconcileFn = (*prog).reconcileNetworkChange // Real accepted-path wiring, with recovery stubbed below. + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + a := sourceTestState("en0", true, "192.0.2.20/24") + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.10/24"), New: a}, true) + b := sourceTestState("en0", true, "192.0.2.20/24") + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: a, New: b}, false) + select { + case <-calls: + case <-time.After(1200 * time.Millisecond): + t.Fatal("ignored delta canceled accepted recovery; no replacement was scheduled") + } +} + +func TestIgnoredDeltaPreservesValidPendingSource(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + ctrld.SetDefaultLocalIPv4(context.Background(), nil) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + networkChangeDefaultRouteIPFn = func(*prog) string { close(entered); <-release; return "192.0.2.20" } + reconciled := make(chan uint64, 2) + networkChangeReconcileFn = func(_ *prog, _ context.Context, id uint64) { reconciled <- id } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + a := sourceTestState("en0", true, "192.0.2.20/24") + go func() { + defer close(done) + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.10/24"), New: a}, true) + }() + <-entered + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: a, New: sourceTestState("en0", true, "192.0.2.20/24")}, false) + close(release) + <-done + if !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.20")) || len(reconciled) != 1 { + t.Fatalf("valid source discarded after ignored delta: source=%v reconciliations=%d", ctrld.GetDefaultLocalIPv4(), len(reconciled)) + } +} + +func TestNewAcceptedDeltaSupersedesPendingAcceptedSource(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var lookups atomic.Int32 + networkChangeDefaultRouteIPFn = func(*prog) string { + if lookups.Add(1) == 1 { + close(entered) + <-release + return "192.0.2.20" + } + return "" + } + reconciled := make(chan uint64, 2) + networkChangeReconcileFn = func(_ *prog, _ context.Context, id uint64) { reconciled <- id } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + a := sourceTestState("en0", true, "192.0.2.20/24") + go func() { + defer close(done) + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.10/24"), New: a}, true) + }() + <-entered + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: a, New: sourceTestState("en0", true, "192.0.2.30/24")}, true) + close(release) + <-done + if !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.30")) || len(reconciled) != 1 || <-reconciled != 2 { + t.Fatal("older accepted delta replaced the newer source or recovery") + } +} + +func TestSourceCommitUsesCurrentMajorEpoch(t *testing.T) { + mon := netmon.NewStatic() + latest := mon.InterfaceState() + minor := *latest + p := sourceTestProg(&prog{networkSourceState: &minor, networkSourceEpoch: latest}) + delta := &netmon.ChangeDelta{Monitor: mon, Old: latest, New: &minor} + if p.sourceCommitState(delta) != &minor { + t.Fatal("discarded the observed minor update within the current epoch") + } + p.networkSourceEpoch = &minor + if p.sourceCommitState(delta) != latest { + t.Fatal("did not refresh from a newer major epoch") + } + if networkSnapshotCurrent(&netmon.ChangeDelta{Monitor: mon, Old: &minor, New: &minor}, false, latest) { + t.Fatal("accepted a minor callback from an obsolete major epoch") + } +} diff --git a/cmd/cli/network_transition_test.go b/cmd/cli/network_transition_test.go new file mode 100644 index 00000000..809e6b7a --- /dev/null +++ b/cmd/cli/network_transition_test.go @@ -0,0 +1,340 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/netip" + "strings" + "sync" + "testing" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +func sourceTestState(route string, up bool, ips ...string) *netmon.State { + flags := net.Flags(0) + if up { + flags = net.FlagUp + } + s := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{"en0": {Interface: &net.Interface{Name: "en0", Flags: flags}}}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, ip := range ips { + s.InterfaceIPs["en0"] = append(s.InterfaceIPs["en0"], netip.MustParsePrefix(ip)) + } + return s +} + +func captureTransitionLogs(t *testing.T) *syncBuffer { + t.Helper() + old := mainLog.Load() + buf := &syncBuffer{} + logger := sourceTestLogger(buf) + mainLog.Store(logger) + t.Cleanup(func() { mainLog.Store(old) }) + return buf +} + +func sourceTestGlobals(t *testing.T) { + t.Helper() + // A transition reads the hardware ports and the virtual adapters, and no + // test may start the commands behind them. + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) + v4, v6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + valid, route := networkChangeValidInterfacesFn, networkChangeDefaultRouteIPFn + reconcile, ignored := networkChangeReconcileFn, networkChangeIgnoredInterceptFn + intercept := dnsIntercept + dnsIntercept = true + networkChangeValidInterfacesFn = func(context.Context) map[string]struct{} { return map[string]struct{}{"en0": {}} } + networkChangeDefaultRouteIPFn = func(*prog) string { return "" } + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(context.Background(), v4) + ctrld.SetDefaultLocalIPv6(context.Background(), v6) + networkChangeValidInterfacesFn, networkChangeDefaultRouteIPFn = valid, route + networkChangeReconcileFn, networkChangeIgnoredInterceptFn = reconcile, ignored + dnsIntercept = intercept + }) +} + +func TestNetworkChangeSourceLifecycle(t *testing.T) { + const v4, v6 = "192.0.2.10", "2001:db8::10" + for _, tt := range []struct { + name string + newState *netmon.State + preferred string + want4, want6 string + outcome, reason4, reason6 string + }{ + {"dhcp_lease_replacement_keeps_ipv6", sourceTestState("en0", true, "192.0.2.20/24", v6+"/64"), "", "192.0.2.20", v6, "accepted", "address_removed", ""}, + {"dhcp_lease_replacement", sourceTestState("en0", true, "192.0.2.20/24"), "", "192.0.2.20", "", "accepted", "address_removed", "address_removed"}, + {"accepted_ipv4_loss", sourceTestState("en0", true, v6+"/64"), "", "", v6, "accepted", "address_removed", ""}, + {"accepted_ipv6_loss", sourceTestState("en0", true, v4+"/24"), v4, v4, "", "accepted", "", "address_removed"}, + {"stale_route_discovery", sourceTestState("en0", true, v6+"/64"), v4, "", v6, "accepted", "address_removed", ""}, + {"ignored_down_retains_addresses", sourceTestState("en0", false, v4+"/24", v6+"/64"), "", "", "", "ignored", "interface_down", "interface_down"}, + {"ignored_address_loss", sourceTestState("en0", true), "", "", "", "ignored", "address_removed", "address_removed"}, + {"skipped_no_active_interface", sourceTestState("", false, v4+"/24", v6+"/64"), "", "", "", "no_active_interface", "interface_down", "interface_down"}, + {"ignored_valid_sources", sourceTestState("en0", true, v4+"/24", v6+"/64"), "", v4, v6, "ignored", "", ""}, + {"accepted_valid_preferred_source", sourceTestState("en0", true, "192.0.2.20/24", v4+"/24", v6+"/64"), v4, v4, v6, "accepted", "", ""}, + } { + t.Run(tt.name, func(t *testing.T) { + sourceTestGlobals(t) + logs := captureTransitionLogs(t) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(context.Background(), net.ParseIP(v6)) + networkChangeDefaultRouteIPFn = func(*prog) string { return tt.preferred } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}) + accepted, ignored := 0, 0 + assertSources := func() { + t.Helper() + if got := ctrld.GetDefaultLocalIPv4(); !got.Equal(net.ParseIP(tt.want4)) { + t.Errorf("IPv4 = %v, want %s", got, tt.want4) + } + if got := ctrld.GetDefaultLocalIPv6(); !got.Equal(net.ParseIP(tt.want6)) { + t.Errorf("IPv6 = %v, want %s", got, tt.want6) + } + } + // These seams precede all DNS/PF writes and recovery timers, on every OS. + networkChangeReconcileFn = func(_ *prog, _ context.Context, id uint64) { + accepted++ + if id != 1 { + t.Errorf("transition ID = %d", id) + } + assertSources() + } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { ignored++; assertSources() } + p.handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: sourceTestState("en0", true, v4+"/24", v6+"/64"), New: tt.newState}, true) + assertSources() + wantAccepted, wantIgnored := 0, 0 + if tt.outcome == "accepted" { + wantAccepted = 1 + } + if tt.outcome == "ignored" { + wantIgnored = 1 + } + if accepted != wantAccepted || ignored != wantIgnored { + t.Fatalf("mutation boundaries: accepted=%d ignored=%d; want %d/%d", accepted, ignored, wantAccepted, wantIgnored) + } + var transition, warning map[string]any + for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") { + var event map[string]any + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatal(err) + } + switch event["message"] { + case "Network transition": + transition = event + case "Removed stale resolver source": + warning = event + } + } + if transition["outcome"] != tt.outcome || transition["transition_id"] != float64(1) { + t.Fatalf("transition: %v", transition) + } + wantWarning := tt.reason4 != "" || tt.reason6 != "" + if (warning != nil) != wantWarning { + t.Fatalf("warning = %v, want warning %v", warning, wantWarning) + } + if wantWarning && (warning["ipv4_clear_reason"] != tt.reason4 || warning["ipv6_clear_reason"] != tt.reason6 || warning["level"] != "warn") { + t.Fatalf("warning reasons/level: %v", warning) + } + }) + } +} + +func TestNetworkChangeDoesNotRestoreDownDefaultRouteSources(t *testing.T) { + sourceTestGlobals(t) + const v4, v6 = "192.0.2.10", "2001:db8::10" + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(context.Background(), net.ParseIP(v6)) + networkChangeDefaultRouteIPFn = func(*prog) string { return v4 } + old := sourceTestState("en1", true, "2001:db8:1::10/64") + old.Interface["en1"] = netmon.Interface{Interface: &net.Interface{Name: "en1", Flags: net.FlagUp}} + old.InterfaceIPs["en1"] = []netip.Prefix{netip.MustParsePrefix(v4 + "/24"), netip.MustParsePrefix(v6 + "/64")} + newState := sourceTestState("en1", true, "2001:db8:1::20/64") + newState.Interface["en1"] = netmon.Interface{Interface: &net.Interface{Name: "en1"}} + newState.InterfaceIPs["en1"] = old.InterfaceIPs["en1"] + called := 0 + networkChangeReconcileFn = func(_ *prog, _ context.Context, _ uint64) { + called++ + if ctrld.GetDefaultLocalIPv4() != nil || ctrld.GetDefaultLocalIPv6() != nil { + t.Error("down-interface sources reintroduced before recovery") + } + } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { t.Fatal("accepted delta was ignored") } + (sourceTestProg(&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}})).handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: old, New: newState}, true) + if called != 1 { + t.Fatalf("reconciliation called %d times", called) + } +} + +func TestNetworkChangeSourceMovedToUpInterface(t *testing.T) { + sourceTestGlobals(t) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) + state := sourceTestState("en0", false, "192.0.2.10/24") + state.Interface["utun0"] = netmon.Interface{Interface: &net.Interface{Name: "utun0", Flags: net.FlagUp}} + state.InterfaceIPs["utun0"] = state.InterfaceIPs["en0"] + called := 0 + networkChangeReconcileFn = func(_ *prog, _ context.Context, _ uint64) { t.Fatal("unexpected recovery") } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { called++ } + (sourceTestProg(&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}})).handleNetworkChange(context.Background(), &netmon.ChangeDelta{Old: state, New: state}, false) + if called != 1 || !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.10")) { + t.Fatal("valid source on another up interface was cleared") + } +} + +func TestRecoveryTransitionDiagnostics(t *testing.T) { + logs := captureTransitionLogs(t) + stubHeaderSnapshotSources(t) + original, intercept := recoveryResetDNSFn, dnsIntercept + dnsIntercept = false + t.Cleanup(func() { recoveryResetDNSFn = original; dnsIntercept = intercept }) + for _, superseded := range []bool{false, true} { + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + calls := 0 + recoveryResetDNSFn = func(p *prog, start, restore bool) { + calls++ + if start || restore { + t.Error("changed reset flags") + } + if superseded { + p.beginRecovery(RecoveryReasonNetworkChange) + } else { + p.recoveryCancel() + } + } + p.networkAcceptedGen.Store(42) + p.handleRecoveryForTransition(RecoveryReasonNetworkChange, 42) + if calls != 1 { + t.Fatalf("resetDNS seam called %d times", calls) + } + if p.recoveryCancel != nil { + p.recoveryCancel() + } + } + for _, expected := range []string{`"transition_id":42`, `"recovery_generation":1`, `"message":"Recovery begin"`, `"outcome":"canceled"`, `"outcome":"superseded"`, `"message":"Recovery end"`} { + if !strings.Contains(logs.String(), expected) { + t.Errorf("missing %s in %s", expected, logs.String()) + } + } +} + +func TestRecoveryDebouncePreservesTransitionID(t *testing.T) { + original := handleRecoveryForTransitionFn + t.Cleanup(func() { handleRecoveryForTransitionFn = original }) + ids := make(chan uint64, 2) + handleRecoveryForTransitionFn = func(_ *prog, reason RecoveryReason, id uint64) { + if reason != RecoveryReasonNetworkChange { + t.Error("wrong recovery reason") + } + ids <- id + } + p := sourceTestProg(&prog{cfg: &ctrld.Config{}}) + p.networkAcceptedGen.Store(11) + p.debounceRecovery(11) + p.networkAcceptedGen.Store(12) + p.debounceRecovery(12) + p.debounceRecovery(11) // a delayed older callback must not replace the new timer + select { + case id := <-ids: + if id != 12 { + t.Fatalf("recovery transition = %d, want 12", id) + } + case <-time.After(3 * time.Second): + t.Fatal("debounced recovery did not run") + } +} + +func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { + sourceTestGlobals(t) + old := mainLog.Load() + debug, warnings := newLogWriterWithSize(2048), newSmallLogWriter() + // Runtime summaries occur after initialization, outside the preserved prefix. + debug.Write([]byte(logWriterInitEndMarker)) + config := zap.NewProductionEncoderConfig() + config.MessageKey = "message" + logger := &ctrld.Logger{Logger: zap.New(zapcore.NewTee( + zapcore.NewCore(zapcore.NewJSONEncoder(config), zapcore.AddSync(debug), zap.DebugLevel), + newJournalCore(warnings), + ))} + mainLog.Store(logger) + t.Cleanup(func() { mainLog.Store(old) }) + ctrld.SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(context.Background(), nil) + state := sourceTestState("en0", false, "192.0.2.10/24") + for range 100 { + validateDefaultLocalIPsFromDelta(ctrld.LoggerCtx(context.Background(), mainLog.Load()), state, 7) + } + d := &recoveryDiagnostic{transitionID: 7, generation: 9, started: time.Now()} + var wg sync.WaitGroup + for range 100 { + wg.Add(1) + go func() { defer wg.Done(); d.failure(errors.New("private.customer.example secret payload")) }() + } + wg.Wait() + d.end("completed") + (&recoveryDiagnostic{transitionID: 8, generation: 10, started: time.Now()}).end("completed") + journal(logger.Info()).Str("interface", "en0").Msg("Network snapshot") + logger.Info().Msg("plain info line") + for range 100 { + logger.Debug().Msg(strings.Repeat("noise", 100)) + } + text := warnings.buf.String() + for _, msg := range []string{"Removed stale resolver source", "Recovery waiting for upstream; first failure"} { + if strings.Count(text, msg) != 1 { + t.Fatalf("expected one retained %q: %s", msg, text) + } + if strings.Contains(debug.buf.String(), msg) { + t.Fatalf("debug buffer did not rotate %q", msg) + } + } + if strings.Count(text, "Recovery end") != 2 || !strings.Contains(text, `"outcome":"completed"`) { + t.Fatal("recovery completion must survive rotation beside its retained failure") + } + if strings.Contains(text, "private.customer") || strings.Contains(text, "secret payload") { + t.Fatalf("raw payload or normal debug elevated: %s", text) + } + if strings.Count(text, "Network snapshot") != 1 { + t.Fatalf("expected one retained marked info line: %s", text) + } + if strings.Contains(text, "plain info line") { + t.Fatalf("unmarked info line must not be retained: %s", text) + } +} + +func TestRecoveryWaitLogsFirstFailure(t *testing.T) { + logs := captureTransitionLogs(t) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + d := &recoveryDiagnostic{transitionID: 8, generation: 2, started: time.Now()} + // Invalid resolver type fails construction before any network access. + _, err := (sourceTestProg(&prog{})).waitForUpstreamRecovery(ctx, map[string]*ctrld.UpstreamConfig{"invalid": {Type: "invalid"}}, d) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wait result = %v", err) + } + if strings.Count(logs.String(), "Recovery waiting for upstream; first failure") != 1 { + t.Fatalf("missing first-failure summary: %s", logs.String()) + } +} + +// sourceTestProg adapts the source fixtures to master's per-program logger. +func sourceTestProg(p *prog) *prog { + p.logger.Store(mainLog.Load()) + return p +} + +func sourceTestLogger(buf *syncBuffer) *ctrld.Logger { + config := zap.NewProductionEncoderConfig() + config.MessageKey = "message" + return &ctrld.Logger{Logger: zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(config), zapcore.AddSync(buf), zap.DebugLevel))} +} diff --git a/cmd/cli/nextdns.go b/cmd/cli/nextdns.go index f4fed479..53e0492a 100644 --- a/cmd/cli/nextdns.go +++ b/cmd/cli/nextdns.go @@ -8,11 +8,12 @@ import ( const nextdnsURL = "https://dns.nextdns.io" +// generateNextDNSConfig generates NextDNS configuration for the given UID func generateNextDNSConfig(uid string) { if uid == "" { return } - mainLog.Load().Info().Msg("generating ctrld config for NextDNS resolver") + mainLog.Load().Info().Msg("Generating ctrld config for NextDNS resolver") cfg = ctrld.Config{ Listener: map[string]*ctrld.ListenerConfig{ "0": { diff --git a/cmd/cli/nrpt_external_gp.go b/cmd/cli/nrpt_external_gp.go new file mode 100644 index 00000000..62c57f86 --- /dev/null +++ b/cmd/cli/nrpt_external_gp.go @@ -0,0 +1,63 @@ +package cli + +import ( + "errors" + "net/netip" + "strings" +) + +const nrptRuleName = `CtrldCatchAll` + +// errGPNRPTVerified marks an intercept startup failure that happened while an externally +// managed (Group Policy) NRPT catch-all was proved - by probe, not by registry shape +// alone - to be routing DNS to this listener. It is the difference between "intercept +// failed but DNS still reaches ctrld" and "intercept failed and nothing is filtering", +// which is what decides whether the interface-DNS fallback must run. +// +// Only the Windows path produces it, but setDNS is shared, so the sentinel and its +// predicate live here with the other platform-neutral NRPT helpers. +var errGPNRPTVerified = errors.New("GP-managed NRPT verified routing to ctrld") + +// errGPNRPTIneffective marks a startup that ends with externally managed NRPT owning the +// namespace while no probe has proved it routes to ctrld. DNS is not reaching ctrld, but +// adapter DNS was deliberately preserved and no ctrld rule may be written beside an +// administrator's catch-all - so this is a failed start that must not take the +// interface-DNS fallback either. +var errGPNRPTIneffective = errors.New("GP-managed NRPT owns the namespace but no probe reached ctrld") + +// interceptFailedWithVerifiedExternalDNS reports whether an intercept startup failure +// happened while externally managed DNS policy was verified to be routing to ctrld. +func interceptFailedWithVerifiedExternalDNS(err error) bool { + return errors.Is(err, errGPNRPTVerified) +} + +// interceptFailedUnderExternalDNSPolicy reports whether an intercept startup failure +// happened while externally managed DNS policy owned the namespace, whether or not it was +// proved to route. Either way the interface-DNS fallback must not run: adapter DNS was +// preserved on purpose, and rewriting it would violate the policy ctrld just deferred to. +// Only the verified case is a successful start. +func interceptFailedUnderExternalDNSPolicy(err error) bool { + return errors.Is(err, errGPNRPTVerified) || errors.Is(err, errGPNRPTIneffective) +} + +// isExternalGPCatchAll recognizes only a single catch-all namespace that is not +// ctrld's deterministic GP key. Registry access stays in the Windows file; this +// pure classifier is shared with host-runnable tests. +func isExternalGPCatchAll(ruleName string, namespaces []string) bool { + return ruleName != "" && !strings.EqualFold(ruleName, nrptRuleName) && len(namespaces) == 1 && strings.TrimSpace(namespaces[0]) == "." +} + +func isMatchingGPNRPTRule(ruleName string, namespaces []string, dnsServers, listenerIP string) bool { + if !isExternalGPCatchAll(ruleName, namespaces) { + return false + } + server, err := netip.ParseAddr(strings.TrimSpace(dnsServers)) + if err != nil { + return false + } + listener, err := netip.ParseAddr(strings.TrimSpace(listenerIP)) + if err != nil { + return false + } + return server.Unmap() == listener.Unmap() +} diff --git a/cmd/cli/nrpt_external_gp_test.go b/cmd/cli/nrpt_external_gp_test.go new file mode 100644 index 00000000..277581ba --- /dev/null +++ b/cmd/cli/nrpt_external_gp_test.go @@ -0,0 +1,129 @@ +package cli + +import ( + "errors" + "fmt" + "testing" +) + +func TestIsMatchingGPNRPTRule(t *testing.T) { + tests := []struct { + name string + ruleName string + namespaces []string + servers string + listener string + want bool + }{ + { + name: "exact IPv4 catch-all", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "127.0.0.1", + listener: "127.0.0.1", + want: true, + }, + { + name: "normalized IPv4-mapped listener", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "::ffff:127.0.0.1", + listener: "127.0.0.1", + want: true, + }, + { + name: "ctrld GP key is not external", + ruleName: "ctrldcatchall", + namespaces: []string{"."}, + servers: "127.0.0.1", + listener: "127.0.0.1", + }, + { + name: "partial namespace", + ruleName: "{A1B2C3D4}", + namespaces: []string{"corp.example"}, + servers: "127.0.0.1", + listener: "127.0.0.1", + }, + { + name: "multiple namespaces", + ruleName: "{A1B2C3D4}", + namespaces: []string{".", "corp.example"}, + servers: "127.0.0.1", + listener: "127.0.0.1", + }, + { + name: "wrong listener", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "127.0.0.2", + listener: "127.0.0.1", + }, + { + name: "multiple nameservers", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "127.0.0.1;127.0.0.2", + listener: "127.0.0.1", + }, + { + name: "malformed nameserver", + ruleName: "{A1B2C3D4}", + namespaces: []string{"."}, + servers: "localhost", + listener: "127.0.0.1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isMatchingGPNRPTRule(tt.ruleName, tt.namespaces, tt.servers, tt.listener); got != tt.want { + t.Fatalf("isMatchingGPNRPTRule() = %t, want %t", got, tt.want) + } + }) + } +} + +func TestIsExternalGPCatchAll(t *testing.T) { + tests := []struct { + name string + ruleName string + namespaces []string + want bool + }{ + {name: "external catch-all", ruleName: "{GP-RULE}", namespaces: []string{"."}, want: true}, + {name: "ctrld key", ruleName: nrptRuleName, namespaces: []string{"."}}, + {name: "partial namespace", ruleName: "{GP-RULE}", namespaces: []string{"corp.example"}}, + {name: "multiple namespaces", ruleName: "{GP-RULE}", namespaces: []string{".", "corp.example"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isExternalGPCatchAll(tt.ruleName, tt.namespaces); got != tt.want { + t.Fatalf("isExternalGPCatchAll() = %t, want %t", got, tt.want) + } + }) + } +} + +// TestInterceptFailedWithVerifiedExternalDNS covers the distinction the interface-DNS +// fallback turns on. "A GP rule exists" is not enough: if it is not actually routing and +// intercept failed too, skipping the fallback leaves the machine with no NRPT, no WFP and +// no adapter DNS - that is, unfiltered. Only a probe-verified route earns the skip. +func TestInterceptFailedWithVerifiedExternalDNS(t *testing.T) { + wfpErr := errors.New("FwpmEngineOpen0 failed: HRESULT 0x5") + + verified := fmt.Errorf("dns intercept: WFP setup failed: %w: %w", wfpErr, errGPNRPTVerified) + if !interceptFailedWithVerifiedExternalDNS(verified) { + t.Error("a failure carrying errGPNRPTVerified must skip the interface-DNS fallback") + } + if !errors.Is(verified, wfpErr) { + t.Error("the underlying cause must stay inspectable for logs and callers") + } + + if interceptFailedWithVerifiedExternalDNS(fmt.Errorf("dns intercept: WFP setup failed: %w", wfpErr)) { + t.Error("an unverified failure must take the interface-DNS fallback rather than leave the machine unfiltered") + } + if interceptFailedWithVerifiedExternalDNS(nil) { + t.Error("no error must not read as a verified external route") + } +} diff --git a/cmd/cli/nrpt_external_gp_windows_test.go b/cmd/cli/nrpt_external_gp_windows_test.go new file mode 100644 index 00000000..3d402d78 --- /dev/null +++ b/cmd/cli/nrpt_external_gp_windows_test.go @@ -0,0 +1,20 @@ +//go:build windows + +package cli + +import "testing" + +func TestWFPStateNRPTPolicyOwner(t *testing.T) { + state := &wfpState{} + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Fatalf("owner = %v, rule = %q", owner, ruleName) + } + + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + owner, ruleName = state.nrptPolicyOwner() + if owner != nrptRuleOwnerCtrld || ruleName != "" { + t.Fatalf("owner = %v, rule = %q", owner, ruleName) + } +} diff --git a/cmd/cli/nrpt_handback_windows_test.go b/cmd/cli/nrpt_handback_windows_test.go new file mode 100644 index 00000000..7bea7cbd --- /dev/null +++ b/cmd/cli/nrpt_handback_windows_test.go @@ -0,0 +1,1701 @@ +//go:build windows + +package cli + +import ( + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// fakeNRPTOps records the Windows side effects a transition would cause and serves +// scripted probe results, so the ownership decisions can be driven without touching the +// registry or the DNS Client. +type fakeNRPTOps struct { + mu sync.Mutex + + // probeResults is consumed in order; the last value repeats once exhausted. + probeResults []bool + probeCalls int + flushCalls int + + gpRule string // rule name findGPRule reports, "" for none + gpConflict bool // whether an external catch-all targets another resolver + parentEmpty bool // whether the GP parent key reads as present but empty + cleanCalls int + gpConflictCalls int + ctrldRule bool // whether ctrld's own catch-all exists in the local store + // ctrldGPRule is whether ctrld's own catch-all also exists in the GP store. The two + // stores are modelled separately because that is where the duplicate lives: the + // production writer adds ctrld's GP rule whenever any other GP rule is present, so a + // fake with a single "ctrld rule exists" flag cannot see a sibling being written + // beside an administrator's catch-all. + ctrldGPRule bool + // otherGPRules models GP rules that are not catch-alls. Production + // otherGPRulesExist is true for any GP rule at all, so the full writer places + // ctrld's GP rule for these too - a shape a gpRule-only fake cannot express. + otherGPRules bool + dropGPCalls int + existsCalls int + addCalls int + restoreLocalCalls int + removeCalls int + signalCalls int + + addErr error + removeErr error + + // beforeAdd runs inside addRule, while the transition lock is held. + beforeAdd func() + // onProbe runs after the nth probe (1-based) is answered, so a test can change the + // GP store mid-probe the way a Group Policy refresh would. + onProbe func(call int) + // onGPConflicts runs after each gpConflicts check, which is the last ops call before + // the two-phase re-add queues for the transition lock. + onGPConflicts func(call int) + // waitHook stands in for a cancellable wait: returning false models the intercept + // being retired mid-wait. + waitHook func() bool +} + +func (f *fakeNRPTOps) ops() *nrptOps { + return &nrptOps{ + probe: func(*wfpState) bool { + f.mu.Lock() + f.probeCalls++ + call := f.probeCalls + result := false + if len(f.probeResults) > 0 { + result = f.probeResults[0] + if len(f.probeResults) > 1 { + f.probeResults = f.probeResults[1:] + } + } + onProbe := f.onProbe + f.mu.Unlock() + if onProbe != nil { + onProbe(call) + } + return result + }, + ruleExists: func() bool { + f.mu.Lock() + defer f.mu.Unlock() + f.existsCalls++ + // nrptCatchAllRuleExists reads the local GUID key and the GP CtrldCatchAll + // key, so a store holding only the sibling still reports "a rule exists". + return f.ctrldRule || f.ctrldGPRule + }, + addRule: func(string) error { + f.mu.Lock() + beforeAdd, err := f.beforeAdd, f.addErr + f.addCalls++ + if err == nil { + f.ctrldRule = true + // Mirror addNRPTCatchAllRule: with any other GP rule present it also + // writes ctrld's GP-path rule, otherwise it cleans the GP path. + f.ctrldGPRule = f.gpRule != "" || f.gpConflict || f.otherGPRules + } + f.mu.Unlock() + if beforeAdd != nil { + beforeAdd() + } + return err + }, + restoreLocalRule: func(string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.restoreLocalCalls++ + if f.addErr != nil { + return f.addErr + } + f.ctrldRule = true // local store only; the GP store is left as found + return nil + }, + removeRule: func() error { + f.mu.Lock() + defer f.mu.Unlock() + f.removeCalls++ + if f.removeErr != nil { + return f.removeErr + } + f.ctrldRule = false + f.ctrldGPRule = false + return nil + }, + signal: func() { + f.mu.Lock() + defer f.mu.Unlock() + f.signalCalls++ + }, + dropGPRule: func() { + f.mu.Lock() + defer f.mu.Unlock() + f.dropGPCalls++ + f.ctrldGPRule = false // the local store and the administrator's rule are untouched + }, + findGPRule: func(string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.gpRule + }, + gpRuleMatches: func(ruleName, _ string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return ruleName != "" && ruleName == f.gpRule + }, + gpConflicts: func(*wfpState, string) bool { + f.mu.Lock() + f.gpConflictCalls++ + call := f.gpConflictCalls + conflict, hook := f.gpConflict, f.onGPConflicts + f.mu.Unlock() + if hook != nil { + hook(call) + } + return conflict + }, + loopback: func(*wfpState) error { return nil }, + // No real waiting in tests; report "still live" unless a test says otherwise. + wait: func(*wfpState, time.Duration) bool { + f.mu.Lock() + hook := f.waitHook + f.mu.Unlock() + if hook != nil { + return hook() + } + return true + }, + startWFP: func(*wfpState) error { return nil }, + flush: func() { + f.mu.Lock() + defer f.mu.Unlock() + f.flushCalls++ + }, + parentEmpty: func(string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.parentEmpty + }, + cleanParent: func() bool { + f.mu.Lock() + defer f.mu.Unlock() + f.cleanCalls++ + wasEmpty := f.parentEmpty + f.parentEmpty = false + return wasEmpty + }, + } +} + +func (f *fakeNRPTOps) counts() (add, remove, signal, probe int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.addCalls, f.removeCalls, f.signalCalls, f.probeCalls +} + +func (f *fakeNRPTOps) ruleExistsCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.existsCalls +} + +func (f *fakeNRPTOps) flushCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.flushCalls +} + +// setGPRule replaces what the fake GP store reports, as a policy refresh would. +func (f *fakeNRPTOps) setGPRule(ruleName string, conflicting bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.gpRule = ruleName + f.gpConflict = conflicting +} + +// hasCtrldRule reports whether ctrld's catch-all is currently installed. +func (f *fakeNRPTOps) hasCtrldRule() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.ctrldRule +} + +// hasCtrldGPSibling reports whether ctrld's own rule is sitting in the GP store, which +// beside an administrator's catch-all is the second GP catch-all that an administrator +// did not write. +func (f *fakeNRPTOps) hasCtrldGPSibling() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.ctrldGPRule +} + +// writes reports every way this file can put ctrld's catch-all back, so a "nothing was +// written" assertion cannot pass by checking one writer while the other ran. +func (f *fakeNRPTOps) writes() (add, restoreLocal int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.addCalls, f.restoreLocalCalls +} + +// dropGPCount is how many times ctrld's GP-store key was deleted on its own. +func (f *fakeNRPTOps) dropGPCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.dropGPCalls +} + +// restoreLocalCount is how many times the local-store-only restore ran. +func (f *fakeNRPTOps) restoreLocalCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.restoreLocalCalls +} + +// installedFakeNRPTOps is the fake currently standing in for the production NRPT +// operations, so fixtures can hand it to tests and can tell whether one is installed at +// all. +var installedFakeNRPTOps *fakeNRPTOps + +// installFakeNRPTOps swaps in the fake for the duration of the test. +func installFakeNRPTOps(t *testing.T, f *fakeNRPTOps) { + t.Helper() + nrptOpsForTest = f.ops() + installedFakeNRPTOps = f + t.Cleanup(func() { + nrptOpsForTest = nil + installedFakeNRPTOps = nil + }) +} + +// fakeNRPTSeed is the starting state a test wants the fixture's fake to have. It is a +// separate type so the fake - which carries a mutex - is never copied. +type fakeNRPTSeed struct { + probeResults []bool + gpRule string + gpConflict bool + parentEmpty bool + ctrldRule bool + ctrldGPRule bool + otherGPRules bool + addErr error + removeErr error +} + +// configure re-arms the fake the fixture installed with a test's own starting state, so +// the fixture keeps ownership of installation - and therefore of the production-call guard. +func (f *fakeNRPTOps) configure(seed fakeNRPTSeed) { + f.mu.Lock() + defer f.mu.Unlock() + f.probeResults = seed.probeResults + f.gpRule = seed.gpRule + f.gpConflict = seed.gpConflict + f.parentEmpty = seed.parentEmpty + f.ctrldRule = seed.ctrldRule + f.ctrldGPRule = seed.ctrldGPRule + f.otherGPRules = seed.otherGPRules + f.addErr = seed.addErr + f.removeErr = seed.removeErr +} + +// requireFakeNRPTOpsInstalled proves the fake is the table production code will actually +// consult, and fails the test immediately if it is not. +// +// Identity is not enough on its own, and neither is asserting side-effect counters after +// the fact: a fixture that never installed the fake, or a seam that stopped consulting the +// override, leaves those counters at zero and every "no side effects" assertion passes +// while the real registry and signalling functions run. So this calls through +// prog.nrptOps() and requires the call to land on the fake. The probe is only made once an +// override is known to exist, so it can never reach the host itself. +func requireFakeNRPTOpsInstalled(t *testing.T, f *fakeNRPTOps) { + t.Helper() + if err := checkFakeNRPTOpsInstalled(f); err != nil { + t.Fatal(err) + } +} + +// checkFakeNRPTOpsInstalled is the precondition itself, as a predicate so it can be tested +// like any other logic - see TestFakeNRPTOpsPreconditionDetectsBypass. It returns nil only +// when f is the table prog.nrptOps() hands out. +func checkFakeNRPTOpsInstalled(f *fakeNRPTOps) error { + if nrptOpsForTest == nil { + return errors.New("fake NRPT operations are not installed: production registry and signalling functions would run") + } + if installedFakeNRPTOps != f { + return errors.New("the installed fake is not the one this fixture returned") + } + before := f.ruleExistsCount() + _ = (&prog{}).nrptOps().ruleExists() + if f.ruleExistsCount() == before { + return errors.New("prog.nrptOps() did not route to the installed fake: the seam is not in effect") + } + return nil +} + +// fakeNRPTOpsForTest returns the installed fake, installing a default one if the test has +// not already done so. +// +// Every fixture goes through this before publishing state, so no test can reach the +// production registry and signalling functions. That matters because those are not +// read-only: on a host where ctrld's deterministic key exists, one unfaked call can delete +// live NRPT policy and force a Group Policy refresh, a Dnscache paramchange and a cache +// flush. A clean runner is not a safety boundary. +func fakeNRPTOpsForTest(t *testing.T) *fakeNRPTOps { + t.Helper() + if installedFakeNRPTOps != nil { + return installedFakeNRPTOps + } + f := &fakeNRPTOps{} + installFakeNRPTOps(t, f) + return f +} + +func newHandbackTestProg(t *testing.T) (*prog, *wfpState, *fakeNRPTOps) { + t.Helper() + // Never let a test fall through to the production NRPT operations, and prove it here + // rather than trusting that installation happened. + f := fakeNRPTOpsForTest(t) + requireFakeNRPTOpsInstalled(t, f) + state := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + p := &prog{} + p.dnsInterceptState = state + return p, state, f +} + +// TestHandbackRestoresFallbackWhenGPCannotRoute is the behaviour test for the handback +// contract. The first probe runs while ctrld's fallback is still installed, so that +// fallback can satisfy it; only a probe taken after ctrld's keys are gone says anything +// about the GP rule. With results [true, false] - the GP child looks fine until ctrld +// steps aside - the transition must put ctrld's rule back and keep ctrld ownership. +// +// Getting this wrong deletes the last working route and declares external ownership: in +// hard mode a machine-wide DNS outage, because WFP keeps blocking outbound DNS with +// nothing redirecting it to ctrld. +// +// How it comes back matters just as much, and is the #576 owned-lab regression: the +// administrator's child is still on disk, so restoring through addNRPTCatchAllRule leaves +// CtrldCatchAll sitting in the GP store beside it - a machine that started with one GP +// catch-all ends the failed handback with two, both naming the same listener. Only the +// local store may be rewritten here. +func TestHandbackRestoresFallbackWhenGPCannotRoute(t *testing.T) { + // The lab fixture: ctrld holds only its local rule, and the GP store holds only the + // administrator's exact catch-all - ctrld has no GP rule to begin with. + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.tryAdoptMatchingGPNRPT(state); got { + t.Error("tryAdoptMatchingGPNRPT() = true for a GP rule that cannot route without ctrld's rule") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerCtrld: ownership must not move to a rule that failed the post-removal probe", owner) + } + add, remove, _, _ := f.counts() + if remove != 1 { + t.Errorf("removeRule calls = %d, want 1: the handback probe must run with ctrld's keys gone", remove) + } + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1: the ctrld fallback must be restored after the probe fails", f.restoreLocalCount()) + } + if add != 0 { + t.Errorf("addRule calls = %d, want 0: the full writer adds ctrld's GP rule beside the administrator's child", add) + } + if !f.hasCtrldRule() { + t.Error("ctrld's NRPT rule is missing after a failed handback; the host has no working route") + } + if f.hasCtrldGPSibling() { + t.Error("the failed handback left ctrld's rule in the GP store beside the administrator's catch-all") + } +} + +// TestHandbackRestoreDropsAPreexistingGPSibling covers the machine that is already in the +// forbidden state: ctrld wrote its GP rule while only unrelated GP rules existed, and an +// administrator catch-all arrived afterwards. The failed handback must not preserve that +// duplicate on the way out. +// +// It does not, and gets there without a special case. removeRule takes all of ctrld's keys +// for the probe - the local GUID key, the legacy local name and the GP CtrldCatchAll - and +// the local-only write puts back just the first, so the store converges to a single +// catch-all, the administrator's. What is asserted is the shape of the store, not that DNS +// is flowing: this branch exists because the probe failed. +func TestHandbackRestoreDropsAPreexistingGPSibling(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + ctrldGPRule: true, // the duplicate this run inherited + }) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if f.hasCtrldGPSibling() { + t.Error("the inherited GP sibling survived the handback; the store still holds two catch-alls") + } + if !f.hasCtrldRule() { + t.Error("ctrld's local-store rule was not restored; the transition dropped ctrld's own key instead of putting it back") + } +} + +// TestHandbackAcceptsGPWhenItRoutesWithoutCtrld is the other half of the contract: when +// the post-removal probe passes, external policy really is carrying DNS, so ctrld hands +// ownership over and leaves its keys off the machine. +func TestHandbackAcceptsGPWhenItRoutesWithoutCtrld(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, true}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if !p.tryAdoptMatchingGPNRPT(state) { + t.Fatal("tryAdoptMatchingGPNRPT() = false for a GP rule that routes on its own") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}", owner, ruleName) + } + add, remove, _, _ := f.counts() + if remove != 1 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, addRule = %d, restoreLocalRule = %d, want 1/0/0: ctrld's rule must be removed and not restored", remove, add, f.restoreLocalCount()) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is still installed beside the adopted GP catch-all") + } +} + +// TestIneffectiveGPRuleTriggersNoSignalling holds ctrld to #576's contract for a +// present-but-ineffective external rule: warn and retry WFP-only, with no policy +// refresh, no Dnscache paramchange and no cache flush. signalNRPTChange is all three at +// once, so on a permanently dead GP rule a signalling loop would force machine Group +// Policy on a schedule. +func TestIneffectiveGPRuleTriggersNoSignalling(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + + // The verification path a health tick and the heal cycle both take. + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if p.healBlockedLoopbackDNS(state, "test") { + t.Error("healBlockedLoopbackDNS() = true although the probe never succeeds") + } + + add, remove, signal, _ := f.counts() + if signal != 0 { + t.Errorf("signal calls = %d, want 0: ctrld must not force GP refresh, paramchange or a cache flush while external policy owns NRPT", signal) + } + if add != 0 || remove != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, want 0/0/0: external policy must be left untouched", + add, f.restoreLocalCount(), remove) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy { + t.Errorf("owner = %v, want nrptRuleOwnerGroupPolicy: ctrld must not seize a namespace an administrator owns", owner) + } +} + +// TestConcurrentFallbackActivationWritesOnce drives two health paths - the monitor and a +// delayed recheck - into activation at the same moment. Without a transition lock they +// both observe "rule missing" and both write and signal. +func TestConcurrentFallbackActivationWritesOnce(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true}} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + // Hold both callers at the barrier until each is inside activateCtrldNRPTFallback. + start := make(chan struct{}) + var wg sync.WaitGroup + results := make([]bool, 2) + for i := range results { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + results[i] = p.activateCtrldNRPTFallback(state, "concurrent test") + }(i) + } + close(start) + wg.Wait() + + add, _, signal, _ := f.counts() + if add != 1 { + t.Errorf("addRule calls = %d, want 1: concurrent activations must not both write the catch-all", add) + } + if signal != 1 { + t.Errorf("signal calls = %d, want 1: the DNS Client must be signalled once per transition", signal) + } + if results[0] == results[1] { + t.Errorf("both callers reported %v; exactly one transition should report having written the rule", results[0]) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// waitFor polls cond until it holds, failing the test if it never does. +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } +} + +// TestActivationLosingRaceWithStopWritesNothing is the interleaving a revocation check +// alone cannot catch: the activation passes its check, and the stop then revokes the +// state, removes NRPT and finishes before the write lands. Windows would be left routing +// every query to a listener that is gone. +// +// The test asserts the transition lock is load-bearing, in three steps. The barrier sits +// inside addRule, so the stop starts while a transition is mid-write; the stop is then +// shown to be unable to progress past its own NRPT cleanup while that transition holds +// the lock; and once the transition releases, the stop must actually have removed the +// rule before a later activation attempt is refused. Dropping the lock on either side - +// activation or stop cleanup - makes the second step fail, because the stop's removeRule +// then runs while the transition is still in flight. +func TestActivationLosingRaceWithStopWritesNothing(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true}} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + stopDone := make(chan struct{}) + f.beforeAdd = func() { + // Runs with the transition lock held, from the test goroutine. + go func() { + defer close(stopDone) + _ = p.stopDNSIntercept() + }() + + // The stop closes stopCh before it reaches the transition lock, so once that is + // closed the stop can only be waiting on this transition. Waiting on stopCh + // rather than interceptStateRevoked matters: the latter also reports the + // stop-requested flag, which is set before the stop has done anything. + waitFor(t, "the stop to revoke the intercept state", func() bool { + select { + case <-state.stopCh: + return true + default: + return false + } + }) + + // Give it a window in which it would visibly proceed if the lock were not held, + // then prove it did not: no NRPT removal, and the stop has not finished. + time.Sleep(100 * time.Millisecond) + if _, remove, _, _ := f.counts(); remove != 0 { + t.Errorf("stop removed NRPT (removeRule calls = %d) while a transition held the lock", remove) + } + select { + case <-stopDone: + t.Error("stop completed while an NRPT transition was still in flight") + default: + } + } + + if !p.activateCtrldNRPTFallback(state, "activation racing a stop") { + t.Fatal("the in-flight activation should have completed its transition") + } + <-stopDone + + // The stop ran after the transition released the lock, so its cleanup must have + // removed the rule this transition wrote. + if f.hasCtrldRule() { + t.Error("ctrld's NRPT rule survived the stop: the machine is left pointing at a listener that is gone") + } + if _, remove, _, _ := f.counts(); remove != 1 { + t.Errorf("removeRule calls = %d, want 1: the stop must clean up the rule written by the racing transition", remove) + } + if p.dnsInterceptState != nil { + t.Error("the stop did not complete after the in-flight transition released the lock") + } + + // A second attempt models the other health path arriving after the stop finished. + addBefore, _, signalBefore, _ := f.counts() + if p.activateCtrldNRPTFallback(state, "activation after the stop completed") { + t.Error("activateCtrldNRPTFallback() = true after shutdown completed") + } + addAfter, _, signalAfter, _ := f.counts() + if addAfter != addBefore || signalAfter != signalBefore { + t.Errorf("addRule %d->%d, signal %d->%d: a post-shutdown transition wrote NRPT policy for a dead listener", + addBefore, addAfter, signalBefore, signalAfter) + } + if f.hasCtrldRule() { + t.Error("a post-shutdown transition re-created ctrld's NRPT rule") + } +} + +// TestOwnedRecoveryDefersToIneffectiveExternalPolicy is the caller-level case for the +// ownership boundary: state says ctrld-owned, ctrld's key has disappeared, and an exact +// GP child is present but not routing. +// +// The handback records Group Policy ownership and reports Unverified. That is terminal: +// an administrator's catch-all owns the namespace whether or not it currently routes, so +// the owned-recovery flow must not go on to signal the DNS Client or delete and recreate +// ctrld's rule beside it. Only loopback WFP protect is allowed, and it touches no policy. +func TestOwnedRecoveryDefersToIneffectiveExternalPolicy(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, gpRule: "{GP-RULE}", ctrldRule: false} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + add, remove, signal, _ := f.counts() + if add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0/0: owned recovery must stop once external policy owns the namespace", + add, remove, signal, f.restoreLocalCount()) + } + if f.hasCtrldRule() { + t.Error("owned recovery recreated ctrld's catch-all beside the administrator's rule") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}", owner, ruleName) + } +} + +// TestHandbackAbortedWhenRuleCannotBeRemoved keeps a failed registry step from being +// read as a verdict about external policy. +func TestHandbackAbortedWhenRuleCannotBeRemoved(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, true}, + gpRule: "{GP-RULE}", + ctrldRule: true, + removeErr: errors.New("access denied"), + }) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackAborted { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackAborted", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld: a failed removal is not proof about external policy", owner) + } +} + +// TestHandbackRestoresFallbackWhenGPChildDisappearsMidProbe covers a Group Policy refresh +// landing during the post-removal probe: the child ctrld was testing is simply gone. There +// is no external policy left to hand ownership to, so ctrld's rule has to come back. +func TestHandbackRestoresFallbackWhenGPChildDisappearsMidProbe(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("", false) // the administrator removed the catch-all + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule was not restored after the GP child disappeared; the host has no route") + } + // A freed namespace is the one case the full writer is still correct for: with no + // external catch-all left to sit beside, it may place ctrld's GP rule where unrelated + // GP rules would otherwise hide the local store, or drop an empty parent to leave GP + // mode. Narrowing this case to a local-only write would leave ctrld invisible there. + if add, remove, _, _ := f.counts(); add != 1 || remove != 1 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, want 1/0/1", add, f.restoreLocalCount(), remove) + } +} + +// TestHandbackWritesNoSiblingWhenGPChildTurnsConflicting is the case the same-child +// re-read exists for. ctrld removes its keys to test an exact catch-all, and during the +// probe Group Policy replaces it with one that targets another resolver (or a malformed +// one). Restoring ctrld's rule would create exactly the competing sibling beside +// administrator policy that this must never write, so the rule stays off and ownership +// goes to nobody. +func TestHandbackWritesNoSiblingWhenGPChildTurnsConflicting(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + // The child now names another resolver: not a match for this listener, and + // reported as a conflict by the GP classifier. + f.setGPRule("", true) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) + } + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: restoring here writes a competing rule beside administrator policy", add, restoreLocal) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is installed beside a GP catch-all that targets another resolver") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("owner = %v, want nrptRuleOwnerNone: ctrld owns nothing once external policy points elsewhere", owner) + } +} + +// TestHandbackWithoutCtrldRuleClassifiesAfterFailedProbe covers the other short-circuit: +// with no ctrld rule to remove, a failed probe must still be classified before anything is +// recorded. A child that changed into a conflicting catch-all is not an ineffective ctrld +// candidate, and must not be recorded as one. +func TestHandbackWithoutCtrldRuleClassifiesAfterFailedProbe(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 1 { + f.setGPRule("", true) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) + } + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0/0", add, remove, signal, f.restoreLocalCount()) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerNone || ruleName != "" { + t.Errorf("owner = %v, rule = %q, want nrptRuleOwnerNone/\"\": a conflicting child must not be recorded as ctrld's external owner", owner, ruleName) + } +} + +// TestHandbackGivesReplacementChildItsOwnPass keeps an unproved child from inheriting a +// verdict that was measured against a different one. With no ctrld rule installed the +// replacement is simply probed on its own pass; failing that pass records it as external +// policy that is not routing, and writes nothing either way. +func TestHandbackGivesReplacementChildItsOwnPass(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false, false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 1 { + f.setGPRule("{NEW-GP-RULE}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0/0", add, remove, signal, f.restoreLocalCount()) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{NEW-GP-RULE}: the verdict must name the child it was measured against", owner, ruleName) + } +} + +// TestHandbackWritesNoSiblingWhenReplacementChildTakesOver is the case where ctrld's rule +// was removed for the probe and a *different* exact catch-all took the namespace while it +// was off. +// +// Restoring ctrld's rule here is not a return to the status quo: the replacement was not +// there when the transition started, and addNRPTCatchAllRule writes ctrld's own GP +// catch-all whenever another GP rule exists, so the restore would plant a sibling beside +// the administrator's new rule. The replacement gets its own probe instead, and a failed +// one leaves NRPT to Group Policy with nothing of ctrld's written. +func TestHandbackWritesNoSiblingWhenReplacementChildTakesOver(t *testing.T) { + // [pre-probe true, post-removal probe false, replacement's own pass false] + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("{NEW-GP-RULE}", false) // a policy refresh swaps the child + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if add, remove, _, _ := f.counts(); add != 0 || remove != 1 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, want 0/0/1: ctrld's rule must not be restored beside a replacement catch-all", + add, f.restoreLocalCount(), remove) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is installed beside an administrator catch-all that took the namespace") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{NEW-GP-RULE}", owner, ruleName) + } +} + +// TestHandbackAdoptsReplacementChildThatRoutes is the same swap where the replacement does +// carry DNS on its own: it is adopted, still with no ctrld write. +func TestHandbackAdoptsReplacementChildThatRoutes(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, true}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("{NEW-GP-RULE}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackVerified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackVerified", got) + } + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0", add, restoreLocal) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{NEW-GP-RULE}", owner, ruleName) + } +} + +// TestHandbackChurnFailsSafeToCurrentExternalOwner bounds the extra pass without letting +// exhaustion become a licence to write. +// +// When the probe budget runs out with the store still changing, the transition classifies +// the store as it stands and proves no route. Reporting "undecided" instead would be +// unsafe: that disposition is not terminal external ownership, so startup and owned +// recovery fall through to writing ctrld's rule, and addNRPTCatchAllRule puts it in the GP +// path beside whichever exact catch-all is there now. +func TestHandbackChurnFailsSafeToCurrentExternalOwner(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("{GP-RULE-3}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test") + if got != nrptHandbackUnverified { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) + } + if !nrptExternalOwns(got) { + t.Error("the churn disposition is not terminal external ownership; callers would fall through and write a sibling") + } + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: no sibling write while the GP store is churning", add, restoreLocal) + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE-3}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE-3}: exhaustion must record the store as it stands", owner, ruleName) + } +} + +// TestHandbackChurnFailsSafeToConflict covers exhaustion where the store has settled on a +// catch-all that does not target ctrld: still terminal, still no write. +func TestHandbackChurnFailsSafeToConflict(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("", true) // now an administrator catch-all for another resolver + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) + } + if add, _, _, _ := f.counts(); add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, want 0/0", add, f.restoreLocalCount()) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("owner = %v, want nrptRuleOwnerNone", owner) + } +} + +// TestHandbackChurnRestoresFallbackWhenNamespaceFreed is the third exhaustion outcome: the +// store ended up with no external catch-all at all, so the namespace is free and ctrld's +// rule is the one that belongs there. +func TestHandbackChurnRestoresFallbackWhenNamespaceFreed(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("", false) // the administrator removed the policy entirely + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule was not restored although no external catch-all remains") + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestActivationWritesNoSiblingWhileGPStoreChurns is the caller-level closure for the churn +// path. activateCtrldNRPTFallback is the function both startup fallback and owned recovery +// reach when ctrld's rule is missing, and it is where a non-terminal churn disposition +// would turn into addNRPTCatchAllRule writing a GP sibling beside the current catch-all. +func TestActivationWritesNoSiblingWhileGPStoreChurns(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false, false}, gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + f.onProbe = func(call int) { + switch call { + case 1: + f.setGPRule("{GP-RULE-2}", false) + case 2: + f.setGPRule("{GP-RULE-3}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + // Startup and the owner-None retry both arrive here with no ownership recorded. + state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") + + if p.activateCtrldNRPTFallback(state, "churn test") { + t.Error("activateCtrldNRPTFallback() = true while an external catch-all owns the namespace") + } + add, remove, signal, _ := f.counts() + if add != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0: ctrld must not write beside the administrator's catch-all", add, signal, f.restoreLocalCount()) + } + if remove != 0 { + t.Errorf("removeRule calls = %d, want 0: there was no ctrld rule to remove", remove) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy { + t.Errorf("owner = %v, want nrptRuleOwnerGroupPolicy: the current external catch-all owns the namespace", owner) + } +} + +// TestStopLeavesGPManagedPolicyAlone is the central promise of this work: an +// administrator's catch-all is a deployment contract that must survive service stop, +// restart and uninstall. A regression that removed NRPT for every owner would delete that +// rule and take the fleet's DNS policy with it. +func TestStopLeavesGPManagedPolicyAlone(t *testing.T) { + f := &fakeNRPTOps{gpRule: "{GP-RULE}"} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + add, remove, signal, _ := f.counts() + if remove != 0 || signal != 0 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, restoreLocalRule = %d, want 0/0/0/0: shutdown must not touch externally owned NRPT policy", + remove, signal, add, f.restoreLocalCount()) + } + if f.flushCount() != 0 { + t.Errorf("flush calls = %d, want 0: no cache flush is owed for a rule ctrld does not own", f.flushCount()) + } + if p.dnsInterceptState != nil { + t.Error("dnsInterceptState survived shutdown") + } +} + +// TestStopRemovesOrphanWhileLeavingGPPolicy is the same shutdown with a ctrld rule left +// behind by an earlier unclean exit. External policy still stays, but the orphan must go: +// while GP mode hides the local store this stop is the last chance anything looks there. +func TestStopRemovesOrphanWhileLeavingGPPolicy(t *testing.T) { + f := &fakeNRPTOps{gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + + if err := p.stopDNSIntercept(); err != nil { + t.Fatalf("stopDNSIntercept() = %v", err) + } + + add, remove, signal, _ := f.counts() + if remove != 1 || signal != 1 { + t.Errorf("removeRule = %d, signal = %d, want 1/1: the orphaned ctrld rule must be removed on the way out", remove, signal) + } + if add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0", add, f.restoreLocalCount()) + } + if f.hasCtrldRule() { + t.Error("the orphaned ctrld rule survived shutdown; a later GP removal would activate it") + } + if f.flushCount() != 0 { + t.Errorf("flush calls = %d, want 0: the ctrld-owned branch is what owes a flush, not the GP branch", f.flushCount()) + } +} + +// TestRemoveOrphanedCtrldNRPTRule covers the sweep directly, including that it is a no-op +// when there is nothing of ctrld's on disk - it runs on shutdown paths where the common +// case is a clean machine. +func TestRemoveOrphanedCtrldNRPTRule(t *testing.T) { + t.Run("removes and signals when a ctrld rule exists", func(t *testing.T) { + f := &fakeNRPTOps{ctrldRule: true} + installFakeNRPTOps(t, f) + + p, _, _ := newHandbackTestProg(t) + p.removeOrphanedCtrldNRPTRule("test") + + add, remove, signal, _ := f.counts() + if remove != 1 || signal != 1 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, restoreLocalRule = %d, want 1/1/0/0", remove, signal, add, f.restoreLocalCount()) + } + if f.hasCtrldRule() { + t.Error("the ctrld rule is still installed") + } + }) + + t.Run("does nothing when no ctrld rule exists", func(t *testing.T) { + f := &fakeNRPTOps{} + installFakeNRPTOps(t, f) + + p, _, _ := newHandbackTestProg(t) + p.removeOrphanedCtrldNRPTRule("test") + + add, remove, signal, _ := f.counts() + if remove != 0 || signal != 0 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, restoreLocalRule = %d, want 0/0/0/0: nothing to sweep must cost no registry writes and no signalling", + remove, signal, add, f.restoreLocalCount()) + } + }) +} + +// TestHandbackThrottleBlocksTheSecondAttempt exercises the throttle through the +// transition, not just its accessor. Each attempt removes the live catch-all for a probe, +// so a second attempt inside the window must cost nothing at all: no probe, no removal. +func TestHandbackThrottleBlocksTheSecondAttempt(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "first"); got != nrptHandbackKeptCtrld { + t.Fatalf("first nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + _, removeAfterFirst, _, probesAfterFirst := f.counts() + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "second"); got != nrptHandbackAborted { + t.Fatalf("second nrptHandbackToExternal() = %v, want nrptHandbackAborted", got) + } + _, remove, _, probes := f.counts() + if probes != probesAfterFirst { + t.Errorf("probe calls went from %d to %d: a throttled attempt must not probe", probesAfterFirst, probes) + } + if remove != removeAfterFirst { + t.Errorf("removeRule calls went from %d to %d: a throttled attempt must not remove the live rule", removeAfterFirst, remove) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestHealLadderRunsRetriesThenTwoPhaseRecovery drives the ctrld-owned heal cycle end to +// end: the immediate probe, the signal-and-backoff retries, then the two-phase delete and +// re-add, with the last probe finally succeeding. Each nrptTransition phase is asserted +// through its side effects, since these are its only call sites. +func TestHealLadderRunsRetriesThenTwoPhaseRecovery(t *testing.T) { + // Every probe fails until the one after the re-add. + f := &fakeNRPTOps{ + probeResults: []bool{false, false, false, false, true}, + ctrldRule: true, + } + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + add, remove, signal, probes := f.counts() + if remove != 1 { + t.Errorf("removeRule calls = %d, want 1: phase one of the two-phase recovery deletes ctrld's rule once", remove) + } + if add != 1 { + t.Errorf("addRule calls = %d, want 1: phase two re-adds it once", add) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule is missing after the two-phase recovery") + } + // Three backoff rounds signal, then the delete phase and the re-add phase. + if signal < 5 { + t.Errorf("signal calls = %d, want at least 5 (three retries plus both recovery phases)", signal) + } + if probes < 5 { + t.Errorf("probe calls = %d, want at least 5", probes) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestHealLadderStopsWhenGPAppearsBeforeTwoPhase covers a Group Policy catch-all landing +// during the backoff retries. The ladder must hand back and stop before the destructive +// delete-and-re-add phase, and must not write ctrld's rule beside the new policy. +func TestHealLadderStopsWhenGPAppearsBeforeTwoPhase(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false}, ctrldRule: true} + installFakeNRPTOps(t, f) + // The administrator's rule appears while the fourth probe is in flight, which is the + // last one before the two-phase recovery. + f.onProbe = func(call int) { + if call == 4 { + f.setGPRule("{GP-RULE}", false) + } + } + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + add, remove, _, _ := f.counts() + if remove != 0 { + t.Errorf("removeRule calls = %d, want 0: the ladder must stop before the delete half of the two-phase recovery", remove) + } + if add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: the ladder must not recreate ctrld's rule beside a new GP catch-all", add, f.restoreLocalCount()) + } + if !f.hasCtrldRule() { + t.Error("ctrld's rule was deleted after an administrator catch-all appeared") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}: the ladder must defer rather than continue", owner, ruleName) + } +} + +// TestHealLadderCleansEmptyGPParentBeforeRetrying pins the empty-GP-parent shortcut: an +// empty parent key puts the DNS Client in GP mode where ctrld's local rule is invisible, so +// signalling retries cannot succeed until it is deleted. +func TestHealLadderCleansEmptyGPParentBeforeRetrying(t *testing.T) { + f := &fakeNRPTOps{probeResults: []bool{false, true}, ctrldRule: true, parentEmpty: true} + installFakeNRPTOps(t, f) + + p, state, _ := newHandbackTestProg(t) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + p.nrptProbeAndHeal(state) + + if f.cleanCalls != 1 { + t.Errorf("cleanParent calls = %d, want 1: the empty GP parent must be removed before burning retries", f.cleanCalls) + } + if add, remove, _, _ := f.counts(); add != 0 || remove != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, restoreLocalRule = %d, want 0/0/0: the probe passed after the cleanup, so no recovery was needed", add, remove, f.restoreLocalCount()) + } +} + +// TestRepairMissingWFPRetriesHardModeWithNoEngine covers the entry point a startup WFP +// failure depends on. In hard mode a closed engine means nothing is being blocked, so the +// health monitor must keep retrying; in dns mode a closed engine is the normal state and +// must not trigger anything. +func TestRepairMissingWFPRetriesHardModeWithNoEngine(t *testing.T) { + originalRebuild, originalHard := rebuildDNSInterceptFn, hardIntercept + t.Cleanup(func() { rebuildDNSInterceptFn, hardIntercept = originalRebuild, originalHard }) + + var reasons []string + rebuildDNSInterceptFn = func(_ *prog, _ *wfpState, reason string) interceptRebuildResult { + reasons = append(reasons, reason) + return interceptRebuildDone + } + + t.Run("hard mode retries when the engine never opened", func(t *testing.T) { + reasons = nil + hardIntercept = true + p, state, _ := newHandbackTestProg(t) + + if !p.repairMissingWFP(state) { + t.Error("repairMissingWFP() = false; the monitor must hand over to the rebuilt intercept") + } + if len(reasons) != 1 { + t.Fatalf("rebuild requests = %d, want 1: hard mode with no WFP engine must be retried", len(reasons)) + } + if !strings.Contains(reasons[0], "no WFP engine") { + t.Errorf("rebuild reason = %q, want it to name the missing engine", reasons[0]) + } + }) + + t.Run("dns mode leaves a closed engine alone", func(t *testing.T) { + reasons = nil + hardIntercept = false + p, state, _ := newHandbackTestProg(t) + + if p.repairMissingWFP(state) { + t.Error("repairMissingWFP() = true in dns mode; a closed engine is normal there") + } + if len(reasons) != 0 { + t.Errorf("rebuild requests = %d, want 0", len(reasons)) + } + }) +} + +// TestPhaseTwoRevalidatesOwnershipAfterWaitingForTheLock is the ordering a pre-lock check +// cannot cover. Phase one removes ctrld's rule and releases the transition lock; the +// pre-re-add checks see no GP rule; another health path then takes the lock, hands the +// namespace to a GP catch-all that has just arrived, and records GroupPolicy. Phase two +// acquires the lock afterwards. +// +// Without a re-read inside the locked closure, phase two re-adds ctrld's rule beside the +// administrator's catch-all and stamps ctrld ownership over theirs - exactly the competing +// policy this work exists to prevent. +func TestPhaseTwoRevalidatesOwnershipAfterWaitingForTheLock(t *testing.T) { + p, state, f := newHandbackTestProg(t) + // Every probe fails, so the ladder runs to the two-phase recovery. + f.configure(fakeNRPTSeed{probeResults: []bool{false}, ctrldRule: true}) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + // The last ops call before phase two queues for the lock. Hold the lock from another + // goroutine and complete the competing transition while phase two waits for it. + f.onGPConflicts = func(call int) { + if call != 1 { + return + } + locked := make(chan struct{}) + go func() { + p.nrptTransitionMu.Lock() + close(locked) + f.setGPRule("{GP-RULE}", false) + state.setNRPTPolicyOwner(nrptRuleOwnerGroupPolicy, "{GP-RULE}") + // Hold it long enough that phase two must queue behind this transition. + time.Sleep(100 * time.Millisecond) + p.nrptTransitionMu.Unlock() + }() + <-locked + } + + p.nrptProbeAndHeal(state) + + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: phase two re-added ctrld's rule beside a GP catch-all that took ownership while it waited for the lock", add, restoreLocal) + } + if f.hasCtrldRule() { + t.Error("ctrld's rule is installed beside the administrator's catch-all") + } + owner, ruleName := state.nrptPolicyOwner() + if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}: phase two overwrote a newer ownership transition", owner, ruleName) + } +} + +// TestRetiredHealDoesNotDeleteSuccessorRule covers the other side of a rebuild. The heal +// cycle re-adds ctrld's rule, then its final settle wait observes that its own state was +// retired - while a successor intercept is already running with the same deterministic key. +// +// The key is process-global, so any cleanup from here would delete the successor's route +// and leave hard mode blocking DNS with nothing redirecting it. Teardown of the retired +// state owns that cleanup instead. +func TestRetiredHealDoesNotDeleteSuccessorRule(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{probeResults: []bool{false}, ctrldRule: true}) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + successor := &wfpState{stopCh: make(chan struct{}), listenerIP: "127.0.0.1"} + // Retire this cycle's state during the final wait - the one after the re-add, which is + // the only wait where both a removal and an add have happened - and publish a successor + // that owns the same deterministic key. + f.waitHook = func() bool { + if add, remove, _, _ := f.counts(); add == 0 || remove == 0 { + return true + } + close(state.stopCh) // a rebuild retired the old state + p.dnsInterceptState = successor + successor.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + return false + } + + p.nrptProbeAndHeal(state) + + if !f.hasCtrldRule() { + t.Error("the successor's NRPT rule was deleted by a retired heal cycle") + } + if owner, _ := successor.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("successor owner = %v, want nrptRuleOwnerCtrld", owner) + } +} + +// TestStartupReportsFailureWhenExternalPolicyNeverRoutes is the readiness direction of the +// GP-managed contract, driven through the real startup control flow. +// +// An exact GP child is present and no probe ever reaches ctrld. Startup must keep the +// recovery state and the monitor - the rule may start routing later, and only external +// policy can fix it - must leave adapter DNS alone, and must not write an owned fallback. +// What it must not do is return success: setDNS records readiness from a nil error, so +// reporting ready here publishes "service healthy" while the DNS Client is not delivering +// queries to ctrld, and in hard mode WFP is blocking every other resolver at the same time. +func TestStartupReportsFailureWhenExternalPolicyNeverRoutes(t *testing.T) { + originalCfg, originalMode, originalIntercept, originalHard := cfg, interceptMode, dnsIntercept, hardIntercept + t.Cleanup(func() { + cfg, interceptMode, dnsIntercept, hardIntercept = originalCfg, originalMode, originalIntercept, originalHard + }) + cfg = ctrld.Config{} + cfg.Listener = map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + interceptMode, dnsIntercept, hardIntercept = "dns", true, false + + p, _, f := newHandbackTestProg(t) + p.cfg = &cfg + p.dnsInterceptState = nil // startup publishes its own state + // An exact GP child that never answers a probe. + f.configure(fakeNRPTSeed{probeResults: []bool{false}, gpRule: "{GP-RULE}"}) + + err := p.startDNSInterceptLocked() + if err == nil { + t.Fatal("startDNSInterceptLocked() = nil: startup reported success while no probe reached ctrld, so setDNS would mark the service ready") + } + if !interceptFailedUnderExternalDNSPolicy(err) { + t.Errorf("err = %v; the failure must tell setDNS to leave adapter DNS untouched", err) + } + if interceptFailedWithVerifiedExternalDNS(err) { + t.Errorf("err = %v; this route was never verified, so it must not read as the verified case", err) + } + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, signal = %d, want 0/0/0/0: no owned fallback may be written beside an administrator catch-all", + add, f.restoreLocalCount(), remove, signal) + } + + // Recovery must stay live so the rule can be re-tested. + state, ok := p.dnsInterceptState.(*wfpState) + if !ok || state == nil { + t.Fatal("intercept state was not published; nothing would keep re-testing the external rule") + } + if owner, ruleName := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE}" { + t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}", owner, ruleName) + } + // Stop the monitor startup launched. + _ = p.stopDNSIntercept() +} + +// TestFakeNRPTOpsPreconditionDetectsBypass guards the guard. The fixtures' promise is that +// no test can reach the production registry and signalling functions, and that promise is +// only as good as this precondition: if it stops detecting a bypass, every "no side +// effects" assertion in the Windows suite silently becomes vacuous while the real host +// policy is at risk. +func TestFakeNRPTOpsPreconditionDetectsBypass(t *testing.T) { + f := &fakeNRPTOps{} + + // The exact bypass to catch: a fixture that returns a fake it never installed. + if err := checkFakeNRPTOpsInstalled(f); err == nil { + t.Error("an uninstalled fake passed the precondition; a test could fall through to production NRPT operations") + } + + installFakeNRPTOps(t, f) + if err := checkFakeNRPTOpsInstalled(f); err != nil { + t.Errorf("an installed fake failed the precondition: %v", err) + } + + // A different fake than the installed one must not pass either: side-effect counters + // would be read from an object nothing consults. + if err := checkFakeNRPTOpsInstalled(&fakeNRPTOps{}); err == nil { + t.Error("a fake that is not the installed one passed the precondition") + } +} + +// startupInterceptMode puts the globals startDNSInterceptLocked reads into dns mode with a +// single loopback listener, and puts them back afterwards. +func startupInterceptMode(t *testing.T) { + t.Helper() + originalCfg, originalMode, originalIntercept, originalHard := cfg, interceptMode, dnsIntercept, hardIntercept + t.Cleanup(func() { + cfg, interceptMode, dnsIntercept, hardIntercept = originalCfg, originalMode, originalIntercept, originalHard + }) + cfg = ctrld.Config{} + cfg.Listener = map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + interceptMode, dnsIntercept, hardIntercept = "dns", true, false +} + +// TestHandbackAbortsWhenTheLocalWriteFails covers the write failure on the put-back path. +// Nothing else exercises it: every other test lets restoreLocalRule succeed. +// +// The rule is gone by then - removeRule ran before the probe - so a failed put-back leaves +// the host with no ctrld route at all. What the transition owes is an accurate report: +// ownership goes back to none so the health monitor treats it as unowned and retries, and +// no signal is sent for a change that did not happen. +func TestHandbackAbortsWhenTheLocalWriteFails(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + addErr: errors.New("registry write failed"), + }) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "local write fails"); got != nrptHandbackAborted { + t.Errorf("nrptHandbackToExternal() = %v, want nrptHandbackAborted", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerNone: a failed put-back owns nothing", owner) + } + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1", f.restoreLocalCount()) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: the failing writer must not be retried through the full writer", add) + } + if f.hasCtrldRule() { + t.Error("ctrld's local rule reads as present after the write that was supposed to create it failed") + } + if f.hasCtrldGPSibling() { + t.Error("a failed put-back left ctrld's rule in the GP store") + } +} + +// TestHandbackKeepsCtrldWhenTheStoreReturnsToTheInitialRule covers the spent-budget branch +// of externalAfterRemoval when the churn ends where it began. +// +// The three existing churn tests all end somewhere else - a third rule, a rule targeting +// another resolver, or a free namespace - so none of them reaches this line with the +// original rule name still current. That combination selects the writer: the class is +// gpChildSameExact, the administrator's catch-all is on disk, and only the local key may +// be written. +func TestHandbackKeepsCtrldWhenTheStoreReturnsToTheInitialRule(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + }) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("{GP-RULE}", false) // back to the rule the transition started on + } + } + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "churn returns to the initial rule"); got != nrptHandbackKeptCtrld { + t.Errorf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerCtrld", owner) + } + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1", f.restoreLocalCount()) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: the spent-budget path must not write a sibling either", add) + } + if f.hasCtrldGPSibling() { + t.Error("the spent-budget path left ctrld's rule in the GP store beside the administrator's catch-all") + } +} + +// TestGPChildGoneWritesTheGPRuleWhenOtherGPRulesRemain is the counterpart to the +// no-sibling tests: with the administrator's catch-all gone but unrelated GP rules still +// present, the full writer is correct and must place ctrld's GP rule, or DNS Client's GP +// mode hides the local store and ctrld routes nothing. +func TestGPChildGoneWritesTheGPRuleWhenOtherGPRulesRemain(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + otherGPRules: true, // unrelated GP policy that keeps DNS Client in GP mode + }) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("", false) // the administrator's catch-all disappears mid-probe + } + } + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "child gone with other GP rules"); got != nrptHandbackKeptCtrld { + t.Errorf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if add, _, _, _ := f.counts(); add != 1 { + t.Errorf("addRule calls = %d, want 1: a free namespace is what the full writer is for", add) + } + if f.restoreLocalCount() != 0 { + t.Errorf("restoreLocalRule calls = %d, want 0", f.restoreLocalCount()) + } + if !f.hasCtrldGPSibling() { + t.Error("ctrld's GP rule is missing while unrelated GP rules keep DNS Client in GP mode; the local store is hidden and nothing routes") + } +} + +// TestStartupKeepsCtrldRuleWhenExternalPolicyCannotRoute is the owned-lab regression as +// startup runs it: a service start with ctrld's rule from an earlier run and a matching +// administrator catch-all that cannot carry DNS alone. +// +// The existing startup test has no ctrld rule and a single failing probe, so it never +// reaches the put-back. This one does, and the assertion that matters is which writer ran. +// +// Ownership is deliberately not asserted: startup launches nrptProbeAndHeal, whose own +// handback is throttled and whose undecided branch moves the owner to Group Policy within +// seconds, so any owner assertion here is a race. +func TestStartupKeepsCtrldRuleWhenExternalPolicyCannotRoute(t *testing.T) { + startupInterceptMode(t) + + p, _, f := newHandbackTestProg(t) + p.cfg = &cfg + p.dnsInterceptState = nil // startup publishes its own state + f.configure(fakeNRPTSeed{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true}) + + if err := p.startDNSInterceptLocked(); err != nil { + t.Fatalf("startDNSInterceptLocked() = %v, want nil: ctrld's own route was put back, so startup succeeded", err) + } + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1: startup must put back only the local key", f.restoreLocalCount()) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: the full writer would place ctrld's GP rule beside the administrator's", add) + } + if !f.hasCtrldRule() { + t.Error("ctrld's local rule is missing after startup; the host has no route") + } + if f.hasCtrldGPSibling() { + t.Error("startup left ctrld's rule in the GP store beside the administrator's catch-all") + } + _ = p.stopDNSIntercept() +} + +// TestStartupDropsAnInheritedGPSibling covers the machine that boots already holding the +// forbidden pair: ctrld's GP rule from an earlier run beside an administrator catch-all. +// +// Adoption used to inherit it. Nothing downstream removes it: every handback from here +// stops at the pre-probe while the administrator's rule stays unproven, the monitor calls +// removeRule only inside a handback, and the orphan sweep runs at stop. The second +// catch-all would therefore stand for the life of the process, through the very outage the +// operator is trying to diagnose. +// +// The GP-key-only case is the one that proves the cleanup is not self-defeating. Dropping +// the sibling empties the store of ctrld keys, so startup has to write its rule after all - +// and the full writer would put the sibling straight back, beside the rule it was just +// removed from. +func TestStartupDropsAnInheritedGPSibling(t *testing.T) { + for _, tc := range []struct { + name string + ctrldRule bool + // wantRestoreLocal is whether startup had to write ctrld's rule itself, which it + // may only ever do into the local store while an administrator rule is present. + wantRestoreLocal int + }{ + {name: "local and GP keys", ctrldRule: true, wantRestoreLocal: 0}, + {name: "GP key only", ctrldRule: false, wantRestoreLocal: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + startupInterceptMode(t) + + p, _, f := newHandbackTestProg(t) + p.cfg = &cfg + p.dnsInterceptState = nil + f.configure(fakeNRPTSeed{ + probeResults: []bool{false}, // the pre-probe never answers, so the handback decides nothing + gpRule: "{GP-RULE}", + ctrldRule: tc.ctrldRule, + ctrldGPRule: true, + }) + + err := p.startDNSInterceptLocked() + if f.hasCtrldGPSibling() { + t.Error("startup left ctrld's rule in the GP store beside the administrator's catch-all; nothing later removes it") + } + if f.dropGPCount() != 1 { + t.Errorf("dropGPRule calls = %d, want 1", f.dropGPCount()) + } + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != tc.wantRestoreLocal { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/%d: the full writer re-creates the sibling", + add, restoreLocal, tc.wantRestoreLocal) + } + if !f.hasCtrldRule() { + t.Error("no ctrld rule after startup; the host has no route") + } + if err == nil { + _ = p.stopDNSIntercept() + } + }) + } +} + +// TestHandbackPutBackSettlesToGroupPolicyOwnership pins the registry and ownership state a +// failed handback actually leaves behind, which is what QA will find on the machine. +// +// After the put-back the health monitor keeps running: its handback is throttled, so the +// heal reaches deferToExternalCatchAll undecided and records Group Policy ownership. The +// end state is ctrld's local key on disk, the administrator's rule on disk, no sibling, +// and no further writes - and stop then removes the local key as an orphan. +func TestHandbackPutBackSettlesToGroupPolicyOwnership(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true}) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "probe finds no route"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + _, _, signalBefore, _ := f.counts() + restoreBefore := f.restoreLocalCount() + + p.nrptProbeAndHeal(state) + + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerGroupPolicy: the throttled heal ends undecided and defers", owner) + } + addAfter, _, signalAfter, _ := f.counts() + // Absolute, not "unchanged": the whole sequence must never reach the full writer, + // so a first call that had already happened would be a defect too. + if addAfter != 0 { + t.Errorf("addRule calls = %d, want 0: neither the handback nor the heal may write a sibling", addAfter) + } + if signalAfter != signalBefore { + t.Errorf("signal calls = %d, want %d: nothing changed, so nothing is signalled", signalAfter, signalBefore) + } + if f.restoreLocalCount() != restoreBefore { + t.Errorf("restoreLocalRule calls = %d, want %d: the rule is already back", f.restoreLocalCount(), restoreBefore) + } + if !f.hasCtrldRule() { + t.Error("ctrld's local key is missing; the put-back did not survive the heal") + } + if f.hasCtrldGPSibling() { + t.Error("the heal wrote ctrld's rule into the GP store") + } + + _, removeBefore, _, _ := f.counts() + _ = p.stopDNSIntercept() + if _, removeAfter, _, _ := f.counts(); removeAfter != removeBefore+1 { + t.Errorf("removeRule calls = %d, want %d: stop must sweep the local key it left on disk", removeAfter, removeBefore+1) + } + if f.hasCtrldRule() { + t.Error("stop left ctrld's local key behind") + } +} diff --git a/cmd/cli/nrpt_recovery_limiter_windows.go b/cmd/cli/nrpt_recovery_limiter_windows.go new file mode 100644 index 00000000..64b6526a --- /dev/null +++ b/cmd/cli/nrpt_recovery_limiter_windows.go @@ -0,0 +1,101 @@ +//go:build windows + +package cli + +import ( + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +const ( + // Default to current behavior: keep recovering indefinitely unless configured. + defaultNRPTRecoveryMaxAttempts = 0 + defaultNRPTRecoveryCooldown = 30 * time.Minute + + // Require more than one good health tick before clearing the circuit. A probe can + // pass briefly after delete/re-add even when another agent recreates broken NRPT state. + nrptRecoveryStableSuccessesToReset = 2 +) + +type nrptRecoveryLimiter struct { + mu sync.Mutex + attempts int + stableSuccesses int + cooldownUntil time.Time + lastSkipLog time.Time +} + +func nrptRecoveryMaxAttempts(cfg *ctrld.Config) int { + if cfg != nil && cfg.Service.NRPTRecoveryMaxAttempts != nil { + return *cfg.Service.NRPTRecoveryMaxAttempts + } + return defaultNRPTRecoveryMaxAttempts +} + +func nrptRecoveryCooldown(cfg *ctrld.Config) time.Duration { + if cfg != nil && cfg.Service.NRPTRecoveryCooldown != nil { + return *cfg.Service.NRPTRecoveryCooldown + } + return defaultNRPTRecoveryCooldown +} + +func (l *nrptRecoveryLimiter) allow(now time.Time, cfg *ctrld.Config) (bool, time.Duration) { + maxAttempts := nrptRecoveryMaxAttempts(cfg) + if maxAttempts <= 0 { + return true, 0 + } + + l.mu.Lock() + defer l.mu.Unlock() + + if now.Before(l.cooldownUntil) { + return false, l.cooldownUntil.Sub(now) + } + return true, 0 +} + +func (l *nrptRecoveryLimiter) recordRecoveryFlow(now time.Time, cfg *ctrld.Config) { + maxAttempts := nrptRecoveryMaxAttempts(cfg) + if maxAttempts <= 0 { + return + } + + cooldown := nrptRecoveryCooldown(cfg) + if cooldown <= 0 { + cooldown = defaultNRPTRecoveryCooldown + } + + l.mu.Lock() + defer l.mu.Unlock() + + l.stableSuccesses = 0 + l.attempts++ + if l.attempts >= maxAttempts { + l.cooldownUntil = now.Add(cooldown) + } +} + +func (l *nrptRecoveryLimiter) recordStableSuccess() { + l.mu.Lock() + defer l.mu.Unlock() + + l.stableSuccesses++ + if l.stableSuccesses >= nrptRecoveryStableSuccessesToReset { + l.attempts = 0 + l.cooldownUntil = time.Time{} + l.lastSkipLog = time.Time{} + } +} + +func (l *nrptRecoveryLimiter) shouldLogSkip(now time.Time) bool { + l.mu.Lock() + defer l.mu.Unlock() + + if l.lastSkipLog.IsZero() || now.Sub(l.lastSkipLog) >= 5*time.Minute { + l.lastSkipLog = now + return true + } + return false +} diff --git a/cmd/cli/nrpt_recovery_limiter_windows_test.go b/cmd/cli/nrpt_recovery_limiter_windows_test.go new file mode 100644 index 00000000..d0974a54 --- /dev/null +++ b/cmd/cli/nrpt_recovery_limiter_windows_test.go @@ -0,0 +1,74 @@ +//go:build windows + +package cli + +import ( + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +func TestNRPTRecoveryLimiterCooldownAndStableReset(t *testing.T) { + maxAttempts := 2 + cooldown := 10 * time.Minute + cfg := &ctrld.Config{} + cfg.Service.NRPTRecoveryMaxAttempts = &maxAttempts + cfg.Service.NRPTRecoveryCooldown = &cooldown + + limiter := &nrptRecoveryLimiter{} + now := time.Unix(100, 0) + + if ok, wait := limiter.allow(now, cfg); !ok || wait != 0 { + t.Fatalf("initial allow = %v, %v; want true, 0", ok, wait) + } + + limiter.recordRecoveryFlow(now, cfg) + if ok, wait := limiter.allow(now.Add(time.Second), cfg); !ok || wait != 0 { + t.Fatalf("allow after first flow = %v, %v; want true, 0", ok, wait) + } + + limiter.recordRecoveryFlow(now.Add(2*time.Second), cfg) + if ok, wait := limiter.allow(now.Add(3*time.Second), cfg); ok || wait <= 0 { + t.Fatalf("allow after max flows = %v, %v; want false, positive wait", ok, wait) + } + + limiter.recordStableSuccess() + if ok, _ := limiter.allow(now.Add(4*time.Second), cfg); ok { + t.Fatal("one stable success cleared cooldown; want cooldown to remain") + } + + limiter.recordStableSuccess() + if ok, wait := limiter.allow(now.Add(5*time.Second), cfg); !ok || wait != 0 { + t.Fatalf("allow after stable reset = %v, %v; want true, 0", ok, wait) + } +} + +func TestNRPTRecoveryLimiterDefaultIsUnlimited(t *testing.T) { + cfg := &ctrld.Config{} + limiter := &nrptRecoveryLimiter{} + now := time.Unix(100, 0) + + for i := 0; i < 10; i++ { + limiter.recordRecoveryFlow(now.Add(time.Duration(i)*time.Second), cfg) + } + if ok, wait := limiter.allow(now.Add(time.Hour), cfg); !ok || wait != 0 { + t.Fatalf("default allow after recovery flows = %v, %v; want true, 0", ok, wait) + } +} + +func TestNRPTRecoveryLimiterUnlimited(t *testing.T) { + maxAttempts := 0 + cfg := &ctrld.Config{} + cfg.Service.NRPTRecoveryMaxAttempts = &maxAttempts + + limiter := &nrptRecoveryLimiter{} + now := time.Unix(100, 0) + + for i := 0; i < 10; i++ { + limiter.recordRecoveryFlow(now.Add(time.Duration(i)*time.Second), cfg) + } + if ok, wait := limiter.allow(now.Add(time.Hour), cfg); !ok || wait != 0 { + t.Fatalf("unlimited allow = %v, %v; want true, 0", ok, wait) + } +} diff --git a/cmd/cli/os_darwin.go b/cmd/cli/os_darwin.go index 4c358b0e..7421aee9 100644 --- a/cmd/cli/os_darwin.go +++ b/cmd/cli/os_darwin.go @@ -8,26 +8,31 @@ import ( "os/exec" "strings" - "github.com/Control-D-Inc/ctrld/internal/resolvconffile" + "github.com/Control-D-Inc/ctrld" ) -// allocate loopback ip +// allocateIP allocates an IP address on the specified interface // sudo ifconfig lo0 alias 127.0.0.2 up func allocateIP(ip string) error { + mainLog.Load().Debug().Str("ip", ip).Msg("Allocating IP address") cmd := exec.Command("ifconfig", "lo0", "alias", ip, "up") if err := cmd.Run(); err != nil { - mainLog.Load().Error().Err(err).Msg("allocateIP failed") + mainLog.Load().Error().Err(err).Msg("AllocateIP failed") return err } + mainLog.Load().Debug().Str("ip", ip).Msg("IP address allocated successfully") return nil } +// deAllocateIP deallocates an IP address from the specified interface func deAllocateIP(ip string) error { + mainLog.Load().Debug().Str("ip", ip).Msg("Deallocating IP address") cmd := exec.Command("ifconfig", "lo0", "-alias", ip) if err := cmd.Run(); err != nil { - mainLog.Load().Error().Err(err).Msg("deAllocateIP failed") + mainLog.Load().Error().Err(err).Msg("DeAllocateIP failed") return err } + mainLog.Load().Debug().Str("ip", ip).Msg("IP address deallocated successfully") return nil } @@ -47,6 +52,8 @@ func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) e // networksetup -setdnsservers Wi-Fi 8.8.8.8 1.1.1.1 // TODO(cuonglm): use system API func setDNS(iface *net.Interface, nameservers []string) error { + mainLog.Load().Debug().Str("interface", iface.Name).Strs("nameservers", nameservers).Msg("Setting DNS configuration") + // Note that networksetup won't modify search domains settings, // This assignment is just a placeholder to silent linter. _ = searchDomains @@ -56,6 +63,8 @@ func setDNS(iface *net.Interface, nameservers []string) error { if out, err := exec.Command(cmd, args...).CombinedOutput(); err != nil { return fmt.Errorf("%v: %w", string(out), err) } + + mainLog.Load().Debug().Str("interface", iface.Name).Msg("DNS configuration set successfully") return nil } @@ -73,25 +82,30 @@ func resetDnsIgnoreUnusableInterface(iface *net.Interface) error { // TODO(cuonglm): use system API func resetDNS(iface *net.Interface) error { + mainLog.Load().Debug().Str("interface", iface.Name).Msg("Resetting DNS configuration") + cmd := "networksetup" args := []string{"-setdnsservers", iface.Name, "empty"} if out, err := exec.Command(cmd, args...).CombinedOutput(); err != nil { return fmt.Errorf("%v: %w", string(out), err) } + + mainLog.Load().Debug().Str("interface", iface.Name).Msg("DNS configuration reset successfully") return nil } // restoreDNS restores the DNS settings of the given interface. // this should only be executed upon turning off the ctrld service. func restoreDNS(iface *net.Interface) (err error) { - if ns := savedStaticNameservers(iface); len(ns) > 0 { + if ns := ctrld.SavedStaticNameservers(iface); len(ns) > 0 { err = setDNS(iface, ns) } return err } +// currentDNS returns the current DNS servers for the specified interface func currentDNS(_ *net.Interface) []string { - return resolvconffile.NameServers() + return ctrld.CurrentNameserversFromResolvconf() } // currentStaticDNS returns the current static DNS settings of given interface. diff --git a/cmd/cli/os_freebsd.go b/cmd/cli/os_freebsd.go index d66e4bff..76ac998e 100644 --- a/cmd/cli/os_freebsd.go +++ b/cmd/cli/os_freebsd.go @@ -9,27 +9,32 @@ import ( "tailscale.com/health" "tailscale.com/util/dnsname" + "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/dns" - "github.com/Control-D-Inc/ctrld/internal/resolvconffile" ) -// allocate loopback ip +// allocateIP allocates an IP address on the specified interface // sudo ifconfig lo0 127.0.0.53 alias func allocateIP(ip string) error { + mainLog.Load().Debug().Str("ip", ip).Msg("Allocating IP address") cmd := exec.Command("ifconfig", "lo0", ip, "alias") if err := cmd.Run(); err != nil { mainLog.Load().Error().Err(err).Msg("allocateIP failed") return err } + mainLog.Load().Debug().Str("ip", ip).Msg("IP address allocated successfully") return nil } +// deAllocateIP deallocates an IP address from the specified interface func deAllocateIP(ip string) error { + mainLog.Load().Debug().Str("ip", ip).Msg("Deallocating IP address") cmd := exec.Command("ifconfig", "lo0", ip, "-alias") if err := cmd.Run(); err != nil { mainLog.Load().Error().Err(err).Msg("deAllocateIP failed") return err } + mainLog.Load().Debug().Str("ip", ip).Msg("IP address deallocated successfully") return nil } @@ -40,9 +45,11 @@ func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) e // set the dns server for the provided network interface func setDNS(iface *net.Interface, nameservers []string) error { + mainLog.Load().Debug().Str("interface", iface.Name).Strs("nameservers", nameservers).Msg("Setting DNS configuration") + r, err := dns.NewOSConfigurator(logf, &health.Tracker{}, &controlknobs.Knobs{}, iface.Name) if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to create DNS OS configurator") + mainLog.Load().Error().Err(err).Msg("Failed to create DNS OS configurator") return err } @@ -58,13 +65,15 @@ func setDNS(iface *net.Interface, nameservers []string) error { if sds, err := searchDomains(); err == nil { osConfig.SearchDomains = sds } else { - mainLog.Load().Debug().Err(err).Msg("failed to get search domains list") + mainLog.Load().Debug().Err(err).Msg("Failed to get search domains list") } if err := r.SetDNS(osConfig); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to set DNS") + mainLog.Load().Error().Err(err).Msg("Failed to set DNS") return err } + + mainLog.Load().Debug().Str("interface", iface.Name).Msg("DNS configuration set successfully") return nil } @@ -73,17 +82,22 @@ func resetDnsIgnoreUnusableInterface(iface *net.Interface) error { return resetDNS(iface) } +// resetDNS resets DNS servers for the specified interface func resetDNS(iface *net.Interface) error { + mainLog.Load().Debug().Str("interface", iface.Name).Msg("Resetting DNS configuration") + r, err := dns.NewOSConfigurator(logf, &health.Tracker{}, &controlknobs.Knobs{}, iface.Name) if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to create DNS OS configurator") + mainLog.Load().Error().Err(err).Msg("Failed to create DNS OS configurator") return err } if err := r.Close(); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to rollback DNS setting") + mainLog.Load().Error().Err(err).Msg("Failed to rollback DNS setting") return err } + + mainLog.Load().Debug().Str("interface", iface.Name).Msg("DNS configuration reset successfully") return nil } @@ -93,8 +107,9 @@ func restoreDNS(iface *net.Interface) (err error) { return err } +// currentDNS returns the current DNS servers for the specified interface func currentDNS(_ *net.Interface) []string { - return resolvconffile.NameServers() + return ctrld.CurrentNameserversFromResolvconf() } // currentStaticDNS returns the current static DNS settings of given interface. diff --git a/cmd/cli/os_linux.go b/cmd/cli/os_linux.go index 8caad63c..3d6762d2 100644 --- a/cmd/cli/os_linux.go +++ b/cmd/cli/os_linux.go @@ -21,30 +21,36 @@ import ( "tailscale.com/health" "tailscale.com/util/dnsname" + "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/dns" ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" - "github.com/Control-D-Inc/ctrld/internal/resolvconffile" ) const resolvConfBackupFailedMsg = "open /etc/resolv.pre-ctrld-backup.conf: read-only file system" +type getDNS func(iface string) []string + // allocate loopback ip // sudo ip a add 127.0.0.2/24 dev lo func allocateIP(ip string) error { + mainLog.Load().Debug().Str("ip", ip).Msg("Allocating IP address") cmd := exec.Command("ip", "a", "add", ip+"/24", "dev", "lo") if out, err := cmd.CombinedOutput(); err != nil { - mainLog.Load().Error().Err(err).Msgf("allocateIP failed: %s", string(out)) + mainLog.Load().Error().Err(err).Msgf("AllocateIP failed: %s", string(out)) return err } + mainLog.Load().Debug().Str("ip", ip).Msg("IP address allocated successfully") return nil } func deAllocateIP(ip string) error { + mainLog.Load().Debug().Str("ip", ip).Msg("Deallocating IP address") cmd := exec.Command("ip", "a", "del", ip+"/24", "dev", "lo") if err := cmd.Run(); err != nil { - mainLog.Load().Error().Err(err).Msg("deAllocateIP failed") + mainLog.Load().Error().Err(err).Msg("DeAllocateIP failed") return err } + mainLog.Load().Debug().Str("ip", ip).Msg("IP address deallocated successfully") return nil } @@ -56,9 +62,11 @@ func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) e } func setDNS(iface *net.Interface, nameservers []string) error { + mainLog.Load().Debug().Str("interface", iface.Name).Strs("nameservers", nameservers).Msg("Setting DNS configuration") + r, err := dns.NewOSConfigurator(logf, &health.Tracker{}, &controlknobs.Knobs{}, iface.Name) if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to create DNS OS configurator") + mainLog.Load().Error().Err(err).Msg("Failed to create dns os configurator") return err } @@ -72,9 +80,17 @@ func setDNS(iface *net.Interface, nameservers []string) error { SearchDomains: []dnsname.FQDN{}, } if sds, err := searchDomains(); err == nil { - osConfig.SearchDomains = sds + // Filter the root domain, since it's not allowed by systemd. + // See https://github.com/systemd/systemd/issues/9515 + filteredSds := slices.DeleteFunc(sds, func(s dnsname.FQDN) bool { + return s == "" || s == "." + }) + if len(filteredSds) != len(sds) { + mainLog.Load().Debug().Msg(`Removed root domain "." from search domains list`) + } + osConfig.SearchDomains = filteredSds } else { - mainLog.Load().Debug().Err(err).Msg("failed to get search domains list") + mainLog.Load().Debug().Err(err).Msg("Failed to get search domains list") } trySystemdResolve := false if err := r.SetDNS(osConfig); err != nil { @@ -117,6 +133,8 @@ systemdResolve: } mainLog.Load().Debug().Msg("DNS was not set for some reason") } + + mainLog.Load().Debug().Str("interface", iface.Name).Msg("DNS configuration set successfully") return nil } @@ -126,6 +144,8 @@ func resetDnsIgnoreUnusableInterface(iface *net.Interface) error { } func resetDNS(iface *net.Interface) (err error) { + mainLog.Load().Debug().Str("interface", iface.Name).Msg("Resetting DNS configuration") + defer func() { if err == nil { return @@ -137,7 +157,7 @@ func resetDNS(iface *net.Interface) (err error) { if r, oerr := dns.NewOSConfigurator(logf, &health.Tracker{}, &controlknobs.Knobs{}, iface.Name); oerr == nil { _ = r.SetDNS(dns.OSConfig{}) if err := r.Close(); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to rollback DNS setting") + mainLog.Load().Error().Err(err).Msg("Failed to rollback dns setting") return } err = nil @@ -165,18 +185,18 @@ func resetDNS(iface *net.Interface) (err error) { } // TODO(cuonglm): handle DHCPv6 properly. - mainLog.Load().Debug().Msg("checking for IPv6 availability") + mainLog.Load().Debug().Msg("Checking for ipv6 availability") if ctrldnet.IPv6Available(ctx) { c := client6.NewClient() conversation, err := c.Exchange(iface.Name) if err != nil && !errAddrInUse(err) { - mainLog.Load().Debug().Err(err).Msg("could not exchange DHCPv6") + mainLog.Load().Debug().Err(err).Msg("Could not exchange dhcpv6") } for _, packet := range conversation { if packet.Type() == dhcpv6.MessageTypeReply { msg, err := packet.GetInnerMessage() if err != nil { - mainLog.Load().Debug().Err(err).Msg("could not get inner DHCPv6 message") + mainLog.Load().Debug().Err(err).Msg("Could not get inner dhcpv6 message") return nil } nameservers := msg.Options.DNS() @@ -201,7 +221,7 @@ func restoreDNS(iface *net.Interface) (err error) { } func currentDNS(iface *net.Interface) []string { - resolvconfFunc := func(_ string) []string { return resolvconffile.NameServers() } + resolvconfFunc := func(_ string) []string { return ctrld.CurrentNameserversFromResolvconf() } for _, fn := range []getDNS{getDNSByResolvectl, getDNSBySystemdResolved, getDNSByNmcli, resolvconfFunc} { if ns := fn(iface.Name); len(ns) > 0 { return ns diff --git a/cmd/cli/os_others.go b/cmd/cli/os_others.go index 45edf0a9..64b9709c 100644 --- a/cmd/cli/os_others.go +++ b/cmd/cli/os_others.go @@ -2,12 +2,12 @@ package cli -// TODO(cuonglm): implement. +// allocateIP allocates an IP address on the specified interface func allocateIP(ip string) error { return nil } -// TODO(cuonglm): implement. +// deAllocateIP deallocates an IP address from the specified interface func deAllocateIP(ip string) error { return nil } diff --git a/cmd/cli/os_windows.go b/cmd/cli/os_windows.go index 7ebc54a8..d67ca06c 100644 --- a/cmd/cli/os_windows.go +++ b/cmd/cli/os_windows.go @@ -1,21 +1,18 @@ package cli import ( - "bytes" "errors" "fmt" "net" "net/netip" - "os" - "os/exec" "slices" "strings" - "sync" "golang.org/x/sys/windows" "golang.org/x/sys/windows/registry" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" + "github.com/Control-D-Inc/ctrld" ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) @@ -24,11 +21,6 @@ const ( v6InterfaceKeyPathFormat = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces\` ) -var ( - setDNSOnce sync.Once - resetDNSOnce sync.Once -) - // setDnsIgnoreUnusableInterface likes setDNS, but return a nil error if the interface is not usable. func setDnsIgnoreUnusableInterface(iface *net.Interface, nameservers []string) error { return setDNS(iface, nameservers) @@ -39,49 +31,7 @@ func setDNS(iface *net.Interface, nameservers []string) error { if len(nameservers) == 0 { return errors.New("empty DNS nameservers") } - setDNSOnce.Do(func() { - // If there's a Dns server running, that means we are on AD with Dns feature enabled. - // Configuring the Dns server to forward queries to ctrld instead. - if hasLocalDnsServerRunning() { - mainLog.Load().Debug().Msg("Local DNS server detected, configuring forwarders") - - file := absHomeDir(windowsForwardersFilename) - mainLog.Load().Debug().Msgf("Using forwarders file: %s", file) - - oldForwardersContent, err := os.ReadFile(file) - if err != nil { - mainLog.Load().Debug().Err(err).Msg("Could not read existing forwarders file") - } else { - mainLog.Load().Debug().Msgf("Existing forwarders content: %s", string(oldForwardersContent)) - } - - hasLocalIPv6Listener := needLocalIPv6Listener() - mainLog.Load().Debug().Bool("has_ipv6_listener", hasLocalIPv6Listener).Msg("IPv6 listener status") - forwarders := slices.DeleteFunc(slices.Clone(nameservers), func(s string) bool { - if !hasLocalIPv6Listener { - return false - } - return s == "::1" - }) - mainLog.Load().Debug().Strs("forwarders", forwarders).Msg("Filtered forwarders list") - - if err := os.WriteFile(file, []byte(strings.Join(forwarders, ",")), 0600); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not save forwarders settings") - } else { - mainLog.Load().Debug().Msg("Successfully wrote new forwarders file") - } - - oldForwarders := strings.Split(string(oldForwardersContent), ",") - mainLog.Load().Debug().Strs("old_forwarders", oldForwarders).Msg("Previous forwarders") - - if err := addDnsServerForwarders(forwarders, oldForwarders); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not set forwarders settings") - } else { - mainLog.Load().Debug().Msg("Successfully configured DNS server forwarders") - } - } - }) luid, err := winipcfg.LUIDFromIndex(uint32(iface.Index)) if err != nil { return fmt.Errorf("setDNS: %w", err) @@ -125,25 +75,8 @@ func resetDnsIgnoreUnusableInterface(iface *net.Interface) error { return resetDNS(iface) } -// TODO(cuonglm): should we use system API? +// resetDNS resets DNS servers for the specified interface func resetDNS(iface *net.Interface) error { - resetDNSOnce.Do(func() { - // See corresponding comment in setDNS. - if hasLocalDnsServerRunning() { - file := absHomeDir(windowsForwardersFilename) - content, err := os.ReadFile(file) - if err != nil { - mainLog.Load().Error().Err(err).Msg("could not read forwarders settings") - return - } - nameservers := strings.Split(string(content), ",") - if err := removeDnsServerForwarders(nameservers); err != nil { - mainLog.Load().Error().Err(err).Msg("could not remove forwarders settings") - return - } - } - }) - luid, err := winipcfg.LUIDFromIndex(uint32(iface.Index)) if err != nil { return fmt.Errorf("resetDNS: %w", err) @@ -161,7 +94,7 @@ func resetDNS(iface *net.Interface) error { // restoreDNS restores the DNS settings of the given interface. // this should only be executed upon turning off the ctrld service. func restoreDNS(iface *net.Interface) (err error) { - if nss := savedStaticNameservers(iface); len(nss) > 0 { + if nss := ctrld.SavedStaticNameservers(iface); len(nss) > 0 { v4ns := make([]string, 0, 2) v6ns := make([]string, 0, 2) for _, ns := range nss { @@ -178,24 +111,24 @@ func restoreDNS(iface *net.Interface) (err error) { } if len(v4ns) > 0 { - mainLog.Load().Debug().Msgf("restoring IPv4 static DNS for interface %q: %v", iface.Name, v4ns) + mainLog.Load().Debug().Msgf("Restoring IPv4 static DNS for interface %q: %v", iface.Name, v4ns) if err := setDNS(iface, v4ns); err != nil { return fmt.Errorf("restoreDNS (IPv4): %w", err) } } else { - mainLog.Load().Debug().Msgf("restoring IPv4 DHCP for interface %q", iface.Name) + mainLog.Load().Debug().Msgf("Restoring IPv4 DHCP for interface %q", iface.Name) if err := luid.SetDNS(windows.AF_INET, nil, nil); err != nil { return fmt.Errorf("restoreDNS (IPv4 clear): %w", err) } } if len(v6ns) > 0 { - mainLog.Load().Debug().Msgf("restoring IPv6 static DNS for interface %q: %v", iface.Name, v6ns) + mainLog.Load().Debug().Msgf("Restoring IPv6 static DNS for interface %q: %v", iface.Name, v6ns) if err := setDNS(iface, v6ns); err != nil { return fmt.Errorf("restoreDNS (IPv6): %w", err) } } else { - mainLog.Load().Debug().Msgf("restoring IPv6 DHCP for interface %q", iface.Name) + mainLog.Load().Debug().Msgf("Restoring IPv6 DHCP for interface %q", iface.Name) if err := luid.SetDNS(windows.AF_INET6, nil, nil); err != nil { return fmt.Errorf("restoreDNS (IPv6 clear): %w", err) } @@ -204,15 +137,16 @@ func restoreDNS(iface *net.Interface) (err error) { return err } +// currentDNS returns the current DNS servers for the specified interface func currentDNS(iface *net.Interface) []string { luid, err := winipcfg.LUIDFromIndex(uint32(iface.Index)) if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to get interface LUID") + mainLog.Load().Error().Err(err).Msg("Failed to get interface LUID") return nil } nameservers, err := luid.DNS() if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to get interface DNS") + mainLog.Load().Error().Err(err).Msg("Failed to get interface DNS") return nil } ns := make([]string, 0, len(nameservers)) @@ -240,7 +174,7 @@ func currentStaticDNS(iface *net.Interface) ([]string, error) { interfaceKeyPath := path + guid.String() k, err := registry.OpenKey(registry.LOCAL_MACHINE, interfaceKeyPath, registry.QUERY_VALUE) if err != nil { - mainLog.Load().Debug().Err(err).Msgf("failed to open registry key %q for interface %q; trying next key", interfaceKeyPath, iface.Name) + mainLog.Load().Debug().Err(err).Msgf("Failed to open registry key %q for interface %q; trying next key", interfaceKeyPath, iface.Name) continue } func() { @@ -248,11 +182,11 @@ func currentStaticDNS(iface *net.Interface) ([]string, error) { for _, keyName := range []string{"NameServer", "ProfileNameServer"} { value, _, err := k.GetStringValue(keyName) if err != nil && !errors.Is(err, registry.ErrNotExist) { - mainLog.Load().Debug().Err(err).Msgf("error reading %s registry key", keyName) + mainLog.Load().Debug().Err(err).Msgf("Error reading %s registry key", keyName) continue } if len(value) > 0 { - mainLog.Load().Debug().Msgf("found static DNS for interface %q: %s", iface.Name, value) + mainLog.Load().Debug().Msgf("Found static DNS for interface %q: %s", iface.Name, value) parsed := parseDNSServers(value) for _, pns := range parsed { if !slices.Contains(ns, pns) { @@ -264,7 +198,7 @@ func currentStaticDNS(iface *net.Interface) ([]string, error) { }() } if len(ns) == 0 { - mainLog.Load().Debug().Msgf("no static DNS values found for interface %q", iface.Name) + mainLog.Load().Debug().Msgf("No static DNS values found for interface %q", iface.Name) } return ns, nil } @@ -284,49 +218,3 @@ func parseDNSServers(val string) []string { } return servers } - -// addDnsServerForwarders adds given nameservers to DNS server forwarders list, -// and also removing old forwarders if provided. -func addDnsServerForwarders(nameservers, old []string) error { - newForwardersMap := make(map[string]struct{}) - newForwarders := make([]string, len(nameservers)) - for i := range nameservers { - newForwardersMap[nameservers[i]] = struct{}{} - newForwarders[i] = fmt.Sprintf("%q", nameservers[i]) - } - oldForwarders := old[:0] - for _, fwd := range old { - if _, ok := newForwardersMap[fwd]; !ok { - oldForwarders = append(oldForwarders, fwd) - } - } - // NOTE: It is important to add new forwarder before removing old one. - // Testing on Windows Server 2022 shows that removing forwarder1 - // then adding forwarder2 sometimes ends up adding both of them - // to the forwarders list. - cmd := fmt.Sprintf("Add-DnsServerForwarder -IPAddress %s", strings.Join(newForwarders, ",")) - if len(oldForwarders) > 0 { - cmd = fmt.Sprintf("%s ; Remove-DnsServerForwarder -IPAddress %s -Force", cmd, strings.Join(oldForwarders, ",")) - } - if out, err := powershell(cmd); err != nil { - return fmt.Errorf("%w: %s", err, string(out)) - } - return nil -} - -// removeDnsServerForwarders removes given nameservers from DNS server forwarders list. -func removeDnsServerForwarders(nameservers []string) error { - for _, ns := range nameservers { - cmd := fmt.Sprintf("Remove-DnsServerForwarder -IPAddress %s -Force", ns) - if out, err := powershell(cmd); err != nil { - return fmt.Errorf("%w: %s", err, string(out)) - } - } - return nil -} - -// powershell runs the given powershell command. -func powershell(cmd string) ([]byte, error) { - out, err := exec.Command("powershell", "-Command", cmd).CombinedOutput() - return bytes.TrimSpace(out), err -} diff --git a/cmd/cli/os_windows_test.go b/cmd/cli/os_windows_test.go index 40be5ed2..054b77cc 100644 --- a/cmd/cli/os_windows_test.go +++ b/cmd/cli/os_windows_test.go @@ -1,8 +1,10 @@ package cli import ( + "bytes" "fmt" "net" + "os/exec" "slices" "strings" "testing" @@ -66,3 +68,9 @@ func currentStaticDnsPowershell(iface *net.Interface) ([]string, error) { } return ns, nil } + +// powershell runs the given powershell command. +func powershell(cmd string) ([]byte, error) { + out, err := exec.Command("powershell", "-Command", cmd).CombinedOutput() + return bytes.TrimSpace(out), err +} diff --git a/cmd/cli/pf_diagnostic.go b/cmd/cli/pf_diagnostic.go new file mode 100644 index 00000000..14969e67 --- /dev/null +++ b/cmd/cli/pf_diagnostic.go @@ -0,0 +1,192 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "io" + "os/exec" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +const ( + pfDiagnosticInterval = time.Minute + pfDiagnosticTimeout = 500 * time.Millisecond + pfDiagnosticMaxBytes = 64 << 10 +) + +// Only these two exact, ctrld-generated rules are identified. PF's @N indexes +// are not stable identifiers. No rule text or other anchor contents are logged. +var pfDiagnosticRule = regexp.MustCompile(`^@[0-9]+ block drop out quick on ! ?lo0 inet6 proto (udp|tcp) from any to any port = 53$`) +var pfDiagnosticCounters = regexp.MustCompile(`^\[ Evaluations: +([0-9]+) +Packets: +([0-9]+) +Bytes: +([0-9]+) +States: +([0-9]+) +\]$`) + +type pfDiagnosticRuleCount struct { + Installed bool + Known bool + Packets uint64 +} + +type pfDiagnosticSnapshot struct { + Code string + UDP, TCP pfDiagnosticRuleCount + RouteV4, RouteV6 string + AddressCode string + NativeIPv4, CLAT bool +} + +// A missing or unrecognized rule is not a zero counter. A counter describes all +// traffic matching that rule since its last reset, not delivery of our UDP4 probe. +func parsePFDiagnostic(data []byte) pfDiagnosticSnapshot { + s := pfDiagnosticSnapshot{Code: "rules_not_observed"} + var current *pfDiagnosticRuleCount + duplicates := false + for _, raw := range strings.Split(string(data), "\n") { + line := strings.TrimSpace(raw) + if strings.HasPrefix(line, "@") { + current = nil + if m := pfDiagnosticRule.FindStringSubmatch(line); m != nil { + current = &s.UDP + if m[1] == "tcp" { + current = &s.TCP + } + if current.Installed { + duplicates = true + } + current.Installed = true + } + continue + } + if current == nil { + continue + } + if strings.HasPrefix(line, "[ Evaluations:") { + m := pfDiagnosticCounters.FindStringSubmatch(line) + if m != nil { + valid := true + for _, value := range m[1:] { + if _, err := strconv.ParseUint(value, 10, 64); err != nil { + valid = false + } + } + if valid { + current.Packets, _ = strconv.ParseUint(m[2], 10, 64) + current.Known = true + } + } + current = nil // never attach a later rule's counters to this rule + } + } + switch { + case duplicates: + s.Code = "ambiguous_rules" + s.UDP.Known, s.TCP.Known = false, false + case s.UDP.Known && s.TCP.Known: + s.Code = "ok" + case s.UDP.Installed || s.TCP.Installed: + s.Code = "partial_or_malformed" + } + return s +} + +var errPFDiagnosticLimit = errors.New("PF diagnostic output limit") + +type pfDiagnosticBuffer struct { + data bytes.Buffer // do not embed: promoted ReadFrom would bypass Write's cap + limited bool +} + +func (b *pfDiagnosticBuffer) Len() int { return b.data.Len() } + +func (b *pfDiagnosticBuffer) Write(p []byte) (int, error) { + if len(p) > pfDiagnosticMaxBytes-b.Len() { + b.limited = true + return 0, errPFDiagnosticLimit + } + return b.data.Write(p) +} + +// Stdout is bounded, stderr is discarded, and no raw command error escapes. +// WaitDelay also bounds inherited stdout pipes if a helper fails to close them. +func runPFDiagnosticCommand(ctx context.Context, cmd *exec.Cmd) pfDiagnosticSnapshot { + var out pfDiagnosticBuffer + cmd.Stdout, cmd.Stderr = &out, io.Discard + cmd.WaitDelay = 100 * time.Millisecond + err := cmd.Run() + code := "" + switch { + case ctx.Err() != nil: + code = pfProbeContextCode(ctx) + case out.limited: + code = "output_limit" + case err != nil: + code = "read_failed" + } + if code != "" { + return pfDiagnosticSnapshot{Code: code} + } + return parsePFDiagnostic(out.data.Bytes()) +} + +type pfDiagnosticEvent struct { + Snapshot pfDiagnosticSnapshot + Observation pfProbeObservation + Generation uint64 + Age time.Duration + Cached bool + UDPDelta, TCPDelta uint64 + UDPDeltaCode, TCPDeltaCode string +} + +// Deltas are shared-traffic observations, never per-probe attribution. Even an +// increasing counter may span an unobserved reload, so it is not an exact total. +func pfDiagnosticDelta(before, after pfDiagnosticRuleCount) (uint64, string) { + if !before.Known || !after.Known { + return 0, "unknown" + } + if after.Packets < before.Packets { + return 0, "reset" + } + return after.Packets - before.Packets, "shared_traffic" +} + +type pfDiagnosticState struct { + mu sync.Mutex + active bool + captured time.Time + generation uint64 + snapshot pfDiagnosticSnapshot + lastCondition pfProbeCondition +} + +// Healthy polls do no work. Failures capture at most once a minute; a restoration +// inside that interval names the cached capture and its age, not a fresh reading. +// Identical outcomes and snapshots are quiet. This state never controls repair. +func (s *pfDiagnosticState) observe(now time.Time, o pfProbeObservation, capture func() pfDiagnosticSnapshot) (pfDiagnosticEvent, bool) { + if o.code == "canceled" || !s.mu.TryLock() { + return pfDiagnosticEvent{}, false + } + defer s.mu.Unlock() + if o.result == pfProbeIntercepted && !s.active { + return pfDiagnosticEvent{}, false + } + condition := pfProbeCondition{o.result, o.stage, o.code, o.target} + e := pfDiagnosticEvent{Observation: o, Cached: true, UDPDeltaCode: "unknown", TCPDeltaCode: "unknown"} + changed := condition != s.lastCondition + if s.captured.IsZero() || now.Sub(s.captured) >= pfDiagnosticInterval { + next := capture() + changed = changed || next != s.snapshot + e.UDPDelta, e.UDPDeltaCode = pfDiagnosticDelta(s.snapshot.UDP, next.UDP) + e.TCPDelta, e.TCPDeltaCode = pfDiagnosticDelta(s.snapshot.TCP, next.TCP) + s.snapshot, s.captured = next, now + s.generation++ + e.Cached = false + } + s.active = o.result != pfProbeIntercepted + s.lastCondition = condition + e.Snapshot, e.Generation, e.Age = s.snapshot, s.generation, now.Sub(s.captured) + return e, changed +} diff --git a/cmd/cli/pf_diagnostic_darwin.go b/cmd/cli/pf_diagnostic_darwin.go new file mode 100644 index 00000000..51c57526 --- /dev/null +++ b/cmd/cli/pf_diagnostic_darwin.go @@ -0,0 +1,50 @@ +//go:build darwin + +package cli + +import ( + "context" + "os/exec" + "time" +) + +// This is the only command added by diagnostics: read only the exact ctrld +// anchor, never recurse or dump system rules/states. Tests replace this seam +// before driving the real probe, including on privileged native runners. +var capturePFDiagnosticFn = capturePFDiagnostic + +func capturePFDiagnostic() pfDiagnosticSnapshot { + ctx, cancel := context.WithTimeout(context.Background(), pfDiagnosticTimeout) + defer cancel() + s := runPFDiagnosticCommand(ctx, exec.CommandContext(ctx, "/sbin/pfctl", "-a", pfAnchorName, "-vvsr")) + v4, v6 := defaultRoutesFn() + s.RouteV4, s.RouteV6 = v4.Interface, v6.Interface + state, err := readNetworkSourceStateFn() + s.AddressCode = "read_failed" + if err == nil && state != nil { + s.CLAT = dns64StateHasCLAT(state) + s.AddressCode = "default_v4_unavailable" + if iface, ok := state.Interface[v4.Interface]; ok && iface.Interface != nil && iface.IsUp() { + s.AddressCode = "ok" + for _, prefix := range state.InterfaceIPs[v4.Interface] { + addr := prefix.Addr().Unmap() + if addr.Is4() && !clatPrefix.Contains(addr) && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() && !addr.IsUnspecified() && !addr.IsMulticast() { + s.NativeIPv4 = true + } + } + } + } + return s +} + +func (p *prog) logPFIPv6Diagnostic(o pfProbeObservation, family string) { + state, ok := p.dnsInterceptState.(*pfState) + if !ok || state == nil { + return + } + e, emit := state.ipv6Diagnostic.observe(time.Now(), o, capturePFDiagnosticFn) + if !emit { + return + } + logPFDiagnostic(e, family) +} diff --git a/cmd/cli/pf_diagnostic_darwin_test.go b/cmd/cli/pf_diagnostic_darwin_test.go new file mode 100644 index 00000000..c1ff7648 --- /dev/null +++ b/cmd/cli/pf_diagnostic_darwin_test.go @@ -0,0 +1,63 @@ +//go:build darwin + +package cli + +import ( + "encoding/json" + "strings" + "testing" +) + +func stubPFDiagnosticCapture(t *testing.T) *int { + t.Helper() + old := capturePFDiagnosticFn + t.Cleanup(func() { capturePFDiagnosticFn = old }) + calls := 0 + capturePFDiagnosticFn = func() pfDiagnosticSnapshot { + calls++ + s := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + s.RouteV4, s.RouteV6, s.AddressCode = "en0", "en0", "ok" + s.CLAT = true + return s + } + return &calls +} + +// Exercise the production probe -> diagnostic handoff, without executing a PF +// command or sending DNS. Removing that handoff must make this test fail. +func TestPFDiagnosticProbeWiringAndPrivacy(t *testing.T) { + logs := captureTransitionLogs(t) + reads := stubPFDiagnosticCapture(t) + old := pfProbeNameservers + t.Cleanup(func() { pfProbeNameservers = old; pfProbeLogs = pfProbeLogState{} }) + pfProbeNameservers = func() []string { return []string{"[fe80::1%en0]:53", "76.76.2.0:53"} } + p := &prog{dnsInterceptState: &pfState{}} + p.recoveryGen.Store(7) + + p.probePFIntercept() + p.probePFIntercept() + p.logPFIPv6Diagnostic(pfProbeObservation{result: pfProbeIntercepted, recoveryGeneration: 7, stage: "received", target: "192.0.2.1:53"}, "ipv4") + p.logPFIPv6Diagnostic(pfProbeObservation{result: pfProbeIntercepted}, "ipv4") + var events []map[string]any + for _, line := range strings.Split(logs.String(), "\n") { + var e map[string]any + if json.Unmarshal([]byte(line), &e) != nil || e["message"] != "DNS intercept IPv6 block diagnostic" { + continue + } + events = append(events, e) + for _, forbidden := range []string{"_pf-probe", "Owner", "Inserted", "Bytes:", "from any", "probe_id", "client_address"} { + if strings.Contains(line, forbidden) { + t.Fatalf("raw payload escaped: %s", forbidden) + } + } + if e["counter_scope"] != "shared_traffic_not_probe_attribution" || e["ipv6_dns_udp_packets"] != float64(812) || e["recovery_generation"] != float64(7) { + t.Fatalf("missing evidence: %v", e) + } + } + if *reads != 1 || len(events) != 2 { + t.Fatalf("reads=%d events=%v", *reads, events) + } + if events[0]["outcome"] != "indeterminate" || events[1]["outcome"] != "intercepted" || events[1]["capture_cached"] != true { + t.Fatalf("missing restore: %v", events) + } +} diff --git a/cmd/cli/pf_diagnostic_log.go b/cmd/cli/pf_diagnostic_log.go new file mode 100644 index 00000000..1bf1651d --- /dev/null +++ b/cmd/cli/pf_diagnostic_log.go @@ -0,0 +1,47 @@ +package cli + +import ctrld "github.com/Control-D-Inc/ctrld" + +func logPFDiagnosticRule(e *ctrld.LogEvent, id string, r pfDiagnosticRuleCount, delta uint64, deltaCode string) { + // false means this exact rule was not observed, not proof no IPv6 block exists. + e.Bool(id+"_rule_observed", r.Installed).Bool(id+"_counter_known", r.Known) + if r.Known { + e.Uint64(id+"_packets", r.Packets) + } + e.Str(id+"_delta_status", deltaCode) + if deltaCode == "shared_traffic" { + e.Uint64(id+"_packet_delta", delta) + } +} + +func logPFDiagnostic(e pfDiagnosticEvent, family string) { + o := e.Observation + s := e.Snapshot + // Recovery generation and target associate this snapshot with the existing + // probe-result event without repeating the probe's DNS query name. + probeFamily := "none" + if o.stage != "target" && o.target != "" && family == "ipv4" { + probeFamily = "udp4" + } + log := mainLog.Load().Debug(). + Uint64("capture_generation", e.Generation). + Int64("capture_age_ms", e.Age.Milliseconds()).Bool("capture_cached", e.Cached). + Uint64("recovery_generation", o.recoveryGeneration). + Str("resolver_target", o.target).Str("resolver_family", family). + Str("probe_transport", probeFamily).Str("outcome", o.result.String()). + Str("stage", o.stage).Str("probe_error_code", o.code). + Str("pf_read_status", s.Code). + Str("counter_scope", "shared_traffic_not_probe_attribution"). + Str("default_route_v4_interface", s.RouteV4). + Str("default_route_v6_interface", s.RouteV6). + Str("address_status", s.AddressCode) + if s.AddressCode == "ok" || s.AddressCode == "default_v4_unavailable" { + log.Bool("clat_address_present", s.CLAT) + } + if s.AddressCode == "ok" { + log.Bool("native_ipv4_on_default_interface", s.NativeIPv4) + } + logPFDiagnosticRule(log, "ipv6_dns_udp", s.UDP, e.UDPDelta, e.UDPDeltaCode) + logPFDiagnosticRule(log, "ipv6_dns_tcp", s.TCP, e.TCPDelta, e.TCPDeltaCode) + log.Msg("DNS intercept IPv6 block diagnostic") +} diff --git a/cmd/cli/pf_diagnostic_log_test.go b/cmd/cli/pf_diagnostic_log_test.go new file mode 100644 index 00000000..9ccf90a3 --- /dev/null +++ b/cmd/cli/pf_diagnostic_log_test.go @@ -0,0 +1,47 @@ +package cli + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestPFDiagnosticLogAllowlist(t *testing.T) { + for _, code := range []string{"ok", "read_failed", "partial_or_malformed"} { + t.Run(code, func(t *testing.T) { + s := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + s.Code, s.AddressCode = code, "read_failed" + if code != "ok" { + s.UDP.Known, s.TCP.Known = false, false + } + e := pfDiagnosticEvent{Snapshot: s, Generation: 3, UDPDeltaCode: "unknown", TCPDeltaCode: "unknown", + Observation: pfProbeObservation{result: pfProbeNotIntercepted, probeID: "private-query.invalid", target: "192.0.2.1:53", recoveryGeneration: 7, stage: "delivery", code: "timeout"}} + output := captureDebugMainLog(t) + logPFDiagnostic(e, "ipv4") + line := strings.TrimSpace(output.String()) + var event map[string]any + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"private-query", "probe_id", "Owner", "Inserted", "from any", "Bytes:"} { + if strings.Contains(line, forbidden) { + t.Fatalf("raw payload escaped: %s", forbidden) + } + } + if event["pf_read_status"] != code || event["probe_transport"] != "udp4" || event["capture_generation"] != float64(3) || event["recovery_generation"] != float64(7) || event["counter_scope"] != "shared_traffic_not_probe_attribution" { + t.Fatalf("missing evidence: %v", event) + } + for _, field := range []string{"ipv6_dns_udp_packets", "ipv6_dns_tcp_packets"} { + _, exists := event[field] + if exists != (code == "ok") { + t.Fatalf("unknown counters serialized as zero: %v", event) + } + } + for _, field := range []string{"ipv6_dns_udp_packet_delta", "ipv6_dns_tcp_packet_delta", "clat_address_present", "native_ipv4_on_default_interface"} { + if _, exists := event[field]; exists { + t.Fatalf("unknown evidence serialized: %s", field) + } + } + }) + } +} diff --git a/cmd/cli/pf_diagnostic_test.go b/cmd/cli/pf_diagnostic_test.go new file mode 100644 index 00000000..2dcd12f0 --- /dev/null +++ b/cmd/cli/pf_diagnostic_test.go @@ -0,0 +1,208 @@ +package cli + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// Native-format provenance: macOS pfctl's @N / Owner / Evaluations / Inserted +// layout is published at https://apple.stackexchange.com/questions/335019 +// (pfctl -vvv -s all); the counter layout is also reported on macOS 26.3 at +// https://apple.stackexchange.com/questions/486150 (pfctl -s rules -v). +// This is an ADAPTED fixture, not a capture from a ctrld host: headers use the +// two expanded forms of generatePFRules' IPv6 DNS block, and numbers are test +// values. Keep the native spacing/metadata; do not use pf.conf input as output. +const pfDiagnosticFixture = `@30 block drop out quick on ! lo0 inet6 proto udp from any to any port = 53 + [ Owner : nil Priority : 0 ] + [ Evaluations: 574091 Packets: 812 Bytes: 51968 States: 0 ] + [ Inserted: uid 0 pid 68 ] +@31 block drop out quick on ! lo0 inet6 proto tcp from any to any port = 53 + [ Owner : nil Priority : 0 ] + [ Evaluations: 573678 Packets: 210 Bytes: 13152 States: 0 ] + [ Inserted: uid 0 pid 68 ] +` + +func TestPFDiagnosticNativeFormat(t *testing.T) { + s := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + if s.Code != "ok" || !s.UDP.Installed || !s.TCP.Installed || !s.UDP.Known || !s.TCP.Known || s.UDP.Packets != 812 || s.TCP.Packets != 210 { + t.Fatalf("native-format fixture: %+v", s) + } + // IDs and metadata do not become logged identity, and need not be stable. + changed := strings.NewReplacer("@30", "@1", "@31", "@2", "! lo0", "!lo0", "pid 68", "pid 99").Replace(pfDiagnosticFixture) + if got := parsePFDiagnostic([]byte(changed)); got != s { + t.Fatalf("unstable identity: %+v", got) + } +} + +func TestPFDiagnosticUnknownIsNotZero(t *testing.T) { + for _, tc := range []struct { + name, input, code string + installed bool + }{ + {"empty", "", "rules_not_observed", false}, + {"unrelated", strings.ReplaceAll(pfDiagnosticFixture, "inet6", "inet"), "rules_not_observed", false}, + {"other interface", strings.ReplaceAll(pfDiagnosticFixture, "! lo0", "en0"), "rules_not_observed", false}, + {"not exact", strings.ReplaceAll(pfDiagnosticFixture, "port = 53", "port = 53 label private"), "rules_not_observed", false}, + {"malformed", strings.ReplaceAll(pfDiagnosticFixture, "Packets:", "Packets: invalid"), "partial_or_malformed", true}, + {"overflow", strings.ReplaceAll(pfDiagnosticFixture, "Packets:", "Packets: 18446744073709551616"), "partial_or_malformed", true}, + {"duplicate", pfDiagnosticFixture + pfDiagnosticFixture, "ambiguous_rules", true}, + {"no counters", strings.Split(pfDiagnosticFixture, "\n")[0], "partial_or_malformed", true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := parsePFDiagnostic([]byte(tc.input)) + if s.Code != tc.code || s.UDP.Installed != tc.installed || s.UDP.Known || s.TCP.Known { + t.Fatalf("unknown misreported: %+v", s) + } + }) + } + zero := strings.NewReplacer("812", "0", "210", "0").Replace(pfDiagnosticFixture) + if s := parsePFDiagnostic([]byte(zero)); s.Code != "ok" || !s.UDP.Known || s.UDP.Packets != 0 { + t.Fatalf("real zero: %+v", s) + } + // A missing UDP counter must not consume the following unrelated rule's one. + input := strings.Split(pfDiagnosticFixture, "\n")[0] + "\n@32 pass out all\n [ Evaluations: 1 Packets: 900 Bytes: 100 States: 0 ]\n" + if s := parsePFDiagnostic([]byte(input)); s.UDP.Known { + t.Fatalf("borrowed unrelated counter: %+v", s) + } +} + +func TestPFDiagnosticProcess(t *testing.T) { + switch os.Getenv("CTRLD_PF_DIAGNOSTIC_TEST") { + case "": + return + case "ok": + fmt.Print(pfDiagnosticFixture) + case "failure": + fmt.Print(pfDiagnosticFixture) + fmt.Fprint(os.Stderr, "private rules/client-address") + os.Exit(1) + case "limit": + fmt.Print(strings.Repeat("x", pfDiagnosticMaxBytes+1)) + case "hang": + time.Sleep(time.Minute) + } + os.Exit(0) +} + +func pfDiagnosticTestCommand(ctx context.Context, mode string) *exec.Cmd { + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPFDiagnosticProcess$") + cmd.Env = append(os.Environ(), "CTRLD_PF_DIAGNOSTIC_TEST="+mode) + return cmd +} + +func TestPFDiagnosticCommandBounds(t *testing.T) { + for _, tc := range []struct{ mode, code string }{{"ok", "ok"}, {"failure", "read_failed"}, {"limit", "output_limit"}, {"hang", "timeout"}} { + t.Run(tc.mode, func(t *testing.T) { + d := 5 * time.Second + if tc.mode == "hang" { + d = 100 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + start := time.Now() + s := runPFDiagnosticCommand(ctx, pfDiagnosticTestCommand(ctx, tc.mode)) + if s.Code != tc.code { + t.Fatalf("got %+v, want %s", s, tc.code) + } + if tc.mode != "ok" && (s.UDP.Known || s.TCP.Known || s.UDP.Installed) { + t.Fatalf("failed read trusted: %+v", s) + } + if tc.mode == "hang" && time.Since(start) > time.Second { + t.Fatal("deadline not bounded") + } + }) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got := runPFDiagnosticCommand(ctx, pfDiagnosticTestCommand(ctx, "ok")); got.Code != "canceled" { + t.Fatalf("cancellation: %+v", got) + } + if got := runPFDiagnosticCommand(context.Background(), exec.Command(t.TempDir()+"/missing")); got.Code != "read_failed" { + t.Fatalf("start failure: %+v", got) + } + var b pfDiagnosticBuffer + _, _ = b.Write(make([]byte, pfDiagnosticMaxBytes)) + if _, err := b.Write([]byte{1}); err == nil || b.Len() != pfDiagnosticMaxBytes { + t.Fatal("buffer is unbounded") + } +} + +func TestPFDiagnosticCadenceTransitionsAndReset(t *testing.T) { + var s pfDiagnosticState + now := time.Unix(1, 0) + reads := 0 + fixture := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + capture := func() pfDiagnosticSnapshot { reads++; return fixture } + o := pfProbeObservation{result: pfProbeIntercepted, target: "192.0.2.1:53"} + if _, emit := s.observe(now, o, capture); emit || reads != 0 { + t.Fatal("healthy poll did work") + } + o.result, o.stage, o.code = pfProbeNotIntercepted, "delivery", "timeout" + e, emit := s.observe(now, o, capture) + if !emit || e.Cached || e.Generation != 1 || e.UDPDeltaCode != "unknown" { + t.Fatalf("first failure: %+v %v", e, emit) + } + for i := 0; i < 100; i++ { + o.probeID = fmt.Sprint(i) + o.recoveryGeneration++ + if _, emit := s.observe(now.Add(time.Second), o, capture); emit { + t.Fatal("duplicate failure noisy") + } + } + if reads != 1 { + t.Fatalf("unbounded reads: %d", reads) + } + if _, emit := s.observe(now.Add(time.Minute), o, capture); emit || reads != 2 { + t.Fatal("unchanged refresh noisy") + } + fixture.UDP.Packets++ + e, emit = s.observe(now.Add(2*time.Minute), o, capture) + if !emit || e.UDPDelta != 1 || e.UDPDeltaCode != "shared_traffic" { + t.Fatalf("delta: %+v", e) + } + fixture.UDP.Packets = 0 + e, _ = s.observe(now.Add(3*time.Minute), o, capture) + if e.UDPDeltaCode != "reset" || e.UDPDelta != 0 { + t.Fatalf("reset: %+v", e) + } + o.result = pfProbeIntercepted + e, emit = s.observe(now.Add(3*time.Minute+time.Second), o, capture) + if !emit || !e.Cached || e.Age != time.Second || e.Generation != 4 || e.UDPDeltaCode != "unknown" { + t.Fatalf("restored: %+v", e) + } + if _, emit := s.observe(now.Add(4*time.Minute), o, capture); emit || reads != 4 { + t.Fatal("healthy repeats did work") + } + o.code = "canceled" + o.result = pfProbeIndeterminate + if _, emit := s.observe(now.Add(5*time.Minute), o, capture); emit || reads != 4 { + t.Fatal("shutdown did work") + } + fixture.Code = "read_failed" + fixture.UDP.Known = false + fixture.TCP.Known = false + o.code = "unavailable" + e, emit = s.observe(now.Add(6*time.Minute), o, capture) + if !emit || e.UDPDeltaCode != "unknown" { + t.Fatalf("failed read: %+v", e) + } +} + +func TestPFDiagnosticConcurrentCaptureDoesNotBlock(t *testing.T) { + var s pfDiagnosticState + started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + s.observe(time.Now(), pfProbeObservation{}, func() pfDiagnosticSnapshot { close(started); <-release; return pfDiagnosticSnapshot{} }) + }() + <-started + defer func() { close(release); <-done }() + if _, emit := s.observe(time.Now(), pfProbeObservation{}, func() pfDiagnosticSnapshot { t.Error("overlapping capture"); return pfDiagnosticSnapshot{} }); emit { + t.Fatal("concurrent capture should stand down") + } +} diff --git a/cmd/cli/pf_probe.go b/cmd/cli/pf_probe.go new file mode 100644 index 00000000..83d93a75 --- /dev/null +++ b/cmd/cli/pf_probe.go @@ -0,0 +1,233 @@ +package cli + +import ( + "bufio" + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os/exec" + "strings" + "sync" + "syscall" + "time" + + "tailscale.com/net/tsaddr" +) + +type pfProbeResult uint8 + +const ( + pfProbeIndeterminate pfProbeResult = iota + pfProbeIntercepted + pfProbeNotIntercepted +) + +func (r pfProbeResult) String() string { + switch r { + case pfProbeIntercepted: + return "intercepted" + case pfProbeNotIntercepted: + return "not_intercepted" + default: + return "indeterminate" + } +} + +type pfProbeObservation struct { + result pfProbeResult + stage string + code string + probeID string + recoveryGeneration uint64 + target string +} + +// Conditions exclude attempt IDs and generations so repeated failures stay bounded. +type pfProbeCondition struct { + result pfProbeResult + stage, code, target string +} + +type pfProbeLogState struct { + mu sync.Mutex + previous pfProbeCondition + initialized bool + repeats uint64 +} + +// change returns whether to retain a transition and the preceding repeat count. +// A restored receipt is retained too, so an old warning does not imply an outage. +func (s *pfProbeLogState) change(o pfProbeObservation) (retain bool, repeats uint64) { + s.mu.Lock() + defer s.mu.Unlock() + condition := pfProbeCondition{o.result, o.stage, o.code, o.target} + if s.initialized && condition == s.previous { + s.repeats++ + return false, s.repeats + } + retain = o.result != pfProbeIntercepted || (s.initialized && s.previous.result != pfProbeIntercepted) + repeats = s.repeats + s.previous, s.initialized, s.repeats = condition, true, 0 + return retain, repeats +} + +// OsResolverNameservers groups LAN entries before public entries. Preserve a usable +// first target, including public DNS. Otherwise scan only the LAN group. Stop at +// the public boundary, not at a particular IP: the public group can be synthetic. +func pfProbeTarget(servers []string) string { + for i, server := range servers { + host, _, err := net.SplitHostPort(server) + if err != nil { + return "" + } + ip, err := netip.ParseAddr(host) + if err != nil { + return "" + } + lan := ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || tsaddr.CGNATRange().Contains(ip) + usable := ip.Is4() && !ip.IsLoopback() && !ip.IsUnspecified() && !ip.IsMulticast() + if usable && (i == 0 || lan) { + return host + } + if !lan { + return "" + } + } + return "" +} + +// Keep the helper protocol bounded and free of resolver IDs, queries and raw OS errors. +func pfProbeErrorCode(err error) string { + switch { + case errors.Is(err, syscall.ENETUNREACH), errors.Is(err, syscall.EHOSTUNREACH): + return "unreachable" + case errors.Is(err, syscall.EADDRNOTAVAIL): + return "source_unavailable" + case errors.Is(err, syscall.EACCES), errors.Is(err, syscall.EPERM): + return "permission" + default: + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + return "timeout" + } + return "io" + } +} + +func sendPFProbe(host, hexPacket string, dial func(string, string, time.Duration) (net.Conn, error), status io.Writer) error { + fail := func(stage string, err error) error { + _, _ = fmt.Fprintf(status, "%s:%s\n", stage, pfProbeErrorCode(err)) + return err + } + packet, err := hex.DecodeString(hexPacket) + if err != nil { + _, _ = io.WriteString(status, "decode:invalid_argument\n") + return err + } + conn, err := dial("udp4", net.JoinHostPort(host, "53"), time.Second) + if err != nil { + return fail("dial", err) + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(time.Second)); err != nil { + return fail("deadline", err) + } + n, err := conn.Write(packet) + if err == nil && n != len(packet) { + err = io.ErrShortWrite + } + if err != nil { + return fail("write", err) + } + if _, err := io.WriteString(status, "sent\n"); err != nil { + return err + } + // Keep the socket alive for the local handler's response. A read failure does + // not undo the successful send acknowledged above. + _, _ = conn.Read(make([]byte, 512)) + return nil +} + +func parsePFProbeStatus(status string) pfProbeObservation { + if status == "sent\n" { + return pfProbeObservation{stage: "sent"} + } + stage, code, ok := strings.Cut(strings.TrimSuffix(status, "\n"), ":") + if ok && (stage == "decode" || stage == "dial" || stage == "deadline" || stage == "write") { + switch code { + case "unreachable", "source_unavailable", "permission", "timeout", "io", "invalid_argument": + return pfProbeObservation{stage: stage, code: code} + } + } + return pfProbeObservation{stage: "helper", code: "invalid_status"} +} + +func pfProbeContextCode(ctx context.Context) string { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return "timeout" + } + return "canceled" +} + +// Receipt by the local handler is conclusive. A missing receipt is a failed +// interception test only after the child confirms a successful packet send. +func runPFProbe(ctx context.Context, cmd *exec.Cmd, received <-chan struct{}, deliveryTimeout time.Duration) pfProbeObservation { + stdout, err := cmd.StdoutPipe() + if err != nil { + return pfProbeObservation{stage: "start", code: pfProbeErrorCode(err)} + } + if err := cmd.Start(); err != nil { + _ = stdout.Close() + return pfProbeObservation{stage: "start", code: pfProbeErrorCode(err)} + } + statusCh := make(chan pfProbeObservation, 1) + done := make(chan struct{}) + go func() { + line, _ := bufio.NewReader(io.LimitReader(stdout, 128)).ReadString('\n') + statusCh <- parsePFProbeStatus(line) + _ = cmd.Wait() + close(done) + }() + defer func() { + _ = cmd.Process.Kill() + <-done + }() + + var sent pfProbeObservation + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + case <-ctx.Done(): + return pfProbeObservation{stage: "helper", code: pfProbeContextCode(ctx)} + case sent = <-statusCh: + } + if sent.stage != "sent" { + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + default: + return sent + } + } + + timer := time.NewTimer(deliveryTimeout) + defer timer.Stop() + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + case <-ctx.Done(): + return pfProbeObservation{stage: "delivery", code: pfProbeContextCode(ctx)} + case <-timer.C: + // Prefer receipt when notification and deadline became ready together. + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + default: + return pfProbeObservation{result: pfProbeNotIntercepted, stage: "delivery", code: "timeout"} + } + } +} diff --git a/cmd/cli/pf_probe_darwin_test.go b/cmd/cli/pf_probe_darwin_test.go new file mode 100644 index 00000000..5d5cb8b4 --- /dev/null +++ b/cmd/cli/pf_probe_darwin_test.go @@ -0,0 +1,216 @@ +//go:build darwin + +package cli + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "reflect" + "strings" + "testing" + "time" +) + +func TestPFProbeDefaultCommand(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cmd := newPFProbeCommand(ctx, "192.0.2.1", "0102") + if !reflect.DeepEqual(cmd.Args, []string{os.Args[0], "pf-probe-send", "192.0.2.1", "0102"}) { + t.Fatalf("args=%q", cmd.Args) + } + if cmd.SysProcAttr == nil || cmd.SysProcAttr.Credential == nil { + t.Fatal("missing credentials") + } + cred := cmd.SysProcAttr.Credential + if cred.Uid != 0 || cred.Gid != 0 || cred.NoSetGroups || len(cred.Groups) != 0 { + t.Fatalf("unexpected credentials: %+v", cred) + } + if cmd.Cancel == nil { + t.Fatal("command is not context-bound") + } + // Do not start this command: credential changes require native privileged QA. +} + +func TestPFProbeMonitorIndeterminateContinues(t *testing.T) { + oldProbe, oldReload, oldDelays := probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays + t.Cleanup(func() { + probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays = oldProbe, oldReload, oldDelays + }) + pfInterceptMonitorDelays = []time.Duration{0, 0, 0} + probes, reloads := 0, 0 + probePFInterceptFn = func(*prog) pfProbeObservation { + probes++ + if probes == 2 { + return pfProbeObservation{result: pfProbeIntercepted} + } + return pfProbeObservation{stage: "dial", code: "unreachable"} + } + forceReloadPFInterceptFn = func(*prog) bool { reloads++; return true } + p := sourceTestProg(&prog{dnsInterceptState: &pfState{}}) + p.pfInterceptMonitor() + if probes != 3 || reloads != 0 || p.pfMonitorRunning.Load() { + t.Fatalf("probes=%d reloads=%d owner=%v", probes, reloads, p.pfMonitorRunning.Load()) + } +} + +func TestPFProbeRepairCorrelation(t *testing.T) { + for _, caller := range []string{"post_stabilization", "post_wake", "watchdog", "monitor"} { + for _, performed := range []bool{false, true} { + t.Run(caller+"/"+map[bool]string{false: "not_run", true: "completed"}[performed], func(t *testing.T) { + oldProbe, oldReload, oldDelays := probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays + t.Cleanup(func() { + probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays = oldProbe, oldReload, oldDelays + }) + logs := captureTransitionLogs(t) + stubWakeProbeSchedule(t, 1) + pfInterceptMonitorDelays = []time.Duration{0} + p := sourceTestProg(&prog{dnsInterceptState: &pfState{}, stopCh: make(chan struct{})}) + probes, reloads := 0, 0 + probePFInterceptFn = func(*prog) pfProbeObservation { + probes++ + if probes == 1 { + return pfProbeObservation{result: pfProbeNotIntercepted, probeID: "cause-probe", recoveryGeneration: 7, stage: "delivery", code: "timeout", target: "192.0.2.1:53"} + } + return pfProbeObservation{result: pfProbeIndeterminate, probeID: "confirmation-probe", recoveryGeneration: 8, stage: "dial", code: "unreachable"} + } + forceReloadPFInterceptFn = func(*prog) bool { reloads++; p.recoveryGen.Store(99); return performed } + switch caller { + case "post_stabilization": + p.verifyInterceptAfterStabilization() + case "post_wake": + p.runWakeProbeAttempt(0, 0) + case "watchdog": + p.checkPFWatchdogProbe(pfAnchorCheckIntact) + case "monitor": + p.pfInterceptMonitor() + } + expected := "not_run" + if performed { + expected = "reload_completed" + } + found, confirmation := false, false + for _, line := range strings.Split(logs.String(), "\n") { + var event map[string]interface{} + if json.Unmarshal([]byte(line), &event) != nil || event["message"] != "PF repair" { + continue + } + if event["probe_id"] != "cause-probe" || event["recovery_generation"] != float64(7) || event["caller"] != caller { + t.Fatalf("lost cause: %v", event) + } + if event["outcome"] == expected { + found = true + } + if event["outcome"] == "indeterminate" && event["confirmation_probe_id"] == "confirmation-probe" { + confirmation = true + } + } + if reloads != 1 || !found { + t.Fatalf("reloads=%d outcome found=%v", reloads, found) + } + if performed && caller != "watchdog" && !confirmation { + t.Fatal("missing confirmation outcome") + } + }) + } + } +} + +func TestPFProbeMissingTargetLogTransitions(t *testing.T) { + logs := captureTransitionLogs(t) + stubPFDiagnosticCapture(t) + old := pfProbeNameservers + t.Cleanup(func() { pfProbeNameservers = old; pfProbeLogs = pfProbeLogState{} }) + pfProbeLogs = pfProbeLogState{} + first := "[fe80::1%en0]:53" + pfProbeNameservers = func() []string { return []string{first, "76.76.2.0:53"} } + p := sourceTestProg(&prog{dnsInterceptState: &pfState{}}) + p.probePFIntercept() + p.probePFIntercept() + first = "127.0.0.1:53" + p.probePFIntercept() + var levels []string + for _, line := range strings.Split(logs.String(), "\n") { + var event map[string]interface{} + if json.Unmarshal([]byte(line), &event) != nil || event["message"] != "DNS intercept probe result" { + continue + } + levels = append(levels, event["level"].(string)) + if len(levels) == 3 && event["repeated_results"] != float64(1) { + t.Fatal("missing suppressed count") + } + } + if !reflect.DeepEqual(levels, []string{"warn", "debug", "warn"}) { + t.Fatalf("levels=%v", levels) + } +} + +func TestPFProbeUnsentDoesNotReload(t *testing.T) { + logs := captureTransitionLogs(t) + stubPFDiagnosticCapture(t) + oldServers, oldCommand, oldReload := pfProbeNameservers, newPFProbeCommand, forceReloadPFInterceptFn + t.Cleanup(func() { + pfProbeNameservers, newPFProbeCommand, forceReloadPFInterceptFn = oldServers, oldCommand, oldReload + }) + pfProbeNameservers = func() []string { return []string{"192.0.2.1:53"} } + newPFProbeCommand = func(ctx context.Context, host, packet string) *exec.Cmd { return pfProbeTestCommand(ctx, "dial") } + reloads := 0 + forceReloadPFInterceptFn = func(*prog) bool { reloads++; return true } + p := sourceTestProg(&prog{dnsInterceptState: &pfState{}, stopCh: make(chan struct{})}) + if got := p.checkPFWatchdogProbe(pfAnchorCheckIntact); got != pfAnchorCheckDeferred { + t.Fatalf("watchdog result=%v", got) + } + p.verifyInterceptAfterStabilization() + stubWakeProbeSchedule(t, 1) + if restored, repaired := p.runWakeProbeAttempt(0, 0); restored || repaired { + t.Fatalf("unsent wake probe: restored=%v repaired=%v", restored, repaired) + } + if reloads != 0 { + t.Fatalf("unsent probes caused %d reloads", reloads) + } + output := logs.String() + for _, field := range []string{`"probe_id":`, `"recovery_generation":`, `"resolver_target":"192.0.2.1:53"`, `"stage":"dial"`, `"error_code":"unreachable"`, `"outcome":"indeterminate"`, `"repair_eligible":false`} { + if !strings.Contains(output, field) { + t.Errorf("missing %s in logs", field) + } + } + if strings.Contains(output, "interception is translating") || strings.Contains(output, "anchor was missing") { + t.Fatal("unknown probe claimed PF health or missing rules") + } +} + +func TestPFProbeWatchdogMeaningfulFailureRepairsWithoutMissingAnchorClaim(t *testing.T) { + _, reloads := stubStabilizationProbe(t, []bool{false}, true) + p := sourceTestProg(&prog{dnsInterceptState: &pfState{}}) + if got := p.checkPFWatchdogProbe(pfAnchorCheckIntact); got != pfAnchorCheckDeferred { + t.Fatalf("probe repair must not be counted as missing-anchor restoration: %v", got) + } + if *reloads != 1 { + t.Fatalf("reloads=%d", *reloads) + } +} + +func TestPFProbeWatchdogSkipsIndeterminateAnchor(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, nil, true) + p := sourceTestProg(&prog{dnsInterceptState: &pfState{}}) + for _, r := range []pfAnchorCheckResult{pfAnchorCheckDeferred, pfAnchorCheckSkipped, pfAnchorCheckFailed, pfAnchorCheckRestored} { + if got := p.checkPFWatchdogProbe(r); got != r { + t.Fatalf("changed outcome %v -> %v", r, got) + } + } + if *probes != 0 || *reloads != 0 { + t.Fatal("probed without intact rule evidence") + } +} + +func TestPFProbeNoIPv4TargetIsIndeterminate(t *testing.T) { + stubPFDiagnosticCapture(t) + old := pfProbeNameservers + t.Cleanup(func() { pfProbeNameservers = old }) + pfProbeNameservers = func() []string { return []string{"[fe80::1%en0]:53", "76.76.2.0:53"} } + p := sourceTestProg(&prog{dnsInterceptState: &pfState{}}) + if got := p.probePFIntercept(); got.result != pfProbeIndeterminate { + t.Fatalf("got %v", got) + } +} diff --git a/cmd/cli/pf_probe_test.go b/cmd/cli/pf_probe_test.go new file mode 100644 index 00000000..6afae2ad --- /dev/null +++ b/cmd/cli/pf_probe_test.go @@ -0,0 +1,292 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "os" + "os/exec" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +func TestPFProbeProcess(t *testing.T) { + mode := os.Getenv("CTRLD_PF_PROBE_TEST") + if mode == "" { + return + } + switch mode { + case "main_dispatch": + os.Args = []string{os.Args[0], "pf-probe-send", "127.0.0.1", "zz"} + Main() + os.Exit(99) // Production dispatch must exit on invalid input. + case "sent": + _, _ = os.Stdout.WriteString("sent\n") + case "dial": + _, _ = os.Stdout.WriteString("dial:unreachable\n") + os.Exit(1) + case "write": + _, _ = os.Stdout.WriteString("write:permission\n") + os.Exit(1) + case "malformed": + _, _ = os.Stdout.WriteString(strings.Repeat("private-payload", 1000)) + case "hang": + time.Sleep(time.Minute) + case "silent": + os.Exit(1) + } + os.Exit(0) +} + +func pfProbeTestCommand(ctx context.Context, mode string) *exec.Cmd { + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPFProbeProcess$") + cmd.Env = append(os.Environ(), "CTRLD_PF_PROBE_TEST="+mode) + return cmd +} + +func TestPFProbeProcessClassification(t *testing.T) { + for _, tc := range []struct { + mode string + want pfProbeResult + stage string + }{ + {"dial", pfProbeIndeterminate, "dial"}, + {"write", pfProbeIndeterminate, "write"}, + {"silent", pfProbeIndeterminate, "helper"}, + {"malformed", pfProbeIndeterminate, "helper"}, + {"sent", pfProbeNotIntercepted, "delivery"}, + } { + t.Run(tc.mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + got := runPFProbe(ctx, pfProbeTestCommand(ctx, tc.mode), make(chan struct{}), 10*time.Millisecond) + if got.result != tc.want || got.stage != tc.stage { + t.Fatalf("got %+v, want %v/%s", got, tc.want, tc.stage) + } + if strings.Contains(got.code, "private") { + t.Fatal("helper output escaped into diagnostic") + } + }) + } +} + +func TestPFProbeStartFailureAndCancellation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + got := runPFProbe(ctx, exec.Command(t.TempDir()+"/missing-helper"), make(chan struct{}), time.Second) + if got.result != pfProbeIndeterminate || got.stage != "start" { + t.Fatalf("start failure: %+v", got) + } + ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + got = runPFProbe(ctx, pfProbeTestCommand(ctx, "hang"), make(chan struct{}), time.Second) + if got.result != pfProbeIndeterminate { + t.Fatalf("canceled helper must not request repair: %+v", got) + } +} + +func TestPFProbeReceiptWins(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + received := make(chan struct{}) + close(received) + got := runPFProbe(ctx, pfProbeTestCommand(ctx, "sent"), received, time.Millisecond) + if got.result != pfProbeIntercepted { + t.Fatalf("local receipt is conclusive: %+v", got) + } +} + +type pfProbeErrorConn struct { + net.Conn + writeErr error + short bool +} + +func (c pfProbeErrorConn) SetDeadline(time.Time) error { return nil } +func (c pfProbeErrorConn) Close() error { return nil } +func (c pfProbeErrorConn) Write(b []byte) (int, error) { + if c.short { + return len(b) - 1, nil + } + return 0, c.writeErr +} + +func TestPFProbeSendFailures(t *testing.T) { + for _, tc := range []struct { + name, packet, want string + dial func(string, string, time.Duration) (net.Conn, error) + }{ + {"decode", "zz", "decode:invalid_argument\n", nil}, + {"dial", "0102", "dial:unreachable\n", func(string, string, time.Duration) (net.Conn, error) { + return nil, &net.OpError{Op: "dial", Err: syscall.ENETUNREACH} + }}, + {"write", "0102", "write:permission\n", func(string, string, time.Duration) (net.Conn, error) { + return pfProbeErrorConn{writeErr: syscall.EACCES}, nil + }}, + {"short", "0102", "write:io\n", func(string, string, time.Duration) (net.Conn, error) { return pfProbeErrorConn{short: true}, nil }}, + } { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + err := sendPFProbe("192.0.2.1", tc.packet, tc.dial, &out) + if err == nil || out.String() != tc.want { + t.Fatalf("status=%q err=%v", out.String(), err) + } + if strings.Contains(out.String(), "sent") { + t.Fatal("failed write claimed success") + } + }) + } +} + +func TestPFProbeSendRealUDP(t *testing.T) { + listener, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + received := make(chan []byte, 1) + go func() { + b := make([]byte, 100) + n, addr, e := listener.ReadFrom(b) + if e != nil { + return + } + received <- append([]byte(nil), b[:n]...) + _, _ = listener.WriteTo([]byte{0}, addr) + }() + var out bytes.Buffer + err = sendPFProbe("127.0.0.1", "010203", func(network, address string, timeout time.Duration) (net.Conn, error) { + if network != "udp4" || address != "127.0.0.1:53" { + t.Errorf("unexpected target %s %s", network, address) + } + return net.DialTimeout(network, listener.LocalAddr().String(), timeout) + }, &out) + if err != nil || out.String() != "sent\n" { + t.Fatalf("status=%q error=%v", out.String(), err) + } + select { + case b := <-received: + if !bytes.Equal(b, []byte{1, 2, 3}) { + t.Fatalf("wrong packet %v", b) + } + case <-time.After(time.Second): + t.Fatal("helper reported sent without UDP receipt") + } +} + +func TestPFProbeTarget(t *testing.T) { + for _, first := range []string{"[fe80::1%en0]:53", "[2001:db8::1]:53", "127.0.0.53:53", "0.0.0.0:53", "224.0.0.1:53", "bad"} { + // Resolver construction can append this public fallback. No later entry + // may become a new probe recipient when the original first target fails. + if got := pfProbeTarget([]string{first, "76.76.2.0:53"}); got != "" { + t.Fatalf("first=%s selected new recipient=%s", first, got) + } + } + for _, first := range []string{"192.0.2.10", "76.76.2.0"} { + if got := pfProbeTarget([]string{first + ":53", "192.0.2.11:53"}); got != first { + t.Fatalf("changed original first target %s to %s", first, got) + } + } + if got := pfProbeTarget(nil); got != "" { + t.Fatalf("empty list selected %s", got) + } +} + +func TestPFProbeMainDispatch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + out, err := pfProbeTestCommand(ctx, "main_dispatch").Output() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 || string(out) != "decode:invalid_argument\n" { + t.Fatalf("production Main dispatch: output=%q error=%v", out, err) + } +} + +func TestPFProbeTargetLANScan(t *testing.T) { + for _, tc := range []struct { + servers []string + want string + }{ + {[]string{"[fe80::1%en0]:53", "192.168.1.1:53", "76.76.2.0:53"}, "192.168.1.1"}, + {[]string{"127.0.0.1:53", "100.64.0.1:53", "76.76.2.0:53"}, "100.64.0.1"}, + {[]string{"[fd00::1]:53", "169.254.1.1:53", "76.76.2.0:53"}, "169.254.1.1"}, + {[]string{"[fe80::1%en0]:53", "76.76.2.0:53"}, ""}, + {[]string{"[fe80::1%en0]:53", "8.8.8.8:53", "192.168.1.1:53"}, ""}, + {[]string{"[2001:db8::1]:53", "192.168.1.1:53"}, ""}, + {[]string{"76.76.2.0:53"}, "76.76.2.0"}, + {[]string{"8.8.8.8:53", "192.168.1.1:53"}, "8.8.8.8"}, + } { + if got := pfProbeTarget(tc.servers); got != tc.want { + t.Errorf("%v: got %q want %q", tc.servers, got, tc.want) + } + } +} + +func TestPFProbeLogConcurrentRepeats(t *testing.T) { + var state pfProbeLogState + observed := pfProbeObservation{stage: "dial", code: "unreachable"} + state.change(observed) + var wg sync.WaitGroup + for range 20 { + wg.Add(1) + go func() { defer wg.Done(); state.change(observed) }() + } + wg.Wait() + observed.result, observed.stage, observed.code = pfProbeIntercepted, "received", "" + if retained, repeats := state.change(observed); !retained || repeats != 20 { + t.Fatalf("retained=%v repeats=%d", retained, repeats) + } +} + +func TestPFProbeLogConditionChanges(t *testing.T) { + var state pfProbeLogState + missing := pfProbeObservation{stage: "target", code: "unavailable", target: "[fe80::1%en0]:53"} + if retain, repeats := state.change(missing); !retain || repeats != 0 { + t.Fatal("first missing target not retained") + } + missing.probeID, missing.recoveryGeneration = "new-attempt", 2 + if retain, repeats := state.change(missing); retain || repeats != 1 { + t.Fatal("attempt IDs defeat dedupe") + } + for _, change := range []func(*pfProbeObservation){ + func(o *pfProbeObservation) { o.target = "127.0.0.1:53" }, + func(o *pfProbeObservation) { o.stage = "dial" }, + func(o *pfProbeObservation) { o.code = "unreachable" }, + func(o *pfProbeObservation) { o.result = pfProbeNotIntercepted }, + } { + change(&missing) + if retain, repeats := state.change(missing); !retain || repeats != 1 { + t.Fatalf("change not retained with prior repeats: %v %d", retain, repeats) + } + if retain, repeats := state.change(missing); retain || repeats != 1 { + t.Fatal("repeat not coalesced") + } + } + healthy := pfProbeObservation{result: pfProbeIntercepted, stage: "received", target: missing.target} + if retain, repeats := state.change(healthy); !retain || repeats != 1 { + t.Fatal("restoration not retained") + } + if retain, _ := state.change(healthy); retain { + t.Fatal("normal receipt retained") + } +} + +func TestPFProbeStatusIsBounded(t *testing.T) { + if got := parsePFProbeStatus("decode:invalid_argument\n"); got.stage != "decode" || got.code != "invalid_argument" || got.result != pfProbeIndeterminate { + t.Fatalf("invalid argument classification: %+v", got) + } + for _, s := range []string{"dial:secret-token\n", "secret:io\n", strings.Repeat("private", 1000)} { + got := parsePFProbeStatus(s) + if got.result != pfProbeIndeterminate || got.stage != "helper" || got.code != "invalid_status" { + t.Fatalf("untrusted status leaked: %+v", got) + } + } + if pfProbeErrorCode(errors.New("secret")) != "io" || pfProbeErrorCode(io.ErrShortWrite) != "io" { + t.Fatal("unbounded diagnostic") + } +} diff --git a/cmd/cli/pf_ruleset.go b/cmd/cli/pf_ruleset.go new file mode 100644 index 00000000..73d5df6f --- /dev/null +++ b/cmd/cli/pf_ruleset.go @@ -0,0 +1,79 @@ +package cli + +import ( + "fmt" + "strings" +) + +// pfNoRulesMarker is what pfctl prints for a ruleset that contains nothing. +const pfNoRulesMarker = "(no rules)" + +// pfFilterRuleLines reduces pfctl output to the lines that are actually pf rules. +// +// It exists because every pfctl reader here uses CombinedOutput, and pfctl on macOS +// writes "No ALTQ support in kernel" and "ALTQ related functions disabled" to stderr on +// essentially every show command, so raw output is never a clean rule list. An empty +// ruleset can also report "(no rules)", which is a status line rather than a rule. +// +// Two consequences follow from getting this wrong, and both have bitten this file: +// callers that test the output for emptiness can never see empty, and callers that feed +// the lines back into "pfctl -f -" would splice non-rule text into a ruleset and have +// the reload rejected. +// +// Registry access and platform specifics stay elsewhere; this is pure string handling +// so it can be tested on any host. +func pfFilterRuleLines(output string) []string { + var rules []string + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + // pfctl stderr warnings, merged in by CombinedOutput. + if strings.Contains(line, "ALTQ") { + continue + } + // Status line for an empty ruleset, not a rule. + if line == pfNoRulesMarker { + continue + } + rules = append(rules, line) + } + return rules +} + +// pfRulesetEmpty reports whether pfctl output describes a ruleset with no rules. +// +// Use this rather than testing the raw output for emptiness: the merged stderr warnings +// described above mean a raw test is always false, so the condition it guards - an +// anchor whose contents were flushed - would never be detected. +func pfRulesetEmpty(output string) bool { + return len(pfFilterRuleLines(output)) == 0 +} + +// pfContainsRule checks if any line in the slice contains the given rule string. +// Uses substring matching because pfctl may append extra tokens like " all" to rules +// (e.g., `rdr-anchor "com.controld.ctrld" all`), which would fail exact matching. +func pfContainsRule(lines []string, rule string) bool { + for _, line := range lines { + if strings.Contains(line, rule) { + return true + } + } + return false +} + +// pfAnchorReferencesPresent reports whether ctrld's anchor references appear in the +// running ruleset, given the output of "pfctl -sn" and "pfctl -sr". +// +// Removing the references means reloading the entire main ruleset, and that reload +// carries no options section - so it resets system-wide pf options, including any +// third-party "set skip" directives. Doing that when there is nothing of ours to +// remove is pure collateral damage, which is what a startup rollback would otherwise +// cause after failing before the references were ever added. +func pfAnchorReferencesPresent(natOutput, filterOutput, anchorName string) bool { + rdrAnchorRef := fmt.Sprintf("rdr-anchor %q", anchorName) + anchorRef := fmt.Sprintf("anchor %q", anchorName) + return pfContainsRule(pfFilterRuleLines(natOutput), rdrAnchorRef) || + pfContainsRule(pfFilterRuleLines(filterOutput), anchorRef) +} diff --git a/cmd/cli/pf_ruleset_test.go b/cmd/cli/pf_ruleset_test.go new file mode 100644 index 00000000..d84252d2 --- /dev/null +++ b/cmd/cli/pf_ruleset_test.go @@ -0,0 +1,157 @@ +package cli + +import "testing" + +// altqNoise is what macOS pfctl writes to stderr on show commands. Because every +// pfctl reader here uses CombinedOutput, it lands in the middle of the data being +// parsed — which is why these helpers exist. +const altqNoise = "No ALTQ support in kernel\nALTQ related functions disabled\n" + +// TestPFRulesetEmpty is the regression guard for a flushed anchor being undetectable. +// +// The anchor-content checks in verifyPFState and ensurePFAnchorActive decide whether pf +// still has ctrld's rules. Testing the raw pfctl output for emptiness can never be true +// on macOS, because the merged ALTQ warnings are always present — so a genuinely flushed +// anchor reads as healthy and neither the startup gate nor the watchdog restore fires. +func TestPFRulesetEmpty(t *testing.T) { + tests := []struct { + name string + output string + want bool + }{ + { + // The case that was broken: nothing but merged stderr. + name: "only ALTQ warnings", + output: altqNoise, + want: true, + }, + { + // As captured on macOS 26.6 from "pfctl -sn -a com.controld.ctrld". + name: "ALTQ warnings plus the empty-ruleset marker", + output: altqNoise + "(no rules)\n", + want: true, + }, + { + name: "empty output", + output: "", + want: true, + }, + { + name: "whitespace only", + output: "\n \n\t\n", + want: true, + }, + { + name: "a real rdr rule behind the warnings", + output: altqNoise + "rdr on lo0 inet proto udp from any to ! 127.0.0.1 port = 53 -> 127.0.0.1 port 5354\n", + want: false, + }, + { + name: "a real filter rule behind the warnings", + output: altqNoise + "pass in quick on lo0 reply-to lo0 inet proto udp from any to 127.0.0.1 port = 5354\n", + want: false, + }, + { + name: "rule with no warnings at all", + output: "anchor \"com.controld.ctrld\" all\n", + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := pfRulesetEmpty(tc.output); got != tc.want { + t.Errorf("pfRulesetEmpty() = %v, want %v\noutput:\n%s", got, tc.want, tc.output) + } + }) + } +} + +// TestPFFilterRuleLines checks what survives filtering, since these lines are fed back +// into "pfctl -f -" by the ruleset-rebuild paths. Splicing a warning or the +// empty-ruleset marker into a ruleset would have the reload rejected outright. +func TestPFFilterRuleLines(t *testing.T) { + got := pfFilterRuleLines(altqNoise + "(no rules)\nrdr-anchor \"com.controld.ctrld\" all\n\nanchor \"com.controld.ctrld\" all\n") + want := []string{ + `rdr-anchor "com.controld.ctrld" all`, + `anchor "com.controld.ctrld" all`, + } + if len(got) != len(want) { + t.Fatalf("got %d lines %q, want %d %q", len(got), got, len(want), want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("line %d = %q, want %q", i, got[i], want[i]) + } + } + + if lines := pfFilterRuleLines(altqNoise); lines != nil { + t.Errorf("warnings alone must yield no rule lines, got %q", lines) + } +} + +// TestPFAnchorReferencesPresent guards when the main ruleset may be rewritten. +// +// Removing our anchor references means reloading the whole main ruleset, and that +// reload carries no options section — so it resets system-wide pf options, including +// third-party "set skip" directives. Startup rollback runs after failures that happen +// before the references were ever added, so without this check it would reset another +// application's pf options while removing nothing of ours. +func TestPFAnchorReferencesPresent(t *testing.T) { + const anchor = "com.controld.ctrld" + const otherAppRules = "scrub-anchor \"com.apple/*\" all fragment reassemble\nanchor \"com.vendor.vpn\" all\n" + + tests := []struct { + name string + nat string + filter string + want bool + }{ + { + name: "both references present", + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + filter: altqNoise + "anchor \"com.controld.ctrld\" all\n", + want: true, + }, + { + // pfctl appends tokens like " all", so matching is substring-based. + name: "rdr reference only", + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + filter: altqNoise + otherAppRules, + want: true, + }, + { + name: "filter reference only", + nat: altqNoise, + filter: altqNoise + "anchor \"com.controld.ctrld\"\n", + want: true, + }, + { + // The rollback case: we failed before adding anything, and another + // application owns the ruleset. Rewriting it would be pure collateral. + name: "someone else's ruleset, none of ours", + nat: altqNoise, + filter: altqNoise + otherAppRules, + want: false, + }, + { + name: "empty ruleset", + nat: altqNoise + "(no rules)\n", + filter: altqNoise + "(no rules)\n", + want: false, + }, + { + // A different anchor whose name merely contains ours must not count. + name: "another anchor with a similar name", + nat: altqNoise, + filter: altqNoise + "anchor \"com.vendor.controld-shim\" all\n", + want: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := pfAnchorReferencesPresent(tc.nat, tc.filter, anchor); got != tc.want { + t.Errorf("pfAnchorReferencesPresent() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/pf_state.go b/cmd/cli/pf_state.go new file mode 100644 index 00000000..2b10cc6b --- /dev/null +++ b/cmd/cli/pf_state.go @@ -0,0 +1,82 @@ +package cli + +import ( + "bufio" + "io" + "maps" + "slices" + "strings" +) + +// parsePFAnchorNames returns the anchor names that the running ruleset refers to, +// unique and sorted. It takes both pfctl show commands because each one holds half +// of the list: "pfctl -sr" the scrub and filter anchors, "pfctl -sn" the +// translation anchors. +func parsePFAnchorNames(rules, nat io.Reader) []string { + unique := make(map[string]struct{}) + // A read that stopped early gives a shorter list, and a shorter list reads + // as an anchor that went away. + if !collectPFAnchorNames(rules, unique) || !collectPFAnchorNames(nat, unique) { + return nil + } + return slices.Sorted(maps.Keys(unique)) +} + +// collectPFAnchorNames reports whether it read the whole output. +func collectPFAnchorNames(r io.Reader, into map[string]struct{}) bool { + if r == nil { + return true + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + name := pfAnchorNameOf(scanner.Text()) + if name == "" { + continue + } + into[name] = struct{}{} + } + return scanner.Err() == nil +} + +// pfAnchorNameOf returns the anchor name of one pfctl rule line. The rule verb is +// anchor, nat-anchor, rdr-anchor, scrub-anchor or dummynet-anchor, and the quoted +// name follows it. Every other line, including the merged ALTQ warnings, gives "". +func pfAnchorNameOf(line string) string { + verb, rest, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok || !strings.HasSuffix(verb, "anchor") { + return "" + } + _, quoted, ok := strings.Cut(rest, `"`) + if !ok { + return "" + } + name, _, ok := strings.Cut(quoted, `"`) + if !ok { + return "" + } + return name +} + +// parsePFStatus reads "pfctl -si" output and reports whether pf runs, with the +// uptime text that pfctl prints beside it. +func parsePFStatus(r io.Reader) (enabled bool, since string) { + if r == nil { + return false, "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + status, ok := strings.CutPrefix(strings.TrimSpace(scanner.Text()), "Status:") + if !ok { + continue + } + // pfctl pads the status into a column and prints the debug level next to it. + status, _, _ = strings.Cut(status, "Debug:") + state, uptime, _ := strings.Cut(strings.TrimSpace(status), " for ") + if state != "Enabled" { + return false, "" + } + return true, strings.TrimSpace(uptime) + } + // A read that stopped early may have stopped before the status line. + return false, "" +} diff --git a/cmd/cli/pf_state_darwin.go b/cmd/cli/pf_state_darwin.go new file mode 100644 index 00000000..b89dcfe6 --- /dev/null +++ b/cmd/cli/pf_state_darwin.go @@ -0,0 +1,119 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os/exec" + "time" +) + +// pfStateReadTimeout bounds each state read. These reads run on the network change +// path, so a pfctl that waits on a busy kernel lock must not hold it. +const pfStateReadTimeout = 3 * time.Second + +// pfStateReadMaxBytes bounds the output of one state read. A ruleset or an +// interface dump of this size is not the state ctrld reads, and these reads run +// on the network change path. +const pfStateReadMaxBytes = 1 << 20 + +// errPFStateOutputTooLarge marks output that passed the bound. The bytes kept +// are a prefix of the real output, so the caller must not parse them. +var errPFStateOutputTooLarge = errors.New("pf state output passed its bound") + +// boundedBuffer keeps at most limit bytes and remembers that it dropped the +// rest. +type boundedBuffer struct { + bytes.Buffer + limit int + exceeded bool +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + room := b.limit - b.Len() + if len(p) > room { + b.exceeded = true + p = p[:max(room, 0)] + } + if _, err := b.Buffer.Write(p); err != nil { + return 0, err + } + return len(p), nil +} + +// pfStateRunCommand runs one read-only state command. A var so tests replace it +// instead of shelling out. +var pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), pfStateReadTimeout) + defer cancel() + out := &boundedBuffer{limit: pfStateReadMaxBytes} + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = out + cmd.Stderr = out + err := cmd.Run() + if out.exceeded { + return nil, errPFStateOutputTooLarge + } + return out.Bytes(), err +} + +// pfRuleDump carries pfctl rule output that the caller already read. A nil +// field means that the caller holds no bytes for that flag, so the read runs +// here. +type pfRuleDump struct { + rules []byte // "pfctl -sr" + nat []byte // "pfctl -sn" +} + +// pfAnchorNames returns the anchor names of the running ruleset. A failed read +// gives no names, because a partial list reads as an anchor that went away. +func (p *prog) pfAnchorNames(dump pfRuleDump) []string { + rules, ok := p.pfRuleBytes(dump.rules, "-sr", "list filter anchors") + if !ok { + return nil + } + nat, ok := p.pfRuleBytes(dump.nat, "-sn", "list translation anchors") + if !ok { + return nil + } + return parsePFAnchorNames(bytes.NewReader(rules), bytes.NewReader(nat)) +} + +// pfRuleBytes returns the rule output of one pfctl flag. It starts pfctl only +// when the caller holds no bytes, and a failed read feeds the backoff that +// keeps ctrld off an exhausted host. +func (p *prog) pfRuleBytes(held []byte, flag, operation string) ([]byte, bool) { + if held != nil { + return held, true + } + out, err := pfStateRunCommand("pfctl", flag) + if err != nil { + p.pfBackoffResourceExhaustion(err, out, operation) + return nil, false + } + return out, true +} + +// pfStatus reports whether pf runs and for how long. A failed read gives false +// and "". +func (p *prog) pfStatus() (bool, string) { + out, err := pfStateRunCommand("pfctl", "-si") + if err != nil { + p.pfBackoffResourceExhaustion(err, out, "read pf status") + return false, "" + } + return parsePFStatus(bytes.NewReader(out)) +} + +// tunnelOwner returns the network extension that owns a tunnel interface, or "" +// when the read fails or no extension owns it. +func tunnelOwner(iface string) string { + if iface == "" { + return "" + } + out, err := pfStateRunCommand("ifconfig", "-v", iface) + if err != nil { + return "" + } + return parseIfconfigAgent(bytes.NewReader(out)) +} diff --git a/cmd/cli/pf_state_test.go b/cmd/cli/pf_state_test.go new file mode 100644 index 00000000..b7c47237 --- /dev/null +++ b/cmd/cli/pf_state_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "slices" + "strings" + "testing" +) + +// pfStatusEnabled is "pfctl -si" output from a Mac with pf enabled. +const pfStatusEnabled = `Status: Enabled for 0 days 02:11:05 Debug: Urgent +` + +// Captured from macOS 26.6 with ctrld installed. "pfctl -sr" prints the scrub and +// filter anchors, "pfctl -sn" prints the translation anchors, and both carry the +// ALTQ warnings because every reader here uses CombinedOutput. +const pfShowRules = altqNoise + `scrub-anchor "com.apple/*" all fragment reassemble +anchor "com.apple/*" all +anchor "com.controld.ctrld" all +pass in quick on lo0 reply-to lo0 inet proto udp from any to 127.0.0.1 port = 5354 +` + +const pfShowNAT = altqNoise + `nat-anchor "com.apple/*" all +rdr-anchor "com.apple/*" all +rdr-anchor "com.controld.ctrld" all +` + +func Test_parsePFAnchorNames(t *testing.T) { + tests := []struct { + name string + rules string + nat string + want []string + }{ + { + name: "a real ruleset with ctrld installed", + rules: pfShowRules, + nat: pfShowNAT, + want: []string{"com.apple/*", "com.controld.ctrld"}, + }, + { + // The names arrive out of order, so the result proves the sort. + name: "a VPN adds its own anchors", + rules: altqNoise + `anchor "windscribe" all +anchor "com.apple/*" all +`, + nat: altqNoise + `dummynet-anchor "com.apple/*" all +rdr-anchor "windscribe" all +`, + want: []string{"com.apple/*", "windscribe"}, + }, + { + name: "anchors in the translation output only", + rules: altqNoise, + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + want: []string{"com.controld.ctrld"}, + }, + { + name: "a ruleset with no anchor", + rules: altqNoise + "block drop in all\n", + nat: altqNoise + "(no rules)\n", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parsePFAnchorNames(strings.NewReader(tc.rules), strings.NewReader(tc.nat)) + if !slices.Equal(got, tc.want) { + t.Fatalf("parsePFAnchorNames() = %v, want %v", got, tc.want) + } + }) + } +} + +func Test_parsePFStatus(t *testing.T) { + tests := []struct { + name string + output string + wantEnabled bool + wantSince string + }{ + { + name: "enabled with an uptime", + output: altqNoise + `Status: Enabled for 3 days 02:11:05 Debug: Urgent + +Interface Stats for en0 IPv4 IPv6 + Bytes In 123456789 0 +`, + wantEnabled: true, + wantSince: "3 days 02:11:05", + }, + { + name: "disabled", + output: altqNoise + "Status: Disabled Debug: Urgent\n\n", + }, + { + name: "enabled without an uptime", + output: "Status: Enabled Debug: Urgent\n", + wantEnabled: true, + }, + { + name: "no status line", + output: altqNoise, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(tc.output)) + if enabled != tc.wantEnabled || since != tc.wantSince { + t.Fatalf("parsePFStatus() = %v, %q, want %v, %q", enabled, since, tc.wantEnabled, tc.wantSince) + } + }) + } +} + +// oversizedLine is longer than the largest line a scanner holds, so the read +// of it stops with an error. +var oversizedLine = strings.Repeat("a", 70<<10) + "\n" + +// Test_parsePFAnchorNamesRejectsAnOversizedLine covers a truncated read. The +// names collected before the error read as a ruleset that lost its anchors, +// and that is the state the watchdog acts on. +func Test_parsePFAnchorNamesRejectsAnOversizedLine(t *testing.T) { + if got := parsePFAnchorNames(strings.NewReader(pfShowRules+oversizedLine), strings.NewReader(pfShowNAT)); got != nil { + t.Fatalf("anchor names of a truncated rule list = %v, want none", got) + } + if got := parsePFAnchorNames(strings.NewReader(pfShowRules), strings.NewReader(pfShowNAT+oversizedLine)); got != nil { + t.Fatalf("anchor names of a truncated translation list = %v, want none", got) + } +} + +// Test_parsePFStatusRejectsAnOversizedLine covers a truncated status read. +func Test_parsePFStatusRejectsAnOversizedLine(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(oversizedLine + pfStatusEnabled)) + if enabled || since != "" { + t.Fatalf("pf status of a truncated read = %v, %q, want false and an empty uptime", enabled, since) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 76f7c366..62b17d42 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "io/fs" "math/rand" "net" @@ -24,7 +25,6 @@ import ( "github.com/Masterminds/semver/v3" "github.com/kardianos/service" - "github.com/rs/zerolog" "github.com/spf13/viper" "golang.org/x/sync/singleflight" "tailscale.com/net/netmon" @@ -34,8 +34,8 @@ import ( "github.com/Control-D-Inc/ctrld/internal/clientinfo" "github.com/Control-D-Inc/ctrld/internal/controld" "github.com/Control-D-Inc/ctrld/internal/dnscache" - "github.com/Control-D-Inc/ctrld/internal/router" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" + "github.com/Control-D-Inc/ctrld/internal/firewall" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) const ( @@ -48,7 +48,23 @@ const ( upstreamOS = upstreamPrefix + "os" upstreamOSLocal = upstreamOS + ".local" dnsWatchdogDefaultInterval = 20 * time.Second - ctrldServiceName = "ctrld" + ctrldServiceName = "ctrld-client" + // ctrldServiceDisplayName must differ from the v1 service's display name + // ("Control-D Helper Service"). Windows requires display names to be unique + // across all installed services and fails registration with + // ERROR_DUPLICATE_SERVICE_NAME otherwise, so reusing v1's name would make + // "ctrld-client start" unable to install on any host that still has the v1 + // service. It moves with ctrldServiceName: both identify this service. + ctrldServiceDisplayName = "Control-D Client Service" +) + +// Service-manager paths derived from ctrldServiceName. Every init system names +// its unit after the service identifier, so these must move with it: a rename +// that missed one would leave ctrld managing a unit it no longer installs. +const ( + systemdUnitFile = "/etc/systemd/system/" + ctrldServiceName + ".service" + sysVInitScript = "/etc/init.d/" + ctrldServiceName + launchdPlistFile = "/Library/LaunchDaemons/" + ctrldServiceName + ".plist" ) // RecoveryReason provides context for why we are waiting for recovery. @@ -82,15 +98,18 @@ var logf = func(format string, args ...any) { //lint:ignore U1000 use in newLoopbackOSConfigurator var noopLogf = func(format string, args ...any) {} -var svcConfig = &service.Config{ - Name: ctrldServiceName, - DisplayName: "Control-D Helper Service", - Description: "A highly configurable, multi-protocol DNS forwarding proxy", - Option: service.KeyValue{}, -} - var useSystemdResolved = false +type pfAnchorCheckResult uint8 + +const ( + pfAnchorCheckSkipped pfAnchorCheckResult = iota + pfAnchorCheckIntact + pfAnchorCheckRestored + pfAnchorCheckDeferred + pfAnchorCheckFailed +) + type prog struct { mu sync.Mutex waitCh chan struct{} @@ -101,8 +120,10 @@ type prog struct { apiReloadCh chan *ctrld.Config apiForceReloadCh chan struct{} apiForceReloadGroup singleflight.Group - logConn net.Conn + logConn io.WriteCloser + logConnMu sync.Mutex cs *controlServer + logger atomic.Pointer[ctrld.Logger] csSetDnsDone chan struct{} csSetDnsOk bool dnsWg sync.WaitGroup @@ -119,24 +140,45 @@ type prog struct { sema semaphore ciTable *clientinfo.Table um *upstreamMonitor - router router.Router ptrLoopGuard *loopGuard lanLoopGuard *loopGuard metricsQueryStats atomic.Bool queryFromSelfMap sync.Map initInternalLogWriterOnce sync.Once internalLogWriter *logWriter - internalWarnLogWriter *logWriter - internalLogSent time.Time + internalJournalWriter *logWriter + querySampler errorSampler + internalDomainProbes internalDomainProber + lastNetworkState atomic.Pointer[netmon.State] + logUpload logUploadGate + logSendMu sync.Mutex + logSendStopped bool + logSend *logSendServer runningIface string requiredMultiNICsConfig bool - adDomain string - runningOnDomainController bool - selfUninstallMu sync.Mutex - refusedQueryCount int - canSelfUninstall atomic.Bool - checkingSelfUninstall bool + // The network journal state. The value fields here hold a mutex, so no + // code may copy a prog. Every user of a prog holds a pointer to it. + networkNoise noiseCoalescer + wake wakeReporter + repeats repeatLogger //lint:ignore U1000 used on darwin + dnsConfig *dnsConfigPoller + health *queryHealth + snapshotWrites snapshotLimiter + lastNAT64Mu sync.Mutex + lastNAT64Logged string + + selfUninstallMu sync.Mutex + refusedQueryCount int + canSelfUninstall atomic.Bool + + // startedInAPIMaintenance records that this run began on the configuration + // already on disk because the API was in maintenance, so it holds no answer + // from the API at all. The reload loop retries faster while it is set, and + // clears it once a recovered configuration has been applied - not when a + // fetch returns, because the reload receiver fetches again and can fail. + startedInAPIMaintenance atomic.Bool + checkingSelfUninstall bool loopMu sync.Mutex loop map[string]bool @@ -144,28 +186,347 @@ type prog struct { recoveryCancelMu sync.Mutex recoveryCancel context.CancelFunc recoveryRunning atomic.Bool + recoveryGen atomic.Uint64 + + // OS-only failures can refresh the resolver without owning global recovery. + osRecoveryRefreshMu sync.Mutex + osRecoveryRefreshAt time.Time + osRecoverySkipLogAt time.Time + osRecoverySkipUpstream string + + // All deltas get an ID. Only accepted deltas replace recovery ownership. + networkTransitionGen atomic.Uint64 + networkAcceptedGen atomic.Uint64 + networkSourceMu sync.Mutex + networkSourceState *netmon.State + networkSourceEpoch *netmon.State + networkSourceReadFailed bool + // netmon hands a minor callback the cached major snapshot in Old, so the + // diff keeps the state that the callback before it reported. + networkDeltaState *netmon.State + networkDeltaEpoch *netmon.State + + // recoveryDebounceTimer coalesces rapid NetworkChange recovery triggers + // into a single handleRecovery call. Only handleRecovery is debounced — + // all other state updates (IP, pf anchor, VPN DNS) run immediately. + recoveryDebounceMu sync.Mutex + recoveryDebounceTimer *time.Timer + // recoveryBypass is set when dns-intercept mode enters recovery. + // While true, proxy() forwards all queries to the OS/DHCP resolver + // instead of the configured upstreams. This allows captive portal + // authentication without tearing down WFP/pf filters. + recoveryBypass atomic.Bool + + // dns64 tracks DNS64/NAT64 synthesis state for IPv6-only networks + // without client-side 464XLAT. See cmd/cli/dns64.go. + dns64 dns64State + + // interceptDNSTargetService names the macOS network service on which + // ctrld set a temporary DNS target; interceptDNSTargetSetValue records the + // exact value. Both are guarded by interceptDNSTargetMu. + interceptDNSTargetMu sync.Mutex //lint:ignore U1000 used on darwin + interceptDNSTargetService string //lint:ignore U1000 used on darwin + interceptDNSTargetSetValue string //lint:ignore U1000 used on darwin + interceptDNSTargetLoaded bool //lint:ignore U1000 used on darwin + // interceptIPv6OnlySeen and interceptIPv6OnlyTimer track an IPv6-only + // observation that has not yet held long enough to admit a target + // (issue #629). Guarded by interceptDNSTargetMu. + interceptIPv6OnlySeen ipv6OnlyObservation //lint:ignore U1000 used on darwin + interceptIPv6OnlyTimer *time.Timer //lint:ignore U1000 used on darwin + + // interceptTargetMirror holds the same value as interceptDNSTargetSetValue + // for readers that must not wait: the mutex above covers networksetup + // calls that take seconds. + interceptTargetMirror atomic.Value // holds string + + // DNS intercept mode state (platform-specific). + // On Windows: *wfpState, on macOS: *pfState, nil on other platforms. + dnsInterceptState any + + // dnsInterceptMu serializes DNS intercept lifecycle transitions - start, stop and + // the health monitor's rebuild - and guards every write to dnsInterceptState, so a + // service stop can never interleave with a monitor-driven rebuild. + dnsInterceptMu sync.Mutex //lint:ignore U1000 used on windows + + // dnsInterceptStopRequested is set while a stop waits for dnsInterceptMu. The + // health and recovery flows read it as a shutdown signal and abandon their work, + // rather than making the stop wait out their probe backoffs. + dnsInterceptStopRequested atomic.Bool //lint:ignore U1000 used on windows + + // nrptTransitionMu makes one NRPT ownership transition - observe, mutate, signal, + // record owner - atomic against shutdown and against another transition. It is + // deliberately finer-grained than dnsInterceptMu: it is taken for the duration of a + // single transition, never across the recovery flows' probe backoffs. + nrptTransitionMu sync.Mutex //lint:ignore U1000 used on windows + + // lastTunnelIfaces tracks the tunnel set included in the last successfully loaded + // pf anchor. Pending tunnel state is kept separately so failed PF work is retried + // instead of being mistaken for an applied update. Protected by mu. + lastTunnelIfaces []string //lint:ignore U1000 used on darwin + // lastLoggedTunnelIfaces is the tunnel set of the last logged event. The + // reconcile retries a set until it succeeds, so a diff of the applied set + // repeats one event at every retry. + lastLoggedTunnelIfaces []string //lint:ignore U1000 used on darwin + pendingTunnelIfaces []string //lint:ignore U1000 used on darwin + hasPendingTunnelIfaces bool //lint:ignore U1000 used on darwin + + // pfStabilizing is true while we're waiting for a VPN's pf ruleset to settle. + // While true, the watchdog and network change callbacks do NOT restore our rules. + pfStabilizing atomic.Bool + + // pfStabilizeCancel cancels the active stabilization goroutine, if any. + // Protected by mu. + pfStabilizeCancel context.CancelFunc //lint:ignore U1000 used on darwin + + // pfLastRestoreTime records when we last restored our anchor (unix millis). + // Used to detect immediate re-wipes (VPN reconnect cycle). + pfLastRestoreTime atomic.Int64 //lint:ignore U1000 used on darwin + + // pfBackoffMultiplier tracks exponential backoff for stabilization. + // Resets to 0 when rules survive for >60s. + pfBackoffMultiplier atomic.Int32 //lint:ignore U1000 used on darwin + + // pfMonitorRunning ensures only one pfInterceptMonitor goroutine runs at a time. + // When an interface appears/disappears, we spawn a monitor that probes pf + // interception with exponential backoff and auto-heals if broken. + pfMonitorRunning atomic.Bool //lint:ignore U1000 used on darwin + + // pfEnsureRunning ensures only one pf validation or mutation runs at a time. + // Network callbacks, VPN exemption updates, delayed rechecks, probes, and the + // watchdog can converge during macOS churn; concurrent pfctl/scutil work can + // exhaust process/file limits or interleave anchor snapshots. + pfEnsureRunning atomic.Bool //lint:ignore U1000 used on darwin + + // pfExecBackoffUntil suppresses pf anchor validation after pfctl/scutil execs + // fail due host resource exhaustion (fork unavailable, too many open files). + pfExecBackoffUntil atomic.Int64 //lint:ignore U1000 used on darwin + + // pfDelayedRecheckTimers coalesces delayed DNS-intercept rechecks after noisy + // network changes. Protected by pfDelayedRecheckMu. + pfDelayedRecheckMu sync.Mutex //lint:ignore U1000 used on darwin + pfDelayedRecheckTimers []*time.Timer //lint:ignore U1000 used on darwin + + // pfSettleFollowupTimer is the pending post-settle VPN DNS refresh, if any. + // Tracked so teardown can cancel it and a later stabilization replaces it + // instead of stacking another copy of the same refresh. Protected by + // pfDelayedRecheckMu. + pfSettleFollowupTimer *time.Timer //lint:ignore U1000 used on darwin + + // pfIgnoredChangeLastReconcile bounds immediate pf/VPN-DNS work for noisy + // ignored macOS network deltas. Tunnel changes bypass this limit, and the + // existing delayed checks provide a trailing reconciliation after churn. + pfIgnoredChangeLastReconcile atomic.Int64 //lint:ignore U1000 used on darwin + + // interceptProbes maps the domain of each pending interception probe to the channel + // that probe waits on. A probe verifies that interception is actually translating or + // redirecting packets, not merely present in rule text: the DNS handler looks up + // incoming queries here and signals the matching waiter. + // + // It holds one entry per in-flight probe rather than a single slot, because probes do + // overlap - the health monitor, a handback and a heal cycle can each have one out at + // the same time - and a single slot means the last registration wins and the loser + // waits out its timeout for a query that was answered. A false failure then triggers + // recovery work that was not needed. + // + // Registrations are rare and lookups happen on every query, so the map is stored as + // an immutable snapshot behind an atomic: readers never take a lock, writers copy + // under interceptProbeMu. + interceptProbes atomic.Value // map[string]chan struct{} + interceptProbeMu sync.Mutex //lint:ignore U1000 written only by registerInterceptProbe, used on darwin/windows + + // VPN DNS manager for split DNS routing when intercept mode is active. + vpnDNS *vpnDNSManager + // Serializes journal reads with publication after an intercept startup retry. + vpnDNSJournalMu sync.Mutex + + // rejectedDestinationsKey is the signature of the organization allowed + // destination entries that were last reported as unusable, so re-parsing an + // unchanged list on every refresh does not repeat the warning. Protected by mu. + rejectedDestinationsKey string + + // wideDestinationsKey is the same signature for the accepted entries that were + // last reported as covering a very wide range. Protected by mu. + wideDestinationsKey string + + // appliedDestinations is the organization allowed destination set that + // platform enforcement (pf/WFP) has actually accepted, which is not always the + // set the API last sent: a failed pfctl call or WFP filter operation leaves + // this behind the desired set, and reconcileAllowedDestinations retries the + // difference until they agree. Protected by destinationsMu, which is separate + // from mu because the mirror it guards runs subprocesses. + appliedDestinations []netip.Prefix + destinationsMu sync.Mutex + + // destinationsNeedResync marks that platform enforcement holds state ctrld + // cannot describe - a pf persist table inherited from a previous run, or a + // WFP session that holds nothing yet - so the next reconcile must replace its + // whole set instead of applying a delta. Cleared only once that replace has + // succeeded. Protected by destinationsMu. + destinationsNeedResync bool + + // firewallGen identifies the current run's firewall enforcement. It advances + // on every Firewall Mode start, reload and teardown, so a maintenance worker + // left over from an earlier run can tell that the enforcement it was given is + // no longer the enforcement in place, and stop touching it. Advanced under + // destinationsMu, which is also held across the platform mirror, so teardown + // and a worker's reconcile cannot interleave. + firewallGen atomic.Uint64 + + // allowList tracks IPs resolved by ctrld for firewall mode enforcement. + // When firewall_mode is "on", only IPs in this list (plus permanent entries) + // are allowed for outbound connections. nil when firewall mode is off. + allowList *firewall.AllowList + + // platformFirewallState stores the OS-specific firewall state used to keep + // platform enforcement synchronized with allowList. + // On Windows: *wfpFirewallState. On macOS: *pfFirewallState. + platformFirewallState any //lint:ignore U1000 used on darwin/windows started chan struct{} onStartedDone chan struct{} onStarted []func() onStopped []func() + + netMonitorMu sync.Mutex + netMonitor networkChangeMonitor + netMonitorClosed bool + netMonitorStopCh chan struct{} + restoreOnce sync.Once + restoreErr error + releaseOnce sync.Once + csStopOnce sync.Once + runDone chan struct{} + runAbortCh chan struct{} + listenerWg sync.WaitGroup + apiReloadWg sync.WaitGroup + osStateMu sync.Mutex + osStateRestored bool + + // netMonitorWG tracks admitted callbacks and their recovery work. Admission + // is serialized with shutdown under netMonitorMu; no Add races with Wait. + netMonitorWG sync.WaitGroup + netMonitorCloseOnce sync.Once + // Published by the joined initializer; read only after runDone. + waitNetworkJournal func() } -func (p *prog) Start(s service.Service) error { +// setNetMonitor publishes mon as the active network monitor, closing any +// predecessor. It reports false if shutdown already ran, in which case the +// caller owns mon and must not start it. +func (p *prog) setNetMonitor(mon networkChangeMonitor) bool { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + if p.netMonitorClosed { + return false + } + old := p.netMonitor + p.netMonitor = mon + if old != nil { + _ = old.Close() + } + return true +} + +// beginNetworkActivity admits a callback or recovery before shutdown. Every +// successful admission must be paired with netMonitorWG.Done. +func (p *prog) beginNetworkActivity() bool { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + if p.netMonitorClosed { + return false + } + p.netMonitorWG.Add(1) + return true +} + +// retryVPNDNSRefresh returns the refresh that a Tailscale route retry of m +// runs. Like other background refreshes it is admitted through the network +// activity gate, and it does nothing once m is no longer the live manager. +func (p *prog) retryVPNDNSRefresh(m *vpnDNSManager) func() { + return func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + p.vpnDNSJournalMu.Lock() + live := p.vpnDNS == m + p.vpnDNSJournalMu.Unlock() + if live { + m.Refresh(m.backgroundCtx(), true) + } + } +} + +func (p *prog) networkActivityClosed() bool { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + return p.netMonitorClosed +} + +// networkActivityDone cancels admitted work before service Stop closes stopCh. +func (p *prog) networkActivityDone() <-chan struct{} { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + if p.netMonitorStopCh == nil { + p.netMonitorStopCh = make(chan struct{}) + if p.netMonitorClosed { + close(p.netMonitorStopCh) + } + } + return p.netMonitorStopCh +} + +// closeNetMonitor fences publications and callbacks, cancels recovery, and +// drains admitted work before OS state restoration. netmon.Close alone does +// not join change callbacks. Never hold an admission/state mutex while waiting: +// callbacks and canceled recoveries need those mutexes to finish. +func (p *prog) closeNetMonitor() { + p.netMonitorCloseOnce.Do(func() { + p.netMonitorMu.Lock() + mon := p.netMonitor + p.netMonitor = nil + p.netMonitorClosed = true + if p.netMonitorStopCh != nil { + close(p.netMonitorStopCh) + } + p.netMonitorMu.Unlock() + + p.recoveryDebounceMu.Lock() + if p.recoveryDebounceTimer != nil { + p.recoveryDebounceTimer.Stop() + p.recoveryDebounceTimer = nil + } + p.recoveryDebounceMu.Unlock() + + p.recoveryCancelMu.Lock() + if p.recoveryCancel != nil { + p.recoveryCancel() + } + p.recoveryCancelMu.Unlock() + + if mon != nil { + _ = mon.Close() + p.Debug().Msg("network monitor stopped") + } + p.netMonitorWG.Wait() + }) +} + +func (p *prog) Start(_ service.Service) error { go p.runWait() return nil } // runWait runs ctrld components, waiting for signal to reload. func (p *prog) runWait() { - p.mu.Lock() - p.cfg = &cfg - p.mu.Unlock() + if p.runDone != nil { + defer close(p.runDone) + } reloadSigCh := make(chan os.Signal, 1) + defer stopNotifyReloadSigCh(reloadSigCh) notifyReloadSigCh(reloadSigCh) reload := false - logger := mainLog.Load() for { reloadCh := make(chan struct{}) done := make(chan struct{}) @@ -178,13 +539,18 @@ func (p *prog) runWait() { var newCfg *ctrld.Config select { case sig := <-reloadSigCh: - logger.Notice().Msgf("got signal: %s, reloading...", sig.String()) + p.Notice().Msgf("Got signal: %s, reloading...", sig.String()) case <-p.reloadCh: - logger.Notice().Msg("reloading...") + p.Notice().Msg("Reloading...") case apiCfg := <-p.apiReloadCh: newCfg = apiCfg case <-p.stopCh: close(reloadCh) + <-done + return + case <-p.runAbortCh: + close(reloadCh) + <-done return } @@ -203,18 +569,19 @@ func (p *prog) runWait() { } v.SetConfigFile(confFile) if err := v.ReadInConfig(); err != nil { - logger.Err(err).Msg("could not read new config") + p.Error().Err(err).Msg("Could not read new config") waitOldRunDone() continue } if err := v.Unmarshal(&newCfg); err != nil { - logger.Err(err).Msg("could not unmarshal new config") + p.Error().Err(err).Msg("Could not unmarshal new config") waitOldRunDone() continue } if cdUID != "" { - if rc, err := processCDFlags(newCfg); err != nil { - logger.Err(err).Msg("could not fetch ControlD config") + rc, err := p.fetchCDConfigBoundedByLifetime(newCfg) + if err != nil { + p.Error().Err(err).Msg("Could not fetch controld config") waitOldRunDone() continue } else { @@ -225,6 +592,10 @@ func (p *prog) runWait() { } } + // Though the log configuration could not be changed during reloading, we still need to + // process the current flags here, so runtime internal logs can be used correctly. + processLogAndCacheFlags(v, newCfg) + waitOldRunDone() p.mu.Lock() @@ -244,183 +615,385 @@ func (p *prog) runWait() { } } if err := validateConfig(newCfg); err != nil { - logger.Err(err).Msg("invalid config") + p.Error().Err(err).Msg("Invalid config") continue } addExtraSplitDnsRule(newCfg) - if err := writeConfigFile(newCfg); err != nil { - logger.Err(err).Msg("could not write new config") - } + persistManagedConfig(newCfg, p.logger.Load()) // This needs to be done here, otherwise, the DNS handler may observe an invalid // upstream config because its initialization function have not been called yet. - mainLog.Load().Debug().Msg("setup upstream with new config") + p.Debug().Msg("Setup upstream with new config") p.setupUpstream(newCfg) + // The configuration is in effect now. Every failure above continues the + // loop without applying anything, so a run that started in maintenance + // stays pending and keeps retrying until one gets this far. + p.maintenanceConfigApplied(p.logger.Load()) p.mu.Lock() + oldUpstreams := p.cfg.Upstream *p.cfg = *newCfg + // In DNS-intercept mode on macOS, the DNS listener is bound once at startup and is + // NOT re-bound on reload (see prog.run: serveDNS is started only when !reload). When + // the configured/generated port (e.g. 127.0.0.1:53) is unavailable at startup because + // mDNSResponder owns *:53, ctrld falls back to an alternate local port (e.g. 5354). + // The on-disk config still declares 53, so adopting it here would revert p.cfg to a + // port nothing is listening on, and the pf rdr rules/probes rebuilt from p.cfg would + // target a dead port. Since a reload cannot move the running listener anyway, keep + // p.cfg pointing at the actual bound listener. The on-disk config (written above) is + // left unchanged. See #551. + if dnsIntercept && runtime.GOOS == "darwin" { + preserveBoundListeners(p.cfg.Listener, curListener) + } p.mu.Unlock() - logger.Notice().Msg("reloading config successfully") + closeReplacedUpstreams(oldUpstreams, newCfg.Upstream) + p.applyDebugLogBudget(debugLogBudget(&newCfg.Service)) + // The header names the mode, the listeners, and the upstreams, so the + // open files take the values of the config that now runs. + p.refreshLogHeader() + + p.Notice().Msg("Reloading config successfully") select { case p.reloadDoneCh <- struct{}{}: + p.Debug().Msg("Reload done signal sent") default: } } } +// preserveBoundListeners overrides the IP/Port of each listener in newListeners with the +// actual bound address from curListeners when they differ, logging the divergence. It is used +// on config reload in DNS-intercept mode where the running listener is never re-bound, so a +// port change on disk (e.g. reverting a fallback 5354 back to the generated 53) must not be +// applied to the in-memory config that drives pf rdr rules and probes. +// +// Preservation is limited to fallback-eligible (default/unset, i.e. 127.0.0.1:53) listeners. +// An explicit, non-default listener in the reloaded config is an intentional change that must +// be applied: tryUpdateListenerConfigIntercept binds explicit listeners exactly (no fallback), +// and the control-server reload handler detects the IP/port diff to trigger a restart that +// re-binds. Reverting an explicit change here would make that comparison return 200 instead of +// 201, silently dropping the new listener. See #551. +func preserveBoundListeners(newListeners, curListeners map[string]*ctrld.ListenerConfig) { + for n, curLc := range curListeners { + newLc := newListeners[n] + if newLc == nil || curLc == nil { + continue + } + if newLc.IP == curLc.IP && newLc.Port == curLc.Port { + continue + } + if isExplicitInterceptListener(newLc.IP, newLc.Port) { + continue + } + mainLog.Load().Info(). + Str("configured", net.JoinHostPort(newLc.IP, strconv.Itoa(newLc.Port))). + Str("actual", net.JoinHostPort(curLc.IP, strconv.Itoa(curLc.Port))). + Msg("DNS intercept: preserving actual bound listener across reload; on-disk config port not applied to running listener") + newLc.IP = curLc.IP + newLc.Port = curLc.Port + } +} + func (p *prog) preRun() { - if iface == "auto" { + if iface == autoIface { iface = defaultIfaceName() p.requiredMultiNICsConfig = requiredMultiNICsConfig() } p.runningIface = iface + p.logger.Store(mainLog.Load()) } func (p *prog) postRun() { if !service.Interactive() { - if runtime.GOOS == "windows" { - isDC, roleInt := isRunningOnDomainController() - p.runningOnDomainController = isDC - mainLog.Load().Debug().Msgf("running on domain controller: %t, role: %d", p.runningOnDomainController, roleInt) - } - p.resetDNS(false, false) - ns := ctrld.InitializeOsResolver(false) - mainLog.Load().Debug().Msgf("initialized OS resolver with nameservers: %v", ns) - p.setDNS() + // A Windows organization can install a GP-owned NRPT catch-all before + // starting ctrld. Detect that policy before resetDNS touches adapter DNS; + // startDNSIntercept will then prove the rule functionally before adopting it. + if !p.skipInitialDNSReset() { + p.resetDNS(false, false) + } + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + ns, systemNameservers := initializeOsResolverWithSystemNameserversFn(loggerCtx, false, osResolverReasonStart) + p.Debug().Msgf("Initialized os resolver with nameservers: %v", ns) + p.setDNS(systemNameservers) + if p.allowList != nil { + p.initPlatformFirewall() + } p.csSetDnsDone <- struct{}{} close(p.csSetDnsDone) - p.logInterfacesState() } } +// fetchResolverConfigFn fetches the resolver config for a configuration refresh. +// Indirected so the refresh loop itself - its ticker and its forced-reload path - +// can be driven in tests without an API server, rather than only the handler it +// calls. +var fetchResolverConfigFn = controld.FetchResolverConfig + +// startAPIConfigReload starts one program-lifetime worker, not one per reload. +func (p *prog) startAPIConfigReload() { + p.apiReloadWg.Add(1) + go func() { + defer p.apiReloadWg.Done() + p.apiConfigReload() + }() +} + // apiConfigReload calls API to check for latest config update then reload ctrld if necessary. func (p *prog) apiConfigReload() { if cdUID == "" { return } - ticker := time.NewTicker(timeDurationOrDefault(p.cfg.Service.RefetchTime, 3600) * time.Second) + ctx, cancel := context.WithCancel(context.Background()) + watcherDone := make(chan struct{}) + defer func() { + cancel() + <-watcherDone + }() + go func() { + defer close(watcherDone) + select { + case <-p.stopCh: + case <-p.runAbortCh: + case <-ctx.Done(): + } + cancel() + }() + + p.mu.Lock() + refetchInterval := timeDurationOrDefault(p.cfg.Service.RefetchTime, 3600) * time.Second + p.mu.Unlock() + ticker := time.NewTicker(refetchInterval) defer ticker.Stop() - logger := mainLog.Load().With().Str("mode", "api-reload").Logger() - logger.Debug().Msg("starting custom config reload timer") + logger := p.logger.Load().With().Str("mode", "api-reload") + logger.Debug().Msg("Starting custom config reload timer") lastUpdated := time.Now().Unix() curVerStr := curVersion() curVer, err := semver.NewVersion(curVerStr) isStable := curVer != nil && curVer.Prerelease() == "" if err != nil || !isStable { - l := mainLog.Load().Warn() + l := p.Warn() if err != nil { l = l.Err(err) } - l.Msgf("current version is not stable, skipping self-upgrade: %s", curVerStr) + l.Msgf("Current version is not stable, skipping self-upgrade: %s", curVerStr) } - doReloadApiConfig := func(forced bool, logger zerolog.Logger) { - resolverConfig, err := controld.FetchResolverConfig(cdUID, rootCmd.Version, cdDev) + doReloadApiConfig := func(forced bool, logger *ctrld.Logger) { + if ctx.Err() != nil { + return + } + loggerCtx := ctrld.LoggerCtx(ctx, p.logger.Load()) + req := &controld.ResolverConfigRequest{ + RawUID: cdUID, + Version: appVersion, + Metadata: ctrld.SystemMetadataRuntime(loggerCtx), + } + resolverConfig, err := fetchResolverConfigFn(loggerCtx, req, cdDev) + if ctx.Err() != nil { + return + } selfUninstallCheck(err, p, logger) if err != nil { - logger.Warn().Err(err).Msg("could not fetch resolver config") + logResolverConfigFetchFailure(logger, err) return } // Performing self-upgrade check for production version. if isStable { - _ = selfUpgradeCheck(resolverConfig.Ctrld.VersionTarget, curVer, &logger) + _ = selfUpgradeCheck(resolverConfig.Ctrld.VersionTarget, curVer, logger) } - if resolverConfig.DeactivationPin != nil { - newDeactivationPin := *resolverConfig.DeactivationPin - curDeactivationPin := cdDeactivationPin.Load() - switch { - case curDeactivationPin != defaultDeactivationPin: - logger.Debug().Msg("saving deactivation pin") - case curDeactivationPin != newDeactivationPin: - logger.Debug().Msg("update deactivation pin") + lastUpdated = p.applyFetchedResolverConfig(loggerCtx, logger, resolverConfig, forced, lastUpdated) + } + // A run that started in maintenance holds no answer from the API, so it + // waits out the whole refetch interval before it has one. Retry faster + // until it does, without touching refetch_time: that interval governs + // every endpoint, and shortening it fleet-wide would multiply load on an + // API that is by definition already struggling. + var maintenanceRetry *time.Timer + var maintenanceRetryC <-chan time.Time + if p.startedInAPIMaintenance.Load() { + maintenanceRetry = time.NewTimer(maintenanceReloadDelay()) + defer maintenanceRetry.Stop() + maintenanceRetryC = maintenanceRetry.C + logger.Debug().Msg("Started on the configuration on disk, retrying the API faster until it answers") + } + for { + select { + case <-p.apiForceReloadCh: + doReloadApiConfig(true, logger.With().Bool("forced", true)) + case <-ticker.C: + doReloadApiConfig(false, logger) + case <-maintenanceRetryC: + // The normal ticker or a forced reload may have recovered first. + if !p.startedInAPIMaintenance.Load() { + maintenanceRetryC = nil + continue } - cdDeactivationPin.Store(newDeactivationPin) - } else { - cdDeactivationPin.Store(defaultDeactivationPin) + doReloadApiConfig(false, logger.With().Bool("maintenance_retry", true)) + if p.startedInAPIMaintenance.Load() { + maintenanceRetry.Reset(maintenanceReloadDelay()) + continue + } + maintenanceRetryC = nil + case <-ctx.Done(): + return } + } +} - p.mu.Lock() - rc := p.rc - p.rc = resolverConfig - p.mu.Unlock() - noCustomConfig := resolverConfig.Ctrld.CustomConfig == "" - noExcludeListChanged := true - if rc != nil { - slices.Sort(rc.Exclude) - slices.Sort(resolverConfig.Exclude) - noExcludeListChanged = slices.Equal(rc.Exclude, resolverConfig.Exclude) - } - if noCustomConfig && noExcludeListChanged { - return +// applyFetchedResolverConfig applies a freshly fetched resolver config, and is +// the whole of what a configuration refresh does with one: the deactivation pin, +// the organization's allowed destinations, and the decision whether the change +// warrants reloading ctrld. Returns the lastUpdated watermark to carry into the +// next refresh. +// +// Split out of apiConfigReload's fetch loop so both refresh paths - the scheduled +// tick and a forced reload - can be exercised without an API server, including +// the early return taken when neither the custom config nor the exclusion list +// changed. That case is the one where a destination change would be easiest to +// drop, because nothing else about the refresh has any effect. +func (p *prog) applyFetchedResolverConfig( + loggerCtx context.Context, + logger *ctrld.Logger, + resolverConfig *controld.ResolverConfig, + forced bool, + lastUpdated int64, +) int64 { + if loggerCtx.Err() != nil { + return lastUpdated + } + // The answer settles the PIN either way: a response without one means the + // API says there is none, which is not the same as never having asked. + deactivationPinKnown.Store(true) + if resolverConfig.DeactivationPin != nil { + newDeactivationPin := *resolverConfig.DeactivationPin + curDeactivationPin := cdDeactivationPin.Load() + switch { + case curDeactivationPin != defaultDeactivationPin: + logger.Debug().Msg("Saving deactivation pin") + case curDeactivationPin != newDeactivationPin: + logger.Debug().Msg("Update deactivation pin") } + cdDeactivationPin.Store(newDeactivationPin) + } else { + cdDeactivationPin.Store(defaultDeactivationPin) + } - if noCustomConfig && !noExcludeListChanged { - logger.Debug().Msg("exclude list changes detected, reloading...") - p.apiReloadCh <- nil - return + p.mu.Lock() + rc := p.rc + p.rc = resolverConfig + p.mu.Unlock() + + // Apply the organization's Allowed Destination IP list before the early + // returns below: adds and removals must take effect on every refresh, + // scheduled or forced, whether or not anything else changed. It needs no + // ctrld reload - the set is enforced directly. + p.applyAllowedDestinations(p.firewallAllowList(), resolverConfig.DestinationIPs) + + noCustomConfig := resolverConfig.Ctrld.CustomConfig == "" + noExcludeListChanged := true + // Regenerate Internal Domains on add, removal, domain, mode or resolver change. + noInternalDomainsChanged := true + // A run that started on the configuration on disk has applied no answer + // from the API yet, so nothing here is known to be current and the + // response has to reach the apply path instead of being compared + // against a configuration that was never applied. This holds until the + // apply lands, not until a fetch returns: the receiver fetches again + // and can fail, and p.rc is already replaced above, so comparing would + // make every later response equal the stored one and leave the excludes + // and Internal Domains of the previous run in place for good. + pendingRecovery := p.startedInAPIMaintenance.Load() + if rc != nil && !pendingRecovery { + slices.Sort(rc.Exclude) + slices.Sort(resolverConfig.Exclude) + noExcludeListChanged = slices.Equal(rc.Exclude, resolverConfig.Exclude) + noInternalDomainsChanged = internalDomainsEqual(rc.SplitDNS, resolverConfig.SplitDNS) + } else { + noExcludeListChanged, noInternalDomainsChanged = false, false + } + if noCustomConfig && noExcludeListChanged && noInternalDomainsChanged { + return lastUpdated + } + + if noCustomConfig && (!noExcludeListChanged || !noInternalDomainsChanged) { + if !noExcludeListChanged { + logger.Debug().Msg("Exclude list changes detected, reloading...") } + if !noInternalDomainsChanged { + logger.Info().Msg("Internal domain changes detected, reloading...") + } + p.firewallOnConfigReload() + select { + case p.apiReloadCh <- nil: + case <-loggerCtx.Done(): + } + return lastUpdated + } - if resolverConfig.Ctrld.CustomLastUpdate > lastUpdated || forced { - lastUpdated = time.Now().Unix() - cfg := &ctrld.Config{} - var cfgErr error - if cfgErr = validateCdRemoteConfig(resolverConfig, cfg); cfgErr == nil { - setListenerDefaultValue(cfg) - setNetworkDefaultValue(cfg) - cfgErr = validateConfig(cfg) - } - if cfgErr != nil { - logger.Warn().Err(err).Msg("skipping invalid custom config") - if _, err := controld.UpdateCustomLastFailed(cdUID, rootCmd.Version, cdDev, true); err != nil { - logger.Error().Err(err).Msg("could not mark custom last update failed") - } - return + if resolverConfig.Ctrld.CustomLastUpdate > lastUpdated || forced || pendingRecovery { + lastUpdated = time.Now().Unix() + cfg := &ctrld.Config{} + var cfgErr error + if cfgErr = validateCdRemoteConfig(resolverConfig, cfg); cfgErr == nil { + setListenerDefaultValue(cfg) + setNetworkDefaultValue(cfg) + cfgErr = validateConfig(cfg) + } + if cfgErr != nil { + logger.Warn().Err(cfgErr).Msg("Skipping invalid custom config") + if _, err := controld.UpdateCustomLastFailed(loggerCtx, cdUID, appVersion, cdDev, true); err != nil { + logger.Error().Err(err).Msg("Could not mark custom last update failed") } - logger.Debug().Msg("custom config changes detected, reloading...") - p.apiReloadCh <- cfg - } else { - logger.Debug().Msg("custom config does not change") + return lastUpdated } - } - for { + logger.Debug().Msg("Custom config changes detected, reloading...") + // Firewall mode: flush allowlist so DNS queries against the new + // config repopulate it with IPs allowed under the updated policy. + p.firewallOnConfigReload() select { - case <-p.apiForceReloadCh: - doReloadApiConfig(true, logger.With().Bool("forced", true).Logger()) - case <-ticker.C: - doReloadApiConfig(false, logger) - case <-p.stopCh: - return + case p.apiReloadCh <- cfg: + case <-loggerCtx.Done(): } + } else { + logger.Debug().Msg("Custom config does not change") } + return lastUpdated } func (p *prog) setupUpstream(cfg *ctrld.Config) { localUpstreams := make([]string, 0, len(cfg.Upstream)) ptrNameservers := make([]string, 0, len(cfg.Upstream)) isControlDUpstream := false + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) for n := range cfg.Upstream { uc := cfg.Upstream[n] sdns := uc.Type == ctrld.ResolverTypeSDNS - uc.Init() + uc.Init(loggerCtx) if sdns { - mainLog.Load().Debug().Msgf("initialized DNS Stamps with endpoint: %s, type: %s", uc.Endpoint, uc.Type) + p.Debug().Msgf("Initialized dns stamps with endpoint: %s, type: %s", uc.Endpoint, uc.Type) } isControlDUpstream = isControlDUpstream || uc.IsControlD() + // Generated resolver bootstrap addresses are private organization data. + bootstrapEvent := func() *ctrld.LogEvent { + if strings.HasPrefix(n, internalDomainUpstreamPrefix) { + return p.Debug() + } + return p.Info() + } if uc.BootstrapIP == "" { - uc.SetupBootstrapIP() - mainLog.Load().Info().Msgf("bootstrap IPs for upstream.%s: %q", n, uc.BootstrapIPs()) + uc.SetupBootstrapIP(ctrld.LoggerCtx(context.Background(), p.logger.Load())) + bootstrapEvent().Msgf("Bootstrap ips for upstream.%s: %q", n, uc.BootstrapIPs()) } else { - mainLog.Load().Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("using bootstrap IP for upstream.%s", n) + bootstrapEvent().Str("bootstrap_ip", uc.BootstrapIP).Msgf("Using bootstrap ip for upstream.%s", n) } uc.SetCertPool(rootCertPool) - go uc.Ping() + go uc.Ping(loggerCtx) if canBeLocalUpstream(uc.Domain) { localUpstreams = append(localUpstreams, upstreamPrefix+n) @@ -430,13 +1003,30 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { } } // Self-uninstallation is ok If there is only 1 ControlD upstream, and no remote config. - if len(cfg.Upstream) == 1 && isControlDUpstream { - p.canSelfUninstall.Store(true) + // Generated Internal Domain resolvers are part of the managed configuration, + // not independent user upstreams. Recompute eligibility on every setup/reload; + // uninstall still requires API confirmation that the device is gone. + managedUpstreams := 0 + for _, uc := range cfg.Upstream { + if !isGeneratedInternalDomainUpstream(uc) { + managedUpstreams++ + } } + p.canSelfUninstall.Store(managedUpstreams == 1 && isControlDUpstream) p.localUpstreams = localUpstreams p.ptrNameservers = ptrNameservers } +// reportServeDNSFailure classifies a listener that bound successfully - the +// LISTENER_* codes already rule out a bind conflict - but failed to actually +// serve DNS. No dedicated code exists for this, so it falls back to +// UNCLASSIFIED. notifyExitToLogServer unblocks a waiting "ctrld start" before +// the process exits. +func (p *prog) reportServeDNSFailure(listenerNum string, err error) { + msg := fmt.Sprintf("unable to start dns proxy on listener.%s: %v", listenerNum, err) + failRunUnclassified(p.Error().Err(err), msg, p.notifyExitToLogServer) +} + // run runs the ctrld main components. // // The reload boolean indicates that the function is run when ctrld first start @@ -447,7 +1037,16 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { // so all listeners could be terminated and re-spawned again. func (p *prog) run(reload bool, reloadCh chan struct{}) { // Wait the caller to signal that we can do our logic. - <-p.waitCh + select { + case <-p.waitCh: + case <-p.stopCh: + return + case <-p.runAbortCh: + return + } + if stopRequested(p.stopCh) || stopRequested(p.runAbortCh) { + return + } if !reload { p.preRun() } @@ -458,10 +1057,11 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { p.csSetDnsDone = make(chan struct{}, 1) p.registerControlServerHandler() if err := p.cs.start(); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not start control server") + p.Warn().Err(err).Msg("Could not start control server") } - mainLog.Load().Debug().Msgf("control server started: %s", p.cs.addr) + p.Debug().Msgf("Control server started: %s", p.cs.addr) } + p.startLogSendServer() } p.onStartedDone = make(chan struct{}) p.loop = make(map[string]bool) @@ -469,10 +1069,28 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { p.ptrLoopGuard = newLoopGuard() p.cacheFlushDomainsMap = nil p.metricsQueryStats.Store(p.cfg.Service.MetricsQueryStats) + + // context for managing spawned goroutines. Firewall mode needs it before + // listeners start so its TTL reaper can run from the first DNS response. + ctx, cancelFunc := context.WithCancel(context.Background()) + var wg sync.WaitGroup + defer func() { cancelFunc(); wg.Wait() }() + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-p.stopCh: + case <-p.runAbortCh: + case <-reloadCh: + case <-ctx.Done(): + } + cancelFunc() + }() + if p.cfg.Service.CacheEnable { cacher, err := dnscache.NewLRUCache(p.cfg.Service.CacheSize) if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to create cacher, caching is disabled") + p.Error().Err(err).Msg("Failed to create cacher, caching is disabled") } else { p.cache = cacher p.cacheFlushDomainsMap = make(map[string]struct{}, 256) @@ -481,26 +1099,22 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } } } - if domain, err := getActiveDirectoryDomain(); err == nil && domain != "" && hasLocalDnsServerRunning() { - mainLog.Load().Debug().Msgf("active directory domain: %s", domain) - p.adDomain = domain - } - var wg sync.WaitGroup - wg.Add(len(p.cfg.Listener)) + // Synchronize firewall mode before listeners process DNS responses. + p.syncFirewallMode(ctx) for _, nc := range p.cfg.Network { for _, cidr := range nc.Cidrs { _, ipNet, err := net.ParseCIDR(cidr) if err != nil { - mainLog.Load().Error().Err(err).Str("network", nc.Name).Str("cidr", cidr).Msg("invalid cidr") + p.Error().Err(err).Str("network", nc.Name).Str("cidr", cidr).Msg("Invalid cidr") continue } nc.IPNets = append(nc.IPNets, ipNet) } } - p.um = newUpstreamMonitor(p.cfg) + p.replaceUpstreamMonitor() if !reload { p.sema = &chanSemaphore{ready: make(chan struct{}, defaultSemaphoreCap)} @@ -513,13 +1127,9 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } } p.setupUpstream(p.cfg) - p.setupClientInfoDiscover(defaultRouteIP()) + p.setupClientInfoDiscover() } - // context for managing spawn goroutines. - ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() - // Newer versions of android and iOS denies permission which breaks connectivity. if !isMobile() && !reload { wg.Add(1) @@ -527,54 +1137,57 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { defer wg.Done() p.runClientInfoDiscover(ctx) }() - go p.watchLinkState(ctx) } if !reload { + p.newNetworkJournal() + wg.Add(1) go func() { - // Start network monitoring - if err := p.monitorNetworkChanges(); err != nil { - mainLog.Load().Error().Err(err).Msg("Failed to start network monitoring") - } + defer wg.Done() + p.waitNetworkJournal = p.startNetworkMonitorAndJournal(ctx) }() } for listenerNum := range p.cfg.Listener { p.cfg.Listener[listenerNum].Init() if !reload { + p.listenerWg.Add(1) go func(listenerNum string) { + defer p.listenerWg.Done() listenerConfig := p.cfg.Listener[listenerNum] upstreamConfig := p.cfg.Upstream[listenerNum] if upstreamConfig == nil { - mainLog.Load().Warn().Msgf("no default upstream for: [listener.%s]", listenerNum) + p.Warn().Msgf("No default upstream for: [listener.%s]", listenerNum) } addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)) - mainLog.Load().Info().Msgf("starting DNS server on listener.%s: %s", listenerNum, addr) - if err := p.serveDNS(listenerNum); err != nil { - mainLog.Load().Fatal().Err(err).Msgf("unable to start dns proxy on listener.%s", listenerNum) + p.Info().Msgf("Starting dns server on listener.%s: %s", listenerNum, addr) + // serveCtx uses Background() context so listeners survive between reloads. + // Changes to listeners config require a service restart, not just reload. + serveCtx := context.Background() + if err := serveDNSFn(p, serveCtx, listenerNum); err != nil { + p.reportServeDNSFailure(listenerNum, err) } - mainLog.Load().Debug().Msgf("end of serveDNS listener.%s: %s", listenerNum, addr) + p.Debug().Msgf("End of serveDNS listener.%s: %s", listenerNum, addr) }(listenerNum) } - go func() { - defer func() { - cancelFunc() - wg.Done() - }() - select { - case <-p.stopCh: - case <-ctx.Done(): - case <-reloadCh: - } - }() } if !reload { for i := 0; i < numListeners; i++ { - <-p.started + select { + case <-p.started: + case <-p.stopCh: + return + case <-p.runAbortCh: + return + } } - for _, f := range p.onStarted { - f() + if !p.startOSState(func() { + for _, f := range p.onStarted { + f() + } + }) { + return } } @@ -598,31 +1211,144 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { if !reload { // Stop writing log to unix socket. - consoleWriter.Out = os.Stdout + consoleWriter = newHumanReadableZapCore(os.Stdout, consoleWriterLevel) p.initLogging(false) - if p.logConn != nil { - _ = p.logConn.Close() - } - go p.apiConfigReload() - p.postRun() + p.closeLogConn() + p.startAPIConfigReload() + p.startOSState(func() { postRunFn(p) }) } wg.Wait() } +// dnsConfigChangedMessage names the journal event of the resolver table of the +// host. A log tool selects every resolver change of a run by this message. +const dnsConfigChangedMessage = "DNS configuration changed" + +// runSCUtilDNSFn reads the resolver table of the host. A test replaces it, +// because the configuration daemon of the host is not a fixture. +var runSCUtilDNSFn = runSCUtilDNS + +// replaceUpstreamMonitor puts a new upstream monitor in place of the one of +// the run before. The new monitor starts with every upstream up, so the old one +// closes its open outages first. Without this the journal holds a down event +// that no up event ever follows. +func (p *prog) replaceUpstreamMonitor() { + p.mu.Lock() + defer p.mu.Unlock() + if p.um != nil { + p.um.retire() + } + p.um = newUpstreamMonitor(p.cfg, p.logger.Load()) +} + +// upstreamMonitorNow reads the monitor of this run under the lock that a +// replacement takes, because a reload swaps it while the journal loops run. +func (p *prog) upstreamMonitorNow() *upstreamMonitor { + p.mu.Lock() + defer p.mu.Unlock() + return p.um +} + +// newNetworkJournal creates the trackers that follow the host network. The +// query path reads them without a lock, so they exist before a listener starts. +func (p *prog) newNetworkJournal() { + p.health = newQueryHealth() + p.querySampler.onFailure = p.health.countFailure + if runtime.GOOS == "darwin" { + p.dnsConfig = newDNSConfigPoller(runSCUtilDNSFn) + } +} + +// startNetworkMonitorAndJournal starts the network monitor and then opens the +// journal. The trackers do not depend on the monitor, so a monitor that cannot +// start leaves them running. It passes on the wait function of the journal. +func (p *prog) startNetworkMonitorAndJournal(ctx context.Context) func() { + if err := monitorNetworkChangesFn(p, ctx); err != nil { + p.Error().Err(err).Msg("Failed to start network monitoring") + } + return p.startNetworkJournal() +} + +// startNetworkJournal runs the trackers and opens the journal of this run with +// one snapshot. Support reads the grade of the query path and the resolver +// table of each moment after an incident. It returns a function that waits for +// both loops, so a caller that closed the stop channel knows when they ended. +func (p *prog) startNetworkJournal() func() { + // Journal trackers persist across reloads, but not a stop or startup abort. + stop := make(chan struct{}) + go func() { + select { + case <-p.stopCh: + case <-p.runAbortCh: + } + close(stop) + }() + healthDone := p.health.startLoop(stop, func() (int, bool) { + return p.upstreamMonitorNow().countDownExcept(p.isInternalDomainUpstream), p.recoveryBypass.Load() + }) + var configDone <-chan struct{} + if p.dnsConfig != nil { + configDone = p.dnsConfig.startLoop(stop, p.logDNSConfigChanges) + } + p.logNetworkSnapshot("start") + return func() { + <-healthDone + if configDone != nil { + <-configDone + } + } +} + +// logDNSConfigChanges puts each changed resolver of the host in the journal. A +// resolver that vanished comes without a nameserver, so the reader sees the +// drop. The search suffixes name the organization of the endpoint, so the +// journal holds their number only. +func (p *prog) logDNSConfigChanges(entries []dnsResolverEntry) { + // The poller emits one batch per changed snapshot (at most once per poll), + // not one discovery per resolver. Late DNS publication on an unchanged + // tunnel otherwise has no network event to refresh split routing. + // Join the existing shutdown fence before touching the manager or PF/WFP: + // closeNetMonitor drains this callback before restoring the host's DNS. + if len(entries) > 0 && p.beginNetworkActivity() { + defer p.netMonitorWG.Done() + // setDNS publishes the manager before signalling completion. The journal + // starts earlier; never race initial manager publication or initialize a + // manager in traditional/hard intercept mode. + select { + case <-p.csSetDnsDone: + p.vpnDNSJournalMu.Lock() + manager := p.vpnDNS + p.vpnDNSJournalMu.Unlock() + if manager != nil { + manager.Refresh(ctrld.LoggerCtx(context.Background(), p.logger.Load()), true) + } + default: + } + } + for _, entry := range entries { + journal(mainLog.Load().Info()). + Strs("nameservers", entry.Nameservers). + Int("if_index", entry.IfIndex). + Str("interface", entry.Interface). + Bool("scoped", entry.Scoped). + Str("action", entry.Action). + Str("flags", entry.Flags). + Int("search_domain_count", len(entry.SearchDomains)). + Int("order", entry.Order). + Str("reachable", entry.Reachable). + Msg(dnsConfigChangedMessage) + } +} + // setupClientInfoDiscover performs necessary works for running client info discover. -func (p *prog) setupClientInfoDiscover(selfIP string) { - p.ciTable = clientinfo.NewTable(&cfg, selfIP, cdUID, p.ptrNameservers) +func (p *prog) setupClientInfoDiscover() { + selfIP := p.defaultRouteIP() + p.ciTable = clientinfo.NewTable(&cfg, selfIP, cdUID, p.ptrNameservers, p.logger.Load()) if leaseFile := p.cfg.Service.DHCPLeaseFile; leaseFile != "" { - mainLog.Load().Debug().Msgf("watching custom lease file: %s", leaseFile) + p.Debug().Msgf("Watching custom lease file: %s", leaseFile) format := ctrld.LeaseFileFormat(p.cfg.Service.DHCPLeaseFileFormat) p.ciTable.AddLeaseFile(leaseFile, format) } - if leaseFiles := dnsmasq.AdditionalLeaseFiles(); len(leaseFiles) > 0 { - mainLog.Load().Debug().Msgf("watching additional lease files: %v", leaseFiles) - for _, leaseFile := range leaseFiles { - p.ciTable.AddLeaseFile(leaseFile, ctrld.Dnsmasq) - } - } } // runClientInfoDiscover runs the client info discover. @@ -636,20 +1362,141 @@ func (p *prog) metricsEnabled() bool { return p.cfg.Service.MetricsQueryStats || p.cfg.Service.MetricsListener != "" } -func (p *prog) Stop(s service.Service) error { - p.stopDnsWatchers() - mainLog.Load().Debug().Msg("dns watchers stopped") - for _, f := range p.onStopped { - f() +// finishRun also cancels workers waiting for startup when preflight returns early. +// The caller's stop channel belongs to the mobile controller, not to this cleanup. +func (p *prog) finishRun() { + // Keep the files open through final diagnostics, including mobile exits + // that never invoke the service Stop callback. + defer p.closeInternalLogs() + close(p.runAbortCh) + select { + case <-p.runDone: + case <-time.After(shutdownTimeout): + p.Warn().Msg("timeout waiting for ctrld components to stop") + // A timeout is not proof that workers stopped. Never release resources + // still owned by the run, or allow a mobile restart to overlap it. + <-p.runDone + } + // Listeners live across config reloads, so they belong to prog rather + // than the per-run wait group. runDone closes after their final Add. + p.apiReloadWg.Wait() + p.listenerWg.Wait() + if p.waitNetworkJournal != nil { + p.waitNetworkJournal() + } + if err := p.shutdown(); err != nil { + p.Warn().Err(err).Msg("error during shutdown") + } +} + +// startOSState serializes startup's OS mutations with restoration. A stop +// either restores an already-completed action or prevents it from starting. +func (p *prog) startOSState(f func()) bool { + p.osStateMu.Lock() + defer p.osStateMu.Unlock() + if p.osStateRestored || stopRequested(p.stopCh) || stopRequested(p.runAbortCh) { + return false + } + f() + return true +} + +// restoreOSState puts back what ctrld changed outside the process: the DNS +// settings, interception state and allocated listener IPs. It runs +// while the listeners are still serving, so the OS is never left pointing at a +// resolver that is already gone. Safe to call multiple times. +func (p *prog) restoreOSState() error { + p.restoreOnce.Do(func() { + p.stopLogSendServer() + p.osStateMu.Lock() + p.osStateRestored = true + p.osStateMu.Unlock() + p.closeNetMonitor() + p.stopDnsWatchers() + p.Debug().Msg("dns watchers stopped") + for _, f := range p.onStopped { + f() + } + p.Debug().Msg("finish running onStopped functions") + if derr := p.deAllocateIP(); derr != nil { + p.Error().Err(derr).Msg("de-allocate ip failed") + p.restoreErr = derr + } + }) + return p.restoreErr +} + +// releaseResources releases the control server and upstream transports. It must run after +// the listeners stopped, since retiring an upstream a listener still answers +// queries on would fail those queries. Safe to call multiple times. +func (p *prog) releaseResources() { + p.releaseOnce.Do(func() { + p.stopControlServer() + p.closeLogConn() + p.mu.Lock() + upstreams := p.cfg.Upstream + p.mu.Unlock() + closeReplacedUpstreams(upstreams, nil) + }) +} + +// stopControlServer closes the control socket. Stop calls it as soon as the stop +// is accepted, and releaseResources again for the stop paths that never call +// Stop. Safe to call multiple times. +// +// It must not wait for the run to finish. A stopping process that still answers +// on the socket answers for the service that replaces it: `ctrld upgrade` and +// restart take any reply to /started as "the new binary is up", so an upgrade +// to a build that never became ready was reported successful instead of rolled +// back. Closing late also unlinks the socket path after the new process has +// bound its own there. +func (p *prog) stopControlServer() { + p.csStopOnce.Do(func() { + if p.cs == nil { + return + } + if err := p.cs.stop(); err != nil { + p.Warn().Err(err).Msg("could not stop control server") + } + }) +} + +// shutdown runs the full teardown, in the order both halves require. It is +// called once the listeners stopped, on every stop path including the mobile +// one where the OS never terminates the process. +func (p *prog) shutdown() error { + err := p.restoreOSState() + p.releaseResources() + return err +} + +// closeReplacedUpstreams releases the transports of upstreams that cur no longer +// refers to, or all of them when cur is nil. Requests in flight on them fail +// fast and are retried, the same way a re-bootstrap treats connections it +// replaces. +func closeReplacedUpstreams(old, cur map[string]*ctrld.UpstreamConfig) { + inUse := make(map[*ctrld.UpstreamConfig]struct{}, len(cur)) + for _, uc := range cur { + inUse[uc] = struct{}{} + } + for _, uc := range old { + if uc == nil { + continue + } + if _, ok := inUse[uc]; !ok { + uc.CloseTransports() + } } - mainLog.Load().Debug().Msg("finish running onStopped functions") +} + +func (p *prog) Stop(_ service.Service) error { defer func() { - mainLog.Load().Info().Msg("Service stopped") + p.Info().Msg("Service stopped") + // The files take the last line of this run before they close. + p.closeInternalLogs() }() - if err := p.deAllocateIP(); err != nil { - mainLog.Load().Error().Err(err).Msg("de-allocate ip failed") - return err - } + // Restore OS state while listeners still serve; retire resources after their drain. + err := p.restoreOSState() if deactivationPinSet() { select { case <-p.pinCodeValidCh: @@ -659,22 +1506,25 @@ func (p *prog) Stop(s service.Service) error { // No valid pin code was checked, that mean we are stopping // because of OS signal sent directly from someone else. // In this case, restarting ctrld service by ourselves. - mainLog.Load().Debug().Msgf("receiving stopping signal without valid pin code") - mainLog.Load().Debug().Msgf("self restarting ctrld service") + p.Debug().Msgf("Receiving stopping signal without valid pin code") + p.Debug().Msgf("Self restarting ctrld service") if exe, err := os.Executable(); err == nil { cmd := exec.Command(exe, "restart") cmd.SysProcAttr = sysProcAttrForDetachedChildProcess() if err := cmd.Start(); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to run self restart command") + p.Error().Err(err).Msg("Failed to run self restart command") } } else { - mainLog.Load().Error().Err(err).Msg("failed to self restart ctrld service") + p.Error().Err(err).Msg("Failed to self restart ctrld service") } os.Exit(deactivationPinInvalidExitCode) } } + // The pin check above is the last use of the control server. From here on + // this process is going away and must not answer for its successor. + p.stopControlServer() close(p.stopCh) - return nil + return err } func (p *prog) stopDnsWatchers() { @@ -702,19 +1552,168 @@ func (p *prog) deAllocateIP() error { return nil } for _, lc := range p.cfg.Listener { - if err := deAllocateIP(lc.IP); err != nil { + if err := deAllocateIPFn(lc.IP); err != nil { return err } } return nil } -func (p *prog) setDNS() { +// Seams for the intercept-start failure lifecycle. Choosing between the interface-DNS +// fallback and refusing it has side effects - restoring the host's DNS, then +// terminating - which a test has to observe without reconfiguring the host or exiting +// the test binary. The intercept start itself is indirected for the same reason: it is +// the real platform interceptor, which on macOS mutates pf and on Windows installs an +// NRPT rule, so a test of what happens *after* it fails must not be the thing that +// runs it. +var ( + deAllocateIPFn = deAllocateIP + serveDNSFn = (*prog).serveDNS + monitorNetworkChangesFn = (*prog).monitorNetworkChanges + postRunFn = (*prog).postRun + startDNSInterceptFn = (*prog).startDNSIntercept + ensureInterceptDNSTargetFn = (*prog).ensureInterceptDNSTarget + removeInterceptDNSTargetFn = (*prog).removeInterceptDNSTarget + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason + newResolverFn = ctrld.NewResolver + setDnsForRunningIfaceFn = (*prog).setDnsForRunningIface + resetDNSFn = (*prog).resetDNS + // refuseFallbackFatal reports a startup failure the interface-DNS fallback + // cannot safely paper over, then exits. No dedicated code exists for this, + // so it falls back to UNCLASSIFIED. + refuseFallbackFatal = func(p *prog, format string, v ...any) { + failRunUnclassified(p.Error(), fmt.Sprintf(format, v...), p.notifyExitToLogServer) + } +) + +// interfaceDNSFallbackViable reports whether the interface-DNS fallback can actually +// direct queries to ctrld's listener. +// +// Interface DNS names a resolver by IP and has no port field - true of macOS interface +// settings and of Windows NRPT rules - so pointing the system straight at a listener +// that did not bind :53 sends queries to whatever owns :53 instead, and that resolver's +// upstream is ctrld's address: a loop, not a fallback. +// +// A nil or portless listener is treated as viable: the port is resolved elsewhere and +// defaults to 53, so there is nothing to refuse yet. +// +// master no longer supports the router-local forwarding-resolver path, so there is +// no intermediary that can make a non-53 listener reachable from interface DNS. +func interfaceDNSFallbackViable(lc *ctrld.ListenerConfig) bool { + return lc == nil || lc.Port == 0 || lc.Port == 53 +} + +func (p *prog) setDNS(systemNameservers []string) { setDnsOK := false defer func() { p.csSetDnsOk = setDnsOK }() + // Validate and resolve intercept mode: --intercept-mode, then + // service.intercept_mode, then the platform default (resolveInterceptMode). + // Valid values: "off" (interface DNS mode), "dns" (with VPN split routing), + // and "hard" (all DNS through ctrld). + if interceptMode != "" && !validInterceptMode(interceptMode) { + p.Fatal().Msgf("invalid --intercept-mode value %q: must be 'off', 'dns', or 'hard'", interceptMode) + } + if interceptMode == "" { + mode, source := resolveInterceptMode("", p.cfg) + interceptMode, interceptModeSource = mode, source + switch { + case source == interceptModeSourceDefault && mode == "": + if _, reason := defaultInterceptMode(p.cfg); reason != nil { + p.Info().Err(reason).Msg("Intercept mode default not applied; using interface DNS mode") + } + case mode == "" || mode == "off": + case source == interceptModeSourceConfig: + p.Info().Msgf("Intercept mode enabled via config (intercept_mode = %q)", mode) + case source == interceptModeSourceDefault: + p.Info().Msgf("Intercept mode %q is the default on this platform; use --intercept-mode off for interface DNS mode", mode) + } + } else if interceptModeSource == "" { + interceptModeSource = interceptModeSourceFlag + } + + // Derive convenience bools from interceptMode. + switch interceptMode { + case "dns": + dnsIntercept = true + case "hard": + dnsIntercept = true + hardIntercept = true + } + + // DNS intercept mode: use OS-level packet interception (WFP/pf) instead of + // modifying interface DNS settings. This eliminates race conditions with VPN + // software that also manages DNS. See issue #489. + if dnsIntercept { + if err := startDNSInterceptFn(p); err != nil { + removeInterceptDNSTargetFn(p, "DNS intercept unavailable") + // An external GP catch-all still owns the namespace even when its probe + // fails. In either external-owner state, rewriting adapter DNS would violate + // the policy that startup deliberately preserved. Only the verified case has + // working DNS; the ineffective case remains a failed/not-ready start. + if interceptFailedUnderExternalDNSPolicy(err) { + if interceptFailedWithVerifiedExternalDNS(err) { + p.Error().Err(err).Msg("DNS intercept mode failed but externally managed DNS policy is verified routing to ctrld — not falling back to interface DNS settings") + } else { + p.Error().Err(err).Msg("DNS intercept mode failed and externally managed DNS policy is not routing to ctrld — leaving interface DNS settings untouched; the service is not ready") + } + return + } + + // Interface DNS cannot express a port: macOS interface settings and Windows + // NRPT rules both name a resolver by IP alone. So it is only a usable + // fallback when the listener actually bound :53. When something else owns + // :53 - mDNSResponder on macOS, which is the whole reason the :5354 fallback + // exists - pointing the system at 127.0.0.1 hands queries to that other + // resolver, whose own upstream is now ctrld's address. That is a resolution + // loop, not degraded operation: a healthy ctrld listener nothing on the host + // can reach, no working DNS, and no recovery short of stopping the service. + // + // Refuse instead, after putting the host's own DNS back. A visible startup + // failure beats DNS that is broken by design, and it stops a fallback that + // cannot work from quietly undoing the fail-closed verification above. + if lc := cfg.FirstListener(); !interfaceDNSFallbackViable(lc) { + mainLog.Load().Error().Err(err).Msgf("DNS intercept mode failed with the listener on port %d", lc.Port) + // Leave the host resolvable: restore static settings or DHCP rather than + // exiting with an interface still pointed at a ctrld that is not serving. + resetDNSFn(p, false, true) + refuseFallbackFatal(p, "Refusing to fall back to interface DNS: it cannot direct queries to %s:%d, which would leave this host with no working resolver. Free port 53 for ctrld, or resolve the intercept failure, then start again.", lc.IP, lc.Port) + // Unreachable in production - the line above exits - but returning + // explicitly keeps the refusal from depending on that, so nothing can + // fall through to installing the fallback this just rejected. + return + } + p.Error().Err(err).Msg("DNS intercept mode failed — falling back to interface DNS settings") + // Fall through to traditional setDNS behavior. + } else { + ensureInterceptDNSTargetFn(p, systemNameservers) + if hardIntercept { + p.Info().Msg("Hard intercept mode active — all DNS through ctrld, no VPN split routing") + } else { + p.Info().Msg("DNS intercept mode active — skipping interface DNS configuration and watchdog") + + // Initialize VPN DNS manager for split DNS routing. + // Discovers search domains from virtual/VPN interfaces and forwards + // matching queries to the DNS server on that interface. + // Skipped in --intercept-mode hard where all DNS goes through ctrld. + p.vpnDNSJournalMu.Lock() + p.vpnDNS.Close() + p.vpnDNS = newVPNDNSManager(&p.logger, p.exemptVPNDNSServers) + p.vpnDNS.retryRefresh = p.retryVPNDNSRefresh(p.vpnDNS) + p.vpnDNS.Refresh(ctrld.LoggerCtx(context.Background(), p.logger.Load())) + p.vpnDNSJournalMu.Unlock() + } + + setDnsOK = true + return + } + } + if !dnsIntercept { + removeInterceptDNSTargetFn(p, "intercept mode inactive") + } + if cfg.Listener == nil { return } @@ -729,9 +1728,6 @@ func (p *prog) setDNS() { ns = "127.0.0.1" case lc.Port != 53: ns = "127.0.0.1" - if resolver := router.LocalResolverIP(); resolver != "" { - ns = resolver - } default: // If we ever reach here, it means ctrld is running on lc.IP port 53, // so we could just use lc.IP as nameserver. @@ -748,7 +1744,7 @@ func (p *prog) setDNS() { slices.Sort(nameservers) netIfaceName := "" - netIface := p.setDnsForRunningIface(nameservers) + netIface := setDnsForRunningIfaceFn(p, nameservers) if netIface != nil { netIfaceName = netIface.Name } @@ -769,7 +1765,7 @@ func (p *prog) setDNS() { p.dnsWg.Add(1) go func() { defer p.dnsWg.Done() - p.watchResolvConf(netIface, servers, setResolvConf) + p.watchResolvConf(netIface, servers, p.setResolvConf) }() } if p.dnsWatchdogEnabled() { @@ -781,12 +1777,21 @@ func (p *prog) setDNS() { } } +// configuredInterceptMode resolves the service's effective intercept mode without +// mutating package state. An explicit flag value, including "off", takes priority +// over the persisted config value, and the platform default applies when neither +// is set. +func (p *prog) configuredInterceptMode() string { + mode, _ := resolveInterceptMode(interceptMode, p.cfg) + return mode +} + func (p *prog) setDnsForRunningIface(nameservers []string) (runningIface *net.Interface) { if p.runningIface == "" { return } - logger := mainLog.Load().With().Str("iface", p.runningIface).Logger() + logger := p.logger.Load().With().Str("iface", p.runningIface) const maxDNSRetryAttempts = 3 const retryDelay = 1 * time.Second @@ -799,33 +1804,33 @@ func (p *prog) setDnsForRunningIface(nameservers []string) (runningIface *net.In } if attempt < maxDNSRetryAttempts { // Try to find a different working interface - newIface := findWorkingInterface(p.runningIface) + newIface := p.findWorkingInterface() if newIface != p.runningIface { p.runningIface = newIface - logger = mainLog.Load().With().Str("iface", p.runningIface).Logger() - logger.Info().Msg("switched to new interface") + logger = p.logger.Load().With().Str("iface", p.runningIface) + logger.Info().Msg("Switched to new interface") continue } - logger.Warn().Err(err).Int("attempt", attempt).Msg("could not get interface, retrying...") + logger.Warn().Err(err).Int("attempt", attempt).Msg("Could not get interface, retrying...") time.Sleep(retryDelay) continue } - logger.Error().Err(err).Msg("could not get interface after all attempts") + logger.Error().Err(err).Msg("Could not get interface after all attempts") return } - if err := setupNetworkManager(); err != nil { - logger.Error().Err(err).Msg("could not patch NetworkManager") + if err := p.setupNetworkManager(); err != nil { + logger.Error().Err(err).Msg("Could not patch networkmanager") return } runningIface = netIface - logger.Debug().Msg("setting DNS for interface") + logger.Debug().Msg("Setting dns for interface") if err := setDNS(netIface, nameservers); err != nil { - logger.Error().Err(err).Msgf("could not set DNS for interface") + logger.Error().Err(err).Msgf("Could not set dns for interface") return } - logger.Debug().Msg("setting DNS successfully") + logger.Debug().Msg("Setting dns successfully") return } @@ -854,7 +1859,7 @@ func (p *prog) dnsWatchdog(iface *net.Interface, nameservers []string) { return } - mainLog.Load().Debug().Msg("start DNS settings watchdog") + p.Debug().Msg("Start dns settings watchdog") ns := nameservers slices.Sort(ns) @@ -865,34 +1870,34 @@ func (p *prog) dnsWatchdog(iface *net.Interface, nameservers []string) { case <-p.dnsWatcherStopCh: return case <-p.stopCh: - mainLog.Load().Debug().Msg("stop dns watchdog") + p.Debug().Msg("Stop dns watchdog") return case <-ticker.C: if p.recoveryRunning.Load() { return } - if dnsChanged(iface, ns) { - mainLog.Load().Debug().Msg("DNS settings were changed, re-applying settings") + if p.dnsChanged(iface, ns) { + p.Debug().Msg("DNS settings were changed, re-applying settings") // Check if the interface already has static DNS servers configured. // currentStaticDNS is an OS-dependent helper that returns the current static DNS. staticDNS, err := currentStaticDNS(iface) if err != nil { - mainLog.Load().Debug().Err(err).Msgf("failed to get static DNS for interface %s", iface.Name) + p.Debug().Err(err).Msgf("Failed to get static DNS for interface %s", iface.Name) } else if len(staticDNS) > 0 { //filter out loopback addresses staticDNS = slices.DeleteFunc(staticDNS, func(s string) bool { return net.ParseIP(s).IsLoopback() }) // if we have a static config and no saved IPs already, save them - if len(staticDNS) > 0 && len(savedStaticNameservers(iface)) == 0 { + if len(staticDNS) > 0 && len(ctrld.SavedStaticNameservers(iface)) == 0 { // Save these static DNS values so that they can be restored later. if err := saveCurrentStaticDNS(iface); err != nil { - mainLog.Load().Debug().Err(err).Msgf("failed to save static DNS for interface %s", iface.Name) + p.Debug().Err(err).Msgf("Failed to save static DNS for interface %s", iface.Name) } } } if err := setDNS(iface, ns); err != nil { - mainLog.Load().Error().Err(err).Str("iface", iface.Name).Msgf("could not re-apply DNS settings") + p.Error().Err(err).Str("iface", iface.Name).Msgf("Could not re-apply DNS settings") } } if p.requiredMultiNICsConfig { @@ -901,31 +1906,31 @@ func (p *prog) dnsWatchdog(iface *net.Interface, nameservers []string) { ifaceName = iface.Name } withEachPhysicalInterfaces(ifaceName, "", func(i *net.Interface) error { - if dnsChanged(i, ns) { + if p.dnsChanged(i, ns) { // Check if the interface already has static DNS servers configured. // currentStaticDNS is an OS-dependent helper that returns the current static DNS. staticDNS, err := currentStaticDNS(i) if err != nil { - mainLog.Load().Debug().Err(err).Msgf("failed to get static DNS for interface %s", i.Name) + p.Debug().Err(err).Msgf("Failed to get static DNS for interface %s", i.Name) } else if len(staticDNS) > 0 { //filter out loopback addresses staticDNS = slices.DeleteFunc(staticDNS, func(s string) bool { return net.ParseIP(s).IsLoopback() }) // if we have a static config and no saved IPs already, save them - if len(staticDNS) > 0 && len(savedStaticNameservers(i)) == 0 { + if len(staticDNS) > 0 && len(ctrld.SavedStaticNameservers(i)) == 0 { // Save these static DNS values so that they can be restored later. if err := saveCurrentStaticDNS(i); err != nil { - mainLog.Load().Debug().Err(err).Msgf("failed to save static DNS for interface %s", i.Name) + p.Debug().Err(err).Msgf("Failed to save static DNS for interface %s", i.Name) } } } if err := setDnsIgnoreUnusableInterface(i, nameservers); err != nil { - mainLog.Load().Error().Err(err).Str("iface", i.Name).Msgf("could not re-apply DNS settings") + p.Error().Err(err).Str("iface", i.Name).Msgf("Could not re-apply DNS settings") } else { - mainLog.Load().Debug().Msgf("re-applying DNS for interface %q successfully", i.Name) + p.Debug().Msgf("Re-applying DNS for interface %q successfully", i.Name) } } return nil @@ -936,7 +1941,22 @@ func (p *prog) dnsWatchdog(iface *net.Interface, nameservers []string) { } // resetDNS performs a DNS reset for all interfaces. +// In DNS intercept mode, this tears down the WFP/pf filters instead. func (p *prog) resetDNS(isStart bool, restoreStatic bool) { + removeInterceptDNSTargetFn(p, "DNS reset") + if dnsIntercept && p.dnsInterceptState != nil { + if err := p.stopDNSIntercept(); err != nil { + p.Error().Err(err).Msg("Failed to stop DNS intercept mode during reset") + } + + // Clean up VPN DNS manager + p.vpnDNSJournalMu.Lock() + p.vpnDNS.Close() + p.vpnDNS = nil + p.vpnDNSJournalMu.Unlock() + + return + } netIfaceName := "" if netIface := p.resetDNSForRunningIface(isStart, restoreStatic); netIface != nil { netIfaceName = netIface.Name @@ -947,6 +1967,32 @@ func (p *prog) resetDNS(isStart bool, restoreStatic bool) { } } +// The OS boundaries of the DNS reset path, as variables so tests can exercise +// its failure handling without changing the host's DNS or NetworkManager state. +var ( + netInterfaceFn = netInterface + restoreNetworkManagerFn = (*prog).restoreNetworkManager + setIfaceDNSFn = setDNS + resetIfaceDNSFn = resetDNS + savedStaticNameserversFn = ctrld.SavedStaticNameservers +) + +// logIfaceLookupFailure reports a failed lookup of the interface the caller was +// about to work on, naming that work in skipping, e.g. "DNS restoration". An +// interface that no longer exists has nothing left to act on — an unplugged +// adapter or a torn down tether is gone along with the settings ctrld changed — +// so the skip is a debug diagnostic rather than a user-facing error: it +// otherwise makes a successful upgrade look broken. Every other lookup failure +// still says something went wrong and stays at error level, as does a failure +// on an interface that does exist. +func logIfaceLookupFailure(logger *ctrld.Logger, skipping string, err error) { + if errors.Is(err, errInterfaceNotFound) { + logger.Debug().Msgf("Skipping %s: previous interface is no longer present", skipping) + return + } + logger.Error().Err(err).Msg("Could not get interface") +} + // resetDNSForRunningIface performs a DNS reset on the running interface. // The parameter isStart indicates whether this is being called as part of a start (or restart) // command. When true, we check if the current static DNS configuration already differs from the @@ -955,18 +2001,18 @@ func (p *prog) resetDNS(isStart bool, restoreStatic bool) { // Otherwise, we restore the saved configuration (if any) or reset to DHCP. func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runningIface *net.Interface) { if p.runningIface == "" { - mainLog.Load().Debug().Msg("no running interface, skipping resetDNS") + p.Debug().Msg("No running interface, skipping resetDNS") return } - logger := mainLog.Load().With().Str("iface", p.runningIface).Logger() - netIface, err := netInterface(p.runningIface) + logger := p.logger.Load().With().Str("iface", p.runningIface) + netIface, err := netInterfaceFn(p.runningIface) if err != nil { - logger.Error().Err(err).Msg("could not get interface") + logIfaceLookupFailure(logger, "DNS restoration", err) return } runningIface = netIface - if err := restoreNetworkManager(); err != nil { - logger.Error().Err(err).Msg("could not restore NetworkManager") + if err := restoreNetworkManagerFn(p); err != nil { + logger.Error().Err(err).Msg("Could not restore NetworkManager") return } @@ -974,7 +2020,7 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin if isStart { current, err := currentStaticDNS(netIface) if err != nil { - logger.Warn().Err(err).Msg("unable to obtain current static DNS configuration; proceeding to restore saved config") + logger.Warn().Err(err).Msg("Unable to obtain current static DNS configuration; proceeding to restore saved config") } else if len(current) > 0 { // If any static DNS value is not our own listener, assume an admin override. hasManualConfig := false @@ -992,47 +2038,26 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin } // Default logic: if there is a saved static DNS configuration, restore it. - saved := savedStaticNameservers(netIface) + saved := savedStaticNameserversFn(netIface) if len(saved) > 0 && restoreStatic { logger.Debug().Msgf("Restoring interface %q from saved static config: %v", netIface.Name, saved) - if err := setDNS(netIface, saved); err != nil { - logger.Error().Err(err).Msgf("failed to restore static DNS config on interface %q", netIface.Name) + if err := setIfaceDNSFn(netIface, saved); err != nil { + logger.Error().Err(err).Msgf("Failed to restore static DNS config on interface %q", netIface.Name) return } } else { logger.Debug().Msgf("No saved static DNS config for interface %q; resetting to DHCP", netIface.Name) - if err := resetDNS(netIface); err != nil { - logger.Error().Err(err).Msgf("failed to reset DNS to DHCP on interface %q", netIface.Name) + if err := resetIfaceDNSFn(netIface); err != nil { + logger.Error().Err(err).Msgf("Failed to reset DNS to DHCP on interface %q", netIface.Name) return } } return } -func (p *prog) logInterfacesState() { - withEachPhysicalInterfaces("", "", func(i *net.Interface) error { - addrs, err := i.Addrs() - if err != nil { - mainLog.Load().Warn().Str("interface", i.Name).Err(err).Msg("failed to get addresses") - } - nss, err := currentStaticDNS(i) - if err != nil { - mainLog.Load().Warn().Str("interface", i.Name).Err(err).Msg("failed to get DNS") - } - if len(nss) == 0 { - nss = currentDNS(i) - } - mainLog.Load().Debug(). - Any("addrs", addrs). - Strs("nameservers", nss). - Int("index", i.Index). - Msgf("interface state: %s", i.Name) - return nil - }) -} - // findWorkingInterface looks for a network interface with a valid IP configuration -func findWorkingInterface(currentIface string) string { +func (p *prog) findWorkingInterface() string { + currentIface := p.runningIface // Helper to check if IP is valid (not link-local) isValidIP := func(ip net.IP) bool { return ip != nil && @@ -1050,7 +2075,7 @@ func findWorkingInterface(currentIface string) string { addrs, err := iface.Addrs() if err != nil { - mainLog.Load().Debug(). + p.Debug(). Str("interface", iface.Name). Err(err). Msg("failed to get interface addresses") @@ -1069,13 +2094,15 @@ func findWorkingInterface(currentIface string) string { } // Get default route interface + foundDefaultRoute := false defaultRoute, err := netmon.DefaultRoute() if err != nil { - mainLog.Load().Debug(). + p.Debug(). Err(err). Msg("failed to get default route") } else { - mainLog.Load().Debug(). + foundDefaultRoute = true + p.Debug(). Str("default_route_iface", defaultRoute.InterfaceName). Msg("found default route") } @@ -1083,7 +2110,7 @@ func findWorkingInterface(currentIface string) string { // Get all interfaces ifaces, err := net.Interfaces() if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to list network interfaces") + p.Error().Err(err).Msg("Failed to list network interfaces") return currentIface // Return current interface as fallback } @@ -1111,9 +2138,9 @@ func findWorkingInterface(currentIface string) string { } // Found working physical interface - if err == nil && defaultRoute.InterfaceName == iface.Name { + if foundDefaultRoute && defaultRoute.InterfaceName == iface.Name { // Found interface with default route - use it immediately - mainLog.Load().Info(). + p.Info(). Str("old_iface", currentIface). Str("new_iface", iface.Name). Msg("switching to interface with default route") @@ -1134,7 +2161,7 @@ func findWorkingInterface(currentIface string) string { // Return interfaces in order of preference: // 1. Current interface if it's still valid if currentIfaceValid { - mainLog.Load().Debug(). + p.Debug(). Str("interface", currentIface). Msg("keeping current interface") return currentIface @@ -1142,7 +2169,7 @@ func findWorkingInterface(currentIface string) string { // 2. First working interface found if firstWorkingIface != "" { - mainLog.Load().Info(). + p.Info(). Str("old_iface", currentIface). Str("new_iface", firstWorkingIface). Msg("switching to first working physical interface") @@ -1150,9 +2177,9 @@ func findWorkingInterface(currentIface string) string { } // 3. Fall back to current interface if nothing else works - mainLog.Load().Warn(). + p.Warn(). Str("current_iface", currentIface). - Msg("no working physical interface found, keeping current") + Msg("No working physical interface found, keeping current") return currentIface } @@ -1168,28 +2195,6 @@ func randomPort() int { return n } -// runLogServer starts a unix listener, use by startCmd to gather log from runCmd. -func runLogServer(sockPath string) net.Conn { - addr, err := net.ResolveUnixAddr("unix", sockPath) - if err != nil { - mainLog.Load().Warn().Err(err).Msg("invalid log sock path") - return nil - } - ln, err := net.ListenUnix("unix", addr) - if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not listen log socket") - return nil - } - defer ln.Close() - - server, err := ln.Accept() - if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not accept connection") - return nil - } - return server -} - func errAddrInUse(err error) bool { var opErr *net.OpError if errors.As(err, &opErr) { @@ -1200,37 +2205,80 @@ func errAddrInUse(err error) bool { var _ = errAddrInUse +// The unreachable winsock errnos (ENETUNREACH/EHOSTUNREACH) are matched via +// ctrldnet.IsUnreachable, which owns their definitions. +// // https://learn.microsoft.com/en-us/windows/win32/winsock/windows-sockets-error-codes-2 var ( windowsECONNREFUSED = syscall.Errno(10061) - windowsENETUNREACH = syscall.Errno(10051) windowsEINVAL = syscall.Errno(10022) windowsEADDRINUSE = syscall.Errno(10048) - windowsEHOSTUNREACH = syscall.Errno(10065) ) +// errUrlNetworkError reports whether a failed HTTP attempt is worth retrying. +// +// The two-attempt paths compose one *url.Error per attempt - hostname first, then the +// direct-IP fallback - so this walks them in order rather than classifying only the first +// one errors.As happens to find. Each attempt can say one of three things: +// +// - retryable (unreachable, refused, temporary): retry, whichever attempt said it; +// - a name-resolution failure: no verdict. Only the hostname attempt resolves DNS, and +// at boot behind a captive portal or before the router's forwarder is up it fails +// this way while the network is merely not ready yet. Consult the next attempt; +// - anything else, notably a locally denied socket (WSAEACCES from a firewall blocking +// ctrld): definitive. Stop, because retrying cannot clear it - the Firewall Mode +// incident spent 256 retry cycles against filters that were never going to clear. func errUrlNetworkError(err error) bool { - var urlErr *url.Error - if errors.As(err, &urlErr) { - return errNetworkError(urlErr.Err) + for _, attempt := range attemptErrors(err) { + var urlErr *url.Error + if !errors.As(attempt, &urlErr) { + continue + } + switch { + case errNetworkError(urlErr.Err): + return true + case errDNSResolutionFailure(urlErr.Err): + // Neutral; let a later attempt decide. + default: + return false + } } return false } +// attemptErrors returns the per-attempt errors recorded in err, in the order they were +// tried. A composed fallback error wraps one per attempt; anything else is a single +// attempt. +func attemptErrors(err error) []error { + if multi, ok := err.(interface{ Unwrap() []error }); ok { + return multi.Unwrap() + } + return []error{err} +} + +// errDNSResolutionFailure reports whether err is a name-resolution failure. Go marks a +// *net.DNSError as temporary only for socket failures that reached the server, so a +// SERVFAIL or "no such host" answer is not temporary - but it is also not evidence that +// retrying is pointless, which is why callers treat it as no verdict. +func errDNSResolutionFailure(err error) bool { + var dnsErr *net.DNSError + return errors.As(err, &dnsErr) +} + func errNetworkError(err error) bool { var opErr *net.OpError if errors.As(err, &opErr) { if opErr.Temporary() { return true } + if ctrldnet.IsUnreachable(err) { + return true + } switch { case errors.Is(opErr.Err, syscall.ECONNREFUSED), errors.Is(opErr.Err, syscall.EINVAL), - errors.Is(opErr.Err, syscall.ENETUNREACH), - errors.Is(opErr.Err, windowsENETUNREACH), errors.Is(opErr.Err, windowsEINVAL), - errors.Is(opErr.Err, windowsECONNREFUSED), - errors.Is(opErr.Err, windowsEHOSTUNREACH): + errors.Is(opErr.Err, windowsECONNREFUSED): return true } } @@ -1246,6 +2294,16 @@ func errConnectionRefused(err error) bool { return errors.Is(opErr.Err, syscall.ECONNREFUSED) || errors.Is(opErr.Err, windowsECONNREFUSED) } +// errLogServerUnavailable reports whether err indicates the log server is not up yet +// (e.g. socket missing or connection refused). Callers should not log these as errors. +func errLogServerUnavailable(err error) bool { + var opErr *net.OpError + if !errors.As(err, &opErr) { + return false + } + return errors.Is(opErr.Err, syscall.ECONNREFUSED) || errors.Is(opErr.Err, syscall.ENOENT) || errors.Is(opErr.Err, windowsECONNREFUSED) +} + func ifaceFirstPrivateIP(iface *net.Interface) string { if iface == nil { return "" @@ -1272,7 +2330,7 @@ func ifaceFirstPrivateIP(iface *net.Interface) string { } // defaultRouteIP returns private IP string of the default route if present, prefer IPv4 over IPv6. -func defaultRouteIP() string { +func (p *prog) defaultRouteIP() string { dr, err := netmon.DefaultRoute() if err != nil { return "" @@ -1281,9 +2339,9 @@ func defaultRouteIP() string { if err != nil { return "" } - mainLog.Load().Debug().Str("iface", drNetIface.Name).Msg("checking default route interface") + p.Debug().Str("iface", drNetIface.Name).Msg("Checking default route interface") if ip := ifaceFirstPrivateIP(drNetIface); ip != "" { - mainLog.Load().Debug().Str("ip", ip).Msg("found ip with default route interface") + p.Debug().Str("ip", ip).Msg("Found ip with default route interface") return ip } @@ -1308,7 +2366,7 @@ func defaultRouteIP() string { }) if len(addrs) == 0 { - mainLog.Load().Warn().Msg("no default route IP found") + p.Warn().Msg("No default route IP found") return "" } sort.Slice(addrs, func(i, j int) bool { @@ -1316,7 +2374,7 @@ func defaultRouteIP() string { }) ip := addrs[0].String() - mainLog.Load().Debug().Str("ip", ip).Msg("found LAN interface IP") + p.Debug().Str("ip", ip).Msg("Found LAN interface IP") return ip } @@ -1331,8 +2389,8 @@ func canBeLocalUpstream(addr string) bool { // withEachPhysicalInterfaces runs the function f with each physical interfaces, excluding // the interface that matches excludeIfaceName. The context is used to clarify the // log message when error happens. -func withEachPhysicalInterfaces(excludeIfaceName, context string, f func(i *net.Interface) error) { - validIfacesMap := validInterfacesMap() +func withEachPhysicalInterfaces(excludeIfaceName, contextStr string, f func(i *net.Interface) error) { + validIfacesMap := ctrld.ValidInterfaces(ctrld.LoggerCtx(context.Background(), mainLog.Load())) netmon.ForeachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { // Skip loopback/virtual/down interface. if i.IsLoopback() || len(i.HardwareAddr) == 0 { @@ -1344,7 +2402,7 @@ func withEachPhysicalInterfaces(excludeIfaceName, context string, f func(i *net. } netIface := i.Interface if patched, err := patchNetIfaceName(netIface); err != nil { - mainLog.Load().Debug().Err(err).Msg("failed to patch net interface name") + mainLog.Load().Debug().Err(err).Msg("Failed to patch net interface name") return } else if !patched { // The interface is not functional, skipping. @@ -1356,11 +2414,11 @@ func withEachPhysicalInterfaces(excludeIfaceName, context string, f func(i *net. } // TODO: investigate whether we should report this error? if err := f(netIface); err == nil { - if context != "" { - mainLog.Load().Debug().Msgf("Ran %s for interface %q successfully", context, i.Name) + if contextStr != "" { + mainLog.Load().Debug().Msgf("Ran %s for interface %q successfully", contextStr, i.Name) } } else if !errors.Is(err, errSaveCurrentStaticDNSNotSupported) { - mainLog.Load().Err(err).Msgf("%s for interface %q failed", context, i.Name) + mainLog.Load().Err(err).Msgf("%s for interface %q failed", contextStr, i.Name) } }) } @@ -1381,7 +2439,7 @@ var errSaveCurrentStaticDNSNotSupported = errors.New("saving current DNS is not // Only works on Windows and Mac. func saveCurrentStaticDNS(iface *net.Interface) error { if iface == nil { - mainLog.Load().Debug().Msg("could not save current static DNS settings for nil interface") + mainLog.Load().Debug().Msg("Could not save current static DNS settings for nil interface") return nil } switch runtime.GOOS { @@ -1389,14 +2447,14 @@ func saveCurrentStaticDNS(iface *net.Interface) error { default: return errSaveCurrentStaticDNSNotSupported } - file := savedStaticDnsSettingsFilePath(iface) + file := ctrld.SavedStaticDnsSettingsFilePath(iface) ns, err := currentStaticDNS(iface) if err != nil { - mainLog.Load().Warn().Err(err).Msgf("could not get current static DNS settings for %q", iface.Name) + mainLog.Load().Warn().Err(err).Msgf("Could not get current static DNS settings for %q", iface.Name) return err } if len(ns) == 0 { - mainLog.Load().Debug().Msgf("no static DNS settings for %q, removing old static DNS settings file", iface.Name) + mainLog.Load().Debug().Msgf("No static DNS settings for %q, removing old static DNS settings file", iface.Name) _ = os.Remove(file) // removing old static DNS settings return nil } @@ -1411,47 +2469,15 @@ func saveCurrentStaticDNS(iface *net.Interface) error { return nil } if err := os.Remove(file); err != nil && !errors.Is(err, fs.ErrNotExist) { - mainLog.Load().Warn().Err(err).Msgf("could not remove old static DNS settings file: %s", file) + mainLog.Load().Warn().Err(err).Msgf("Could not remove old static DNS settings file: %s", file) } nss := strings.Join(ns, ",") mainLog.Load().Debug().Msgf("DNS settings for %q is static: %v, saving ...", iface.Name, nss) if err := os.WriteFile(file, []byte(nss), 0600); err != nil { - mainLog.Load().Err(err).Msgf("could not save DNS settings for iface: %s", iface.Name) + mainLog.Load().Err(err).Msgf("Could not save DNS settings for iface: %s", iface.Name) return err } - mainLog.Load().Debug().Msgf("save DNS settings for interface %q successfully", iface.Name) - return nil -} - -// savedStaticDnsSettingsFilePath returns the path to saved DNS settings of the given interface. -func savedStaticDnsSettingsFilePath(iface *net.Interface) string { - if iface == nil { - return "" - } - return absHomeDir(".dns_" + iface.Name) -} - -// savedStaticNameservers returns the static DNS nameservers of the given interface. -// -//lint:ignore U1000 use in os_windows.go and os_darwin.go -func savedStaticNameservers(iface *net.Interface) []string { - if iface == nil { - mainLog.Load().Debug().Msg("could not get saved static DNS settings for nil interface") - return nil - } - file := savedStaticDnsSettingsFilePath(iface) - if data, _ := os.ReadFile(file); len(data) > 0 { - saveValues := strings.Split(string(data), ",") - returnValues := []string{} - // check each one, if its in loopback range, remove it - for _, v := range saveValues { - if net.ParseIP(v).IsLoopback() { - continue - } - returnValues = append(returnValues, v) - } - return returnValues - } + mainLog.Load().Debug().Msgf("Save DNS settings for interface %q successfully", iface.Name) return nil } @@ -1459,27 +2485,40 @@ func savedStaticNameservers(iface *net.Interface) []string { // It returns false for a nil iface. // // The caller must sort the nameservers before calling this function. -func dnsChanged(iface *net.Interface, nameservers []string) bool { +func (p *prog) dnsChanged(iface *net.Interface, nameservers []string) bool { if iface == nil { return false } curNameservers, _ := currentStaticDNS(iface) slices.Sort(curNameservers) if !slices.Equal(curNameservers, nameservers) { - mainLog.Load().Debug().Msgf("interface %q current DNS settings: %v, expected: %v", iface.Name, curNameservers, nameservers) + p.Debug().Msgf("Interface %q current DNS settings: %v, expected: %v", iface.Name, curNameservers, nameservers) return true } return false } // selfUninstallCheck checks if the error dues to controld.InvalidConfigCode, perform self-uninstall then. -func selfUninstallCheck(uninstallErr error, p *prog, logger zerolog.Logger) { +// selfUninstallFn is the uninstall itself, indirected so a test can prove what +// does and does not reach it. The real one ends in os.Exit, which a test cannot +// survive to make an assertion about. +var selfUninstallFn = selfUninstall + +func selfUninstallCheck(uninstallErr error, p *prog, logger *ctrld.Logger) { + // Maintenance is the API being away; it establishes nothing about whether + // this device still exists. Guarded here rather than at each caller because + // this is the one place that acts, and acting would uninstall a working + // install over a temporary outage. + if controld.IsMaintenance(uninstallErr) { + logger.Debug().Msg("API is in maintenance, skipping device status check") + return + } var uer *controld.ErrorResponse if errors.As(uninstallErr, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { p.stopDnsWatchers() // Perform self-uninstall now. - selfUninstall(p, logger) + selfUninstallFn(p, logger) } } @@ -1488,9 +2527,9 @@ func selfUninstallCheck(uninstallErr error, p *prog, logger zerolog.Logger) { // // The callers must ensure curVer and logger are non-nil. // Returns true if upgrade is allowed, false otherwise. -func shouldUpgrade(vt string, cv *semver.Version, logger *zerolog.Logger) bool { +func shouldUpgrade(vt string, cv *semver.Version, logger *ctrld.Logger) bool { if vt == "" { - logger.Debug().Msg("no version target set, skipped checking self-upgrade") + logger.Debug().Msg("No version target set, skipped checking self-upgrade") return false } vts := vt @@ -1499,7 +2538,7 @@ func shouldUpgrade(vt string, cv *semver.Version, logger *zerolog.Logger) bool { } targetVer, err := semver.NewVersion(vts) if err != nil { - logger.Warn().Err(err).Msgf("invalid target version, skipped self-upgrade: %s", vt) + logger.Warn().Err(err).Msgf("Invalid target version, skipped self-upgrade: %s", vt) return false } @@ -1508,7 +2547,7 @@ func shouldUpgrade(vt string, cv *semver.Version, logger *zerolog.Logger) bool { logger.Warn(). Str("target", vt). Str("current", cv.String()). - Msgf("major version upgrade not allowed (target: %d, current: %d), skipped self-upgrade", targetVer.Major(), cv.Major()) + Msgf("Major version upgrade not allowed (target: %d, current: %d), skipped self-upgrade", targetVer.Major(), cv.Major()) return false } @@ -1516,28 +2555,40 @@ func shouldUpgrade(vt string, cv *semver.Version, logger *zerolog.Logger) bool { logger.Debug(). Str("target", vt). Str("current", cv.String()). - Msgf("target version is not greater than current one, skipped self-upgrade") + Msgf("Target version is not greater than current one, skipped self-upgrade") return false } return true } +// newUpgradeCmd builds the detached command used to self-upgrade. It is a +// package-level variable so tests can stub it. With the real implementation a +// *test* binary would re-exec itself — os.Executable() is the test binary, and +// because `go test` stops flag parsing at the first positional arg ("upgrade") +// it ignores the args and re-runs the entire suite. That child hits the same +// upgrade test and spawns another child, recursively: a fork bomb of detached +// processes that pins the host and locks the test binary's image file. +var newUpgradeCmd = func(exe string) *exec.Cmd { + cmd := exec.Command(exe, "upgrade", "prod", "-vv") + cmd.SysProcAttr = sysProcAttrForDetachedChildProcess() + return cmd +} + // performUpgrade executes the self-upgrade command. // Returns true if upgrade was initiated successfully, false otherwise. -func performUpgrade(vt string) bool { +func performUpgrade(vt string, logger *ctrld.Logger) bool { exe, err := os.Executable() if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to get executable path, skipped self-upgrade") + logger.Error().Err(err).Msg("Failed to get executable path, skipped self-upgrade") return false } - cmd := exec.Command(exe, "upgrade", "prod", "-vv") - cmd.SysProcAttr = sysProcAttrForDetachedChildProcess() + cmd := newUpgradeCmd(exe) if err := cmd.Start(); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to start self-upgrade") + logger.Error().Err(err).Msg("Failed to start self-upgrade") return false } - mainLog.Load().Debug().Msgf("self-upgrade triggered, version target: %s", vt) + logger.Debug().Msgf("Self-upgrade triggered, version target: %s", vt) return true } @@ -1547,9 +2598,9 @@ func performUpgrade(vt string) bool { // // The callers must ensure curVer and logger are non-nil. // Returns true if upgrade is allowed and should proceed, false otherwise. -func selfUpgradeCheck(vt string, cv *semver.Version, logger *zerolog.Logger) bool { +func selfUpgradeCheck(vt string, cv *semver.Version, logger *ctrld.Logger) bool { if shouldUpgrade(vt, cv, logger) { - return performUpgrade(vt) + return performUpgrade(vt, logger) } return false } @@ -1560,29 +2611,5 @@ func (p *prog) leakOnUpstreamFailure() bool { if ptr := p.cfg.Service.LeakOnUpstreamFailure; ptr != nil { return *ptr } - // Default is false on routers, since this leaking is only useful for devices that move between networks. - if router.Name() != "" { - return false - } - // if we are running on ADDC, we should not leak on upstream failure - if p.runningOnDomainController { - return false - } return true } - -// Domain controller role values from Win32_ComputerSystem -// https://learn.microsoft.com/en-us/windows/win32/cimwin32prov/win32-computersystem -const ( - BackupDomainController = 4 - PrimaryDomainController = 5 -) - -// isRunningOnDomainController checks if the current machine is a domain controller -// by querying the DomainRole property from Win32_ComputerSystem via WMI. -func isRunningOnDomainController() (bool, int) { - if runtime.GOOS != "windows" { - return false, 0 - } - return isRunningOnDomainControllerWindows() -} diff --git a/cmd/cli/prog_darwin.go b/cmd/cli/prog_darwin.go index 9cd57864..a3854703 100644 --- a/cmd/cli/prog_darwin.go +++ b/cmd/cli/prog_darwin.go @@ -4,8 +4,10 @@ import ( "github.com/kardianos/service" ) +// setDependencies sets service dependencies for Darwin func setDependencies(svc *service.Config) {} +// setWorkingDirectory sets the working directory for the service func setWorkingDirectory(svc *service.Config, dir string) { svc.WorkingDirectory = dir } diff --git a/cmd/cli/prog_freebsd.go b/cmd/cli/prog_freebsd.go index 93d737fc..1be94cae 100644 --- a/cmd/cli/prog_freebsd.go +++ b/cmd/cli/prog_freebsd.go @@ -6,9 +6,11 @@ import ( "github.com/kardianos/service" ) +// setDependencies sets service dependencies for FreeBSD func setDependencies(svc *service.Config) { // TODO(cuonglm): remove once https://github.com/kardianos/service/issues/359 fixed. _ = os.MkdirAll("/usr/local/etc/rc.d", 0755) } +// setWorkingDirectory sets the working directory for the service func setWorkingDirectory(svc *service.Config, dir string) {} diff --git a/cmd/cli/prog_intercept_fallback_test.go b/cmd/cli/prog_intercept_fallback_test.go new file mode 100644 index 00000000..40b25360 --- /dev/null +++ b/cmd/cli/prog_intercept_fallback_test.go @@ -0,0 +1,233 @@ +package cli + +import ( + "errors" + "fmt" + "net" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +// TestInterfaceDNSFallbackViable covers when the interface-DNS fallback may be used +// after DNS intercept fails to start. +// +// The fallback names a resolver by IP with no port, so it can only reach a listener on +// :53. Taking it with the listener on a redirect-dependent port produced a total DNS +// outage on macOS: the interface points at 127.0.0.1, mDNSResponder answers there, and +// its upstream is ctrld's own address - a resolution loop with a healthy ctrld listener +// nothing can reach. Intercept startup refuses the fallback in that case rather than +// creating it. +func TestInterfaceDNSFallbackViable(t *testing.T) { + tests := []struct { + name string + lc *ctrld.ListenerConfig + want bool + }{ + { + name: "listener on 53 can be reached by interface DNS", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 53}, + want: true, + }, + { + // The reported outage: no local resolver, so the :5354 fallback port + // cannot be expressed by interface DNS. + name: "listener on the fallback port cannot", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}, + want: false, + }, + { + name: "any other non-53 port cannot", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5300}, + want: false, + }, + + { + // Port is resolved elsewhere and defaults to 53; nothing to refuse yet. + name: "unset port is not refused", + lc: &ctrld.ListenerConfig{IP: "127.0.0.1"}, + want: true, + }, + { + name: "no listener is not refused", + lc: nil, + want: true, + }, + { + // A non-loopback listener on 53 is still reachable by IP. + name: "non-loopback listener on 53", + lc: &ctrld.ListenerConfig{IP: "192.168.1.10", Port: 53}, + want: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := interfaceDNSFallbackViable(tc.lc); got != tc.want { + t.Errorf("interfaceDNSFallbackViable() = %v, want %v", got, tc.want) + } + }) + } +} + +// interceptFallbackHarness drives setDNS() through the intercept-start failure path and +// records the side effects that decide whether the host ends up with a working +// resolver. +// +// Every host-touching step is stubbed, including the intercept start itself: this test +// runs untagged on Linux, macOS and Windows runners, where the real startDNSIntercept +// would set up pf or install an NRPT rule on the machine running the tests. Stubbing it +// also makes the precondition deterministic - the failure under test is injected rather +// than depending on the runner denying a privileged operation. +type interceptFallbackHarness struct { + interceptCalls int + ensureTargetCalls int + ensuredNameservers []string + installedNameservers []string + installCalls int + resetCalls int + removeTargetCalls int + refusals []string +} + +func newInterceptFallbackHarness(t *testing.T, lc *ctrld.ListenerConfig) *interceptFallbackHarness { + t.Helper() + h := &interceptFallbackHarness{} + + origStart, origEnsure, origRemove, origInstall := startDNSInterceptFn, ensureInterceptDNSTargetFn, removeInterceptDNSTargetFn, setDnsForRunningIfaceFn + origReset, origFatal := resetDNSFn, refuseFallbackFatal + origCfg, origMode, origIntercept, origHard := cfg, interceptMode, dnsIntercept, hardIntercept + t.Cleanup(func() { + startDNSInterceptFn, ensureInterceptDNSTargetFn, removeInterceptDNSTargetFn, setDnsForRunningIfaceFn = origStart, origEnsure, origRemove, origInstall + resetDNSFn, refuseFallbackFatal = origReset, origFatal + cfg, interceptMode, dnsIntercept, hardIntercept = origCfg, origMode, origIntercept, origHard + }) + + // Never reach the real interceptor: it would configure pf on macOS and NRPT on + // Windows, on the machine running the tests. + startDNSInterceptFn = func(_ *prog) error { + h.interceptCalls++ + return errors.New("dns intercept: injected start failure") + } + ensureInterceptDNSTargetFn = func(_ *prog, nameservers []string) { + h.ensureTargetCalls++ + h.ensuredNameservers = slices.Clone(nameservers) + } + removeInterceptDNSTargetFn = func(_ *prog, _ string) { h.removeTargetCalls++ } + setDnsForRunningIfaceFn = func(_ *prog, nameservers []string) *net.Interface { + h.installCalls++ + h.installedNameservers = nameservers + return nil + } + resetDNSFn = func(_ *prog, _ bool, _ bool) { h.resetCalls++ } + refuseFallbackFatal = func(_ *prog, format string, v ...any) { + h.refusals = append(h.refusals, fmt.Sprintf(format, v...)) + } + + cfg = ctrld.Config{} + cfg.Service.InterceptMode = "dns" + cfg.Listener = map[string]*ctrld.ListenerConfig{"0": lc} + watchdogOff := false + cfg.Service.DnsWatchdogEnabled = &watchdogOff + interceptMode, dnsIntercept, hardIntercept = "dns", false, false + return h +} + +func (h *interceptFallbackHarness) run(t *testing.T) { + t.Helper() + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.setDNS(nil) +} + +func TestSetDNSEnsuresInterceptTargetAfterSuccessfulStart(t *testing.T) { + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}) + startDNSInterceptFn = func(_ *prog) error { + h.interceptCalls++ + return nil + } + want := []string{"fe80::1"} + p := &prog{cfg: &cfg} + p.logger.Store(mainLog.Load()) + p.setDNS(want) + + if h.interceptCalls != 1 || h.ensureTargetCalls != 1 { + t.Fatalf("intercept calls=%d ensure calls=%d, want 1 each", h.interceptCalls, h.ensureTargetCalls) + } + if !slices.Equal(h.ensuredNameservers, want) { + t.Fatalf("system nameservers = %v, want %v", h.ensuredNameservers, want) + } + if h.installCalls != 0 { + t.Fatalf("interface-DNS fallback installed %d time(s) after successful intercept start", h.installCalls) + } +} + +func TestSetDNSExplicitOffOverridesConfig(t *testing.T) { + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 53}) + interceptMode = "off" + dnsIntercept = false + hardIntercept = false + + h.run(t) + + if h.interceptCalls != 0 { + t.Fatalf("intercept start called %d time(s), want 0: explicit off must override service.intercept_mode", h.interceptCalls) + } + if h.installCalls != 1 { + t.Fatalf("interface DNS installed %d time(s), want 1", h.installCalls) + } + if h.removeTargetCalls != 1 { + t.Fatalf("stale intercept DNS target cleanup called %d time(s), want 1", h.removeTargetCalls) + } +} + +// TestSetDNSRefusesUnreachableFallback is the behaviour test for the reported outage: it +// drives the real setDNS() lifecycle rather than the classification helper alone. +// +// Deleting or bypassing the guard in setDNS makes the first case fail, because interface +// DNS then gets installed pointing at a listener that cannot answer on :53 - which is +// the resolution loop this refuses to create. +func TestSetDNSRefusesUnreachableFallback(t *testing.T) { + t.Run("non-53 listener refuses the fallback and restores DNS", func(t *testing.T) { + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 5354}) + h.run(t) + + if h.interceptCalls != 1 { + t.Fatalf("intercept start called %d time(s) through the seam, want 1 — the real platform interceptor must never run here", h.interceptCalls) + } + if h.installCalls != 0 { + t.Errorf("interface DNS was installed %d time(s) for a listener on :5354 — that is the resolver loop", h.installCalls) + } + if h.resetCalls == 0 { + t.Error("host DNS was not restored before refusing, leaving the interface pointed at a ctrld that is not serving") + } + if h.removeTargetCalls != 1 { + t.Errorf("stale intercept DNS target cleanup called %d time(s), want 1 after intercept failure", h.removeTargetCalls) + } + if len(h.refusals) == 0 { + t.Fatal("refusal was not surfaced: startup must fail loudly rather than silently skip the fallback") + } + if !strings.Contains(h.refusals[0], "5354") { + t.Errorf("refusal does not name the unreachable port: %q", h.refusals[0]) + } + }) + + t.Run("listener on 53 still reaches the interface-DNS fallback", func(t *testing.T) { + h := newInterceptFallbackHarness(t, &ctrld.ListenerConfig{IP: "127.0.0.1", Port: 53}) + h.run(t) + + if h.interceptCalls != 1 { + t.Fatalf("intercept start called %d time(s) through the seam, want 1", h.interceptCalls) + } + if h.installCalls != 1 { + t.Errorf("interface DNS installed %d time(s), want 1: a listener on :53 is reachable, so the fallback must still apply", h.installCalls) + } + if len(h.refusals) != 0 { + t.Errorf("unexpected refusal for a reachable listener: %v", h.refusals) + } + if len(h.installedNameservers) == 0 { + t.Error("fallback installed no nameservers") + } + }) +} diff --git a/cmd/cli/prog_linux.go b/cmd/cli/prog_linux.go index 2e5c7c76..c834b495 100644 --- a/cmd/cli/prog_linux.go +++ b/cmd/cli/prog_linux.go @@ -9,8 +9,6 @@ import ( "strings" "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld/internal/router" ) func init() { @@ -23,6 +21,7 @@ func init() { } } +// setDependencies sets service dependencies for Linux func setDependencies(svc *service.Config) { svc.Dependencies = []string{ "Wants=network-online.target", @@ -37,11 +36,9 @@ func setDependencies(svc *service.Config) { svc.Dependencies = append(svc.Dependencies, "Wants=systemd-networkd-wait-online.service") } } - if routerDeps := router.ServiceDependencies(); len(routerDeps) > 0 { - svc.Dependencies = append(svc.Dependencies, routerDeps...) - } } +// setWorkingDirectory sets the working directory for the service func setWorkingDirectory(svc *service.Config, dir string) { svc.WorkingDirectory = dir } diff --git a/cmd/cli/prog_log.go b/cmd/cli/prog_log.go new file mode 100644 index 00000000..5d8c9db3 --- /dev/null +++ b/cmd/cli/prog_log.go @@ -0,0 +1,50 @@ +package cli + +import "github.com/Control-D-Inc/ctrld" + +// Debug starts a new message with debug level. +func (p *prog) Debug() *ctrld.LogEvent { + return p.logger.Load().Debug() +} + +// Warn starts a new message with warn level. +func (p *prog) Warn() *ctrld.LogEvent { + return p.logger.Load().Warn() +} + +// Info starts a new message with info level. +func (p *prog) Info() *ctrld.LogEvent { + return p.logger.Load().Info() +} + +// Fatal starts a new message with fatal level. +func (p *prog) Fatal() *ctrld.LogEvent { + return p.logger.Load().Fatal() +} + +// Error starts a new message with error level. +func (p *prog) Error() *ctrld.LogEvent { + return p.logger.Load().Error() +} + +// Notice starts a new message with notice level. +func (p *prog) Notice() *ctrld.LogEvent { + return p.logger.Load().Notice() +} + +// notifyExitToLogServer closes this run's connection to the HTTP log server, +// so a waiting "ctrld start" is not left waiting on it after this process +// exits. A nil connection (log server never started) is a no-op. +func (p *prog) notifyExitToLogServer() { + p.closeLogConn() +} + +func (p *prog) closeLogConn() { + p.logConnMu.Lock() + conn := p.logConn + p.logConn = nil + p.logConnMu.Unlock() + if conn != nil { + _ = conn.Close() + } +} diff --git a/cmd/cli/prog_others.go b/cmd/cli/prog_others.go index 9026318b..c1b7f17d 100644 --- a/cmd/cli/prog_others.go +++ b/cmd/cli/prog_others.go @@ -4,8 +4,10 @@ package cli import "github.com/kardianos/service" +// setDependencies sets service dependencies for other platforms func setDependencies(svc *service.Config) {} +// setWorkingDirectory sets the working directory for the service func setWorkingDirectory(svc *service.Config, dir string) { // WorkingDirectory is not supported on Windows. svc.WorkingDirectory = dir diff --git a/cmd/cli/prog_test.go b/cmd/cli/prog_test.go index c4ef5c3b..bb56f76c 100644 --- a/cmd/cli/prog_test.go +++ b/cmd/cli/prog_test.go @@ -1,17 +1,46 @@ package cli import ( - "runtime" + "context" + "net" + "net/url" + "syscall" "testing" "time" "github.com/Masterminds/semver/v3" - "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "go.uber.org/zap" "github.com/Control-D-Inc/ctrld" ) +func TestErrNetworkErrorTreatsNoRouteAsNetworkError(t *testing.T) { + err := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.EHOSTUNREACH} + assert.True(t, errNetworkError(err)) + assert.True(t, errUrlNetworkError(&url.Error{Op: "Get", URL: "https://dns.controld.com", Err: err})) +} + +func TestSleepWithContext(t *testing.T) { + assert.True(t, sleepWithContext(context.Background(), time.Millisecond)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + assert.False(t, sleepWithContext(ctx, time.Minute)) + assert.Less(t, time.Since(start), 100*time.Millisecond) +} + +func TestUnreachableRecoveryBackoff(t *testing.T) { + // Streak starts at the base cadence and doubles each attempt, capped at the max. + assert.Equal(t, checkUpstreamBackoffSleep, unreachableRecoveryBackoff(0)) + assert.Equal(t, checkUpstreamBackoffSleep, unreachableRecoveryBackoff(1)) + assert.Equal(t, 2*checkUpstreamBackoffSleep, unreachableRecoveryBackoff(2)) + assert.Equal(t, 4*checkUpstreamBackoffSleep, unreachableRecoveryBackoff(3)) + assert.Equal(t, checkUpstreamUnreachableBackoffMax, unreachableRecoveryBackoff(100)) +} + func Test_prog_dnsWatchdogEnabled(t *testing.T) { p := &prog{cfg: &ctrld.Config{}} @@ -174,10 +203,10 @@ func Test_shouldUpgrade(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { // Create test logger - testLogger := zerolog.New(zerolog.NewTestWriter(t)).With().Logger() + testLogger := &ctrld.Logger{Logger: zap.NewNop()} // Call the function and capture the result - result := shouldUpgrade(tc.versionTarget, tc.currentVersion, &testLogger) + result := shouldUpgrade(tc.versionTarget, tc.currentVersion, testLogger) // Assert the expected result assert.Equal(t, tc.shouldUpgrade, result, tc.description) @@ -186,10 +215,6 @@ func Test_shouldUpgrade(t *testing.T) { } func Test_selfUpgradeCheck(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("skipped due to Windows file locking issue on Github Action runners") - } - // Helper function to create a version makeVersion := func(v string) *semver.Version { ver, err := semver.NewVersion(v) @@ -226,10 +251,10 @@ func Test_selfUpgradeCheck(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { // Create test logger - testLogger := zerolog.New(zerolog.NewTestWriter(t)).With().Logger() + testLogger := &ctrld.Logger{Logger: zap.NewNop()} // Call the function and capture the result - result := selfUpgradeCheck(tc.versionTarget, tc.currentVersion, &testLogger) + result := selfUpgradeCheck(tc.versionTarget, tc.currentVersion, testLogger) // Assert the expected result assert.Equal(t, tc.shouldUpgrade, result, tc.description) @@ -238,10 +263,6 @@ func Test_selfUpgradeCheck(t *testing.T) { } func Test_performUpgrade(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("skipped due to Windows file locking issue on Github Action runners") - } - tests := []struct { name string versionTarget string @@ -262,11 +283,15 @@ func Test_performUpgrade(t *testing.T) { }, } + // newUpgradeCmd is stubbed in TestMain so performUpgrade does not re-exec + // (and fork-bomb) the test binary; see the comment there. for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { + // Create test logger + testLogger := &ctrld.Logger{Logger: zap.NewNop()} // Call the function and capture the result - result := performUpgrade(tc.versionTarget) + result := performUpgrade(tc.versionTarget, testLogger) assert.Equal(t, tc.expectedResult, result, tc.description) }) } diff --git a/cmd/cli/prog_windows.go b/cmd/cli/prog_windows.go index e4486255..bd5673f6 100644 --- a/cmd/cli/prog_windows.go +++ b/cmd/cli/prog_windows.go @@ -2,12 +2,10 @@ package cli import "github.com/kardianos/service" -func setDependencies(svc *service.Config) { - if hasLocalDnsServerRunning() { - svc.Dependencies = []string{"DNS"} - } -} +// setDependencies sets service dependencies for Windows +func setDependencies(svc *service.Config) {} +// setWorkingDirectory sets the working directory for the service func setWorkingDirectory(svc *service.Config, dir string) { // WorkingDirectory is not supported on Windows. svc.WorkingDirectory = dir diff --git a/cmd/cli/prometheus.go b/cmd/cli/prometheus.go index 9082a58f..90fce209 100644 --- a/cmd/cli/prometheus.go +++ b/cmd/cli/prometheus.go @@ -2,6 +2,8 @@ package cli import "github.com/prometheus/client_golang/prometheus" +// Prometheus metrics label constants for consistent labeling across all metrics +// These ensure standardized metric labeling for monitoring and alerting const ( metricsLabelListener = "listener" metricsLabelClientSourceIP = "client_source_ip" @@ -13,17 +15,21 @@ const ( ) // statsVersion represent ctrld version. +// This metric provides version information for monitoring and debugging var statsVersion = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "ctrld_build_info", Help: "Version of ctrld process.", }, []string{"gitref", "goversion", "version"}) // statsTimeStart represents start time of ctrld service. +// This metric tracks service uptime and helps with monitoring service restarts var statsTimeStart = prometheus.NewGauge(prometheus.GaugeOpts{ Name: "ctrld_time_seconds", Help: "Start time of the ctrld process since unix epoch in seconds.", }) +// statsQueriesCountLabels defines the labels for query count metrics +// These labels provide detailed breakdown of DNS query statistics var statsQueriesCountLabels = []string{ metricsLabelListener, metricsLabelClientSourceIP, @@ -35,6 +41,7 @@ var statsQueriesCountLabels = []string{ } // statsQueriesCount counts total number of queries. +// This provides comprehensive DNS query statistics for monitoring and alerting var statsQueriesCount = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "ctrld_queries_count", Help: "Total number of queries.", @@ -44,12 +51,14 @@ var statsQueriesCount = prometheus.NewCounterVec(prometheus.CounterOpts{ // // The labels "client_source_ip", "client_mac", "client_hostname" are unbounded, // thus this stat is highly inefficient if there are many devices. +// This metric should be used carefully in high-client environments var statsClientQueriesCount = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "ctrld_client_queries_count", Help: "Total number queries of a client.", }, []string{metricsLabelClientSourceIP, metricsLabelClientMac, metricsLabelClientHostname}) // WithLabelValuesInc increases prometheus counter by 1 if query stats is enabled. +// This provides conditional metric collection to avoid performance impact when metrics are disabled func (p *prog) WithLabelValuesInc(c *prometheus.CounterVec, lvs ...string) { if p.metricsQueryStats.Load() { c.WithLabelValues(lvs...).Inc() diff --git a/cmd/cli/provision_boundary_guard_test.go b/cmd/cli/provision_boundary_guard_test.go new file mode 100644 index 00000000..3f298621 --- /dev/null +++ b/cmd/cli/provision_boundary_guard_test.go @@ -0,0 +1,209 @@ +package cli + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "testing" +) + +// packageSourcePath resolves a file in this package's source directory, or +// skips the test when the source is not available. The Windows CI job runs +// pre-built test binaries with no checkout, so neither the working directory +// nor the build-time path recorded by runtime.Caller reaches the source +// there. Skipping is safe for these static checks: they analyze the same +// source on every platform, and the jobs with a checkout still enforce them. +func packageSourcePath(t *testing.T, name string) string { + t.Helper() + if _, err := os.Stat(name); err == nil { + return name + } + if _, thisFile, _, ok := runtime.Caller(0); ok { + p := filepath.Join(filepath.Dir(thisFile), name) + if _, err := os.Stat(p); err == nil { + return p + } + } + t.Skipf("package source for %s not available (pre-built test binary without a checkout); this static check runs where source is present", name) + return "" +} + +// provisionBoundaryFiles are the files this lane's provisioning-hardening +// work audited: every terminal path they can reach must report a classified +// failure (a result file, an output line, a stage-scoped exit code), not a +// bare crash installer tooling cannot read. TestProvisioningTerminalPathAudit +// documents the audit by scenario; this test enforces it structurally so a +// later change cannot reintroduce a bare fatal without either classifying it +// or adding it here with a reason. +// +// Scope: the check sees direct calls (".Fatal()", "os.Exit", "panic") in the +// files listed here. A helper defined in one of these files is caught once, +// at the body that holds the call, so an indirect exit through it needs no +// second rule. A helper defined outside the list, such as +// checkHasElevatedPrivilege in service.go wired as the "start" PreRun, is +// outside the reach of this check. TestProvisioningTerminalPathAudit lists +// those by hand in its exclusion comment. +var provisionBoundaryFiles = []string{ + "cli.go", + "commands_service_start.go", + "commands_run.go", + "prog.go", + "provision_result.go", +} + +// provisionBoundaryFatalAllowlist is every call in the files above that +// still reaches a bare ".Fatal()" (on any logger chain), "os.Exit", or +// "panic", keyed +// by "file.go:FuncName" (FuncName includes the receiver type for methods, +// e.g. "prog.setDNS"). Each entry names why it is not a provisioning +// failure, or why it predates and sits outside the --cd-org boundary this +// lane hardened. +var provisionBoundaryFatalAllowlist = map[string]string{ + "cli.go:RunMobile": `panic on a nil AppConfig is a programming error in the mobile host ` + + `app, at an entry point the CLI command tree never calls. Not reachable from ` + + `"ctrld start --cd-org".`, + + "cli.go:run": `os.Exit(0) is the successful exit of the daemon-respawn launcher, right ` + + `after it starts the real background process. Not a failure.`, + + "cli.go:readConfigFile": `shared CLI config-parsing helper used the same way by every ` + + `invocation mode (config-file, no-config, nextdns, --cd, --cd-org). Predates and is ` + + `orthogonal to the --cd-org provisioning boundary.`, + + "cli.go:processNoConfigFlags": `shared CLI helper enforcing --listen/--primary_upstream in ` + + `no-config mode. Applies uniformly across every invocation mode, not specifically to ` + + `provisioning.`, + + "cli.go:processListenFlag": `shared CLI helper parsing --listen. Applies uniformly across ` + + `every invocation mode, not specifically to provisioning.`, + + "cli.go:readConfigWithNotice": `shared CLI helper (userHomeDir failure while locating the ` + + `default config file). Applies uniformly across every invocation mode, not specifically ` + + `to provisioning.`, + + "cli.go:checkStrFlagEmpty": `shared flag-emptiness helper. Only an explicit empty --cd-org ` + + `is a provisioning-token value, classified separately inside this function; every other ` + + `flag (currently --cd) keeps its bare fatal.`, + + "commands_service_start.go:ServiceCommand.Start": `the deactivation pin check's ` + + `os.Exit(126). Out of scope for this lane by contract ("pin-check 126 untouched").`, + + "prog.go:prog.Stop": `the deactivation pin check's os.Exit(126). Out of scope for this ` + + `lane by contract ("pin-check 126 untouched").`, + + "prog.go:prog.setDNS": `the runtime --intercept-mode validation (validInterceptMode), ` + + `distinct from the early "ctrld start" check that already reports ` + + `INTERCEPT_MODE_INVALID. Left alone per T6's own doc comment.`, +} + +// funcDeclKey names a function declaration the way +// provisionBoundaryFatalAllowlist keys it: "FuncName" for a plain function, +// "ReceiverType.FuncName" for a method. +func funcDeclKey(fn *ast.FuncDecl) string { + if fn.Recv == nil || len(fn.Recv.List) == 0 { + return fn.Name.Name + } + recvType := fn.Recv.List[0].Type + if star, ok := recvType.(*ast.StarExpr); ok { + recvType = star.X + } + if ident, ok := recvType.(*ast.Ident); ok { + return ident.Name + "." + fn.Name.Name + } + return fn.Name.Name +} + +// isBareFatalOrExitCall reports whether call is ".Fatal(...)" on any +// logger chain, "os.Exit(...)", or a bare "panic(...)". +func isBareFatalOrExitCall(call *ast.CallExpr) bool { + if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "panic" { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return false + } + if sel.Sel.Name == "Fatal" { + return true + } + if sel.Sel.Name == "Exit" { + if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == "os" { + return true + } + } + return false +} + +// TestProvisionBoundaryHasNoUnlistedBareFatal parses every provisioning +// boundary file and flags a bare Fatal/os.Exit that is not in the allowlist +// above. A new one appearing here means a change added a crash the +// provisioning boundary's contract does not allow: classify it through +// failProvision/failProvisionUnclassified, or add it to the allowlist with a +// reason if it genuinely sits outside the boundary. +func TestProvisionBoundaryHasNoUnlistedBareFatal(t *testing.T) { + fset := token.NewFileSet() + for _, file := range provisionBoundaryFiles { + node, err := parser.ParseFile(fset, packageSourcePath(t, file), nil, 0) + if err != nil { + t.Fatalf("could not parse %s: %v", file, err) + } + for _, decl := range node.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + key := file + ":" + funcDeclKey(fn) + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok || !isBareFatalOrExitCall(call) { + return true + } + if _, allowed := provisionBoundaryFatalAllowlist[key]; !allowed { + pos := fset.Position(call.Pos()) + t.Errorf("%s:%d: unlisted bare fatal/exit in %s - classify it through "+ + "failProvision/failProvisionUnclassified, or add it to "+ + "provisionBoundaryFatalAllowlist with a reason", file, pos.Line, key) + } + return true + }) + } + } +} + +// TestProvisionBoundaryAllowlistHasNoStaleEntries is the converse check: every +// allowlisted key must still name a real bare Fatal/os.Exit. A stale entry +// (the call was removed or reclassified) would silently widen the allowlist +// and hide a real regression the next time this test runs. +func TestProvisionBoundaryAllowlistHasNoStaleEntries(t *testing.T) { + fset := token.NewFileSet() + found := make(map[string]bool, len(provisionBoundaryFatalAllowlist)) + for _, file := range provisionBoundaryFiles { + node, err := parser.ParseFile(fset, packageSourcePath(t, file), nil, 0) + if err != nil { + t.Fatalf("could not parse %s: %v", file, err) + } + for _, decl := range node.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + key := file + ":" + funcDeclKey(fn) + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok || !isBareFatalOrExitCall(call) { + return true + } + found[key] = true + return true + }) + } + } + for key := range provisionBoundaryFatalAllowlist { + if !found[key] { + t.Errorf("provisionBoundaryFatalAllowlist[%q] no longer matches any bare fatal/exit; remove the stale entry", key) + } + } +} diff --git a/cmd/cli/provision_input_test.go b/cmd/cli/provision_input_test.go new file mode 100644 index 00000000..7914c5bb --- /dev/null +++ b/cmd/cli/provision_input_test.go @@ -0,0 +1,422 @@ +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/spf13/cobra" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// captureMainLog swaps mainLog for a test-local buffer logger and restores +// the previous logger on cleanup. Asserting on the shared logOutput sink is +// order-dependent: a test that ran earlier can re-store mainLog (Windows' +// Test_validInterfaces calls initConsoleLogging), leaving the shared buffer +// stale for every later test. +func captureMainLog(t *testing.T) *syncBuffer { + t.Helper() + buf := &syncBuffer{} + core := zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), + zapcore.AddSync(buf), + zap.DebugLevel, + ) + old := mainLog.Load() + mainLog.Store(&ctrld.Logger{Logger: zap.New(core)}) + t.Cleanup(func() { mainLog.Store(old) }) + return buf +} + +func TestProvisionTokenShapeValid(t *testing.T) { + tests := []struct { + name string + token string + want bool + }{ + {"too short", "abcde", false}, + {"minimum length", "abcdef", true}, + {"maximum length", strings.Repeat("a", 64), true}, + {"too long", strings.Repeat("a", 65), false}, + {"contains space", "org-v1- abc", false}, + {"contains tab", "org-v1-\tabc", false}, + {"contains newline", "org-v1-\nabc", false}, + {"contains control char", "org-v1-\x00abc", false}, + {"sane with prefix", "org-v1-abcdef123456", true}, + {"sane without prefix", "legacytoken123", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := provisionTokenShapeValid(tc.token); got != tc.want { + t.Errorf("provisionTokenShapeValid(%q) = %v, want %v", tc.token, got, tc.want) + } + }) + } +} + +// TestCdUIDFromProvTokenMalformedToken proves a malformed --cd-org value is +// classified before any network attempt: no call reaches fetchResolverUIDFn, +// and the malformed value itself never appears in the persisted message. +func TestCdUIDFromProvTokenMalformedToken(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + const malformed = "ab cd" + cdOrg = malformed + customHostname = "" + + called := false + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + called = true + return nil, nil + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if called { + t.Error("fetchResolverUIDFn was called; malformed token must be rejected before any network attempt") + } + if *exitCode != provisionExitCodeForCode[provisionCodeProvisionTokenMalformed] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeProvisionTokenMalformed]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeProvisionTokenMalformed) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeProvisionTokenMalformed) + } + if r.Stage != string(provisionStageInput) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageInput) + } + if strings.Contains(r.Message, malformed) { + t.Errorf("malformed token leaked into result message: %q", r.Message) + } +} + +// TestCdUIDFromProvTokenPrefixlessTokenProceeds proves a token missing the +// "org-v1-" prefix is not rejected - only warned about - since legacy codes +// may lack it. +func TestCdUIDFromProvTokenPrefixlessTokenProceeds(t *testing.T) { + _, _ = stubProvisionGlobals(t) + logBuf := captureMainLog(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + cdOrg = "legacytoken123" + customHostname = "" + + called := false + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + called = true + return &controld.ResolverConfig{UID: "resolved-uid"}, nil + } + + if got := cdUIDFromProvToken(); got != "resolved-uid" { + t.Errorf("cdUIDFromProvToken() = %q, want resolved-uid", got) + } + if !called { + t.Error("fetchResolverUIDFn was not called; a prefixless-but-sane token must still proceed") + } + if !strings.Contains(logBuf.String(), "org-v1-") { + t.Error("expected a warning mentioning the org-v1- prefix") + } +} + +// TestCdUIDFromProvTokenInvalidCustomHostname proves an invalid +// --custom-hostname value is classified before any network attempt, with a +// message that names the field and format, and never leaks the provision +// token. +func TestCdUIDFromProvTokenInvalidCustomHostname(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + const secretToken = "org-v1-secret-token-42" + cdOrg = secretToken + customHostname = "foo@bar" + + called := false + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + called = true + return nil, nil + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if called { + t.Error("fetchResolverUIDFn was called; invalid custom hostname must be rejected before any network attempt") + } + if *exitCode != provisionExitCodeForCode[provisionCodeCustomHostnameInvalid] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeCustomHostnameInvalid]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeCustomHostnameInvalid) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeCustomHostnameInvalid) + } + if !strings.Contains(r.Message, "allowed format") { + t.Errorf("message = %q, want it to contain the allowed format", r.Message) + } + if strings.Contains(r.Message, secretToken) { + t.Errorf("provision token leaked into result message: %q", r.Message) + } +} + +// TestCdUIDFromProvTokenFoldableHostnameLogsNotice proves a hostname that +// ctrld accepts, but ControlD's device-name formatting would fold or strip, +// gets a notice - not a failure - and provisioning still proceeds. +func TestCdUIDFromProvTokenFoldableHostnameLogsNotice(t *testing.T) { + _, _ = stubProvisionGlobals(t) + logBuf := captureMainLog(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + cdOrg = "org-v1-abcdef123456" + customHostname = "foo.bar" + + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + return &controld.ResolverConfig{UID: "resolved-uid"}, nil + } + + if got := cdUIDFromProvToken(); got != "resolved-uid" { + t.Errorf("cdUIDFromProvToken() = %q, want resolved-uid", got) + } + if !strings.Contains(logBuf.String(), "foo.bar") { + t.Error("expected a notice naming the foldable hostname") + } +} + +func TestValidateInterceptModeFlag(t *testing.T) { + oldMode := interceptMode + t.Cleanup(func() { interceptMode = oldMode }) + + t.Run("valid values proceed", func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + for _, mode := range []string{"", "off", "dns", "hard"} { + interceptMode = mode + if !validateInterceptModeFlag(mode) { + t.Errorf("validateInterceptModeFlag(%q) = false, want true", mode) + } + if *exitCode != -1 { + t.Errorf("mode %q: provisionExit called with %d, want no exit", mode, *exitCode) + } + } + }) + + t.Run("invalid value is classified", func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + interceptMode = "fds" + if validateInterceptModeFlag("fds") { + t.Error("validateInterceptModeFlag(\"fds\") = true, want false") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInterceptModeInvalid] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInterceptModeInvalid]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeInterceptModeInvalid) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInterceptModeInvalid) + } + for _, want := range []string{"off", "dns", "hard"} { + if !strings.Contains(r.Message, want) { + t.Errorf("message = %q, want it to contain %q", r.Message, want) + } + } + }) +} + +// TestCheckStrFlagEmptyClassifiesEmptyCdOrg proves an explicit empty --cd-org +// is classified as a malformed provisioning token rather than a bare fatal. +func TestCheckStrFlagEmptyClassifiesEmptyCdOrg(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + cmd := &cobra.Command{} + cmd.Flags().String(cdOrgFlagName, "", "") + if err := cmd.Flags().Set(cdOrgFlagName, ""); err != nil { + t.Fatal(err) + } + + if checkStrFlagEmpty(cmd, cdOrgFlagName) { + t.Error("checkStrFlagEmpty() = true, want false for an explicit empty --cd-org") + } + if *exitCode != provisionExitCodeForCode[provisionCodeProvisionTokenMalformed] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeProvisionTokenMalformed]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeProvisionTokenMalformed) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeProvisionTokenMalformed) + } + if r.Stage != string(provisionStageInput) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageInput) + } + if !strings.Contains(r.Message, cdOrgFlagName) { + t.Errorf("message = %q, want it to name --%s", r.Message, cdOrgFlagName) + } +} + +// TestCheckStrFlagEmptyProceedsWhenNotChangedOrNotEmpty proves the two cases +// that must not classify: the flag was never set, and it was set to a +// non-empty value. +func TestCheckStrFlagEmptyProceedsWhenNotChangedOrNotEmpty(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + + t.Run("flag never set", func(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().String(cdOrgFlagName, "", "") + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + t.Error("checkStrFlagEmpty() = false, want true when the flag was never set") + } + }) + + t.Run("flag set to a non-empty value", func(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().String(cdOrgFlagName, "", "") + if err := cmd.Flags().Set(cdOrgFlagName, "org-v1-abc"); err != nil { + t.Fatal(err) + } + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + t.Error("checkStrFlagEmpty() = false, want true for a non-empty value") + } + }) + + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } +} + +// TestValidateCdAndNextDNSFlagsClassifiesConflict proves --cd or --cd-org +// combined with --nextdns is classified as INVALID_FLAG_COMBINATION, naming +// every flag involved. +func TestValidateCdAndNextDNSFlagsClassifiesConflict(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldCdUID, oldCdOrg, oldNextdns := cdUID, cdOrg, nextdns + t.Cleanup(func() { cdUID, cdOrg, nextdns = oldCdUID, oldCdOrg, oldNextdns }) + + t.Run("non-conflicting combinations proceed", func(t *testing.T) { + cases := []struct{ cdUID, cdOrg, nextdns string }{ + {"", "", ""}, + {"uid123", "", ""}, + {"", "org-v1-abc", ""}, + {"", "", "nextdns-id"}, + } + for _, tc := range cases { + cdUID, cdOrg, nextdns = tc.cdUID, tc.cdOrg, tc.nextdns + if !validateCdAndNextDNSFlags() { + t.Errorf("validateCdAndNextDNSFlags() = false for cdUID=%q cdOrg=%q nextdns=%q, want true", tc.cdUID, tc.cdOrg, tc.nextdns) + } + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("cd-org with nextdns is classified", func(t *testing.T) { + cdUID, cdOrg, nextdns = "", "org-v1-abc", "nextdns-id" + if validateCdAndNextDNSFlags() { + t.Error("validateCdAndNextDNSFlags() = true, want false") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInvalidFlagCombination] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInvalidFlagCombination]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeInvalidFlagCombination) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInvalidFlagCombination) + } + if r.Stage != string(provisionStageInput) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageInput) + } + for _, want := range []string{cdUidFlagName, cdOrgFlagName, nextdnsFlagName} { + if !strings.Contains(r.Message, want) { + t.Errorf("message = %q, want it to name --%s", r.Message, want) + } + } + }) + + t.Run("cd with nextdns is classified", func(t *testing.T) { + cdUID, cdOrg, nextdns = "uid123", "", "nextdns-id" + if validateCdAndNextDNSFlags() { + t.Error("validateCdAndNextDNSFlags() = true, want false") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInvalidFlagCombination] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInvalidFlagCombination]) + } + }) +} + +// TestValidateCdUpstreamProtocol proves an invalid --proto value is +// classified only once --cd is in play, and that notify runs when given. +func TestValidateCdUpstreamProtocol(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldCdUID, oldProto := cdUID, cdUpstreamProto + t.Cleanup(func() { cdUID, cdUpstreamProto = oldCdUID, oldProto }) + + t.Run("no --cd proceeds regardless of protocol", func(t *testing.T) { + cdUID = "" + cdUpstreamProto = "garbage" + if !validateCdUpstreamProtocol(nil) { + t.Error("validateCdUpstreamProtocol(nil) = false, want true when --cd is not set") + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("valid protocols proceed", func(t *testing.T) { + cdUID = "uid123" + for _, proto := range []string{ctrld.ResolverTypeDOH, ctrld.ResolverTypeDOH3} { + cdUpstreamProto = proto + if !validateCdUpstreamProtocol(nil) { + t.Errorf("validateCdUpstreamProtocol(nil) = false for proto %q, want true", proto) + } + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("invalid protocol is classified and notifies", func(t *testing.T) { + cdUID = "uid123" + cdUpstreamProto = "quic" + notified := false + if validateCdUpstreamProtocol(func() { notified = true }) { + t.Error("validateCdUpstreamProtocol() = true, want false") + } + if !notified { + t.Error("notify not called") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInvalidFlagCombination] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInvalidFlagCombination]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeInvalidFlagCombination) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInvalidFlagCombination) + } + if !strings.Contains(r.Message, "quic") || !strings.Contains(r.Message, "doh") { + t.Errorf("message = %q, want it to name the given and allowed values", r.Message) + } + }) +} diff --git a/cmd/cli/provision_result.go b/cmd/cli/provision_result.go new file mode 100644 index 00000000..19940e18 --- /dev/null +++ b/cmd/cli/provision_result.go @@ -0,0 +1,323 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + "unicode/utf8" + + "github.com/Control-D-Inc/ctrld" +) + +// A terminal provisioning failure reports the same stable code on three +// surfaces: a persisted result file, one fixed-format output line, and a +// stage-scoped process exit code. docs/provisioning-failure-codes.md maps +// each code to its scenario and must stay in sync with the constants below. +// Codes are append-only once released; renaming or reusing one breaks the +// support contract. + +type provisionStage string + +const ( + provisionStageInput provisionStage = "input" + provisionStageBootstrap provisionStage = "bootstrap" + provisionStageListener provisionStage = "listener" + provisionStageService provisionStage = "service" +) + +type provisionFailureCode string + +const ( + provisionCodeProvisionTokenMalformed provisionFailureCode = "PROVISION_TOKEN_MALFORMED" + provisionCodeCustomHostnameInvalid provisionFailureCode = "CUSTOM_HOSTNAME_INVALID" + provisionCodeInterceptModeInvalid provisionFailureCode = "INTERCEPT_MODE_INVALID" + provisionCodeInvalidFlagCombination provisionFailureCode = "INVALID_FLAG_COMBINATION" + provisionCodeAPIUnreachable provisionFailureCode = "API_UNREACHABLE" + provisionCodeAPIRejected provisionFailureCode = "API_REJECTED" + provisionCodeAPIDeviceInvalid provisionFailureCode = "API_DEVICE_INVALID" + provisionCodeAPIMaintenance provisionFailureCode = "API_MAINTENANCE" + provisionCodeTokenInvalid provisionFailureCode = "TOKEN_INVALID" + provisionCodeTokenExpired provisionFailureCode = "TOKEN_EXPIRED" + provisionCodeTokenLimitReached provisionFailureCode = "TOKEN_LIMIT_REACHED" + provisionCodeTokenDisabled provisionFailureCode = "TOKEN_DISABLED" + provisionCodeListenerBindFailed provisionFailureCode = "LISTENER_BIND_FAILED" + provisionCodeListenerAddrUnavail provisionFailureCode = "LISTENER_CONFIGURED_ADDR_UNAVAILABLE" + provisionCodeServiceInstall provisionFailureCode = "SERVICE_INSTALL_FAILED" + provisionCodeServiceStartFailed provisionFailureCode = "SERVICE_START_FAILED" + provisionCodeServiceSelfCheck provisionFailureCode = "SERVICE_SELFCHECK_FAILED" + provisionCodeUnclassified provisionFailureCode = "UNCLASSIFIED" +) + +var allProvisionFailureCodes = []provisionFailureCode{ + provisionCodeProvisionTokenMalformed, + provisionCodeCustomHostnameInvalid, + provisionCodeInterceptModeInvalid, + provisionCodeInvalidFlagCombination, + provisionCodeAPIUnreachable, + provisionCodeAPIRejected, + provisionCodeAPIDeviceInvalid, + provisionCodeAPIMaintenance, + provisionCodeTokenInvalid, + provisionCodeTokenExpired, + provisionCodeTokenLimitReached, + provisionCodeTokenDisabled, + provisionCodeListenerBindFailed, + provisionCodeListenerAddrUnavail, + provisionCodeServiceInstall, + provisionCodeServiceStartFailed, + provisionCodeServiceSelfCheck, + provisionCodeUnclassified, +} + +var provisionStageForCode = map[provisionFailureCode]provisionStage{ + provisionCodeProvisionTokenMalformed: provisionStageInput, + provisionCodeCustomHostnameInvalid: provisionStageInput, + provisionCodeInterceptModeInvalid: provisionStageInput, + provisionCodeInvalidFlagCombination: provisionStageInput, + provisionCodeAPIUnreachable: provisionStageBootstrap, + provisionCodeAPIRejected: provisionStageBootstrap, + provisionCodeAPIDeviceInvalid: provisionStageBootstrap, + provisionCodeAPIMaintenance: provisionStageBootstrap, + provisionCodeTokenInvalid: provisionStageBootstrap, + provisionCodeTokenExpired: provisionStageBootstrap, + provisionCodeTokenLimitReached: provisionStageBootstrap, + provisionCodeTokenDisabled: provisionStageBootstrap, + provisionCodeListenerBindFailed: provisionStageListener, + provisionCodeListenerAddrUnavail: provisionStageListener, + provisionCodeServiceInstall: provisionStageService, + provisionCodeServiceStartFailed: provisionStageService, + provisionCodeServiceSelfCheck: provisionStageService, + provisionCodeUnclassified: provisionStageService, +} + +// Exit codes are grouped by stage (input 20-29, bootstrap 30-39, listener +// 40-49, service 50-59) so the exit code alone names the failed stage. 0-3 +// belong to "ctrld status" and 126 to the deactivation pin check; never +// reuse those. +var provisionExitCodeForCode = map[provisionFailureCode]int{ + provisionCodeProvisionTokenMalformed: 21, + provisionCodeCustomHostnameInvalid: 22, + provisionCodeInterceptModeInvalid: 23, + provisionCodeInvalidFlagCombination: 24, + provisionCodeAPIUnreachable: 30, + provisionCodeAPIRejected: 31, + provisionCodeAPIDeviceInvalid: 32, + provisionCodeAPIMaintenance: 37, + provisionCodeTokenInvalid: 33, + provisionCodeTokenExpired: 34, + provisionCodeTokenLimitReached: 35, + provisionCodeTokenDisabled: 36, + provisionCodeListenerBindFailed: 41, + provisionCodeListenerAddrUnavail: 42, + provisionCodeServiceInstall: 51, + provisionCodeServiceStartFailed: 52, + provisionCodeServiceSelfCheck: 53, + provisionCodeUnclassified: 59, +} + +const ( + provisionResultFileName = "provision_result.json" + // Detail identifies a failure, it is not a log. Caps keep the artifact + // small and predictable. + maxProvisionBindAttempts = 12 + maxProvisionStringLen = 256 +) + +type provisionBindAttempt struct { + Addr string `json:"addr"` + Proto string `json:"proto"` + OSError string `json:"os_error"` +} + +type provisionDetail struct { + Attempts []provisionBindAttempt `json:"attempts,omitempty"` +} + +type provisionResult struct { + Version int `json:"version"` + Timestamp string `json:"timestamp"` + Stage string `json:"stage"` + Code string `json:"code"` + ExitCode int `json:"exit_code"` + Message string `json:"message"` + Detail *provisionDetail `json:"detail,omitempty"` +} + +// provisionResultPath is a var so tests can point it at a temp dir. +var provisionResultPath = func() string { + return absHomeDir(provisionResultFileName) +} + +// provisionExit is a var so tests can observe the exit code instead of dying. +var provisionExit = os.Exit + +// newProvisionResult builds a result with every field bounded and the given +// secrets stripped. The artifact reaches installer logs and support tickets, +// so callers pass every secret in scope (provision token, cd UID). +func newProvisionResult(code provisionFailureCode, message string, attempts []provisionBindAttempt, secrets ...string) *provisionResult { + sanitize := func(s string) string { + s = redactSecrets(s, secrets...) + if len(s) > maxProvisionStringLen { + // Cut on a rune boundary so a localized OS error does not end in + // a broken multi-byte sequence. + cut := maxProvisionStringLen + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + s = s[:cut] + } + return s + } + r := &provisionResult{ + Version: 1, + Timestamp: time.Now().UTC().Format(time.RFC3339), + Stage: string(provisionStageForCode[code]), + Code: string(code), + ExitCode: provisionExitCodeForCode[code], + Message: sanitize(message), + } + if len(attempts) > 0 { + if len(attempts) > maxProvisionBindAttempts { + attempts = attempts[:maxProvisionBindAttempts] + } + detail := &provisionDetail{Attempts: make([]provisionBindAttempt, 0, len(attempts))} + for _, a := range attempts { + detail.Attempts = append(detail.Attempts, provisionBindAttempt{ + Addr: sanitize(a.Addr), + Proto: sanitize(a.Proto), + OSError: sanitize(a.OSError), + }) + } + r.Detail = detail + } + return r +} + +// redactSecretMinLen is the length below which a value is too short to +// redact. Two or three letters also match the keys of a JSON line and the +// words of a message, so a redaction of them corrupts the output. Corrupt +// output costs support more than a short value that the API rejects at the +// next step. It is the shortest token that the API parses. +const redactSecretMinLen = provisionTokenMinLen + +// redactSecrets removes every secret from s that is long enough to redact. +func redactSecrets(s string, secrets ...string) string { + for _, secret := range secrets { + if len(secret) < redactSecretMinLen { + continue + } + s = strings.ReplaceAll(s, secret, "[redacted]") + } + return s +} + +// provisionResultTrusted rejects a result whose code, stage, or exit code is +// not part of the known contract, so a corrupt or planted file cannot drive +// what "ctrld start" logs and exits with. +func provisionResultTrusted(r *provisionResult) bool { + code := provisionFailureCode(r.Code) + stage, ok := provisionStageForCode[code] + if !ok { + return false + } + return r.Stage == string(stage) && r.ExitCode == provisionExitCodeForCode[code] +} + +func (r *provisionResult) failureLine() string { + return fmt.Sprintf("provisioning failed: stage=%s code=%s (exit %d)", r.Stage, r.Code, r.ExitCode) +} + +// writeProvisionResult persists the result atomically (temp file + rename in +// the same directory) so a reader never sees a partial file. +func writeProvisionResult(r *provisionResult) error { + path := provisionResultPath() + buf, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), provisionResultFileName+".tmp*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.Write(buf); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Chmod(tmpName, 0o600); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + _ = os.Remove(tmpName) + return err + } + return nil +} + +func readProvisionResult() (*provisionResult, error) { + return readProvisionResultAt(provisionResultPath()) +} + +func readProvisionResultAt(path string) (*provisionResult, error) { + buf, err := os.ReadFile(path) + if err != nil { + return nil, err + } + r := &provisionResult{} + if err := json.Unmarshal(buf, r); err != nil { + return nil, err + } + return r, nil +} + +// clearProvisionResult removes a stale result once provisioning succeeds, so +// support never diagnoses a healthy install from an old failure. +func clearProvisionResult() { + if err := os.Remove(provisionResultPath()); err != nil && !os.IsNotExist(err) { + mainLog.Load().Debug().Err(err).Msg("could not remove provision result file") + } +} + +// failProvision persists the result, prints the identifier line, unblocks a +// waiting "ctrld start" via notify, then exits with the stage code. The write +// comes first so the file survives even if logging or notify misbehaves. +func failProvision(r *provisionResult, notify func()) { + if err := writeProvisionResult(r); err != nil { + mainLog.Load().Warn().Err(err).Msg("could not persist provision result") + } + mainLog.Load().Error().Msg(r.failureLine()) + if notify != nil { + notify() + } + provisionExit(r.ExitCode) +} + +// failProvisionUnclassified persists an UNCLASSIFIED result (service stage, +// exit 59) and exits. It is the fallback for a terminal path that predates a +// stable code - a config unmarshal, a file-system or environment failure - +// so support still gets a result file and a stage-scoped exit code instead +// of a bare crash with nothing to read. No terminal path on the +// provisioning boundary may bypass this or an existing classified code. +func failProvisionUnclassified(message string, notify func()) { + failProvision(newProvisionResult(provisionCodeUnclassified, message, nil, provisionSecrets()...), notify) +} + +// failRunUnclassified logs msg on ev, then fails provisioning as UNCLASSIFIED +// and unblocks a waiting "ctrld start" via notify (nil if none). Callers +// return immediately after this call: provisionExit is stubbed out under +// test, so nothing stops execution from falling through otherwise. ev +// carries whatever the caller already chained onto it (for example .Err()), +// so each call site keeps its own log fields. +func failRunUnclassified(ev *ctrld.LogEvent, msg string, notify func()) { + ev.Msg(msg) + failProvisionUnclassified(msg, notify) +} diff --git a/cmd/cli/provision_result_test.go b/cmd/cli/provision_result_test.go new file mode 100644 index 00000000..b3afdd1b --- /dev/null +++ b/cmd/cli/provision_result_test.go @@ -0,0 +1,313 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + "unicode/utf8" +) + +func overrideProvisionResultPath(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), provisionResultFileName) + old := provisionResultPath + provisionResultPath = func() string { return path } + t.Cleanup(func() { provisionResultPath = old }) + return path +} + +// The result file must obey the same homedir override as the log and the +// config file, so a daemon started with --homedir writes it next to them. +func TestProvisionResultPathHonorsHomedir(t *testing.T) { + old := homedir + homedir = t.TempDir() + t.Cleanup(func() { homedir = old }) + want := filepath.Join(homedir, provisionResultFileName) + if got := provisionResultPath(); got != want { + t.Errorf("provisionResultPath() = %q, want %q", got, want) + } +} + +func TestProvisionCodesMapToOneStageAndInRangeExit(t *testing.T) { + stageRanges := map[provisionStage][2]int{ + provisionStageInput: {20, 29}, + provisionStageBootstrap: {30, 39}, + provisionStageListener: {40, 49}, + provisionStageService: {50, 59}, + } + reservedExits := map[int]string{ + statusExitRunning: "ctrld status running", + statusExitStopped: "ctrld status stopped", + statusExitUnknown: "ctrld status unknown", + statusExitNotReady: "ctrld status not ready", + deactivationPinInvalidExitCode: "deactivation pin invalid", + } + seenExits := make(map[int]provisionFailureCode) + for _, code := range allProvisionFailureCodes { + stage, ok := provisionStageForCode[code] + if !ok { + t.Fatalf("code %s has no stage", code) + } + exit, ok := provisionExitCodeForCode[code] + if !ok { + t.Fatalf("code %s has no exit code", code) + } + r := stageRanges[stage] + if exit < r[0] || exit > r[1] { + t.Errorf("code %s exit %d outside stage %s range %v", code, exit, stage, r) + } + if owner, ok := reservedExits[exit]; ok { + t.Errorf("code %s exit %d collides with %s", code, exit, owner) + } + if prev, dup := seenExits[exit]; dup { + t.Errorf("codes %s and %s share exit %d", prev, code, exit) + } + seenExits[exit] = code + } + if len(allProvisionFailureCodes) != 18 { + t.Errorf("expected 18 codes, got %d", len(allProvisionFailureCodes)) + } +} + +func TestNewProvisionResultRedactsSecrets(t *testing.T) { + token := "org-secret-token-12345" + cdUIDValue := "abcdef123456" + attempts := []provisionBindAttempt{ + {Addr: "127.0.0.1:53", Proto: "udp", OSError: "bind failed for " + token}, + } + r := newProvisionResult( + provisionCodeListenerBindFailed, + "could not bind, token="+token+" uid="+cdUIDValue, + attempts, + token, cdUIDValue, + ) + raw, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{token, cdUIDValue} { + if strings.Contains(string(raw), secret) { + t.Errorf("serialized result contains secret %q: %s", secret, raw) + } + } +} + +func TestNewProvisionResultBoundsDetail(t *testing.T) { + long := strings.Repeat("x", 1000) + var attempts []provisionBindAttempt + for i := 0; i < 50; i++ { + attempts = append(attempts, provisionBindAttempt{Addr: long, Proto: "udp", OSError: long}) + } + r := newProvisionResult(provisionCodeListenerBindFailed, long, attempts) + if got := len(r.Detail.Attempts); got > maxProvisionBindAttempts { + t.Errorf("attempts not capped: %d > %d", got, maxProvisionBindAttempts) + } + if len(r.Message) > maxProvisionStringLen { + t.Errorf("message not capped: %d", len(r.Message)) + } + for _, a := range r.Detail.Attempts { + if len(a.Addr) > maxProvisionStringLen || len(a.OSError) > maxProvisionStringLen { + t.Error("attempt fields not capped") + } + } +} + +func TestProvisionResultFields(t *testing.T) { + r := newProvisionResult(provisionCodeAPIRejected, "the API rejected this configuration", nil) + if r.Version != 1 { + t.Errorf("version = %d, want 1", r.Version) + } + if r.Stage != string(provisionStageBootstrap) { + t.Errorf("stage = %q, want bootstrap", r.Stage) + } + if r.ExitCode != provisionExitCodeForCode[provisionCodeAPIRejected] { + t.Errorf("exit = %d", r.ExitCode) + } + if _, err := time.Parse(time.RFC3339, r.Timestamp); err != nil { + t.Errorf("timestamp %q not RFC3339: %v", r.Timestamp, err) + } + if r.Detail != nil { + t.Error("nil attempts should give nil detail") + } +} + +func TestProvisionResultTrusted(t *testing.T) { + good := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + if !provisionResultTrusted(good) { + t.Error("constructor-built result must be trusted") + } + bogusCode := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + bogusCode.Code = "TOTALLY_MADE_UP" + if provisionResultTrusted(bogusCode) { + t.Error("unknown code must not be trusted") + } + wrongExit := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + wrongExit.ExitCode = 126 + if provisionResultTrusted(wrongExit) { + t.Error("exit code not matching the contract must not be trusted") + } + wrongStage := newProvisionResult(provisionCodeListenerBindFailed, "x", nil) + wrongStage.Stage = string(provisionStageService) + if provisionResultTrusted(wrongStage) { + t.Error("stage not matching the code must not be trusted") + } +} + +func TestNewProvisionResultTruncatesOnRuneBoundary(t *testing.T) { + msg := strings.Repeat("é", maxProvisionStringLen) // 2 bytes per rune + r := newProvisionResult(provisionCodeListenerBindFailed, msg, nil) + if len(r.Message) > maxProvisionStringLen { + t.Errorf("message not capped: %d bytes", len(r.Message)) + } + if !utf8.ValidString(r.Message) { + t.Error("truncation split a multi-byte rune") + } +} + +func TestFailureCodeDocTableMatchesConstants(t *testing.T) { + buf, err := os.ReadFile(filepath.Join("..", "..", "docs", "provisioning-failure-codes.md")) + if os.IsNotExist(err) { + // The Windows CI runner executes prebuilt test binaries outside the + // repo; the sync guarantee is still enforced on runners with a checkout. + t.Skip("failure-code doc not available in this test environment") + } + if err != nil { + t.Fatalf("could not read the failure-code doc: %v", err) + } + doc := string(buf) + // Package-stage rows document identifiers scripts/pkg/postinstall emits + // on its own; they are not in this binary's failure-code registry, so + // they sit outside the one-row-per-code count below. + rows := 0 + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(line, "| `") && !strings.Contains(line, "| package |") { + rows++ + } + } + if rows != len(allProvisionFailureCodes) { + t.Errorf("doc table has %d non-package code rows, want %d", rows, len(allProvisionFailureCodes)) + } + for _, code := range allProvisionFailureCodes { + row := "| `" + string(code) + "` | " + string(provisionStageForCode[code]) + " | " + strconv.Itoa(provisionExitCodeForCode[code]) + " |" + if !strings.Contains(doc, row) { + t.Errorf("doc table missing row for %s (want prefix %q)", code, row) + } + } +} + +func TestProvisionFailureLineFormat(t *testing.T) { + r := newProvisionResult(provisionCodeListenerBindFailed, "could not find available listen ip and port", nil) + want := "provisioning failed: stage=listener code=LISTENER_BIND_FAILED (exit 41)" + if got := r.failureLine(); got != want { + t.Errorf("failureLine() = %q, want %q", got, want) + } +} + +func TestProvisionResultRoundTrip(t *testing.T) { + overrideProvisionResultPath(t) + in := newProvisionResult(provisionCodeServiceStartFailed, "service failed to start", nil) + if err := writeProvisionResult(in); err != nil { + t.Fatal(err) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != in.Code || out.Stage != in.Stage || out.ExitCode != in.ExitCode || out.Message != in.Message { + t.Errorf("round trip mismatch: in=%+v out=%+v", in, out) + } +} + +func TestWriteProvisionResultOverwritesAtomically(t *testing.T) { + path := overrideProvisionResultPath(t) + first := newProvisionResult(provisionCodeAPIUnreachable, "first", nil) + if err := writeProvisionResult(first); err != nil { + t.Fatal(err) + } + second := newProvisionResult(provisionCodeListenerBindFailed, "second", nil) + if err := writeProvisionResult(second); err != nil { + t.Fatal(err) + } + out, err := readProvisionResult() + if err != nil { + t.Fatal(err) + } + if out.Code != string(provisionCodeListenerBindFailed) || out.Message != "second" { + t.Errorf("overwrite failed: %+v", out) + } + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Errorf("temp files left behind: %v", entries) + } +} + +func TestClearProvisionResult(t *testing.T) { + path := overrideProvisionResultPath(t) + clearProvisionResult() // missing file must not panic or error loudly + if err := writeProvisionResult(newProvisionResult(provisionCodeAPIUnreachable, "x", nil)); err != nil { + t.Fatal(err) + } + clearProvisionResult() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("result file still present after clear: %v", err) + } +} + +func TestReadProvisionResultMissing(t *testing.T) { + overrideProvisionResultPath(t) + if _, err := readProvisionResult(); err == nil { + t.Error("expected error reading missing result file") + } +} + +func TestFailProvisionWritesLogsNotifiesAndExits(t *testing.T) { + overrideProvisionResultPath(t) + exitCode := -1 + oldExit := provisionExit + provisionExit = func(code int) { exitCode = code } + t.Cleanup(func() { provisionExit = oldExit }) + + notified := false + r := newProvisionResult(provisionCodeListenerBindFailed, "no listen addr", nil) + failProvision(r, func() { notified = true }) + + if !notified { + t.Error("notify func not called") + } + if exitCode != provisionExitCodeForCode[provisionCodeListenerBindFailed] { + t.Errorf("exit code = %d", exitCode) + } + out, err := readProvisionResult() + if err != nil { + t.Fatalf("result not persisted: %v", err) + } + if out.Code != string(provisionCodeListenerBindFailed) { + t.Errorf("persisted code = %q", out.Code) + } +} + +// Test_redactSecretsKeepsAShortClientID puts a two-letter client ID in the +// resolver UID. The provisioning messages hold the same letters in ordinary +// words, and a redaction of the client ID would mangle them. +func Test_redactSecretsKeepsAShortClientID(t *testing.T) { + origCdUID, origCdOrg := cdUID, cdOrg + t.Cleanup(func() { cdUID, cdOrg = origCdUID, origCdOrg }) + cdUID, cdOrg = "uid12345678/os", "" + + const in = "failed to reach the host uid12345678/os" + got := redactSecrets(in, provisionSecrets()...) + + if !strings.Contains(got, "failed to reach the host ") { + t.Fatalf("the redaction mangled the words of the message: %s", got) + } + if strings.Contains(got, "uid12345678") { + t.Fatalf("the redaction kept the resolver uid: %s", got) + } +} diff --git a/cmd/cli/provision_terminal_paths_test.go b/cmd/cli/provision_terminal_paths_test.go new file mode 100644 index 00000000..747ca5c2 --- /dev/null +++ b/cmd/cli/provision_terminal_paths_test.go @@ -0,0 +1,277 @@ +package cli + +import ( + "errors" + "strings" + "testing" +) + +// TestFailProvisionUnclassified pins the fallback every terminal path on the +// provisioning boundary uses when it predates a stable code: it persists +// UNCLASSIFIED, calls notify, and exits at the code's exit number. +func TestFailProvisionUnclassified(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + notified := false + + failProvisionUnclassified("something failed with no dedicated code", func() { notified = true }) + + if !notified { + t.Error("notify not called") + } + if *exitCode != provisionExitCodeForCode[provisionCodeUnclassified] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeUnclassified]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeUnclassified) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeUnclassified) + } + if r.Stage != string(provisionStageService) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageService) + } + if !strings.Contains(r.Message, "something failed") { + t.Errorf("message = %q, want it to contain the given message", r.Message) + } +} + +// TestNotifyExitToLogServer pins the method every run()-side failure uses to +// unblock a waiting "ctrld start": it closes the log connection exactly when +// one is set, and is a no-op otherwise. +func TestNotifyExitToLogServer(t *testing.T) { + t.Run("nil connection is a no-op", func(t *testing.T) { + p := &prog{} + p.notifyExitToLogServer() // must not panic + }) + + t.Run("closes a set connection", func(t *testing.T) { + p := &prog{} + fc := &fakeCloser{} + p.logConn = fc + p.notifyExitToLogServer() + if !fc.closed { + t.Error("expected logConn to be closed") + } + }) +} + +type fakeCloser struct{ closed bool } + +func (f *fakeCloser) Write(p []byte) (int, error) { return len(p), nil } +func (f *fakeCloser) Close() error { f.closed = true; return nil } + +// TestValidateFirewallModeFlag covers the shared validator behind both +// --firewall-mode call sites (the early "ctrld start" check and run()'s own +// check on the daemon side). Neither has a dedicated code, so both fall back +// to UNCLASSIFIED. +func TestValidateFirewallModeFlag(t *testing.T) { + t.Run("flag not changed proceeds regardless of value", func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + if !validateFirewallModeFlag(false, "garbage", nil) { + t.Error("validateFirewallModeFlag() = false, want true when the flag was not set") + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("valid values proceed", func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + for _, mode := range []string{"off", "on"} { + if !validateFirewallModeFlag(true, mode, nil) { + t.Errorf("validateFirewallModeFlag(true, %q) = false, want true", mode) + } + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("invalid value is classified UNCLASSIFIED and notifies", func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + notified := false + if validateFirewallModeFlag(true, "bogus", func() { notified = true }) { + t.Error("validateFirewallModeFlag(true, \"bogus\") = true, want false") + } + if !notified { + t.Error("notify not called") + } + if *exitCode != provisionExitCodeForCode[provisionCodeUnclassified] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeUnclassified]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeUnclassified) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeUnclassified) + } + if !strings.Contains(r.Message, "off") || !strings.Contains(r.Message, "on") { + t.Errorf("message = %q, want it to name the allowed values", r.Message) + } + }) +} + +// TestReportServeDNSFailure covers the one prog.go path where a listener +// bound successfully (LISTENER_* already ruled that out) but the proxy +// itself failed to start serving - a distinct failure with no dedicated +// code. +func TestReportServeDNSFailure(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + notified := false + p.logConn = &fakeCloserNotify{fn: func() { notified = true }} + + p.reportServeDNSFailure("0", errors.New("bind lost mid-flight")) + + if !notified { + t.Error("expected the log connection to be closed") + } + if *exitCode != provisionExitCodeForCode[provisionCodeUnclassified] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeUnclassified]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeUnclassified) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeUnclassified) + } + if !strings.Contains(r.Message, "listener.0") { + t.Errorf("message = %q, want it to name the listener", r.Message) + } +} + +type fakeCloserNotify struct{ fn func() } + +func (f *fakeCloserNotify) Write(p []byte) (int, error) { return len(p), nil } +func (f *fakeCloserNotify) Close() error { f.fn(); return nil } + +// TestRefuseFallbackFatalDefault covers the production refuseFallbackFatal +// closure (not the test-harness override other tests install): it classifies +// UNCLASSIFIED and notifies through the given prog's log connection. +func TestRefuseFallbackFatalDefault(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + notified := false + p.logConn = &fakeCloserNotify{fn: func() { notified = true }} + + refuseFallbackFatal(p, "cannot fall back: %s", "port 53 is unavailable") + + if !notified { + t.Error("expected the log connection to be closed") + } + if *exitCode != provisionExitCodeForCode[provisionCodeUnclassified] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeUnclassified]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeUnclassified) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeUnclassified) + } + if !strings.Contains(r.Message, "port 53 is unavailable") { + t.Errorf("message = %q, want it to contain the formatted detail", r.Message) + } +} + +// terminalPathCase documents one terminal path this lane audited under +// "ctrld start --cd-org": where it lives, what triggers it, and the code it +// now reports. reachable names a test that exercises the real call site +// end-to-end (through the seams this package already exposes); a path deep +// inside Start()/run() with no seam to trigger it in isolation is documented +// here with reachable == "" and relies on the shared helper tests above +// (TestFailProvisionUnclassified, TestNotifyExitToLogServer, +// TestValidateFirewallModeFlag, TestReportServeDNSFailure, +// TestRefuseFallbackFatalDefault) to prove the wiring it calls into. +type terminalPathCase struct { + file string + scenario string + code provisionFailureCode + reachable string // name of the test proving this exact site, or "" if only the shared helper is exercised +} + +// TestProvisioningTerminalPathAudit is the committed enumeration required for +// "ctrld start --cd-org": every Fatal/os.Exit found by re-deriving the +// requirement (grepping commands_service_start.go, cli.go's run() and its +// callers cdUIDFromProvToken/handleAPIPreflightFailure, commands_run.go, and +// prog.go's pre-daemonization paths), reconciled against a code. None of them +// may reach a bare Fatal/os.Exit any more; each row's code is asserted +// against the known contract maps so a typo'd code name fails this test. +func TestProvisioningTerminalPathAudit(t *testing.T) { + cases := []terminalPathCase{ + // commands_service_start.go (parent "ctrld start" process) + {"commands_service_start.go", "--intercept-mode is not off/dns/hard", provisionCodeInterceptModeInvalid, "TestValidateInterceptModeFlag"}, + {"commands_service_start.go", "--firewall-mode is not off/on", provisionCodeUnclassified, "TestValidateFirewallModeFlag"}, + {"commands_service_start.go / cli.go RunCobraCommand", "explicit empty --cd-org", provisionCodeProvisionTokenMalformed, "TestCheckStrFlagEmptyClassifiesEmptyCdOrg"}, + {"commands_service_start.go", "--cd/--cd-org used together with --nextdns", provisionCodeInvalidFlagCombination, "TestValidateCdAndNextDNSFlagsClassifiesConflict"}, + {"commands_service_start.go", "initializeServiceManagerWithServiceConfig fails", provisionCodeUnclassified, "TestServiceCommandStartClassifiesServiceManagerInitFailure"}, + {"commands_service_start.go / cli.go run()", "--proto is not doh/doh3 once --cd is set", provisionCodeInvalidFlagCombination, "TestValidateCdUpstreamProtocol"}, + {"commands_service_start.go", "removeServiceFlag fails while upgrading an existing service's intercept mode", provisionCodeUnclassified, ""}, + {"commands_service_start.go", "appendServiceFlag(\"--intercept-mode\") fails during the same upgrade", provisionCodeUnclassified, ""}, + {"commands_service_start.go", "appendServiceFlag(mode) fails during the same upgrade", provisionCodeUnclassified, ""}, + {"commands_service_start.go", "config unmarshal fails restarting an already-installed service", provisionCodeUnclassified, ""}, + {"commands_service_start.go", "doTasksE fails on a non-service-stage task restarting an existing service", provisionCodeUnclassified, ""}, + {"commands_service_start.go", "socketDir() fails restarting an existing service", provisionCodeUnclassified, ""}, + {"commands_service_start.go", "config unmarshal fails on a fresh install", provisionCodeUnclassified, ""}, + {"commands_service_start.go", "doTasksE fails on a non-service-stage task on a fresh install", provisionCodeUnclassified, ""}, + + // cli.go run() (the "ctrld run" child process --cd-org spawns, or a direct "ctrld run --cd-org" invocation) + {"cli.go run()", "called with a nil stop channel", provisionCodeUnclassified, ""}, + {"cli.go run()", "--daemon on windows", provisionCodeUnclassified, ""}, + {"cli.go run()", "newService fails building the background service handle", provisionCodeUnclassified, ""}, + {"cli.go run()", "readBase64Config fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "config unmarshal fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "network is not up", provisionCodeUnclassified, ""}, + {"cli.go run()", "--firewall-mode is not off/on (daemon-side check)", provisionCodeUnclassified, "TestValidateFirewallModeFlag"}, + {"cli.go run()", "writeConfigFile fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "validateConfig fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "os.Executable fails in daemon respawn", provisionCodeUnclassified, ""}, + {"cli.go run()", "os.Getwd fails in daemon respawn", provisionCodeUnclassified, ""}, + {"cli.go run()", "cmd.Start fails in daemon respawn", provisionCodeUnclassified, ""}, + + // prog.go (pre-daemonization: listener startup and DNS-intercept setup) + {"prog.go (p *prog) run()", "serveDNS fails after the listener already bound", provisionCodeUnclassified, "TestReportServeDNSFailure"}, + {"prog.go (p *prog) setDNS()", "DNS intercept fails and the interface-DNS fallback cannot reach a non-53 listener", provisionCodeUnclassified, "TestRefuseFallbackFatalDefault"}, + } + + for _, tc := range cases { + if _, ok := provisionStageForCode[tc.code]; !ok { + t.Errorf("%s: %s: code %s is not a known contract code", tc.file, tc.scenario, tc.code) + } + if tc.reachable != "" { + continue + } + t.Logf("documented (exercised only via shared helper wiring, not a standalone seam): %s: %s -> %s", tc.file, tc.scenario, tc.code) + } + + // Out of scope, left as bare Fatal/os.Exit deliberately, with reasons: + // + // - cli.go run(): stopCh nil check is still bare-Fatal-free (converted + // above) but is genuinely unreachable from any CLI invocation - the + // CLI always constructs a fresh channel. Converted anyway above for + // consistency, not because it is reachable. + // - prog.go (p *prog) setDNS() line ~995's own --intercept-mode check: + // explicitly left alone per T6 - it is the "runtime validation in + // prog.go" validInterceptMode's own doc comment calls out as distinct + // from "the early start command check" that got INTERCEPT_MODE_INVALID. + // - prog.go line ~912 (deactivation pin) and commands_service_start.go's + // own deactivation-pin os.Exit: explicitly out of scope per the lane + // contract ("pin-check 126 untouched"). + // - service.go: checkHasElevatedPrivilege's os.Exit(1) in the "start" + // PreRun. It runs before RunE, in a process that cannot write to + // /etc/controld, so no result file is possible there. The postinstall + // always runs as root, and the message names the fix. + // - cli.go: general CLI/config-parsing helpers (readConfigFile, + // decoderErrorFromTomlFile, processNoConfigFlags, processListenFlag, + // tryReadingConfigWithNotice's userHomeDir failure): these predate and + // apply uniformly across every ctrld invocation mode (config-file + // mode, no-config mode, nextdns mode), not specifically to --cd-org + // provisioning. Out of scope for this lane's --cd-org enumeration. + // - checkStrFlagEmpty for --cd (cdUidFlagName) keeps its bare fatal: + // only the --cd-org case above is a provisioning-token value. +} diff --git a/cmd/cli/query_error_sampler.go b/cmd/cli/query_error_sampler.go new file mode 100644 index 00000000..70262b4f --- /dev/null +++ b/cmd/cli/query_error_sampler.go @@ -0,0 +1,178 @@ +package cli + +import ( + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// Classes of per-query errors. Each class keeps its own budget per upstream. +const ( + sampleClassResolveFailed = "resolve_failed" + sampleClassAllEndpointsFailed = "all_endpoints_failed" + sampleClassSendResponseFailed = "send_response_failed" + // sampleClassInternalDomain names a failure of an Internal Domain resolver. + // It stays apart from the other classes, because an endpoint away from the + // organization network cannot reach such a resolver. + sampleClassInternalDomain = "internal_domain" +) + +const ( + // querySampleLimit is the number of lines that stay at error level in one window. + querySampleLimit = 5 + // querySampleWindow is the length of a window. It starts at the first error of the window. + querySampleWindow = 60 * time.Second +) + +// sampleKey identifies one budget. +type sampleKey struct { + class string + upstream string +} + +// sampleWindow counts the events of one budget between its start and its end. +type sampleWindow struct { + start time.Time + count int + suppressed int + timer *time.Timer +} + +// ended reports whether the window closed at or before now. +func (w *sampleWindow) ended(now time.Time) bool { + return !now.Before(w.start.Add(querySampleWindow)) +} + +// sampleSummary holds what one closed window reports. +type sampleSummary struct { + key sampleKey + count int + suppressed int +} + +// errorSampler bounds the noise of a repeated per-query error. It lowers the +// level of a line, never its presence, so the debug stream stays complete. +// The console and the log_path file show the lowered lines at debug level only. +// The zero value is ready to use. +type errorSampler struct { + mu sync.Mutex + windows map[sampleKey]*sampleWindow + now func() time.Time + after func(time.Duration, func()) *time.Timer + // onFailure counts the event for the query health tracker. The daemon sets + // it once at start, before a listener serves a query. + onFailure func(class string) +} + +// event returns the event a call site logs with. The first querySampleLimit +// events of a window come back at error level, the later ones at debug level. +// The caller adds its own fields and the message. +func (s *errorSampler) event(class, upstream string) *ctrld.LogEvent { + key := sampleKey{class: class, upstream: upstream} + + s.mu.Lock() + now := s.clock() + var summaries []sampleSummary + window := s.windows[key] + if window != nil && window.ended(now) { + summaries = appendWindowSummary(summaries, key, window) + s.dropWindow(key, window) + window = nil + } + if window == nil { + window = s.startWindow(key, now) + } + window.count++ + suppressed := window.count > querySampleLimit + if suppressed { + window.suppressed++ + } + s.mu.Unlock() + + // A lowered line keeps its place in the breakdown of the window. It is one + // event, not one failed query, so it does not change the grade. + if s.onFailure != nil { + s.onFailure(class) + } + s.logSummaries(summaries) + + if suppressed { + return mainLog.Load().Debug() + } + return mainLog.Load().Error() +} + +// closeExpired closes every window that ended at or before now. The window +// timer calls it, so an idle window reports without a new event. +func (s *errorSampler) closeExpired(now time.Time) { + s.mu.Lock() + var summaries []sampleSummary + for key, window := range s.windows { + if !window.ended(now) { + continue + } + summaries = appendWindowSummary(summaries, key, window) + s.dropWindow(key, window) + } + s.mu.Unlock() + + s.logSummaries(summaries) +} + +// startWindow opens a window and arms the timer that closes it. The caller holds the lock. +func (s *errorSampler) startWindow(key sampleKey, start time.Time) *sampleWindow { + if s.windows == nil { + s.windows = make(map[sampleKey]*sampleWindow) + } + window := &sampleWindow{start: start} + s.windows[key] = window + window.timer = s.afterFunc()(querySampleWindow, func() { s.closeExpired(s.clock()) }) + return window +} + +// dropWindow forgets a window and stops its timer. The caller holds the lock. +func (s *errorSampler) dropWindow(key sampleKey, window *sampleWindow) { + if window.timer != nil { + window.timer.Stop() + } + delete(s.windows, key) +} + +// logSummaries reports the closed windows. The caller released the lock first, +// because the logger writes back through code that can take it again. +func (s *errorSampler) logSummaries(summaries []sampleSummary) { + for _, summary := range summaries { + mainLog.Load().Error(). + Str("class", summary.key.class). + Str("upstream", journalUpstreamName(summary.key.upstream)). + Int("count", summary.count). + Int("suppressed", summary.suppressed). + Int("window_s", int(querySampleWindow/time.Second)). + Msg("Per-query errors sampled") + } +} + +// clock reads the time of the sampler. A test sets its own clock, so a window +// closes without a wait. +func (s *errorSampler) clock() time.Time { + if s.now != nil { + return s.now() + } + return time.Now() +} + +func (s *errorSampler) afterFunc() func(time.Duration, func()) *time.Timer { + if s.after != nil { + return s.after + } + return time.AfterFunc +} + +// appendWindowSummary keeps the windows that suppressed at least one line. +func appendWindowSummary(summaries []sampleSummary, key sampleKey, window *sampleWindow) []sampleSummary { + if window.suppressed == 0 { + return summaries + } + return append(summaries, sampleSummary{key: key, count: window.count, suppressed: window.suppressed}) +} diff --git a/cmd/cli/query_error_sampler_test.go b/cmd/cli/query_error_sampler_test.go new file mode 100644 index 00000000..4eca36e6 --- /dev/null +++ b/cmd/cli/query_error_sampler_test.go @@ -0,0 +1,274 @@ +package cli + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +// samplerTestMessage is the message the fake call site writes on every sampled event. +const samplerTestMessage = "Failed to resolve query" + +// samplerSummaryMessage is the message of the window summary. +const samplerSummaryMessage = "Per-query errors sampled" + +// captureDebugMainLog sends mainLog into a JSON buffer at debug level, so the +// test also sees the lines that the code under test drops to debug. It swaps +// a global, so no test of this package runs in parallel. +func captureDebugMainLog(t *testing.T) *syncBuffer { + t.Helper() + return captureJSONMainLog(t) +} + +// jsonLogEvents parses the captured lines as JSON events, in order. An empty +// message takes every line. +func jsonLogEvents(t *testing.T, logs *syncBuffer, message string) []map[string]any { + t.Helper() + var events []map[string]any + for _, line := range strings.Split(logs.String(), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + if message != "" && event["message"] != message { + continue + } + events = append(events, event) + } + return events +} + +// samplerHarness drives an errorSampler on a clock and a timer queue the test owns. +type samplerHarness struct { + sampler *errorSampler + logs *syncBuffer + now time.Time + pending []func() +} + +func newSamplerHarness(t *testing.T) *samplerHarness { + t.Helper() + h := &samplerHarness{ + logs: captureDebugMainLog(t), + now: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + h.sampler = &errorSampler{ + now: func() time.Time { return h.now }, + after: func(_ time.Duration, fn func()) *time.Timer { + h.pending = append(h.pending, fn) + return nil + }, + } + return h +} + +// emit plays a call site: it takes an event from the sampler and adds its own fields. +func (h *samplerHarness) emit(class, upstream string) { + h.sampler.event(class, upstream).Str("class", class).Str("upstream", upstream).Msg(samplerTestMessage) +} + +func (h *samplerHarness) advance(d time.Duration) { + h.now = h.now.Add(d) +} + +func (h *samplerHarness) events(t *testing.T) []map[string]any { + t.Helper() + return jsonLogEvents(t, h.logs, "") +} + +func countSampled(events []map[string]any, level, upstream string) int { + count := 0 + for _, event := range events { + if event["message"] != samplerTestMessage { + continue + } + if event["level"] != level || event["upstream"] != upstream { + continue + } + count++ + } + return count +} + +func samplerSummaries(events []map[string]any) []map[string]any { + var summaries []map[string]any + for _, event := range events { + if event["message"] == samplerSummaryMessage { + summaries = append(summaries, event) + } + } + return summaries +} + +func wantField(t *testing.T, event map[string]any, field string, want any) { + t.Helper() + if got := event[field]; got != want { + t.Fatalf("field %q: got %v, want %v", field, got, want) + } +} + +func Test_errorSamplerLimitsWindowAndSummarizes(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+2; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + + events := h.events(t) + if got := countSampled(events, "error", "upstream.0"); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSampled(events, "debug", "upstream.0"); got != 2 { + t.Fatalf("debug lines: got %d, want 2", got) + } + if got := len(samplerSummaries(events)); got != 0 { + t.Fatalf("summaries before the window closed: got %d, want 0", got) + } + + h.advance(querySampleWindow) + h.emit(sampleClassResolveFailed, "upstream.0") + + events = h.events(t) + summaries := samplerSummaries(events) + if len(summaries) != 1 { + t.Fatalf("summaries after the window closed: got %d, want 1", len(summaries)) + } + summary := summaries[0] + wantField(t, summary, "level", "error") + wantField(t, summary, "class", sampleClassResolveFailed) + wantField(t, summary, "upstream", "upstream.0") + wantField(t, summary, "count", float64(7)) + wantField(t, summary, "suppressed", float64(2)) + wantField(t, summary, "window_s", float64(60)) + + last := events[len(events)-1] + wantField(t, last, "message", samplerTestMessage) + wantField(t, last, "level", "error") +} + +func Test_errorSamplerKeepsBudgetPerUpstream(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + for i := 0; i < querySampleLimit; i++ { + h.emit(sampleClassResolveFailed, "upstream.1") + } + + events := h.events(t) + if got := countSampled(events, "error", "upstream.1"); got != querySampleLimit { + t.Fatalf("error lines for the second upstream: got %d, want %d", got, querySampleLimit) + } + if got := countSampled(events, "debug", "upstream.1"); got != 0 { + t.Fatalf("debug lines for the second upstream: got %d, want 0", got) + } + if got := countSampled(events, "debug", "upstream.0"); got != 1 { + t.Fatalf("debug lines for the first upstream: got %d, want 1", got) + } +} + +func Test_errorSamplerKeepsQuietWindowSilent(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < 3; i++ { + h.emit(sampleClassAllEndpointsFailed, "") + } + + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + if got := len(samplerSummaries(h.events(t))); got != 0 { + t.Fatalf("summaries for a window without suppressed lines: got %d, want 0", got) + } +} + +func Test_errorSamplerCloseExpiredSummarizesOnce(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+2; i++ { + h.emit(sampleClassSendResponseFailed, "") + } + sampled := len(h.events(t)) + + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + events := h.events(t) + if len(events) != sampled+1 { + t.Fatalf("lines after closeExpired: got %d, want %d", len(events), sampled+1) + } + summaries := samplerSummaries(events) + if len(summaries) != 1 { + t.Fatalf("summaries after closeExpired: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "class", sampleClassSendResponseFailed) + wantField(t, summaries[0], "suppressed", float64(2)) + + h.sampler.closeExpired(h.now.Add(querySampleWindow)) + + if got := len(h.events(t)); got != sampled+1 { + t.Fatalf("lines after the second closeExpired: got %d, want %d", got, sampled+1) + } +} + +func Test_errorSamplerTimerClosesIdleWindow(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + if len(h.pending) != 1 { + t.Fatalf("window timers: got %d, want 1", len(h.pending)) + } + + h.advance(querySampleWindow) + h.pending[0]() + + summaries := samplerSummaries(h.events(t)) + if len(summaries) != 1 { + t.Fatalf("summaries after the timer fired: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "suppressed", float64(1)) + wantField(t, summaries[0], "count", float64(6)) +} + +func Test_errorSamplerZeroValueWorks(t *testing.T) { + buf := captureDebugMainLog(t) + + var sampler errorSampler + // The zero value arms a real timer. Close its window after the assertions, + // so no late summary line reaches another test. + t.Cleanup(func() { sampler.closeExpired(time.Now().Add(querySampleWindow)) }) + sampler.event(sampleClassResolveFailed, "upstream.0").Msg(samplerTestMessage) + + if got := strings.Count(buf.String(), samplerTestMessage); got != 1 { + t.Fatalf("lines from the zero value sampler: got %d, want 1", got) + } + if !strings.Contains(buf.String(), `"level":"error"`) { + t.Fatalf("first line of a window is not at error level: %s", buf.String()) + } +} + +// Test_errorSamplerKeepsAnEmptyUpstreamEmpty drives a window of a class that +// names no upstream. The summary must not report the window as a custom +// upstream. +func Test_errorSamplerKeepsAnEmptyUpstreamEmpty(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassAllEndpointsFailed, "") + } + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + summaries := samplerSummaries(h.events(t)) + if len(summaries) != 1 { + t.Fatalf("summaries after the window closed: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "upstream", "") +} diff --git a/cmd/cli/query_health.go b/cmd/cli/query_health.go new file mode 100644 index 00000000..7fd84aa4 --- /dev/null +++ b/cmd/cli/query_health.go @@ -0,0 +1,284 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// Health classes of the query path. The names go into the journal. +const ( + queryHealthHealthy = "healthy" + queryHealthDegraded = "degraded" + queryHealthFailing = "failing" +) + +const ( + // queryHealthWindow is the length of one counting window and the time + // between two heartbeats. + queryHealthWindow = 15 * time.Minute + // queryHealthFailingRatio and queryHealthDegradedRatio grade the failed + // share of a window. + queryHealthFailingRatio = 0.5 + queryHealthDegradedRatio = 0.05 + // queryHealthFailingQueries is the smallest sample that can leave the + // healthy class and the smallest sample that can report a class change. + queryHealthFailingQueries = 10 + // queryHealthClassTicks is the number of ticks in a row that a new class + // must hold before it reports. + queryHealthClassTicks = 2 + // queryHealthMessage names the journal event of the tracker. + queryHealthMessage = "Query health" +) + +// queryHealthTick is the time between two evaluations. A test shortens it. +var queryHealthTick = time.Minute + +// queryHealth grades the query path over a window. Support reads the grade +// after an incident, so the journal needs one class per window instead of a +// line per query. +type queryHealth struct { + mu sync.Mutex + window time.Duration + now func() time.Time + windowStart time.Time + queries uint64 + cacheHits uint64 + // failedQueries counts the client queries that ended without an answer. + // The grade follows this count, and failures holds the sampler classes + // behind it, which raise several events for one failed query. + failedQueries uint64 + failures map[string]uint64 + lastClass string + // pendingClass is the class that the last tick graded, and pendingTicks + // counts how many ticks in a row graded it. A class reports only after it + // held, so one noisy minute opens no event. + pendingClass string + pendingTicks int +} + +// queryHealthEvent holds the class and the counters of one window. +type queryHealthEvent struct { + Class string + WindowSeconds int + Queries uint64 + CacheHits uint64 + FailedQueries uint64 + Failures map[string]uint64 + UpstreamsDown int + BypassActive bool + CacheHitRatio float64 +} + +func newQueryHealth() *queryHealth { + return &queryHealth{ + window: queryHealthWindow, + now: time.Now, + failures: make(map[string]uint64), + } +} + +// A prog outside a run holds no tracker, so every counter of the query path +// and every loop accepts a nil receiver. +func (q *queryHealth) countQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.queries++ +} + +func (q *queryHealth) countCacheHit() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.cacheHits++ +} + +// countFailedQuery counts one client query that ended without an answer. A +// query that failed on several upstreams counts once, because the grade +// measures the service the client got. +func (q *queryHealth) countFailedQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.failedQueries++ +} + +func (q *queryHealth) countFailure(class string) { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + if q.failures == nil { + q.failures = make(map[string]uint64) + } + q.failures[class]++ +} + +// evaluate grades the counters and reports whether the event goes out. The +// report covers the window that just closed, or the running one. The window +// starts at the first evaluate, so the caller owns the clock. +func (q *queryHealth) evaluate(now time.Time, upstreamsDown int, bypassActive bool) (queryHealthEvent, bool) { + if q == nil { + return queryHealthEvent{}, false + } + q.mu.Lock() + defer q.mu.Unlock() + + if q.windowStart.IsZero() { + q.windowStart = now + } + event := queryHealthEvent{ + Class: healthClass(q.queries, q.failedQueries), + WindowSeconds: int(q.window / time.Second), + Queries: q.queries, + CacheHits: q.cacheHits, + FailedQueries: q.failedQueries, + Failures: copyFailures(q.failures), + UpstreamsDown: upstreamsDown, + BypassActive: bypassActive, + CacheHitRatio: shareOf(q.cacheHits, q.queries), + } + // The end of a window always reports, so a quiet daemon stays visible and + // no window is lost after an early report. A class change waits for a + // sample and for a second tick. + windowEnded := !now.Before(q.windowStart.Add(q.window)) + emit := q.lastClass == "" || windowEnded || q.classChangeHolds(event.Class) + if emit { + q.lastClass = event.Class + q.pendingClass, q.pendingTicks = "", 0 + } + if windowEnded { + q.startWindow(now) + } + return event, emit +} + +// classChangeHolds reports whether a new class may open an event. It needs a +// window with enough queries, and the same class on two ticks in a row. The +// caller holds the lock. +func (q *queryHealth) classChangeHolds(class string) bool { + if class == q.lastClass || q.queries < queryHealthFailingQueries { + q.pendingClass, q.pendingTicks = "", 0 + return false + } + if class != q.pendingClass { + q.pendingClass, q.pendingTicks = class, 1 + return false + } + q.pendingTicks++ + return q.pendingTicks >= queryHealthClassTicks +} + +// startWindow drops the counters of the window that closed. Each report covers +// one window, so a quiet hour cannot hide inside the totals of a busy one. The +// caller holds the lock. +func (q *queryHealth) startWindow(now time.Time) { + q.windowStart = now + q.queries = 0 + q.cacheHits = 0 + q.failedQueries = 0 + q.failures = make(map[string]uint64) +} + +func (q *queryHealth) log(event queryHealthEvent) { + if q == nil { + return + } + journal(mainLog.Load().Info()). + Str("class", event.Class). + Int("window_s", event.WindowSeconds). + Uint64("queries", event.Queries). + Uint64("cache_hits", event.CacheHits). + Uint64("failed_queries", event.FailedQueries). + Dict("failures_by_class", failuresDict(event.Failures)). + Int("upstreams_down", event.UpstreamsDown). + Bool("bypass_active", event.BypassActive). + Float64("cache_hit_ratio", event.CacheHitRatio). + Msg(queryHealthMessage) +} + +// startLoop runs the grading loop and returns a channel that closes when the +// loop returns. A caller joins that channel before it reads the log. +func (q *queryHealth) startLoop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + q.loop(stop, inputs) + }() + return done +} + +// loop grades the counters on every tick until the caller stops it. +func (q *queryHealth) loop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) { + if q == nil { + return + } + ticker := time.NewTicker(queryHealthTick) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + upstreamsDown, bypassActive := inputs() + event, emit := q.evaluate(q.now(), upstreamsDown, bypassActive) + if !emit { + continue + } + q.log(event) + } + } +} + +// healthClass grades the failed share of a window. A small sample stays +// healthy, because a few failures at a start are not an outage. +func healthClass(queries, failures uint64) string { + if queries < queryHealthFailingQueries { + return queryHealthHealthy + } + failed := shareOf(failures, queries) + if failed >= queryHealthFailingRatio { + return queryHealthFailing + } + if failed >= queryHealthDegradedRatio { + return queryHealthDegraded + } + return queryHealthHealthy +} + +// shareOf reports the part of the total, and 0 for an empty total. +func shareOf(part, total uint64) float64 { + if total == 0 { + return 0 + } + return float64(part) / float64(total) +} + +// copyFailures gives the report its own map, because the counters keep running +// while the caller holds the report. +func copyFailures(failures map[string]uint64) map[string]uint64 { + copied := make(map[string]uint64, len(failures)) + maps.Copy(copied, failures) + return copied +} + +// failuresDict renders the classes in a stable order, so two lines of the same +// window compare by eye. +func failuresDict(failures map[string]uint64) *ctrld.LogEvent { + dict := ctrld.Dict() + for _, class := range slices.Sorted(maps.Keys(failures)) { + dict = dict.Uint64(class, failures[class]) + } + return dict +} diff --git a/cmd/cli/query_health_prog_test.go b/cmd/cli/query_health_prog_test.go new file mode 100644 index 00000000..fbbee736 --- /dev/null +++ b/cmd/cli/query_health_prog_test.go @@ -0,0 +1,468 @@ +package cli + +import ( + "context" + "errors" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// wantDNSConfigStrings compares one array field of a logged resolver entry. +func wantDNSConfigStrings(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %T, want an array", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q at %d: got %v, want %v", field, i, value, want[i]) + } + } +} + +// Test_queryHealthCountsFailedProxyQueries drives the real query path against a +// refusing upstream: every sampled error reaches the tracker, so the window of +// the daemon grades as failing. +func Test_queryHealthCountsFailedProxyQueries(t *testing.T) { + captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + // A new monitor holds back the failure counts for one second, and its + // timer would mark the upstream down after the test. + p.um.clearRecovered(upstreamOS) + p.um.after = func(time.Duration, func()) {} + p.querySampler.onFailure = p.health.countFailure + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + } + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + for i := 0; i < failedQueries; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + // serveDNS counts every query it serves, and the eight answered ones never + // reach the sampler. + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + + event, emit := p.health.evaluate(time.Now(), 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthFailing { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthFailing) + } + if event.Queries != servedQueries { + t.Fatalf("queries: got %d, want %d", event.Queries, servedQueries) + } + // The grade follows one failure per failed query. Each of those queries + // also raised two sampler events, and the breakdown still shows them. + if event.FailedQueries != failedQueries { + t.Fatalf("failed queries: got %d, want %d", event.FailedQueries, failedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassResolveFailed, got, failedQueries) + } + if got := event.Failures[sampleClassAllEndpointsFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassAllEndpointsFailed, got, failedQueries) + } +} + +// Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade drives the same +// number of failed queries against an Internal Domain resolver. An endpoint +// away from the organization network cannot reach that resolver, so the class +// stays healthy, the failures keep their own name, and the down entry of that +// resolver stays out of upstreams_down. +func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { + captureDebugMainLog(t) + const internalUpstream = upstreamPrefix + internalDomainUpstreamPrefix + "1" + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + internalDomainUpstreamPrefix + "1": { + Name: "dead internal resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + InternalDomain: internalDomainUpstreamOnly, + }, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(internalUpstream) + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + msg := new(dns.Msg) + msg.SetQuestion("internal.corp.example.", dns.TypeA) + for i := 0; i < failedQueries; i++ { + res := p.proxy(ctx, &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{internalUpstream}, srcAddr: "192.168.0.1:1234"}, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: the internal resolver answered", i+1) + } + } + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + p.um.mu.Lock() + p.um.markDown(internalUpstream, 3, "immediate") + p.um.mu.Unlock() + + event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(p.isInternalDomainUpstream), false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassInternalDomain]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassInternalDomain, got, failedQueries) + } + if event.UpstreamsDown != 0 { + t.Fatalf("upstreams_down: got %d, want 0", event.UpstreamsDown) + } + if got := p.um.countDown(); got != 1 { + t.Fatalf("countDown: got %d, want 1", got) + } +} + +// Test_queryHealthCountsProxyCacheHits proves that an answer from the cache +// counts a cache hit, so the ratio of the journal follows the query path. +func Test_queryHealthCountsProxyCacheHits(t *testing.T) { + captureDebugMainLog(t) + p := &prog{cfg: testhelper.SampleConfig(t), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + cache, err := dnscache.NewLRUCache(16) + require.NoError(t, err) + p.cache = cache + + msg := new(dns.Msg) + msg.SetQuestion("cache.test.", dns.TypeA) + answer := new(dns.Msg) + answer.SetRcode(msg, dns.RcodeSuccess) + p.cache.Add(dnscache.NewKey(msg, "upstream.1"), dnscache.NewValue(answer, time.Now().Add(time.Minute))) + + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{"upstream.1"}}, + }) + require.NotNil(t, res) + require.True(t, res.cached, "the answer did not come from the cache") + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.CacheHits != 1 { + t.Fatalf("cache hits: got %d, want 1", event.CacheHits) + } +} + +// Test_logDNSConfigChangesLogsEveryEntry proves that each changed resolver of +// the host reaches the journal with its fields. A resolver that vanished keeps +// an empty nameserver list. +func Test_logDNSConfigChangesLogsEveryEntry(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + + p.logDNSConfigChanges([]dnsResolverEntry{ + { + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + }, + {Order: 101600, IfIndex: 21, Interface: "utun4", Flags: "Scoped"}, + }) + + events := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(events) != 2 { + t.Fatalf("configuration lines: got %d, want 2", len(events)) + } + + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "if_index", float64(16)) + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "flags", "Request A records, Request AAAA records") + wantField(t, events[0], "order", float64(200000)) + wantField(t, events[0], "reachable", "Reachable,Directly Reachable Address") + wantDNSConfigStrings(t, events[0], "nameservers", "192.168.50.200", "192.168.50.1") + // The suffixes of an organization stay out of the journal. Only their + // number goes in. + wantField(t, events[0], "search_domain_count", float64(1)) + if _, ok := events[0]["search_domains"]; ok { + t.Fatal("the journal holds the search domains") + } + + wantField(t, events[1], "journal", true) + wantField(t, events[1], "interface", "utun4") + wantDNSConfigStrings(t, events[1], "nameservers") + wantField(t, events[1], "search_domain_count", float64(0)) +} + +// Test_startNetworkJournalLogsTheStartSnapshot proves that the daemon opens the +// journal with one snapshot and runs the trackers that follow the query path +// and the DNS configuration of the host. +func Test_startNetworkJournalLogsTheStartSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + var polls atomic.Int64 + // The poll loop reads the delay on every pass, so it parks on a long delay + // before the test puts the package value back. + parked := make(chan struct{}, 1) + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + movedResolver := wifiResolverEntry() + movedResolver.Nameservers = []string{"10.0.0.1"} + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + if polls.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{movedResolver}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if polls.Load() < 2 { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), stopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + // The monitor arms a real timer for each upstream it resets. Nothing in + // this test waits for one. + p.um.after = func(time.Duration, func()) {} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.recoveryBypass.Store(true) + p.um.mu.Lock() + p.um.markDown(upstreamOS, 3, "timer") + p.um.mu.Unlock() + + p.newNetworkJournal() + if p.health == nil || p.querySampler.onFailure == nil { + t.Fatal("the trackers do not exist before the loops start") + } + // A counter of the query path works as soon as the trackers exist. + p.health.countQuery() + waitForJournalLoops := p.startNetworkJournal() + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(events)) + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) + health := jsonLogEvents(t, logs, queryHealthMessage) + wantField(t, health[0], "upstreams_down", float64(1)) + wantField(t, health[0], "bypass_active", true) + + if runtime.GOOS != "darwin" { + if p.dnsConfig != nil { + t.Fatal("the DNS configuration poller runs outside macOS") + } + return + } + if p.dnsConfig == nil { + t.Fatal("macOS starts no DNS configuration poller") + } + waitForLogLine(t, logs, dnsConfigChangedMessage) + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the poll loop did not park") + } + resolvers := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(resolvers) != 1 { + t.Fatalf("configuration lines: got %d, want 1", len(resolvers)) + } + wantField(t, resolvers[0], "interface", "en0") + wantDNSConfigStrings(t, resolvers[0], "nameservers", "10.0.0.1") +} + +// Test_startNetworkJournalReadsTheMonitorAcrossAReload replaces the upstream +// monitor while the health loop runs. A reload builds a new monitor, and the +// loop read the field of the prog with no lock, which the race detector +// reports. +func Test_startNetworkJournalReadsTheMonitorAcrossAReload(t *testing.T) { + captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.health = newQueryHealth() + + waitForJournalLoops := p.startNetworkJournal() + for i := 0; i < 30; i++ { + p.replaceUpstreamMonitor() + time.Sleep(time.Millisecond) + } + close(p.stopCh) + waitForJournalLoops() +} + +// Test_startNetworkJournalRunsAfterAFailedMonitorStart proves that a network +// monitor that cannot start leaves the journal running. The error line goes +// out, the start snapshot opens the journal, and the health loop reports. +func Test_startNetworkJournalRunsAfterAFailedMonitorStart(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + origMonitor := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = origMonitor }) + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { + return nil, errors.New("no network monitor here") + } + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.upstreamMonitorNow().after = func(time.Duration, func()) {} + + p.newNetworkJournal() + p.health.countQuery() + p.health.mu.Lock() + queries := p.health.queries + p.health.mu.Unlock() + if queries != 1 { + t.Fatalf("queries after one count: got %d, want 1", queries) + } + + waitForJournalLoops := p.startNetworkMonitorAndJournal(context.Background()) + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + failures := jsonLogEvents(t, logs, "Failed to start network monitoring") + if len(failures) != 1 { + t.Fatalf("monitor failure lines: got %d, want 1", len(failures)) + } + wantField(t, failures[0], "level", "error") + + snapshots := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(snapshots) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(snapshots)) + } + wantField(t, snapshots[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) +} + +// Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers proves that a failure on +// one upstream does not grade the query. The client got an answer, so the +// failure stays in the breakdown only. The DNS64 companion lookup fails through +// the same call, so it cannot grade a query either. +func Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers(t *testing.T) { + captureDebugMainLog(t) + fixture := startDNSFixture(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + "0": {Name: "dead resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: deadUpstreamEndpoint, Timeout: 500}, + "1": {Name: "live resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: fixture.addr, Timeout: 2000}, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), health: newQueryHealth()} + p.logger.Store(mainLog.Load()) + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(upstreamPrefix + "0") + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + msg := new(dns.Msg) + msg.SetQuestion("failover.test.", dns.TypeA) + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{ + upstreams: []string{upstreamPrefix + "0", upstreamPrefix + "1"}, + srcAddr: "192.168.0.1:1234", + }, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatal("the second upstream did not answer") + } + p.health.countQuery() + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("%s failures: got %d, want 1", sampleClassResolveFailed, got) + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } +} diff --git a/cmd/cli/query_health_test.go b/cmd/cli/query_health_test.go new file mode 100644 index 00000000..6a77583b --- /dev/null +++ b/cmd/cli/query_health_test.go @@ -0,0 +1,413 @@ +package cli + +import ( + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// queryHealthBase is the start time of every tracker test. The tracker takes +// the time from the caller, so no test waits for a real clock. +var queryHealthBase = time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + +// feedQueryHealth counts queries, and one failed query for each failure of a +// class. The grade follows the failed queries, so a test that wants a class +// gives the failures that produced it. +func feedQueryHealth(q *queryHealth, queries int, failures map[string]int) { + for i := 0; i < queries; i++ { + q.countQuery() + } + for class, count := range failures { + for i := 0; i < count; i++ { + q.countFailure(class) + q.countFailedQuery() + } + } +} + +// waitForLogLine polls the buffer, because the loop logs from its own goroutine. +func waitForLogLine(t *testing.T, logs *syncBuffer, message string) { + t.Helper() + waitForLogLineOrExplain(t, logs, message, nil) +} + +// waitForLogLineOrExplain is waitForLogLine with extra text for the failure. +func waitForLogLineOrExplain(t *testing.T, logs *syncBuffer, message string, explain func() string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if strings.Contains(logs.String(), message) { + return + } + time.Sleep(time.Millisecond) + } + extra := "" + if explain != nil { + extra = " " + explain() + } + t.Fatalf("no %q line before the deadline (captured %q)%s", message, logs.String(), extra) +} + +// loggerSwapRecord is one change of the main logger. +type loggerSwapRecord struct { + at time.Duration + writer string +} + +// watchLoggerSwaps records every change of the main logger while the test +// runs, so a missing line names what took the logger. +func watchLoggerSwaps(t *testing.T) func() string { + t.Helper() + start := time.Now() + stop := make(chan struct{}) + done := make(chan struct{}) + var mu sync.Mutex + var records []loggerSwapRecord + go func() { + defer close(done) + lastLogger := mainLog.Load() + for { + select { + case <-stop: + return + default: + } + logger := mainLog.Load() + if logger != lastLogger { + mu.Lock() + records = append(records, loggerSwapRecord{time.Since(start), loggerWriterType(logger)}) + mu.Unlock() + lastLogger = logger + } + time.Sleep(50 * time.Microsecond) + } + }() + t.Cleanup(func() { close(stop); <-done }) + return func() string { + mu.Lock() + defer mu.Unlock() + parts := make([]string, 0, len(records)) + for _, r := range slices.Clone(records) { + parts = append(parts, fmt.Sprintf("{at=%s writer=%s}", r.at, r.writer)) + } + return fmt.Sprintf("swaps=%d %s current=%s", len(records), strings.Join(parts, " "), loggerWriterType(mainLog.Load())) + } +} + +// loggerWriterType names the core behind a logger, so a swapped logger tells +// which init function built it. +func loggerWriterType(l *ctrld.Logger) string { + if l == nil || l.Logger == nil { + return "nil" + } + return fmt.Sprintf("%T", l.Core()) +} + +func Test_queryHealthClassThresholds(t *testing.T) { + // The class names go into the journal, so the test pins the text. + tests := []struct { + name string + queries int + failures int + want string + }{ + {"no queries", 0, 0, "healthy"}, + {"two percent", 100, 2, "healthy"}, + {"five percent", 100, 5, "degraded"}, + {"six percent", 100, 6, "degraded"}, + {"half of twenty", 20, 12, "failing"}, + {"half of ten", 10, 5, "failing"}, + {"small sample cannot fail", 9, 9, "healthy"}, + {"small sample cannot degrade", 9, 1, "healthy"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, tc.queries, map[string]int{sampleClassResolveFailed: tc.failures}) + + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != tc.want { + t.Fatalf("class: got %q, want %q", event.Class, tc.want) + } + }) + } +} + +func Test_queryHealthEmitsHeartbeatWithoutClassChange(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + + for _, minutes := range []int{1, 7, 14} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if _, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d without a class change", minutes) + } + } + + event, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no heartbeat at 15 minutes") + } + if event.Class != "healthy" { + t.Fatalf("heartbeat class: got %q, want %q", event.Class, "healthy") + } + if _, emit := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false); emit { + t.Fatal("a second emit follows the heartbeat") + } +} + +func Test_queryHealthEmitsOnClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 100, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != "healthy" { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 50}) + if _, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + + event, emit = q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != "failing" { + t.Fatalf("class: got %q, want %q", event.Class, "failing") + } + + if _, emit := q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false); emit { + t.Fatal("emit without a class change") + } +} + +// Test_queryHealthHoldsAClassChangeForTwoTicks proves that a class which lasts +// one tick reports nothing. A minute of noise must not open an event. +func Test_queryHealthHoldsAClassChangeForTwoTicks(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthHealthy { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 12}) + if event, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatalf("emit after one tick of class %q", event.Class) + } + + // The next tick grades the same window as degraded, so the failing class + // never held two ticks and never reported. + feedQueryHealth(q, 80, nil) + if event, emit := q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false); emit { + t.Fatalf("emit after the class moved again to %q", event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != queryHealthDegraded { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthDegraded) + } +} + +// Test_queryHealthNeedsASampleBeforeAClassChange proves that the empty window +// which follows a heartbeat reports nothing. The counters start at zero, and +// zero queries say nothing about the query path. +func Test_queryHealthNeedsASampleBeforeAClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("window end: emit %v, class %q", emit, event.Class) + } + + for _, minutes := range []int{16, 17} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if event, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d with %d queries and class %q", minutes, event.Queries, event.Class) + } + } +} + +func Test_queryHealthCacheHitRatio(t *testing.T) { + q := newQueryHealth() + event, _ := q.evaluate(queryHealthBase, 0, false) + if event.CacheHitRatio != 0 { + t.Fatalf("ratio without queries: got %v, want 0", event.CacheHitRatio) + } + + feedQueryHealth(q, 20, nil) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, _ = q.evaluate(queryHealthBase.Add(time.Minute), 0, false) + if event.Queries != 20 || event.CacheHits != 5 { + t.Fatalf("counters: got %d queries and %d cache hits, want 20 and 5", event.Queries, event.CacheHits) + } + if event.CacheHitRatio != 0.25 { + t.Fatalf("ratio: got %v, want 0.25", event.CacheHitRatio) + } +} + +func Test_queryHealthResetsCountersAtWindowEnd(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + q.countCacheHit() + + closed, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no emit at the window end") + } + if closed.Queries != 20 || closed.CacheHits != 1 || closed.Failures[sampleClassResolveFailed] != 12 { + t.Fatalf("the closed window lost its counters: %+v", closed) + } + + q.countFailure(sampleClassResolveFailed) + next, _ := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false) + if next.Queries != 0 || next.CacheHits != 0 { + t.Fatalf("the counters survived the window end: %+v", next) + } + if next.Failures[sampleClassResolveFailed] != 1 { + t.Fatalf("failures of the new window: got %d, want 1", next.Failures[sampleClassResolveFailed]) + } +} + +func Test_queryHealthReportKeepsItsOwnFailures(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 10, map[string]int{sampleClassResolveFailed: 1}) + + report, _ := q.evaluate(queryHealthBase, 0, false) + q.countFailure(sampleClassResolveFailed) + + if got := report.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("the report shares the live failure map: got %d, want 1", got) + } +} + +func Test_queryHealthLogsJournalEvent(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{ + sampleClassResolveFailed: 8, + sampleClassAllEndpointsFailed: 4, + }) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, emit := q.evaluate(queryHealthBase, 2, true) + if !emit { + t.Fatal("the first evaluate did not emit") + } + q.log(event) + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) != 1 { + t.Fatalf("health lines: got %d, want 1 (captured %q) %s", len(events), logs.String(), swaps()) + } + logged := events[0] + wantField(t, logged, "journal", true) + wantField(t, logged, "level", "info") + wantField(t, logged, "class", "failing") + wantField(t, logged, "window_s", float64(900)) + wantField(t, logged, "queries", float64(20)) + wantField(t, logged, "cache_hits", float64(5)) + wantField(t, logged, "upstreams_down", float64(2)) + wantField(t, logged, "bypass_active", true) + wantField(t, logged, "cache_hit_ratio", 0.25) + + failures, ok := logged["failures_by_class"].(map[string]any) + if !ok { + t.Fatalf("failures_by_class: got %T, want an object", logged["failures_by_class"]) + } + if len(failures) != 2 { + t.Fatalf("failure classes: got %d, want 2", len(failures)) + } + wantField(t, failures, sampleClassResolveFailed, float64(8)) + wantField(t, failures, sampleClassAllEndpointsFailed, float64(4)) +} + +func Test_queryHealthLoopEmitsUntilStop(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + tick := queryHealthTick + defer func() { queryHealthTick = tick }() + queryHealthTick = time.Millisecond + + q := newQueryHealth() + stop := make(chan struct{}) + done := make(chan struct{}) + var once sync.Once + stopLoop := func() { + once.Do(func() { + close(stop) + <-done + }) + } + defer stopLoop() + + go func() { + defer close(done) + q.loop(stop, func() (int, bool) { return 1, true }) + }() + + waitForLogLineOrExplain(t, logs, queryHealthMessage, swaps) + stopLoop() + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) == 0 { + t.Fatal("the loop logged no health event") + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "upstreams_down", float64(1)) + wantField(t, events[0], "bypass_active", true) +} + +// Test_queryHealthReportsEveryWindowAfterAnEarlyEmit proves that a class +// change in the middle of a window does not make the rest of the window +// vanish. The heartbeat follows the window, not the last report. +func Test_queryHealthReportsEveryWindowAfterAnEarlyEmit(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 20}) + if _, emit := q.evaluate(queryHealthBase.Add(5*time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + event, emit := q.evaluate(queryHealthBase.Add(6*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("class change at 6 minutes: emit %v, class %q", emit, event.Class) + } + feedQueryHealth(q, 7, nil) + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("the end of the window did not emit") + } + if event.Queries != 27 { + t.Fatalf("queries in the window report: got %d, want 27", event.Queries) + } +} diff --git a/cmd/cli/recovery_events_test.go b/cmd/cli/recovery_events_test.go new file mode 100644 index 00000000..edc7796e --- /dev/null +++ b/cmd/cli/recovery_events_test.go @@ -0,0 +1,352 @@ +package cli + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// The two events that bound one recovery pass. +const ( + recoveryBeginMessage = "Recovery begin" + recoveryEndMessage = "Recovery end" +) + +// recoveredUpstreamName is the upstream that the stubbed probe reports. +const recoveredUpstreamName = upstreamPrefix + "0" + +// recoveryEventHarness runs one whole recovery pass. It stubs every seam of +// the pass, because a real pass probes an upstream, reads the host resolvers, +// and rewrites the PF anchor. +type recoveryEventHarness struct { + prog *prog + logs *syncBuffer + dhcpServers []string + target string + recovered string +} + +func newRecoveryEventHarness(t *testing.T) *recoveryEventHarness { + t.Helper() + h := &recoveryEventHarness{ + logs: captureDebugMainLog(t), + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + dhcpServers: []string{"192.168.10.1"}, + target: "127.0.0.53", + recovered: recoveredUpstreamName, + } + h.prog.logger.Store(mainLog.Load()) + stubHeaderSnapshotSources(t) + h.prog.um = newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + h.prog.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.10/24")) + // A stabilizing host defers the DHCP exemption, so the pass rewrites no + // PF anchor. + h.prog.pfStabilizing.Store(true) + + intercept := dnsIntercept + discovery := initializeOsResolverWithSystemNameserversFn + ensure := ensureInterceptDNSTargetFn + wait := waitForUpstreamRecoveryFn + t.Cleanup(func() { + dnsIntercept = intercept + initializeOsResolverWithSystemNameserversFn = discovery + ensureInterceptDNSTargetFn = ensure + waitForUpstreamRecoveryFn = wait + }) + dnsIntercept = true + initializeOsResolverWithSystemNameserversFn = func(context.Context, bool, string) ([]string, []string) { + return effectiveNameserversOf(h.dhcpServers), h.dhcpServers + } + ensureInterceptDNSTargetFn = func(p *prog, _ []string) { + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.interceptDNSTargetSetValue = h.target + p.publishInterceptTarget(h.target) + } + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return h.recovered, nil + } + return h +} + +// effectiveNameserversOf models the OS resolver list that the root package +// builds: every discovered server with a port, and a synthetic public resolver +// when the network gave no public one. +func effectiveNameserversOf(discovered []string) []string { + effective := make([]string, 0, len(discovered)+1) + for _, server := range discovered { + effective = append(effective, server+":53") + } + return append(effective, "76.76.2.0:53") +} + +// oneRecoveryEvent returns the single event with the given message. +func oneRecoveryEvent(t *testing.T, logs *syncBuffer, message string) map[string]any { + t.Helper() + events := jsonLogEvents(t, logs, message) + if len(events) != 1 { + t.Fatalf("%q events: got %d, want 1", message, len(events)) + } + return events[0] +} + +// snapshotTriggers lists the trigger of every snapshot event, in order. +func snapshotTriggers(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +func Test_recoveryCompletedJournalsBeginAndEnd(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + begin := oneRecoveryEvent(t, h.logs, recoveryBeginMessage) + wantField(t, begin, "journal", true) + wantField(t, begin, "level", "info") + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "journal", true) + wantField(t, end, "outcome", "completed") + wantField(t, end, "recovered_upstream", recoveredUpstreamName) + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "set") + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.10.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } + + // The network of this recovery does not change, so the end snapshot equals + // the begin snapshot and the writer drops it. + triggers := snapshotTriggers(t, h.logs) + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } +} + +// Test_recoveryEndReportsTheDiscoveredDHCPNameservers drives the real OS +// resolver initialization. The effective list carries a synthetic public +// resolver, and a reader who counts it reads a DHCP server that the network +// never gave. +func Test_recoveryEndReportsTheDiscoveredDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func(context.Context) []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.1.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } +} + +// Test_recoveryJournalsTheReinitializedOSResolver covers the resolver line of +// a recovered network. The line names the nameservers that the recovery +// installed, and an outage report needs them. +func Test_recoveryJournalsTheReinitializedOSResolver(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func(context.Context) []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + + h.prog.handleRecoveryForTransition(RecoveryReasonNetworkChange, 0) + + event := oneRecoveryEvent(t, h.logs, "Reinitialized OS resolver with nameservers: [192.168.1.1:53 76.76.2.0:53]") + wantField(t, event, "journal", true) + wantField(t, event, "level", "info") +} + +// Test_recoveryEndSpeedsUpTheDNSConfigurationPoll covers the poll cadence +// after a recovery. The resolver table of the host settles late, and a poll +// every five minutes reports that change long after the outage. +func Test_recoveryEndSpeedsUpTheDNSConfigurationPoll(t *testing.T) { + h := newRecoveryEventHarness(t) + now := time.Date(2026, 9, 17, 9, 0, 0, 0, time.UTC) + previousNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = previousNow }) + h.prog.dnsConfig = newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + if got := h.prog.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a recovery = %s, want %s", got, dnsConfigFastInterval) + } +} + +func Test_recoveryJournalsTheRecoveredUpstream(t *testing.T) { + h := newRecoveryEventHarness(t) + h.prog.um.mu.Lock() + h.prog.um.markDown(recoveredUpstreamName, 7, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "Upstream \""+recoveredUpstreamName+"\" recovered; re-applying DNS settings") + wantField(t, event, "journal", true) + wantField(t, event, "upstream", recoveredUpstreamName) + if _, ok := event["down_for_ms"]; !ok { + t.Fatalf("event has no down_for_ms field: %v", event) + } +} + +// Test_recoveryBoundsTheRecoveredUpstreamName covers an upstream key that +// holds a token. The key is operator text, and the journal leaves the host +// for a support case, so no line may carry it. +func Test_recoveryBoundsTheRecoveredUpstreamName(t *testing.T) { + const secret = "SECRET0123" + h := newRecoveryEventHarness(t) + h.recovered = upstreamPrefix + "doh-" + secret + h.prog.um.mu.Lock() + h.prog.um.markDown(h.recovered, 1, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + const boundedName = upstreamPrefix + "custom" + event := oneRecoveryEvent(t, h.logs, "Upstream \""+boundedName+"\" recovered; re-applying DNS settings") + wantField(t, event, "upstream", boundedName) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "recovered_upstream", boundedName) + if strings.Contains(h.logs.String(), secret) { + t.Fatalf("a log line holds the upstream key: %s", h.logs.String()) + } +} + +func Test_recoveryJournalsTheDHCPBypassLines(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + for _, message := range []string{ + "DNS intercept recovery: enabling DHCP bypass (filters stay active)", + "DNS intercept recovery: found DHCP nameservers: [192.168.10.1]", + "DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow", + } { + wantField(t, oneRecoveryEvent(t, h.logs, message), "journal", true) + } +} + +func Test_recoveryJournalsTheMissingDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + h.dhcpServers = nil + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "DNS intercept recovery: no DHCP nameservers found") + wantField(t, event, "journal", true) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "dhcp_server_count", float64(0)) +} + +// wantOneRecoveryEndPerGeneration asserts the two lines that bound one pass: +// one end event and one snapshot beside it. +func wantOneRecoveryEndPerGeneration(t *testing.T, h *recoveryEventHarness) map[string]any { + t.Helper() + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + triggers := snapshotTriggers(t, h.logs) + // Nothing changes between the begin and the end of these passes, so the + // snapshot limiter drops the end snapshot as a repeat of the begin one. + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } + return end +} + +// Test_recoveryCanceledEndReportsTheCanceledPass covers the canceled outcome. +// A canceled pass releases the bypass, and support reads the end event to tell +// a canceled pass from a pass that never ended. +func Test_recoveryCanceledEndReportsTheCanceledPass(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "", context.Canceled + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "canceled") + wantField(t, end, "bypass_active", false) + wantField(t, end, "recovered_upstream", "") +} + +// Test_recoverySupersededEndReportsNoChange covers the superseded outcome. The +// successor owns the target and the bypass flag, so the end event of the +// superseded pass must not report the state of its successor. +func Test_recoverySupersededEndReportsNoChange(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(p *prog, _ context.Context, _ map[string]*ctrld.UpstreamConfig, _ *recoveryDiagnostic) (string, error) { + // A network change took ownership while the probe ran. + p.recoveryGen.Add(1) + return recoveredUpstreamName, nil + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "superseded") + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "unchanged") +} + +func Test_recoveryCanceledWithNoSuccessorIsJournaled(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + p.logger.Store(mainLog.Load()) + setupInterceptRecovery(t, p) + + p.recoveryCanceledCleanup(p.recoveryGen.Add(1)) + + event := oneRecoveryEvent(t, logs, "Recovery canceled with no successor; cleared recovery state and DHCP bypass") + wantField(t, event, "journal", true) +} + +func Test_interceptTargetAction(t *testing.T) { + for _, tc := range []struct { + name string + before, after string + want string + }{ + {"no target at all", "", "", "unchanged"}, + {"target kept", "127.0.0.53", "127.0.0.53", "unchanged"}, + {"target written", "", "127.0.0.53", "set"}, + {"target moved", "127.0.0.53", "127.0.0.54", "set"}, + {"target dropped", "127.0.0.53", "", "removed"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := interceptTargetAction(tc.before, tc.after); got != tc.want { + t.Fatalf("interceptTargetAction(%q, %q) = %q, want %q", tc.before, tc.after, got, tc.want) + } + }) + } +} + +func Test_noteNAT64ResultReportsChangesOnly(t *testing.T) { + p := &prog{} + p.logger.Store(mainLog.Load()) + const prefix = "64:ff9b::/96" + + for _, tc := range []struct { + result string + want bool + }{ + {prefix, true}, + {prefix, false}, + {"", true}, + } { + if got := p.noteNAT64Result(tc.result); got != tc.want { + t.Fatalf("noteNAT64Result(%q) = %v, want %v", tc.result, got, tc.want) + } + } +} diff --git a/cmd/cli/recovery_scope_test.go b/cmd/cli/recovery_scope_test.go new file mode 100644 index 00000000..ecb1e3eb --- /dev/null +++ b/cmd/cli/recovery_scope_test.go @@ -0,0 +1,349 @@ +package cli + +import ( + "context" + "net" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + ctrld "github.com/Control-D-Inc/ctrld" +) + +// Loopback transports exercise the real proxy and recovery loop without host +// DNS changes. The event harness intercepts the first platform mutation. +func scopeTestResolver(t *testing.T, answering *atomic.Bool, calls *atomic.Int64) *ctrld.UpstreamConfig { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + started := make(chan struct{}) + srv := &dns.Server{PacketConn: pc, Net: "udp", NotifyStartedFunc: func() { close(started) }, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { + calls.Add(1) + if answering.Load() { + a := new(dns.Msg) + a.SetReply(m) + _ = w.WriteMsg(a) + } + })} + go func() { _ = srv.ActivateAndServe() }() + <-started + t.Cleanup(func() { _ = srv.Shutdown() }) + return &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, Endpoint: pc.LocalAddr().String(), Timeout: 40} +} + +type scopeTestHarness struct { + *recoveryEventHarness + osAnswers, defaultAnswers atomic.Bool + osCalls, defaultCalls atomic.Int64 + finished chan struct{} +} + +func newScopeTestHarness(t *testing.T) *scopeTestHarness { + t.Helper() + h := &scopeTestHarness{recoveryEventHarness: newRecoveryEventHarness(t), finished: make(chan struct{}, 8)} + // These flows keep interception active (or reject recovery before host work). + // Fence the first platform calls if that contract regresses, not merely a + // later DNS write: startDNSIntercept can install PF/NRPT/WFP policy. + oldStart, oldReset := startDNSInterceptFn, recoveryResetDNSFn + var hostCalls atomic.Int64 + startDNSInterceptFn = func(*prog) error { + hostCalls.Add(1) + t.Fatal("scope recovery attempted to start host interception") + return nil + } + recoveryResetDNSFn = func(*prog, bool, bool) { + hostCalls.Add(1) + t.Fatal("scope recovery attempted to reset host DNS") + } + t.Cleanup(func() { + startDNSInterceptFn, recoveryResetDNSFn = oldStart, oldReset + if got := hostCalls.Load(); got != 0 { + t.Errorf("unexpected first host calls: %d", got) + } + }) + h.dhcpServers = nil + initializeOsResolverWithSystemNameserversFn = func(context.Context, bool, string) ([]string, []string) { return nil, nil } + oldOS, oldQueryRecovery := osUpstreamConfig, queryRecoveryFn + t.Cleanup(func() { osUpstreamConfig, queryRecoveryFn = oldOS, oldQueryRecovery }) + osUpstreamConfig = scopeTestResolver(t, &h.osAnswers, &h.osCalls) + h.defaultAnswers.Store(true) + h.prog.cfg.Upstream = map[string]*ctrld.UpstreamConfig{"0": scopeTestResolver(t, &h.defaultAnswers, &h.defaultCalls)} + leak := true + h.prog.cfg.Service.LeakOnUpstreamFailure = &leak + h.prog.um = newUpstreamMonitor(h.prog.cfg, mainLog.Load()) + h.prog.um.after = func(time.Duration, func()) {} + h.prog.um.clearRecovered(upstreamOS) + h.prog.um.clearRecovered("upstream.0") + queryRecoveryFn = func(p *prog, reason RecoveryReason) { + p.handleRecovery(reason) + h.finished <- struct{}{} + } + return h +} + +func (h *scopeTestHarness) query(t *testing.T, policy bool) *proxyResponse { + t.Helper() + name := "ordinary.example." + ufr := &upstreamForResult{upstreams: []string{"upstream.0"}, matched: true} + if policy { + name = "os-only.example." + lc := listenerWithRules(ctrld.Rule{"os-only.example": []string{}}) + ufr = h.prog.upstreamFor(context.Background(), "0", lc, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 53000}, "", "os-only.example") + if !ufr.matched || len(ufr.upstreams) != 0 { + t.Fatalf("wrong policy fixture: %+v", ufr) + } + } + m := new(dns.Msg) + m.SetQuestion(name, dns.TypeTXT) + return h.prog.proxy(context.Background(), &proxyRequest{msg: m, ufr: ufr}) +} + +func (h *scopeTestHarness) markOSDown(t *testing.T) { + t.Helper() + if got := h.query(t, true).answer.Rcode; got != dns.RcodeServerFailure { + t.Fatalf("OS fixture rcode=%d", got) + } + // Fire the real delayed-down callback without waiting ten seconds. + h.prog.um.markDownAfterDelay(upstreamOS) + if !h.prog.um.isDown(upstreamOS) { + t.Fatal("OS down precondition failed") + } +} + +func scopeWait(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("recovery handler did not return") + } +} + +func TestOSPolicyFailureDoesNotBypassHealthyDefault(t *testing.T) { + h := newScopeTestHarness(t) + if h.query(t, false).answer.Rcode != dns.RcodeSuccess { + t.Fatal("default fixture failed") + } + h.markOSDown(t) + for range 3 { + if h.query(t, true).answer.Rcode != dns.RcodeServerFailure { + t.Fatal("policy changed resolver") + } + scopeWait(t, h.finished) + if h.prog.recoveryBypass.Load() || h.prog.recoveryRunning.Load() { + t.Fatal("policy failure started global recovery") + } + } + before := h.osCalls.Load() + if h.query(t, false).answer.Rcode != dns.RcodeSuccess { + t.Fatal("default request failed") + } + if h.osCalls.Load() != before { + t.Fatal("unrelated request detoured through OS DNS") + } + if len(jsonLogEvents(t, h.logs, recoveryBeginMessage)) != 0 { + t.Fatal("rejected recovery touched host state") + } +} + +func TestQueuedOSFailureRechecksHealthyDefault(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + entered, proceed := make(chan struct{}), make(chan struct{}) + queryRecoveryFn = func(p *prog, reason RecoveryReason) { + close(entered) + <-proceed + p.handleRecovery(reason) + h.finished <- struct{}{} + } + h.query(t, true) + scopeWait(t, entered) + if h.query(t, false).answer.Rcode != dns.RcodeSuccess { + t.Fatal("default did not recover") + } + close(proceed) + scopeWait(t, h.finished) + if len(jsonLogEvents(t, h.logs, recoveryBeginMessage)) != 0 { + t.Fatal("late OS trigger ignored newer default success") + } +} + +func TestOSFailureRecoversThroughConfiguredPool(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + candidates := make(chan map[string]*ctrld.UpstreamConfig, 1) + proceed := make(chan struct{}) + waitForUpstreamRecoveryFn = func(p *prog, ctx context.Context, ups map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { + candidates <- ups + <-proceed + return p.waitForUpstreamRecovery(ctx, ups, diagnostic) + } + h.query(t, true) + select { + case ups := <-candidates: + if len(ups) != 1 || ups["upstream.0"] != h.prog.cfg.Upstream["0"] { + t.Fatalf("wrong recovery pool: %v", ups) + } + case <-time.After(time.Second): + t.Fatal("general outage did not enter recovery") + } + if !h.prog.recoveryBypass.Load() { + t.Fatal("general outage did not enable bypass") + } + close(proceed) + scopeWait(t, h.finished) + if h.prog.recoveryBypass.Load() || h.prog.recoveryRunning.Load() { + t.Fatal("healthy default did not end recovery") + } + if !h.prog.um.isDown(upstreamOS) { + t.Fatal("fixture OS resolver unexpectedly recovered") + } +} + +func TestOSOnlyConfigurationRetainsOSRecovery(t *testing.T) { + h := newScopeTestHarness(t) + h.prog.cfg.Upstream = nil + h.markOSDown(t) + waitForUpstreamRecoveryFn = func(p *prog, ctx context.Context, ups map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { + if len(ups) != 1 || ups[upstreamOS] != osUpstreamConfig { + return "", context.Canceled + } + h.osAnswers.Store(true) + return p.waitForUpstreamRecovery(ctx, ups, diagnostic) + } + h.query(t, true) + scopeWait(t, h.finished) + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "outcome", "completed") + wantField(t, end, "recovered_upstream", upstreamOS) + if h.prog.recoveryBypass.Load() || h.prog.um.isDown(upstreamOS) { + t.Fatal("OS-only recovery did not finish") + } +} + +func TestOSRecoveryRejectedBeforeHostWork(t *testing.T) { + for _, intercept := range []bool{false, true} { + t.Run(map[bool]string{false: "traditional", true: "intercept"}[intercept], func(t *testing.T) { + h := newScopeTestHarness(t) + dnsIntercept = intercept + oldReset := recoveryResetDNSFn + t.Cleanup(func() { recoveryResetDNSFn = oldReset }) + calls := 0 + recoveryResetDNSFn = func(*prog, bool, bool) { calls++ } + refreshes := 0 + initializeOsResolverWithSystemNameserversFn = func(context.Context, bool, string) ([]string, []string) { refreshes++; return nil, nil } + ensureInterceptDNSTargetFn = func(*prog, []string) { calls++ } + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + calls++ + return "", context.Canceled + } + before := h.prog.recoveryGen.Load() + h.prog.handleRecovery(RecoveryReasonOSFailure) + if refreshes != 1 || calls != 0 || h.prog.recoveryGen.Load() != before || h.prog.recoveryRunning.Load() || h.prog.recoveryBypass.Load() { + t.Fatal("rejected recovery failed to refresh only OS resolver discovery") + } + }) + } +} + +func TestOSRecoveryKeepsExistingOwner(t *testing.T) { + h := newScopeTestHarness(t) + ctx, gen, _, ok := h.prog.beginRecovery(RecoveryReasonRegularFailure) + if !ok { + t.Fatal("could not establish prior owner") + } + t.Cleanup(func() { h.prog.completeRecoveryState(gen) }) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if ctx.Err() != nil || h.prog.recoveryGen.Load() != gen || !h.prog.recoveryOwnsState(gen) || !h.prog.recoveryBypass.Load() { + t.Fatal("rejected OS recovery displaced the active owner") + } +} + +func TestOSRecoveryConfiguredPoolDoesNotBlockRegularFailure(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + h.prog.cfg.Upstream["unused"] = &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, Endpoint: "127.0.0.1:1"} + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + h.query(t, true) + scopeWait(t, h.finished) + if len(jsonLogEvents(t, h.logs, recoveryBeginMessage)) != 0 { + t.Fatal("OS admission ignored an unmarked configured candidate") + } + // Ordinary default failure still follows its original admission path. + h.defaultAnswers.Store(false) + h.query(t, false) + scopeWait(t, h.finished) + begin := oneRecoveryEvent(t, h.logs, recoveryBeginMessage) + wantField(t, begin, "recovery_reason", "upstream_failure") +} + +func TestOtherRecoveryPoolsRemainUnchanged(t *testing.T) { + remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH} + internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + p := &prog{cfg: &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{ + "0": remote, "internal_0": internal, "1": {Type: ctrld.ResolverTypeOS}, "2": nil, + }}} + want := map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.internal_0": internal} + for _, reason := range []RecoveryReason{RecoveryReasonRegularFailure, RecoveryReasonNetworkChange} { + if got := p.buildRecoveryUpstreams(reason); !reflect.DeepEqual(got, want) { + t.Fatalf("reason=%v got=%v want=%v", reason, got, want) + } + } +} + +func TestScopeHarnessRestoresRecoveryWait(t *testing.T) { + original := waitForUpstreamRecoveryFn + t.Cleanup(func() { waitForUpstreamRecoveryFn = original }) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "outer", nil + } + t.Run("overridden wait", func(t *testing.T) { + newScopeTestHarness(t) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "inner", nil + } + }) + got, err := waitForUpstreamRecoveryFn(nil, context.Background(), nil, nil) + if err != nil || got != "outer" { + t.Fatalf("composed harness leaked its wait function: got %q, %v", got, err) + } +} + +func TestOSRecoveryCandidates(t *testing.T) { + remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH} + internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, InternalDomain: internalDomainUpstreamFallback} + // A user-defined upstream that copies a generated key carries no marker. + lookalike := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + osConfig := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeOS} + for _, tc := range []struct { + name string + configured map[string]*ctrld.UpstreamConfig + want map[string]*ctrld.UpstreamConfig + }{ + {"configured", map[string]*ctrld.UpstreamConfig{"0": remote, "internal_0": internal, "1": osConfig, "2": nil}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote}}, + {"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": lookalike}}, + {"only_os", map[string]*ctrld.UpstreamConfig{"0": osConfig}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + {"only_internal", map[string]*ctrld.UpstreamConfig{"internal_0": internal}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + {"internal_lookalike", map[string]*ctrld.UpstreamConfig{"internal_0": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.internal_0": lookalike}}, + {"empty", nil, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + } { + t.Run(tc.name, func(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Upstream: tc.configured}} + if got := p.buildRecoveryUpstreams(RecoveryReasonOSFailure); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %v want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/recovery_skip_test.go b/cmd/cli/recovery_skip_test.go new file mode 100644 index 00000000..ab6d6bd9 --- /dev/null +++ b/cmd/cli/recovery_skip_test.go @@ -0,0 +1,153 @@ +package cli + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + ctrld "github.com/Control-D-Inc/ctrld" +) + +const recoverySkipMessage = "Recovery skipped: configured upstream is not marked down" + +func TestOSRecoverySkipRefreshesWithoutNetworkChange(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + var refreshes atomic.Int64 + initializeOsResolverWithSystemNameserversFn = func(_ context.Context, guard bool, reason string) ([]string, []string) { + if !guard || reason != recoveryResolverReason { + t.Error("refresh lost empty-discovery guard or reason") + } + refreshes.Add(1) + h.osAnswers.Store(true) + return nil, nil + } + // This query still fails; the asynchronously triggered refresh repairs + // discovery for the next policy query, without a network-change callback. + h.query(t, true) + scopeWait(t, h.finished) + if refreshes.Load() != 1 || !h.prog.um.isDown(upstreamOS) { + t.Fatal("refresh must run without claiming a successful OS response") + } + if h.query(t, true).answer.Rcode != dns.RcodeSuccess || h.prog.um.isDown(upstreamOS) { + t.Fatal("OS policy did not recover on its own resolver") + } + if h.prog.recoveryBypass.Load() || h.prog.recoveryRunning.Load() || h.prog.recoveryGen.Load() != 0 { + t.Fatal("local refresh acquired global recovery state") + } +} + +func TestOSRecoverySkipRefreshIsBounded(t *testing.T) { + h := newScopeTestHarness(t) + now := time.Unix(1000, 0) + oldNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = oldNow }) + calls := 0 + initializeOsResolverWithSystemNameserversFn = func(context.Context, bool, string) ([]string, []string) { + calls++ + return nil, nil + } + for range 100 { + h.prog.handleRecovery(RecoveryReasonOSFailure) + } + if calls != 1 { + t.Fatalf("burst discovery calls=%d, want 1", calls) + } + now = now.Add(upstreamDownDelay - time.Nanosecond) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if calls != 1 { + t.Fatal("discovery retried before the cooldown") + } + now = now.Add(time.Nanosecond) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if calls != 2 { + t.Fatal("later OS failures cannot retry discovery") + } + event := oneRecoveryEvent(t, h.logs, recoverySkipMessage) + wantField(t, event, "journal", true) + wantField(t, event, "healthy_upstream", "upstream.0") + wantField(t, event, "recovery_reason", "os_resolver_failure") + now = now.Add(5 * time.Minute) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if len(jsonLogEvents(t, h.logs, recoverySkipMessage)) != 2 { + t.Fatal("persistent skips lost the bounded journal heartbeat") + } +} + +func TestOSRecoverySkipCoalescesConcurrentRefresh(t *testing.T) { + h := newScopeTestHarness(t) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var calls atomic.Int64 + initializeOsResolverWithSystemNameserversFn = func(context.Context, bool, string) ([]string, []string) { + if calls.Add(1) == 1 { + close(entered) + } + <-release + return nil, nil + } + go func() { h.prog.handleRecovery(RecoveryReasonOSFailure); close(done) }() + scopeWait(t, entered) + other := make(chan struct{}) + go func() { h.prog.handleRecovery(RecoveryReasonOSFailure); close(other) }() + // Release the first worker even if coalescing regresses, to keep cleanup safe. + select { + case <-other: + case <-time.After(time.Second): + close(release) + scopeWait(t, done) + scopeWait(t, other) + t.Fatal("concurrent failure waited on discovery") + } + close(release) + scopeWait(t, done) + if calls.Load() != 1 { + t.Fatalf("overlapping discovery calls=%d", calls.Load()) + } +} + +func TestOSRecoverySkipJournalsChangedCandidatePrivately(t *testing.T) { + h := newScopeTestHarness(t) + h.prog.cfg.Upstream["private-token.example"] = &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + h.prog.handleRecovery(RecoveryReasonOSFailure) + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + h.prog.handleRecovery(RecoveryReasonOSFailure) + events := jsonLogEvents(t, h.logs, recoverySkipMessage) + if len(events) != 2 { + t.Fatalf("candidate change events=%d", len(events)) + } + wantField(t, events[0], "healthy_upstream", "upstream.0") + wantField(t, events[1], "healthy_upstream", "upstream.custom") + if strings.Contains(h.logs.String(), "private-token.example") { + t.Fatal("operator-provided upstream key leaked") + } +} + +func TestOSRecoverySkipSurvivesDebugTruncation(t *testing.T) { + h := newScopeTestHarness(t) + p, _ := startInternalLogging(t) + h.prog.logger.Store(mainLog.Load()) + useTestSendBudget(t, 1024) + for range 100 { + h.prog.handleRecovery(RecoveryReasonOSFailure) + } + for range 40 { + mainLog.Load().Debug().Msg(strings.Repeat("debug filler", 20)) + } + // Taken before the read renders the upload header: as an argument after + // readLogReader it would be evaluated later, and fail whenever a second + // boundary falls between the two. + start := time.Now() + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start) + if strings.Contains(string(debugPart), recoverySkipMessage) { + t.Fatal("fixture did not truncate the skip from debug") + } + if strings.Count(string(journalPart), recoverySkipMessage) != 1 || !strings.Contains(string(journalPart), `"healthy_upstream":"upstream.0"`) { + t.Fatal("journal did not retain the bounded skip and admission candidate") + } +} diff --git a/cmd/cli/recovery_state.go b/cmd/cli/recovery_state.go new file mode 100644 index 00000000..e3ff6385 --- /dev/null +++ b/cmd/cli/recovery_state.go @@ -0,0 +1,86 @@ +package cli + +import ( + "context" + + "github.com/Control-D-Inc/ctrld" +) + +// beginRecovery atomically transfers ownership of shared recovery state. A +// network change cancels and replaces the current owner without exposing a nil +// recoveryCancel gap; other triggers are coalesced while an owner exists. +func (p *prog) beginRecovery(reason RecoveryReason) (ctx context.Context, gen uint64, intercept bool, ok bool) { + p.recoveryCancelMu.Lock() + defer p.recoveryCancelMu.Unlock() + + // Close publishes the gate before taking recoveryCancelMu to cancel the + // current owner. Thus an admitted worker either publishes its cancel first + // or observes shutdown here; it cannot create an uncancellable late owner. + if p.networkActivityClosed() { + return nil, 0, false, false + } + if reason != RecoveryReasonNetworkChange && p.recoveryCancel != nil { + return nil, 0, false, false + } + if p.recoveryCancel != nil { + p.recoveryCancel() + } + + ctx, cancel := context.WithCancel(context.Background()) + gen = p.recoveryGen.Add(1) + intercept = dnsIntercept && p.dnsInterceptState != nil + p.recoveryCancel = cancel + p.recoveryRunning.Store(true) + p.recoveryBypass.Store(intercept) + return ctx, gen, intercept, true +} + +func (p *prog) recoveryOwnsState(gen uint64) bool { + p.recoveryCancelMu.Lock() + defer p.recoveryCancelMu.Unlock() + return !p.networkActivityClosed() && p.recoveryGen.Load() == gen && p.recoveryCancel != nil +} + +func (p *prog) systemNameserversForInterceptRetry() []string { + loggerCtx := ctrld.LoggerCtx(context.Background(), p.logger.Load()) + _, system := initializeOsResolverWithSystemNameserversFn(loggerCtx, true, recoveryResolverReason) + if system == nil { + return []string{} + } + return system +} + +// completeRecovery releases shared state only if gen still owns it. The bypass +// reset is unconditional because live intercept state can disappear while a +// recovery is running, but a stale true flag still affects proxy routing. +func (p *prog) completeRecoveryState(gen uint64) bool { + p.recoveryCancelMu.Lock() + defer p.recoveryCancelMu.Unlock() + if p.recoveryGen.Load() != gen || p.recoveryCancel == nil { + return false + } + p.recoveryBypass.Store(false) + p.recoveryRunning.Store(false) + p.recoveryCancel = nil + return true +} + +// recoveryCanceledCleanup resets shared recovery state after a canceled or +// failed recovery, but only when the recovery identified by gen was NOT +// superseded by a newer one (issue #597). +// +// A network-change cancellation is normally followed immediately by a new +// handleRecovery that owns recoveryBypass/recoveryRunning/recoveryCancel; +// clearing them here would disable the successor's bypass mid-flight and +// make it uncancellable. But when the canceled recovery is the LAST one +// (e.g. the tail of a network flap burst), nothing else will ever clear the +// flags: the daemon would stay in recovery bypass forever — every query +// detouring to the OS resolver — and the DNS-settings watchdog would stay +// permanently disabled. +func (p *prog) recoveryCanceledCleanup(gen uint64) { + if !p.completeRecoveryState(gen) { + // Superseded: the newer recovery owns the shared state. + return + } + journal(p.Info()).Msg("Recovery canceled with no successor; cleared recovery state and DHCP bypass") +} diff --git a/cmd/cli/recovery_state_test.go b/cmd/cli/recovery_state_test.go new file mode 100644 index 00000000..59b720e6 --- /dev/null +++ b/cmd/cli/recovery_state_test.go @@ -0,0 +1,167 @@ +package cli + +import ( + "context" + "testing" + "time" +) + +// interceptStateStub stands in for the platform pfState/wfpState; the +// recovery cleanup path only checks dnsInterceptState != nil. +type interceptStateStub struct{} + +// setupInterceptRecovery puts p into "intercept-mode recovery in flight" +// state and restores the package-level dnsIntercept flag on cleanup. +func setupInterceptRecovery(t *testing.T, p *prog) { + t.Helper() + oldIntercept := dnsIntercept + dnsIntercept = true + t.Cleanup(func() { dnsIntercept = oldIntercept }) + p.dnsInterceptState = &interceptStateStub{} + p.logger.Store(mainLog.Load()) + p.recoveryBypass.Store(true) + p.recoveryRunning.Store(true) + p.recoveryCancel = func() {} +} + +// TestRecoveryCanceledCleanup_LastRecoveryResetsState pins issue #597: a +// canceled recovery with no successor must clear recoveryBypass and +// recoveryRunning, or the daemon stays in bypass forever (every query +// detours to the OS resolver) and the DNS watchdog stays disabled. +func TestRecoveryCanceledCleanup_LastRecoveryResetsState(t *testing.T) { + p := &prog{} + setupInterceptRecovery(t, p) + gen := p.recoveryGen.Add(1) + + p.recoveryCanceledCleanup(gen) + + if p.recoveryBypass.Load() { + t.Error("recoveryBypass still set after canceled recovery with no successor") + } + if p.recoveryRunning.Load() { + t.Error("recoveryRunning still set after canceled recovery with no successor") + } + p.recoveryCancelMu.Lock() + cancelCleared := p.recoveryCancel == nil + p.recoveryCancelMu.Unlock() + if !cancelCleared { + t.Error("recoveryCancel not cleared after canceled recovery with no successor") + } +} + +// TestRecoveryCanceledCleanup_SupersededKeepsSuccessorState pins the +// captive-portal/network-flap contract: when a newer recovery superseded the +// canceled one, the canceled recovery must NOT clear shared state — the +// successor owns bypass for its own duration. +func TestRecoveryCanceledCleanup_SupersededKeepsSuccessorState(t *testing.T) { + p := &prog{} + setupInterceptRecovery(t, p) + gen := p.recoveryGen.Add(1) + // A successor recovery started. + p.recoveryGen.Add(1) + + p.recoveryCanceledCleanup(gen) + + if !p.recoveryBypass.Load() { + t.Error("superseded canceled recovery cleared recoveryBypass owned by its successor") + } + if !p.recoveryRunning.Load() { + t.Error("superseded canceled recovery cleared recoveryRunning owned by its successor") + } + p.recoveryCancelMu.Lock() + cancelKept := p.recoveryCancel != nil + p.recoveryCancelMu.Unlock() + if !cancelKept { + t.Error("superseded canceled recovery cleared the successor's recoveryCancel") + } +} + +// TestRecoveryCanceledCleanup_NonInterceptResetsRunning covers traditional +// (non-intercept) mode: recoveryRunning must still be reset so watchdogs +// resume, while bypass is untouched (it is never set in that mode). +func TestRecoveryCanceledCleanup_NonInterceptResetsRunning(t *testing.T) { + oldIntercept := dnsIntercept + dnsIntercept = false + t.Cleanup(func() { dnsIntercept = oldIntercept }) + + p := &prog{} + p.logger.Store(mainLog.Load()) + p.recoveryRunning.Store(true) + p.recoveryCancel = func() {} + gen := p.recoveryGen.Add(1) + + p.recoveryCanceledCleanup(gen) + + if p.recoveryRunning.Load() { + t.Error("recoveryRunning still set after canceled non-intercept recovery") + } +} + +func TestBeginRecoveryTransfersOwnershipAtomically(t *testing.T) { + oldIntercept := dnsIntercept + dnsIntercept = true + t.Cleanup(func() { dnsIntercept = oldIntercept }) + + p := &prog{dnsInterceptState: &interceptStateStub{}} + firstCtx, firstGen, _, ok := p.beginRecovery(RecoveryReasonRegularFailure) + if !ok { + t.Fatal("first recovery did not acquire ownership") + } + if _, _, _, ok := p.beginRecovery(RecoveryReasonRegularFailure); ok { + t.Fatal("duplicate upstream recovery acquired ownership") + } + + _, successorGen, intercept, ok := p.beginRecovery(RecoveryReasonNetworkChange) + if !ok || !intercept || successorGen <= firstGen { + t.Fatalf("network recovery did not replace owner: first=%d successor=%d intercept=%v ok=%v", firstGen, successorGen, intercept, ok) + } + select { + case <-firstCtx.Done(): + case <-time.After(time.Second): + t.Fatal("successor did not cancel the previous recovery") + } + + p.recoveryCanceledCleanup(firstGen) + if !p.recoveryRunning.Load() || !p.recoveryBypass.Load() || !p.recoveryOwnsState(successorGen) { + t.Fatal("stale cleanup changed successor-owned recovery state") + } + if !p.completeRecoveryState(successorGen) { + t.Fatal("successor could not complete its own recovery state") + } +} + +func TestRecoveryCleanupClearsBypassAfterInterceptStateDisappears(t *testing.T) { + p := &prog{} + p.logger.Store(mainLog.Load()) + p.recoveryBypass.Store(true) + p.recoveryRunning.Store(true) + p.recoveryCancel = func() {} + gen := p.recoveryGen.Add(1) + + p.recoveryCanceledCleanup(gen) + if p.recoveryBypass.Load() || p.recoveryRunning.Load() { + t.Fatal("cleanup retained recovery flags after intercept state disappeared") + } +} + +func TestSystemNameserversForInterceptRetryNormalizesEmptyDiscovery(t *testing.T) { + original := initializeOsResolverWithSystemNameserversFn + called := false + initializeOsResolverWithSystemNameserversFn = func(_ context.Context, guard bool, _ string) ([]string, []string) { + called = true + if !guard { + t.Error("intercept retry discovery did not guard the existing resolver") + } + return nil, nil + } + t.Cleanup(func() { initializeOsResolverWithSystemNameserversFn = original }) + + p := &prog{} + p.logger.Store(mainLog.Load()) + if got := p.systemNameserversForInterceptRetry(); got == nil || len(got) != 0 { + t.Fatalf("system discovery = %#v, want non-nil empty slice", got) + } + if !called { + t.Fatal("system discovery was not called") + } +} diff --git a/cmd/cli/redacted_args_test.go b/cmd/cli/redacted_args_test.go new file mode 100644 index 00000000..18edeebb --- /dev/null +++ b/cmd/cli/redacted_args_test.go @@ -0,0 +1,56 @@ +package cli + +import ( + "strings" + "testing" +) + +// The arguments of "ctrld start" go into a debug line; neither the +// provision code nor the resolver uid may reach it, in either flag form. +func TestRedactedArgsRemovesProvisionSecrets(t *testing.T) { + origOrg, origUID := cdOrg, cdUID + t.Cleanup(func() { cdOrg, cdUID = origOrg, origUID }) + cdOrg = "org-v1-provisioncode0123" + cdUID = "resolveruid9/client-name" + + got := redactedArgs([]string{"--cd-org", cdOrg, "--cd=" + cdUID, "-vv", "--log", "C:\\ctrld\\ctrld.log"}) + + for _, secret := range []string{cdOrg, cdUID, "resolveruid9", "client-name"} { + if strings.Contains(got, secret) { + t.Fatalf("redactedArgs kept %q: %s", secret, got) + } + } + for _, kept := range []string{"--cd-org [redacted]", "--cd=[redacted]", "-vv", "--log"} { + if !strings.Contains(got, kept) { + t.Fatalf("redactedArgs lost %q: %s", kept, got) + } + } +} + +// The debug line itself, not only the helper it uses, must never carry the +// provision code or the resolver uid (issue #632). +func TestInterceptUpgradeCheckLineRedactsProvisionSecrets(t *testing.T) { + origOrg, origUID := cdOrg, cdUID + t.Cleanup(func() { cdOrg, cdUID = origOrg, origUID }) + cdOrg = "org-v1-provisioncode0123" + cdUID = "resolveruid9/client-name" + + for _, args := range [][]string{ + {"--cd-org", cdOrg, "-vv"}, + {"--cd-org=" + cdOrg, "--intercept-mode", "dns"}, + {"--cd", cdUID, "--log", "/var/log/ctrld.log"}, + {"--cd=" + cdUID}, + } { + line := interceptUpgradeCheckLine(args, false, true, "dns") + for _, secret := range []string{cdOrg, cdUID, "resolveruid9", "client-name"} { + if strings.Contains(line, secret) { + t.Fatalf("line for %q carries %q: %s", args, secret, line) + } + } + for _, kept := range []string{"intercept upgrade check: args=[", "interceptOnly=false", "svcConfigExists=true", `interceptMode="dns"`} { + if !strings.Contains(line, kept) { + t.Fatalf("line lost %q: %s", kept, line) + } + } + } +} diff --git a/cmd/cli/reload_others.go b/cmd/cli/reload_others.go index 0977af90..480e7733 100644 --- a/cmd/cli/reload_others.go +++ b/cmd/cli/reload_others.go @@ -8,10 +8,14 @@ import ( "syscall" ) +// notifyReloadSigCh sends reload signal to the channel func notifyReloadSigCh(ch chan os.Signal) { signal.Notify(ch, syscall.SIGUSR1) } +func stopNotifyReloadSigCh(ch chan os.Signal) { signal.Stop(ch) } + +// sendReloadSignal sends a reload signal to the current process func (p *prog) sendReloadSignal() error { return syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) } diff --git a/cmd/cli/reload_windows.go b/cmd/cli/reload_windows.go index 0e817e46..3a8a8ac5 100644 --- a/cmd/cli/reload_windows.go +++ b/cmd/cli/reload_windows.go @@ -6,8 +6,12 @@ import ( "time" ) +// notifyReloadSigCh is a no-op on Windows platforms func notifyReloadSigCh(ch chan os.Signal) {} +func stopNotifyReloadSigCh(ch chan os.Signal) {} + +// sendReloadSignal sends a reload signal to the program func (p *prog) sendReloadSignal() error { select { case p.reloadCh <- struct{}{}: diff --git a/cmd/cli/resolvconf.go b/cmd/cli/resolvconf.go index 0f3f731a..325d19d0 100644 --- a/cmd/cli/resolvconf.go +++ b/cmd/cli/resolvconf.go @@ -3,59 +3,45 @@ package cli import ( "net" "net/netip" - "os" "path/filepath" - "strings" "time" "github.com/fsnotify/fsnotify" + + "github.com/Control-D-Inc/ctrld/internal/resolvconffile" ) // parseResolvConfNameservers reads the resolv.conf file and returns the nameservers found. // Returns nil if no nameservers are found. +// This function parses the system DNS configuration to understand current nameserver settings func (p *prog) parseResolvConfNameservers(path string) ([]string, error) { - content, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - // Parse the file for "nameserver" lines - var currentNS []string - lines := strings.Split(string(content), "\n") - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "nameserver") { - parts := strings.Fields(trimmed) - if len(parts) >= 2 { - currentNS = append(currentNS, parts[1]) - } - } - } - - return currentNS, nil + return resolvconffile.NameserversFromFile(path) } // watchResolvConf watches any changes to /etc/resolv.conf file, // and reverting to the original config set by ctrld. +// This ensures that DNS settings are not overridden by other applications or system processes func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn func(iface *net.Interface, ns []netip.Addr) error) { resolvConfPath := "/etc/resolv.conf" // Evaluating symbolics link to watch the target file that /etc/resolv.conf point to. + // This handles systems where resolv.conf is a symlink to another location if rp, _ := filepath.EvalSymlinks(resolvConfPath); rp != "" { resolvConfPath = rp } - mainLog.Load().Debug().Msgf("start watching %s file", resolvConfPath) + p.Debug().Msgf("Start watching %s file", resolvConfPath) watcher, err := fsnotify.NewWatcher() if err != nil { - mainLog.Load().Warn().Err(err).Msg("could not create watcher for /etc/resolv.conf") + p.Warn().Err(err).Msg("Could not create watcher for /etc/resolv.conf") return } defer watcher.Close() // We watch /etc instead of /etc/resolv.conf directly, // see: https://github.com/fsnotify/fsnotify#watching-a-file-doesnt-work-well + // This is necessary because some systems don't properly notify on file changes watchDir := filepath.Dir(resolvConfPath) if err := watcher.Add(watchDir); err != nil { - mainLog.Load().Warn().Err(err).Msgf("could not add %s to watcher list", watchDir) + p.Warn().Err(err).Msgf("Could not add %s to watcher list", watchDir) return } @@ -64,7 +50,7 @@ func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn f case <-p.dnsWatcherStopCh: return case <-p.stopCh: - mainLog.Load().Debug().Msgf("stopping watcher for %s", resolvConfPath) + p.Debug().Msgf("Stopping watcher for %s", resolvConfPath) return case event, ok := <-watcher.Events: if p.recoveryRunning.Load() { @@ -77,9 +63,10 @@ func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn f continue } if event.Has(fsnotify.Write) || event.Has(fsnotify.Create) { - mainLog.Load().Debug().Msgf("/etc/resolv.conf changes detected, reading changes...") + p.Debug().Msgf("/etc/resolv.conf changes detected, reading changes...") // Convert expected nameservers to strings for comparison + // This allows us to detect when the resolv.conf has been modified expectedNS := make([]string, len(ns)) for i, addr := range ns { expectedNS[i] = addr.String() @@ -92,18 +79,20 @@ func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn f for retry := 0; retry < maxRetries; retry++ { foundNS, err = p.parseResolvConfNameservers(resolvConfPath) if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to read resolv.conf content") + p.Error().Err(err).Msg("Failed to read resolv.conf content") break } // If we found nameservers, break out of retry loop + // This handles cases where the file is being written but not yet complete if len(foundNS) > 0 { break } // Only retry if we found no nameservers + // This handles temporary file states during updates if retry < maxRetries-1 { - mainLog.Load().Debug().Msgf("resolv.conf has no nameserver entries, retry %d/%d in 2 seconds", retry+1, maxRetries) + p.Debug().Msgf("resolv.conf has no nameserver entries, retry %d/%d in 2 seconds", retry+1, maxRetries) select { case <-p.stopCh: return @@ -113,7 +102,7 @@ func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn f continue } } else { - mainLog.Load().Debug().Msg("resolv.conf remained empty after all retries") + p.Debug().Msg("resolv.conf remained empty after all retries") } } @@ -130,7 +119,7 @@ func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn f } } - mainLog.Load().Debug(). + p.Debug(). Strs("found", foundNS). Strs("expected", expectedNS). Bool("matches", matches). @@ -139,16 +128,16 @@ func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn f // Only revert if the nameservers don't match if !matches { if err := watcher.Remove(watchDir); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to pause watcher") + p.Error().Err(err).Msg("Failed to pause watcher") continue } if err := setDnsFn(iface, ns); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to revert /etc/resolv.conf changes") + p.Error().Err(err).Msg("Failed to revert /etc/resolv.conf changes") } if err := watcher.Add(watchDir); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to continue running watcher") + p.Error().Err(err).Msg("Failed to continue running watcher") return } } @@ -158,7 +147,7 @@ func (p *prog) watchResolvConf(iface *net.Interface, ns []netip.Addr, setDnsFn f if !ok { return } - mainLog.Load().Err(err).Msg("could not get event for /etc/resolv.conf") + p.Error().Err(err).Msg("Could not get event for /etc/resolv.conf") } } } diff --git a/cmd/cli/resolvconf_darwin.go b/cmd/cli/resolvconf_darwin.go index eb70eed6..05c70178 100644 --- a/cmd/cli/resolvconf_darwin.go +++ b/cmd/cli/resolvconf_darwin.go @@ -12,7 +12,7 @@ import ( const resolvConfPath = "/etc/resolv.conf" // setResolvConf sets the content of resolv.conf file using the given nameservers list. -func setResolvConf(iface *net.Interface, ns []netip.Addr) error { +func (p *prog) setResolvConf(iface *net.Interface, ns []netip.Addr) error { servers := make([]string, len(ns)) for i := range ns { servers[i] = ns[i].String() diff --git a/cmd/cli/resolvconf_not_darwin_unix.go b/cmd/cli/resolvconf_not_darwin_unix.go index af335720..6eb52959 100644 --- a/cmd/cli/resolvconf_not_darwin_unix.go +++ b/cmd/cli/resolvconf_not_darwin_unix.go @@ -14,7 +14,7 @@ import ( ) // setResolvConf sets the content of the resolv.conf file using the given nameservers list. -func setResolvConf(iface *net.Interface, ns []netip.Addr) error { +func (p *prog) setResolvConf(iface *net.Interface, ns []netip.Addr) error { r, err := newLoopbackOSConfigurator() if err != nil { return err @@ -27,7 +27,7 @@ func setResolvConf(iface *net.Interface, ns []netip.Addr) error { if sds, err := searchDomains(); err == nil { oc.SearchDomains = sds } else { - mainLog.Load().Debug().Err(err).Msg("failed to get search domains list when reverting resolv.conf file") + p.Debug().Err(err).Msg("Failed to get search domains list when reverting resolv.conf file") } return r.SetDNS(oc) } diff --git a/cmd/cli/resolvconf_test.go b/cmd/cli/resolvconf_test.go new file mode 100644 index 00000000..9d93607c --- /dev/null +++ b/cmd/cli/resolvconf_test.go @@ -0,0 +1,51 @@ +//go:build unix + +package cli + +import ( + "os" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld/internal/dns/resolvconffile" +) + +func oldParseResolvConfNameservers(path string) ([]string, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + // Parse the file for "nameserver" lines + var currentNS []string + lines := strings.Split(string(content), "\n") + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "nameserver") { + parts := strings.Fields(trimmed) + if len(parts) >= 2 { + currentNS = append(currentNS, parts[1]) + } + } + } + + return currentNS, nil +} + +// Test_prog_parseResolvConfNameservers tests the parsing of nameservers from resolv.conf content. +// Note: The previous implementation was removed to reduce code duplication and consolidate +// the resolv.conf handling logic into a single unified approach. All resolv.conf parsing +// is now handled by the resolvconffile package, which provides a consistent interface +// for both reading and modifying resolv.conf files across different platforms. +func Test_prog_parseResolvConfNameservers(t *testing.T) { + oldNss, _ := oldParseResolvConfNameservers(resolvconffile.Path) + p := &prog{} + nss, _ := p.parseResolvConfNameservers(resolvconffile.Path) + slices.Sort(oldNss) + slices.Sort(nss) + if !slices.Equal(oldNss, nss) { + t.Errorf("result mismatched, old: %v, new: %v", oldNss, nss) + } + t.Logf("result: %v", nss) +} diff --git a/cmd/cli/resolvconf_windows.go b/cmd/cli/resolvconf_windows.go index 3e4ba1c0..20a984fe 100644 --- a/cmd/cli/resolvconf_windows.go +++ b/cmd/cli/resolvconf_windows.go @@ -6,7 +6,7 @@ import ( ) // setResolvConf sets the content of resolv.conf file using the given nameservers list. -func setResolvConf(_ *net.Interface, _ []netip.Addr) error { +func (p *prog) setResolvConf(_ *net.Interface, _ []netip.Addr) error { return nil } diff --git a/cmd/cli/restore_lock_test.go b/cmd/cli/restore_lock_test.go new file mode 100644 index 00000000..75850b44 --- /dev/null +++ b/cmd/cli/restore_lock_test.go @@ -0,0 +1,52 @@ +package cli + +import ( + "sync/atomic" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func TestRestoreOSStateReleasesMutexBeforeCallbackDrain(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + entered, resume, callbackDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var startupCalls, restoreCalls atomic.Int32 + p.onStopped = []func(){func() { + select { + case <-callbackDone: + default: + t.Error("OS restoration ran before the callback drained") + } + restoreCalls.Add(1) + }} + go p.networkChangeCallback(func(*netmon.ChangeDelta) { + close(entered) + <-resume + // This callback was admitted before shutdown. It must be able to take + // osStateMu and observe the closed startup gate while shutdown joins it. + if p.startOSState(func() { startupCalls.Add(1) }) { + t.Error("startup was permitted after restoration began") + } + close(callbackDone) + })(nil) + waitNetworkShutdownSignal(t, entered, "callback entry") + restored := make(chan struct{}) + go func() { + if err := p.restoreOSState(); err != nil { + t.Error(err) + } + close(restored) + }() + waitNetworkShutdownGate(t, p) + close(resume) + waitNetworkShutdownSignal(t, callbackDone, "callback acquiring osStateMu") + waitNetworkShutdownSignal(t, restored, "OS restoration") + if err := p.restoreOSState(); err != nil { + t.Fatal(err) + } + if startupCalls.Load() != 0 || restoreCalls.Load() != 1 { + t.Fatalf("startup calls=%d, restore calls=%d", startupCalls.Load(), restoreCalls.Load()) + } +} diff --git a/cmd/cli/route_darwin.go b/cmd/cli/route_darwin.go new file mode 100644 index 00000000..15d81db4 --- /dev/null +++ b/cmd/cli/route_darwin.go @@ -0,0 +1,158 @@ +package cli + +import ( + "net" + "net/netip" + "syscall" + + "golang.org/x/net/route" +) + +// routeInterfaceNameFn names the interface of one route table index. A test +// replaces it, because the interfaces of the host are not a fixture. +var routeInterfaceNameFn = interfaceNameByIndex + +// defaultRoutes reads the default route of each family. It reads the kernel +// route table, because a snapshot follows every network change and a +// subprocess per change costs too much on a busy host. A table that the kernel +// does not give leaves both routes empty. +func defaultRoutes() (v4, v6 defaultRoute) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeRoute, 0) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + messages, err := route.ParseRIB(route.RIBTypeRoute, rib) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + return defaultRoutesFromMessages(messages) +} + +// defaultRoutesFromMessages picks the first default route of each family. The +// next hop and the interface come from one message, so a reader sees the pair +// that the kernel holds. The kernel keeps one unscoped default per family, so +// the first match is the only one. +func defaultRoutesFromMessages(messages []route.Message) (v4, v6 defaultRoute) { + foundV4, foundV6 := false, false + for _, message := range messages { + match, ok := matchDefaultRoute(message) + if !ok { + continue + } + found := defaultRoute{Interface: routeInterfaceNameFn(match.index)} + if match.gateway.IsValid() { + found.Gateway = match.gateway.String() + } + if match.v4 && !foundV4 { + v4, foundV4 = found, true + } + if !match.v4 && !foundV6 { + v6, foundV6 = found, true + } + if foundV4 && foundV6 { + return v4, v6 + } + } + return v4, v6 +} + +// defaultRouteMatch is one default route of the table. A tunnel default has +// no next hop, so gateway can be invalid while the interface is set. +type defaultRouteMatch struct { + v4 bool + gateway netip.Addr + index int +} + +// defaultRouteGateway reports the next hop and the interface index of one +// default route, for the tests of the selector. See matchDefaultRoute. +func defaultRouteGateway(message route.Message) (netip.Addr, int, bool) { + match, ok := matchDefaultRoute(message) + return match.gateway, match.index, ok +} + +// matchDefaultRoute reports whether one message is the default route of the +// host for its family. A route to another destination or to a part of the +// address space gives nothing: a VPN splits the default into two /1 routes, +// and the zero address alone does not make a route the default. A route bound +// to one interface serves that interface only, a host route is no default, and +// a reject or blackhole route delivers nothing. With the gateway flag, the +// next hop must be a host of the same family. Without it, the link itself +// delivers, which is how a tunnel owns the default; the match then has no +// gateway and names the interface. +func matchDefaultRoute(message route.Message) (defaultRouteMatch, bool) { + routeMessage, ok := message.(*route.RouteMessage) + if !ok || len(routeMessage.Addrs) <= syscall.RTAX_NETMASK { + return defaultRouteMatch{}, false + } + if routeMessage.Flags&(syscall.RTF_IFSCOPE|syscall.RTF_HOST|syscall.RTF_REJECT|syscall.RTF_BLACKHOLE) != 0 { + return defaultRouteMatch{}, false + } + destination, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_DST]) + if !ok || !destination.IsUnspecified() || !zeroNetmask(routeMessage.Addrs[syscall.RTAX_NETMASK], destination.Is4()) { + return defaultRouteMatch{}, false + } + match := defaultRouteMatch{v4: destination.Is4(), index: routeMessage.Index} + if routeMessage.Flags&syscall.RTF_GATEWAY == 0 { + if _, link := routeMessage.Addrs[syscall.RTAX_GATEWAY].(*route.LinkAddr); !link { + return defaultRouteMatch{}, false + } + return match, true + } + gateway, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_GATEWAY]) + if !ok || gateway.Is4() != destination.Is4() { + return defaultRouteMatch{}, false + } + match.gateway = gateway + return match, true +} + +// zeroNetmask reports whether the netmask of a route covers the whole address +// space of its family, which only the default route does. +func zeroNetmask(addr route.Addr, v4 bool) bool { + switch mask := addr.(type) { + case *route.Inet4Addr: + return v4 && mask.IP == [4]byte{} + case *route.Inet6Addr: + return !v4 && mask.IP == [16]byte{} + } + return false +} + +func routeMessageAddr(addr route.Addr) (netip.Addr, bool) { + switch address := addr.(type) { + case *route.Inet4Addr: + return netip.AddrFrom4(address.IP), true + case *route.Inet6Addr: + return withInterfaceZone(netip.AddrFrom16(address.IP), address.ZoneID), true + } + return netip.Addr{}, false +} + +// withInterfaceZone names the interface of a link-local next hop. Every link +// repeats the same range, so the address alone names no host. A tunnel gives +// the bare address fe80:: and the interface is the only part that tells the +// links apart. +func withInterfaceZone(address netip.Addr, zoneID int) netip.Addr { + if zoneID == 0 || !address.IsLinkLocalUnicast() { + return address + } + name := routeInterfaceNameFn(zoneID) + if name == "" { + return address + } + return address.WithZone(name) +} + +// interfaceNameByIndex names one interface of the route table. An index that +// no interface holds gives an empty name. +func interfaceNameByIndex(index int) string { + if index == 0 { + return "" + } + iface, err := net.InterfaceByIndex(index) + if err != nil { + return "" + } + return iface.Name +} diff --git a/cmd/cli/route_darwin_test.go b/cmd/cli/route_darwin_test.go new file mode 100644 index 00000000..8384cf11 --- /dev/null +++ b/cmd/cli/route_darwin_test.go @@ -0,0 +1,291 @@ +package cli + +import ( + "net/netip" + "syscall" + "testing" + + "golang.org/x/net/route" +) + +// stubRouteInterfaceNames names the interface of each index, because the +// interfaces of the host are not a fixture. +func stubRouteInterfaceNames(t *testing.T, names map[int]string) { + t.Helper() + orig := routeInterfaceNameFn + t.Cleanup(func() { routeInterfaceNameFn = orig }) + routeInterfaceNameFn = func(index int) string { return names[index] } +} + +// routeAddrs puts one destination, one netmask, and one gateway at the places +// of the route message that the kernel uses. A nil netmask leaves the slot +// empty, the shape of a host route. +func routeAddrs(destination, netmask, gateway route.Addr) []route.Addr { + addrs := make([]route.Addr, syscall.RTAX_NETMASK+1) + addrs[syscall.RTAX_DST] = destination + addrs[syscall.RTAX_GATEWAY] = gateway + addrs[syscall.RTAX_NETMASK] = netmask + return addrs +} + +func inet4(a, b, c, d byte) *route.Inet4Addr { + return &route.Inet4Addr{IP: [4]byte{a, b, c, d}} +} + +func inet6(address string, zoneID int) *route.Inet6Addr { + addr := netip.MustParseAddr(address) + return &route.Inet6Addr{IP: addr.As16(), ZoneID: zoneID} +} + +func Test_defaultRouteGateway(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{8: "utun4"}) + + for _, tt := range []struct { + name string + message route.Message + wantGateway string + wantIndex int + wantOK bool + }{ + { + name: "v4_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + wantGateway: "192.168.1.1", + wantIndex: 5, + wantOK: true, + }, + { + name: "v6_link_local_gateway_with_a_zone", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + wantGateway: "fe80::1%utun4", + wantIndex: 8, + wantOK: true, + }, + { + name: "destination_is_not_the_default_route", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "gateway_names_a_link", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 5, Name: "en0"}), + }, + }, + { + name: "gateway_family_differs_from_the_destination", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet6("fe80::1", 8)), + }, + }, + { + name: "address_list_holds_no_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: []route.Addr{inet4(0, 0, 0, 0)}, + }, + }, + { + name: "message_is_not_a_route", + message: &route.InterfaceMessage{Index: 5}, + }, + { + name: "route_has_no_gateway_flag", + message: &route.RouteMessage{ + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "route_is_bound_to_one_interface", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + gateway, index, ok := defaultRouteGateway(tt.message) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if !ok { + return + } + if gateway.String() != tt.wantGateway { + t.Errorf("gateway = %q, want %q", gateway.String(), tt.wantGateway) + } + if index != tt.wantIndex { + t.Errorf("index = %d, want %d", index, tt.wantIndex) + } + }) + } +} + +// Test_defaultRoutesFromMessages covers the table of a host with a v4 default +// route and a v6 default route on another interface. The v6 route must not +// take the interface of the v4 route. +func Test_defaultRoutesFromMessages(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0", 8: "utun4"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 9, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 0, 0, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(172, 16, 0, 1)), + }, + }) + + if want := (defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}); v4 != want { + t.Errorf("v4 route = %+v, want %+v", v4, want) + } + if want := (defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}); v6 != want { + t.Errorf("v6 route = %+v, want %+v", v6, want) + } +} + +// Test_defaultRoutesFromMessagesWithoutADefaultRoute covers a table that holds +// no default route, for example a host with no link. +func Test_defaultRoutesFromMessagesWithoutADefaultRoute(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }) + + if v4 != (defaultRoute{}) || v6 != (defaultRoute{}) { + t.Errorf("routes = %+v / %+v, want two empty routes", v4, v6) + } +} + +// Test_defaultRouteGatewayNeedsAZeroNetmask covers routes whose destination is +// the zero address but whose prefix is not /0. OpenVPN splits the default into +// two /1 routes, and a host route to 0.0.0.0 carries no mask at all. +func Test_defaultRouteGatewayNeedsAZeroNetmask(t *testing.T) { + for _, tt := range []struct { + name string + message route.Message + }{ + { + name: "half of the v4 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }, + }, + { + name: "host route to the zero address", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_HOST, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), nil, inet4(10, 8, 0, 1)), + }, + }, + { + name: "one eighth of the v6 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("e000::", 0), inet6("fe80::1", 8)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if gateway, _, ok := defaultRouteGateway(tt.message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } + v4, _ := defaultRoutesFromMessages([]route.Message{&route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }}) + if v4.Gateway != "" || v4.Interface != "" { + t.Fatalf("a table with one /1 route reported the default route %+v, want none", v4) + } +} + +// Test_defaultRoutesTakeALinkDefault covers a VPN that owns the default +// route. Its default has no next hop: the gateway is the link itself and the +// gateway flag is off. The snapshot must name that interface, with an empty +// gateway, instead of the interface of the network monitor. +func Test_defaultRoutesTakeALinkDefault(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{28: "utun4", 16: "en0"}) + linkDefault := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_STATIC, + Index: 28, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 28, Name: "utun4"}), + } + scopedEn0 := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 16, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 50, 1)), + } + + v4, _ := defaultRoutesFromMessages([]route.Message{linkDefault, scopedEn0}) + if v4.Interface != "utun4" || v4.Gateway != "" { + t.Fatalf("default route = %+v, want utun4 with no gateway", v4) + } +} + +// Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes covers a kill switch +// that installs a default route to nowhere. Such a route carries the gateway +// flag, but it delivers nothing, so it is not the default route of the host. +func Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes(t *testing.T) { + for _, tt := range []struct { + name string + flags int + }{ + {"reject", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_REJECT}, + {"blackhole", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_BLACKHOLE}, + } { + t.Run(tt.name, func(t *testing.T) { + message := &route.RouteMessage{ + Flags: tt.flags, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 8, 0, 1)), + } + if gateway, _, ok := defaultRouteGateway(message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } +} diff --git a/cmd/cli/route_others.go b/cmd/cli/route_others.go new file mode 100644 index 00000000..65f1330a --- /dev/null +++ b/cmd/cli/route_others.go @@ -0,0 +1,10 @@ +//go:build !darwin + +package cli + +// defaultRoutes gives no route. Only macOS reads the route table in this +// release, so the snapshot of another system holds no gateway and no default +// route of the v6 family. +func defaultRoutes() (v4, v6 defaultRoute) { + return defaultRoute{}, defaultRoute{} +} diff --git a/cmd/cli/search_domains_windows.go b/cmd/cli/search_domains_windows.go index 320a3223..28d1bb97 100644 --- a/cmd/cli/search_domains_windows.go +++ b/cmd/cli/search_domains_windows.go @@ -33,7 +33,7 @@ func searchDomains() ([]dnsname.FQDN, error) { for a := aa.FirstDNSSuffix; a != nil; a = a.Next { d, err := dnsname.ToFQDN(a.String()) if err != nil { - mainLog.Load().Debug().Err(err).Msgf("failed to parse domain: %s", a.String()) + mainLog.Load().Debug().Err(err).Msgf("Failed to parse domain: %s", a.String()) continue } sds = append(sds, d) diff --git a/cmd/cli/self_delete_others.go b/cmd/cli/self_delete_others.go index 02ae9774..826590ec 100644 --- a/cmd/cli/self_delete_others.go +++ b/cmd/cli/self_delete_others.go @@ -4,4 +4,5 @@ package cli var supportedSelfDelete = true +// selfDeleteExe performs self-deletion on non-Windows platforms func selfDeleteExe() error { return nil } diff --git a/cmd/cli/self_delete_windows.go b/cmd/cli/self_delete_windows.go index c2f2719e..c9618a27 100644 --- a/cmd/cli/self_delete_windows.go +++ b/cmd/cli/self_delete_windows.go @@ -33,6 +33,7 @@ type FILE_DISPOSITION_INFO struct { DeleteFile bool } +// dsOpenHandle opens a handle to the specified file with DELETE access func dsOpenHandle(pwPath *uint16) (windows.Handle, error) { handle, err := windows.CreateFile( pwPath, @@ -51,6 +52,7 @@ func dsOpenHandle(pwPath *uint16) (windows.Handle, error) { return handle, nil } +// dsRenameHandle renames a file handle to a stream name func dsRenameHandle(hHandle windows.Handle) error { var fRename FILE_RENAME_INFO DS_STREAM_RENAME, err := windows.UTF16FromString(":deadbeef") @@ -82,6 +84,7 @@ func dsRenameHandle(hHandle windows.Handle) error { return nil } +// dsDepositeHandle marks a file handle for deletion func dsDepositeHandle(hHandle windows.Handle) error { var fDelete FILE_DISPOSITION_INFO fDelete.DeleteFile = true @@ -100,6 +103,7 @@ func dsDepositeHandle(hHandle windows.Handle) error { return nil } +// selfDeleteExe performs self-deletion on Windows platforms func selfDeleteExe() error { var wcPath [windows.MAX_PATH + 1]uint16 var hCurrent windows.Handle diff --git a/cmd/cli/self_kill_others.go b/cmd/cli/self_kill_others.go index e9fb1f8f..4f32d6f8 100644 --- a/cmd/cli/self_kill_others.go +++ b/cmd/cli/self_kill_others.go @@ -5,12 +5,13 @@ package cli import ( "os" - "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" ) -func selfUninstall(p *prog, logger zerolog.Logger) { +// selfUninstall performs self-uninstallation on non-Unix platforms +func selfUninstall(p *prog, logger *ctrld.Logger) { if uninstallInvalidCdUID(p, logger, false) { - logger.Warn().Msgf("service was uninstalled because device %q does not exist", cdUID) + logger.Warn().Msgf("Service was uninstalled because device %q does not exist", cdUID) os.Exit(0) } } diff --git a/cmd/cli/self_kill_unix.go b/cmd/cli/self_kill_unix.go index 157425fd..70c7c08d 100644 --- a/cmd/cli/self_kill_unix.go +++ b/cmd/cli/self_kill_unix.go @@ -9,17 +9,18 @@ import ( "runtime" "syscall" - "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" ) -func selfUninstall(p *prog, logger zerolog.Logger) { +// selfUninstall performs self-uninstallation on Unix platforms +func selfUninstall(p *prog, logger *ctrld.Logger) { if runtime.GOOS == "linux" { selfUninstallLinux(p, logger) } bin, err := os.Executable() if err != nil { - logger.Fatal().Err(err).Msg("could not determine executable") + logger.Fatal().Err(err).Msg("Could not determine executable") } args := []string{"uninstall"} if deactivationPinSet() { @@ -28,18 +29,19 @@ func selfUninstall(p *prog, logger zerolog.Logger) { cmd := exec.Command(bin, args...) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} if err := cmd.Start(); err != nil { - logger.Fatal().Err(err).Msg("could not start self uninstall command") + logger.Fatal().Err(err).Msg("Could not start self uninstall command") } cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - logger.Warn().Msgf("service was uninstalled because device %q does not exist", cdUID) + logger.Warn().Msgf("Service was uninstalled because device %q does not exist", cdUID) _ = cmd.Wait() os.Exit(0) } -func selfUninstallLinux(p *prog, logger zerolog.Logger) { +// selfUninstallLinux performs self-uninstallation on Linux platforms +func selfUninstallLinux(p *prog, logger *ctrld.Logger) { if uninstallInvalidCdUID(p, logger, true) { - logger.Warn().Msgf("service was uninstalled because device %q does not exist", cdUID) + logger.Warn().Msgf("Service was uninstalled because device %q does not exist", cdUID) os.Exit(0) } } diff --git a/cmd/cli/sema.go b/cmd/cli/sema.go index 92b6ce0f..4285eaf4 100644 --- a/cmd/cli/sema.go +++ b/cmd/cli/sema.go @@ -1,24 +1,31 @@ package cli +// semaphore provides a simple synchronization mechanism type semaphore interface { acquire() release() } +// noopSemaphore is a no-operation implementation of semaphore type noopSemaphore struct{} +// acquire performs a no-operation for the noop semaphore func (n noopSemaphore) acquire() {} +// release performs a no-operation for the noop semaphore func (n noopSemaphore) release() {} +// chanSemaphore is a channel-based implementation of semaphore type chanSemaphore struct { ready chan struct{} } +// acquire blocks until a slot is available in the semaphore func (c *chanSemaphore) acquire() { c.ready <- struct{}{} } +// release signals that a slot has been freed in the semaphore func (c *chanSemaphore) release() { <-c.ready } diff --git a/cmd/cli/service.go b/cmd/cli/service.go index f75ee558..7fd47e6a 100644 --- a/cmd/cli/service.go +++ b/cmd/cli/service.go @@ -11,9 +11,6 @@ import ( "github.com/coreos/go-systemd/v22/unit" "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld/internal/router" - "github.com/Control-D-Inc/ctrld/internal/router/openwrt" ) // newService wraps service.New call to return service.Service @@ -24,10 +21,6 @@ func newService(i service.Interface, c *service.Config) (service.Service, error) return nil, err } switch { - case router.IsOldOpenwrt(), router.IsNetGearOrbi(): - return &procd{sysV: &sysV{s}, svcConfig: c}, nil - case router.IsGLiNet(): - return &sysV{s}, nil case s.Platform() == "unix-systemv": return &sysV{s}, nil case s.Platform() == "linux-systemd": @@ -42,13 +35,13 @@ func newService(i service.Interface, c *service.Config) (service.Service, error) // sysV wraps a service.Service, and provide start/stop/status command // base on "/etc/init.d/". // -// Use this on system where "service" command is not available, like GL.iNET router. +// Use this on system where "service" command is not available. type sysV struct { service.Service } func (s *sysV) installed() bool { - fi, err := os.Stat("/etc/init.d/ctrld") + fi, err := os.Stat(sysVInitScript) if err != nil { return false } @@ -60,7 +53,7 @@ func (s *sysV) Start() error { if !s.installed() { return service.ErrNotInstalled } - _, err := exec.Command("/etc/init.d/ctrld", "start").CombinedOutput() + _, err := exec.Command(sysVInitScript, "start").CombinedOutput() return err } @@ -68,7 +61,7 @@ func (s *sysV) Stop() error { if !s.installed() { return service.ErrNotInstalled } - _, err := exec.Command("/etc/init.d/ctrld", "stop").CombinedOutput() + _, err := exec.Command(sysVInitScript, "stop").CombinedOutput() return err } @@ -89,37 +82,6 @@ func (s *sysV) Status() (service.Status, error) { return unixSystemVServiceStatus() } -// procd wraps a service.Service, and provide start/stop command -// base on "/etc/init.d/", status command base on parsing "ps" command output. -// -// Use this on system where "/etc/init.d/ status" command is not available, -// like old GL.iNET Opal router. -type procd struct { - *sysV - svcConfig *service.Config -} - -func (s *procd) Status() (service.Status, error) { - if !s.installed() { - return service.StatusUnknown, service.ErrNotInstalled - } - bin := s.svcConfig.Executable - if bin == "" { - exe, err := os.Executable() - if err != nil { - return service.StatusUnknown, nil - } - bin = exe - } - - // Looking for something like "/sbin/ctrld run ". - shellCmd := fmt.Sprintf("ps | grep -q %q", bin+" [r]un ") - if err := exec.Command("sh", "-c", shellCmd).Run(); err != nil { - return service.StatusStopped, nil - } - return service.StatusRunning, nil -} - // systemd wraps a service.Service, and provide status command to // report the status correctly. type systemd struct { @@ -127,7 +89,7 @@ type systemd struct { } func (s *systemd) Status() (service.Status, error) { - out, _ := exec.Command("systemctl", "status", "ctrld").CombinedOutput() + out, _ := exec.Command("systemctl", "status", ctrldServiceName).CombinedOutput() if bytes.Contains(out, []byte("/FAILURE)")) { return service.StatusStopped, nil } @@ -135,7 +97,6 @@ func (s *systemd) Status() (service.Status, error) { } func (s *systemd) Start() error { - const systemdUnitFile = "/etc/systemd/system/ctrld.service" f, err := os.Open(systemdUnitFile) if err != nil { return err @@ -153,7 +114,7 @@ func (s *systemd) Start() error { if out, err := exec.Command("systemctl", "daemon-reload").CombinedOutput(); err != nil { return fmt.Errorf("systemctl daemon-reload failed: %w\n%s", err, string(out)) } - mainLog.Load().Debug().Msg("set KillMode=process successfully") + mainLog.Load().Debug().Msg("Set KillMode=process successfully") } return s.Service.Start() } @@ -162,8 +123,10 @@ func (s *systemd) Start() error { // This is necessary for running self-upgrade flow. func ensureSystemdKillMode(r io.Reader) (opts []*unit.UnitOption, change bool) { opts, err := unit.DeserializeOptions(r) + // On success the lexer sends nothing and closes the channel, so the receive + // yields a nil error and this branch is not taken. if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to deserialize options") + mainLog.Load().Error().Err(err).Msg("Failed to deserialize options") return } change = true @@ -187,6 +150,7 @@ func ensureSystemdKillMode(r io.Reader) (opts []*unit.UnitOption, change bool) { return opts, change } +// newLaunchd creates a new launchd service wrapper func newLaunchd(s service.Service) *launchd { return &launchd{ Service: s, @@ -216,28 +180,38 @@ type task struct { Name string } -func doTasks(tasks []task) bool { - for _, task := range tasks { - mainLog.Load().Debug().Msgf("Running task %s", task.Name) - if err := task.f(); err != nil { - if task.abortOnError { - mainLog.Load().Error().Msgf("error running task %s: %v", task.Name, err) - return false +// doTasksE runs tasks in order and reports which abortOnError task, if any, +// stopped the run. Use it over doTasks when the failure must be attributed +// to a specific task. +func doTasksE(tasks []task) (failedTaskName string, err error) { + for _, t := range tasks { + mainLog.Load().Debug().Msgf("Running task %s", t.Name) + if taskErr := t.f(); taskErr != nil { + if t.abortOnError { + mainLog.Load().Error().Msgf("Error running task %s: %v", t.Name, taskErr) + return t.Name, taskErr } // if this is darwin stop command, dont print debug // since launchctl complains on every start - if runtime.GOOS != "darwin" || task.Name != "Stop" { - mainLog.Load().Debug().Msgf("error running task %s: %v", task.Name, err) + if runtime.GOOS != "darwin" || t.Name != "Stop" { + mainLog.Load().Debug().Msgf("Error running task %s: %v", t.Name, taskErr) } } } - return true + return "", nil } +// doTasks executes a list of tasks and returns success status +func doTasks(tasks []task) bool { + _, err := doTasksE(tasks) + return err == nil +} + +// checkHasElevatedPrivilege checks if the process has elevated privileges and exits if not func checkHasElevatedPrivilege() { ok, err := hasElevatedPrivilege() if err != nil { - mainLog.Load().Error().Msgf("could not detect user privilege: %v", err) + mainLog.Load().Error().Msgf("Could not detect user privilege: %v", err) return } if !ok { @@ -246,16 +220,10 @@ func checkHasElevatedPrivilege() { } } +// unixSystemVServiceStatus checks the status of a Unix System V service func unixSystemVServiceStatus() (service.Status, error) { - out, err := exec.Command("/etc/init.d/ctrld", "status").CombinedOutput() + out, err := exec.Command(sysVInitScript, "status").CombinedOutput() if err != nil { - // Specific case for openwrt >= 24.10, it returns non-success code - // for above status command, which may not right. - if router.Name() == openwrt.Name { - if string(bytes.ToLower(bytes.TrimSpace(out))) == "inactive" { - return service.StatusStopped, nil - } - } return service.StatusUnknown, nil } diff --git a/cmd/cli/service_args_darwin.go b/cmd/cli/service_args_darwin.go new file mode 100644 index 00000000..65f18ba7 --- /dev/null +++ b/cmd/cli/service_args_darwin.go @@ -0,0 +1,157 @@ +//go:build darwin + +package cli + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +const launchdPlistPath = launchdPlistFile + +// serviceConfigFileExists returns true if the launchd plist for ctrld exists on disk. +// This is more reliable than checking launchctl status, which may report "not found" +// if the service was unloaded but the plist file still exists. +func serviceConfigFileExists() bool { + _, err := os.Stat(launchdPlistPath) + return err == nil +} + +// appendServiceFlag appends a CLI flag (e.g., "--intercept-mode") to the installed +// service's launch arguments. This is used when upgrading an existing installation +// to intercept mode without losing the existing --cd flag and other arguments. +// +// On macOS, this modifies the launchd plist named after ctrldServiceName +// using PlistBuddy for exact array reads and writes. +// +// The function is idempotent: if the flag already exists, it's a no-op. +func appendServiceFlag(flag string) error { + // Read current ProgramArguments from plist. + out, err := exec.Command("/usr/libexec/PlistBuddy", "-c", "Print :ProgramArguments", launchdPlistPath).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to read plist ProgramArguments: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + + // Check exact array entries. A substring match can confuse a mode such as "off" + // with an unrelated path or argument and leave the flag without its value. + if serviceArgumentPresent(out, flag) { + mainLog.Load().Debug().Msgf("Service flag %q already present in plist, skipping", flag) + return nil + } + + // Use PlistBuddy to append the flag to ProgramArguments array. + // PlistBuddy is more reliable than "defaults" for array manipulation. + addCmd := exec.Command( + "/usr/libexec/PlistBuddy", + "-c", fmt.Sprintf("Add :ProgramArguments: string %s", flag), + launchdPlistPath, + ) + if out, err := addCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to append %q to plist ProgramArguments: %w (output: %s)", flag, err, strings.TrimSpace(string(out))) + } + + mainLog.Load().Info().Msgf("Appended %q to service launch arguments", flag) + return nil +} + +// verifyServiceRegistration is a no-op on macOS (launchd plist verification not needed). +func verifyServiceRegistration() error { + return nil +} + +// removeServiceFlag removes both "--flag value" and "--flag=value" forms from the +// installed service's launch arguments. +// +// The function is idempotent: if the flag doesn't exist, it's a no-op. +func removeServiceFlag(flag string) error { + // Read current ProgramArguments to find the index. + out, err := exec.Command("/usr/libexec/PlistBuddy", "-c", "Print :ProgramArguments", launchdPlistPath).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to read plist ProgramArguments: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + + index, hasValue := serviceFlagPosition(plistArrayEntries(out), flag) + + if index < 0 { + mainLog.Load().Debug().Msgf("Service flag %q not present in plist, skipping removal", flag) + return nil + } + + // Delete a separate value first. An inline --flag=value entry is one array item. + if hasValue { + delVal := exec.Command( + "/usr/libexec/PlistBuddy", + "-c", fmt.Sprintf("Delete :ProgramArguments:%d", index+1), + launchdPlistPath, + ) + if out, err := delVal.CombinedOutput(); err != nil { + return fmt.Errorf("failed to remove value for %q from plist: %w (output: %s)", flag, err, strings.TrimSpace(string(out))) + } + } + + // Delete the flag itself. + delCmd := exec.Command( + "/usr/libexec/PlistBuddy", + "-c", fmt.Sprintf("Delete :ProgramArguments:%d", index), + launchdPlistPath, + ) + if out, err := delCmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to remove %q from plist ProgramArguments: %w (output: %s)", flag, err, strings.TrimSpace(string(out))) + } + + mainLog.Load().Info().Msgf("Removed %q from service launch arguments", flag) + return nil +} + +// installedServiceArgs returns the launch arguments of the installed service. +func installedServiceArgs() ([]string, error) { + out, err := exec.Command("/usr/libexec/PlistBuddy", "-c", "Print :ProgramArguments", launchdPlistPath).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to read plist ProgramArguments: %w (output: %s)", err, strings.TrimSpace(string(out))) + } + return plistArrayEntries(out), nil +} + +// plistArrayEntries parses the entries of an array PlistBuddy printed: +// +// Array { +// /path/to/ctrld +// run +// --cd=xxx +// --intercept-mode +// dns +// } +func plistArrayEntries(out []byte) []string { + var entries []string + for _, line := range strings.Split(string(out), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "Array {" || trimmed == "}" || trimmed == "" { + continue + } + entries = append(entries, trimmed) + } + return entries +} + +func serviceArgumentPresent(out []byte, argument string) bool { + for _, line := range strings.Split(string(out), "\n") { + if strings.TrimSpace(line) == argument { + return true + } + } + return false +} + +func serviceFlagPosition(entries []string, flag string) (index int, hasValue bool) { + for i, entry := range entries { + switch { + case entry == flag: + return i, i+1 < len(entries) && !strings.HasPrefix(entries[i+1], "-") + case strings.HasPrefix(entry, flag+"="): + return i, false + } + } + return -1, false +} diff --git a/cmd/cli/service_args_darwin_test.go b/cmd/cli/service_args_darwin_test.go new file mode 100644 index 00000000..1e535f07 --- /dev/null +++ b/cmd/cli/service_args_darwin_test.go @@ -0,0 +1,63 @@ +//go:build darwin + +package cli + +import "testing" + +func TestServiceArgumentPresent(t *testing.T) { + out := []byte("Array {\n /usr/local/bin/ctrld\n run\n --config=/Users/officer/ctrld.toml\n --intercept-mode=dns\n}\n") + if !serviceArgumentPresent(out, "--intercept-mode=dns") { + t.Fatal("exact inline argument was not found") + } + if serviceArgumentPresent(out, "--intercept-mode") { + t.Fatal("inline flag was mistaken for a separate flag argument") + } + if serviceArgumentPresent(out, "off") { + t.Fatal("substring in an unrelated path was mistaken for the off argument") + } + + splitOut := []byte("Array {\n /usr/local/bin/ctrld\n run\n --intercept-mode\n dns\n}\n") + if !serviceArgumentPresent(splitOut, "--intercept-mode") { + t.Fatal("standalone flag argument was not found") + } +} + +func TestServiceFlagPosition(t *testing.T) { + tests := []struct { + name string + entries []string + wantIndex int + wantHasValue bool + }{ + { + name: "split form", + entries: []string{"run", "--cd=uid", "--intercept-mode", "dns"}, + wantIndex: 2, + wantHasValue: true, + }, + { + name: "inline form", + entries: []string{"run", "--cd=uid", "--intercept-mode=dns"}, + wantIndex: 2, + }, + { + name: "flag followed by another flag", + entries: []string{"run", "--intercept-mode", "--config=/etc/ctrld.toml"}, + wantIndex: 1, + }, + { + name: "absent", + entries: []string{"run", "--cd=uid"}, + wantIndex: -1, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + index, hasValue := serviceFlagPosition(tc.entries, "--intercept-mode") + if index != tc.wantIndex || hasValue != tc.wantHasValue { + t.Fatalf("serviceFlagPosition() = (%d, %v), want (%d, %v)", index, hasValue, tc.wantIndex, tc.wantHasValue) + } + }) + } +} diff --git a/cmd/cli/service_args_others.go b/cmd/cli/service_args_others.go new file mode 100644 index 00000000..3ef762e4 --- /dev/null +++ b/cmd/cli/service_args_others.go @@ -0,0 +1,48 @@ +//go:build !darwin && !windows + +package cli + +import ( + "errors" + "os" +) + +// errServiceFlagsUnsupported is returned by the service-argument helpers on +// platforms that do not store service arguments in a file ctrld can rewrite. +var errServiceFlagsUnsupported = errors.New("modifying service flags is not supported on this platform; use intercept_mode in config instead") + +// serviceConfigFileExists checks common service config file locations on Linux. +func serviceConfigFileExists() bool { + // systemd unit file + if _, err := os.Stat(systemdUnitFile); err == nil { + return true + } + // SysV init script + if _, err := os.Stat(sysVInitScript); err == nil { + return true + } + return false +} + +// appendServiceFlag is not yet implemented on this platform. +// Linux services (systemd) store args in unit files; intercept mode +// should be set via the config file (intercept_mode) on these platforms. +func appendServiceFlag(flag string) error { + return errServiceFlagsUnsupported +} + +// verifyServiceRegistration is a no-op on this platform. +func verifyServiceRegistration() error { + return nil +} + +// installedServiceArgs is not implemented on this platform. It has no dns +// intercept default, so installInterceptMode never needs the arguments here. +func installedServiceArgs() ([]string, error) { + return nil, errServiceFlagsUnsupported +} + +// removeServiceFlag is not yet implemented on this platform. +func removeServiceFlag(flag string) error { + return errServiceFlagsUnsupported +} diff --git a/cmd/cli/service_args_windows.go b/cmd/cli/service_args_windows.go new file mode 100644 index 00000000..1149e0a4 --- /dev/null +++ b/cmd/cli/service_args_windows.go @@ -0,0 +1,191 @@ +//go:build windows + +package cli + +import ( + "fmt" + "strings" + + "golang.org/x/sys/windows/svc/mgr" +) + +// serviceConfigFileExists returns true if the ctrld Windows service is registered. +func serviceConfigFileExists() bool { + m, err := mgr.Connect() + if err != nil { + return false + } + defer m.Disconnect() + s, err := m.OpenService(ctrldServiceName) + if err != nil { + return false + } + s.Close() + return true +} + +// appendServiceFlag appends a CLI flag (e.g., "--intercept-mode") to the installed +// Windows service's BinPath arguments. This is used when upgrading an existing +// installation to intercept mode without losing the existing --cd flag. +// +// The function is idempotent: if the flag already exists, it's a no-op. +func appendServiceFlag(flag string) error { + m, err := mgr.Connect() + if err != nil { + return fmt.Errorf("failed to connect to Windows SCM: %w", err) + } + defer m.Disconnect() + + s, err := m.OpenService(ctrldServiceName) + if err != nil { + return fmt.Errorf("failed to open service %q: %w", ctrldServiceName, err) + } + defer s.Close() + + config, err := s.Config() + if err != nil { + return fmt.Errorf("failed to read service config: %w", err) + } + + // Check exact arguments so a short mode such as "off" is not confused with + // an unrelated path or value. + if binaryPathArgumentPresent(config.BinaryPathName, flag) { + mainLog.Load().Debug().Msgf("Service flag %q already present in BinPath, skipping", flag) + return nil + } + + // Append the flag to BinPath. + config.BinaryPathName = strings.TrimSpace(config.BinaryPathName) + " " + flag + + if err := s.UpdateConfig(config); err != nil { + return fmt.Errorf("failed to update service config with %q: %w", flag, err) + } + + mainLog.Load().Info().Msgf("Appended %q to service BinPath", flag) + return nil +} + +// verifyServiceRegistration opens the Windows Service Control Manager and verifies +// that the ctrld service is correctly registered: logs the BinaryPathName, checks +// that --intercept-mode is present if expected, and verifies SERVICE_AUTO_START. +func verifyServiceRegistration() error { + m, err := mgr.Connect() + if err != nil { + return fmt.Errorf("failed to connect to Windows SCM: %w", err) + } + defer m.Disconnect() + + s, err := m.OpenService(ctrldServiceName) + if err != nil { + return fmt.Errorf("failed to open service %q: %w", ctrldServiceName, err) + } + defer s.Close() + + config, err := s.Config() + if err != nil { + return fmt.Errorf("failed to read service config: %w", err) + } + + mainLog.Load().Debug().Msgf("Service registry: BinaryPathName = %q", config.BinaryPathName) + + // If intercept mode is set, verify the flag is present in BinPath. + if interceptMode == "off" || interceptMode == "dns" || interceptMode == "hard" { + if !strings.Contains(config.BinaryPathName, "--intercept-mode") { + return fmt.Errorf("service registry: --intercept-mode flag missing from BinaryPathName (expected mode %q)", interceptMode) + } + mainLog.Load().Debug().Msgf("Service registry: --intercept-mode flag present in BinaryPathName") + } + + // Verify auto-start. mgr.StartAutomatic == 2 == SERVICE_AUTO_START. + if config.StartType != mgr.StartAutomatic { + return fmt.Errorf("service registry: StartType is %d, expected SERVICE_AUTO_START (%d)", config.StartType, mgr.StartAutomatic) + } + + return nil +} + +// removeServiceFlag removes both "--flag value" and "--flag=value" forms from the +// installed Windows service's BinPath. The function is idempotent. +func removeServiceFlag(flag string) error { + m, err := mgr.Connect() + if err != nil { + return fmt.Errorf("failed to connect to Windows SCM: %w", err) + } + defer m.Disconnect() + + s, err := m.OpenService(ctrldServiceName) + if err != nil { + return fmt.Errorf("failed to open service %q: %w", ctrldServiceName, err) + } + defer s.Close() + + config, err := s.Config() + if err != nil { + return fmt.Errorf("failed to read service config: %w", err) + } + + updatedPath, removed := removeBinaryPathFlag(config.BinaryPathName, flag) + if !removed { + mainLog.Load().Debug().Msgf("Service flag %q not present in BinPath, skipping removal", flag) + return nil + } + config.BinaryPathName = updatedPath + + if err := s.UpdateConfig(config); err != nil { + return fmt.Errorf("failed to update service config: %w", err) + } + + mainLog.Load().Info().Msgf("Removed %q from service BinPath", flag) + return nil +} + +// installedServiceArgs returns the launch arguments of the installed service, +// split from its binary path the way the other service-argument helpers do. +func installedServiceArgs() ([]string, error) { + m, err := mgr.Connect() + if err != nil { + return nil, fmt.Errorf("failed to connect to Windows SCM: %w", err) + } + defer m.Disconnect() + + s, err := m.OpenService(ctrldServiceName) + if err != nil { + return nil, fmt.Errorf("failed to open service %q: %w", ctrldServiceName, err) + } + defer s.Close() + + config, err := s.Config() + if err != nil { + return nil, fmt.Errorf("failed to read service config: %w", err) + } + return strings.Fields(config.BinaryPathName), nil +} + +func binaryPathArgumentPresent(binaryPath, argument string) bool { + for _, part := range strings.Fields(binaryPath) { + if part == argument { + return true + } + } + return false +} + +func removeBinaryPathFlag(binaryPath, flag string) (string, bool) { + parts := strings.Fields(binaryPath) + newParts := make([]string, 0, len(parts)) + removed := false + for i := 0; i < len(parts); i++ { + switch { + case parts[i] == flag: + removed = true + if i+1 < len(parts) && !strings.HasPrefix(parts[i+1], "-") { + i++ + } + case strings.HasPrefix(parts[i], flag+"="): + removed = true + default: + newParts = append(newParts, parts[i]) + } + } + return strings.Join(newParts, " "), removed +} diff --git a/cmd/cli/service_args_windows_test.go b/cmd/cli/service_args_windows_test.go new file mode 100644 index 00000000..ed8e9154 --- /dev/null +++ b/cmd/cli/service_args_windows_test.go @@ -0,0 +1,54 @@ +//go:build windows + +package cli + +import "testing" + +func TestBinaryPathArgumentPresent(t *testing.T) { + path := `C:\ControlD\ctrld.exe run --config=C:\Users\officer\ctrld.toml --intercept-mode=dns` + if !binaryPathArgumentPresent(path, "--intercept-mode=dns") { + t.Fatal("exact inline argument was not found") + } + if binaryPathArgumentPresent(path, "--intercept-mode") { + t.Fatal("inline flag was mistaken for a separate flag argument") + } + if binaryPathArgumentPresent(path, "off") { + t.Fatal("substring in an unrelated path was mistaken for the off argument") + } +} + +func TestRemoveBinaryPathFlag(t *testing.T) { + tests := []struct { + name string + binaryPath string + wantPath string + wantRemoved bool + }{ + { + name: "split form", + binaryPath: `ctrld.exe run --cd=uid --intercept-mode dns --config=ctrld.toml`, + wantPath: `ctrld.exe run --cd=uid --config=ctrld.toml`, + wantRemoved: true, + }, + { + name: "inline form", + binaryPath: `ctrld.exe run --cd=uid --intercept-mode=dns --config=ctrld.toml`, + wantPath: `ctrld.exe run --cd=uid --config=ctrld.toml`, + wantRemoved: true, + }, + { + name: "absent", + binaryPath: `ctrld.exe run --cd=uid`, + wantPath: `ctrld.exe run --cd=uid`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + path, removed := removeBinaryPathFlag(tc.binaryPath, "--intercept-mode") + if path != tc.wantPath || removed != tc.wantRemoved { + t.Fatalf("removeBinaryPathFlag() = (%q, %v), want (%q, %v)", path, removed, tc.wantPath, tc.wantRemoved) + } + }) + } +} diff --git a/cmd/cli/service_image_path.go b/cmd/cli/service_image_path.go new file mode 100644 index 00000000..406ce573 --- /dev/null +++ b/cmd/cli/service_image_path.go @@ -0,0 +1,59 @@ +package cli + +import "strings" + +// serviceBinaryFromImagePath extracts the executable path from a Windows service +// ImagePath value, which carries the command line rather than a bare path: it may be +// quoted and is usually followed by arguments, e.g. +// +// "C:\Program Files\Control D\ctrld-client.exe" run --config C:\...\ctrld.toml +// +// It returns "" when no path can be read, which callers must treat as "cannot tell" +// rather than "does not match". +func serviceBinaryFromImagePath(imagePath string) string { + imagePath = strings.TrimSpace(imagePath) + if imagePath == "" { + return "" + } + if imagePath[0] == '"' { + // Quoted form: everything up to the closing quote is the path, so a directory + // containing spaces stays intact. + if end := strings.IndexByte(imagePath[1:], '"'); end >= 0 { + return strings.TrimSpace(imagePath[1 : 1+end]) + } + return strings.TrimSpace(imagePath[1:]) + } + // Unquoted form: the path cannot contain spaces, so the first field is it. + if idx := strings.IndexByte(imagePath, ' '); idx >= 0 { + return strings.TrimSpace(imagePath[:idx]) + } + return imagePath +} + +// sameExecutableDir reports whether two Windows executable paths live in the same +// directory, compared case-insensitively because Windows paths are. +// +// The separator handling is explicit rather than filepath's, because filepath follows the +// *host* rules: off Windows it does not treat "\\" as a separator, so every backslash path +// would reduce to the same directory and any two paths would compare equal. Doing it here +// keeps the comparison correct and testable on any host. +// +// A path with no directory part answers false, which callers read as "cannot tell". +func sameExecutableDir(a, b string) bool { + dirA, dirB := windowsExecutableDir(a), windowsExecutableDir(b) + if dirA == "" || dirB == "" { + return false + } + return strings.EqualFold(dirA, dirB) +} + +// windowsExecutableDir returns the directory part of a Windows path, accepting either +// separator and normalising to a backslash. It returns "" when there is no directory part. +func windowsExecutableDir(path string) string { + path = strings.TrimSpace(path) + idx := strings.LastIndexAny(path, `\/`) + if idx <= 0 { + return "" + } + return strings.ReplaceAll(path[:idx], "/", `\`) +} diff --git a/cmd/cli/service_image_path_others.go b/cmd/cli/service_image_path_others.go new file mode 100644 index 00000000..866308b8 --- /dev/null +++ b/cmd/cli/service_image_path_others.go @@ -0,0 +1,8 @@ +//go:build !windows + +package cli + +// installedServiceDirMatches is Windows-only: it exists because socketDir() there is +// relative to the running executable. Other platforms answer this question through +// hasElevatedPrivilege in readinessVerifiable. +func installedServiceDirMatches() bool { return true } diff --git a/cmd/cli/service_image_path_test.go b/cmd/cli/service_image_path_test.go new file mode 100644 index 00000000..2fb050f5 --- /dev/null +++ b/cmd/cli/service_image_path_test.go @@ -0,0 +1,103 @@ +package cli + +import "testing" + +// TestServiceBinaryFromImagePath covers the ImagePath shapes Windows stores. Getting this +// wrong makes readinessVerifiable compare the wrong directories, and "ctrld status" would +// then report a healthy service as not-ready - the false positive the readiness exit code +// exists to avoid. +func TestServiceBinaryFromImagePath(t *testing.T) { + tests := []struct { + name string + imagePath string + want string + }{ + { + // The installed form: quoted because the directory contains a space, with the + // service arguments following it. + name: "quoted path with arguments", + imagePath: `"C:\Program Files\Control D\ctrld.exe" run --config "C:\ProgramData\Control D\ctrld.toml"`, + want: `C:\Program Files\Control D\ctrld.exe`, + }, + { + name: "quoted path without arguments", + imagePath: `"C:\Program Files\Control D\ctrld.exe"`, + want: `C:\Program Files\Control D\ctrld.exe`, + }, + { + name: "unquoted path with arguments", + imagePath: `C:\ctrld\ctrld.exe run --cd abc123`, + want: `C:\ctrld\ctrld.exe`, + }, + { + name: "unquoted path alone", + imagePath: `C:\ctrld\ctrld.exe`, + want: `C:\ctrld\ctrld.exe`, + }, + { + name: "surrounding whitespace", + imagePath: ` "C:\ctrld\ctrld.exe" run `, + want: `C:\ctrld\ctrld.exe`, + }, + { + // Unterminated quote: take what is there rather than returning nothing, since + // "" means "cannot tell" and would silently disable the check. + name: "unterminated quote", + imagePath: `"C:\ctrld\ctrld.exe run`, + want: `C:\ctrld\ctrld.exe run`, + }, + { + name: "empty", + imagePath: "", + want: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := serviceBinaryFromImagePath(tc.imagePath); got != tc.want { + t.Errorf("serviceBinaryFromImagePath(%q) = %q, want %q", tc.imagePath, got, tc.want) + } + }) + } +} + +// TestSameExecutableDir pins the comparison itself: Windows paths are case-insensitive, and +// an empty side means "cannot tell", which must never read as a match. +func TestSameExecutableDir(t *testing.T) { + tests := []struct { + name string + a string + b string + want bool + }{ + { + name: "same directory", + a: `C:\Program Files\Control D\ctrld.exe`, + b: `C:\Program Files\Control D\ctrld.exe`, + want: true, + }, + { + name: "same directory different case", + a: `C:\Program Files\Control D\ctrld.exe`, + b: `c:\program files\control d\ctrld.exe`, + want: true, + }, + { + // The case the check exists for: a copy run from a download directory + // resolves a different control socket than the installed service. + name: "different directory", + a: `C:\Program Files\Control D\ctrld.exe`, + b: `C:\Users\admin\Downloads\ctrld.exe`, + want: false, + }, + {name: "unknown installed path", a: "", b: `C:\ctrld\ctrld.exe`, want: false}, + {name: "unknown self path", a: `C:\ctrld\ctrld.exe`, b: "", want: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sameExecutableDir(tc.a, tc.b); got != tc.want { + t.Errorf("sameExecutableDir(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + }) + } +} diff --git a/cmd/cli/service_image_path_windows.go b/cmd/cli/service_image_path_windows.go new file mode 100644 index 00000000..a03456c7 --- /dev/null +++ b/cmd/cli/service_image_path_windows.go @@ -0,0 +1,41 @@ +//go:build windows + +package cli + +import ( + "os" + + "golang.org/x/sys/windows/registry" +) + +// installedServiceDirMatches reports whether this executable is the installed service +// binary, by comparing its directory with the one in the service's registered ImagePath. +// +// socketDir() on Windows is relative to the running executable, so a ctrld-client.exe run from +// somewhere else - a download directory, a build tree - looks for the control socket in +// its own directory and never finds the installed daemon's. A failed probe from there +// says nothing about the service's health, and reporting "not ready" for it would tell +// monitoring to restart a healthy service. +// +// Anything unreadable answers true, keeping the previous behaviour: readiness stays +// verifiable unless there is positive evidence of a different install. +func installedServiceDirMatches() bool { + self, err := os.Executable() + if err != nil { + return true + } + key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Services\`+ctrldServiceName, registry.QUERY_VALUE) + if err != nil { + return true + } + defer key.Close() + imagePath, _, err := key.GetStringValue("ImagePath") + if err != nil { + return true + } + installed := serviceBinaryFromImagePath(imagePath) + if installed == "" { + return true + } + return sameExecutableDir(installed, self) +} diff --git a/cmd/cli/service_others.go b/cmd/cli/service_others.go index 954b2287..e5241062 100644 --- a/cmd/cli/service_others.go +++ b/cmd/cli/service_others.go @@ -4,19 +4,19 @@ package cli import ( "os" + "syscall" ) +// hasElevatedPrivilege checks if the current process has elevated privileges func hasElevatedPrivilege() (bool, error) { return os.Geteuid() == 0, nil } +// openLogFile opens a log file. It refuses a symlink, because ctrld often +// writes its logs as root into a directory that other users can write. func openLogFile(path string, flags int) (*os.File, error) { - return os.OpenFile(path, flags, os.FileMode(0o600)) + return os.OpenFile(path, flags|syscall.O_NOFOLLOW, os.FileMode(0o600)) } -// hasLocalDnsServerRunning reports whether we are on Windows and having Dns server running. -func hasLocalDnsServerRunning() bool { return false } - +// ConfigureWindowsServiceFailureActions is a no-op on non-Windows platforms func ConfigureWindowsServiceFailureActions(serviceName string) error { return nil } - -func isRunningOnDomainControllerWindows() (bool, int) { return false, 0 } diff --git a/cmd/cli/service_test.go b/cmd/cli/service_test.go index 155bd3e2..50477971 100644 --- a/cmd/cli/service_test.go +++ b/cmd/cli/service_test.go @@ -1,6 +1,7 @@ package cli import ( + "errors" "strings" "testing" ) @@ -26,3 +27,59 @@ func Test_ensureSystemdKillMode(t *testing.T) { }) } } + +func TestDoTasksESuccess(t *testing.T) { + var ran []string + tasks := []task{ + {func() error { ran = append(ran, "a"); return nil }, false, "a"}, + {func() error { ran = append(ran, "b"); return nil }, true, "b"}, + } + failedTask, err := doTasksE(tasks) + if failedTask != "" || err != nil { + t.Errorf("doTasksE() = (%q, %v), want (\"\", nil)", failedTask, err) + } + if got := strings.Join(ran, ","); got != "a,b" { + t.Errorf("ran tasks %q, want all tasks run in order", got) + } +} + +func TestDoTasksEAbortsOnAbortOnErrorTask(t *testing.T) { + wantErr := errors.New("install failed") + var ran []string + tasks := []task{ + {func() error { ran = append(ran, "Stop"); return nil }, false, "Stop"}, + {func() error { ran = append(ran, "Install"); return wantErr }, true, "Install"}, + {func() error { ran = append(ran, "Start"); return nil }, true, "Start"}, + } + failedTask, err := doTasksE(tasks) + if failedTask != "Install" || !errors.Is(err, wantErr) { + t.Errorf("doTasksE() = (%q, %v), want (\"Install\", %v)", failedTask, err, wantErr) + } + if got := strings.Join(ran, ","); got != "Stop,Install" { + t.Errorf("ran tasks %q, want the run to stop right after the abort", got) + } +} + +func TestDoTasksENonAbortFailureContinues(t *testing.T) { + var ran []string + tasks := []task{ + {func() error { ran = append(ran, "a"); return errors.New("a failed") }, false, "a"}, + {func() error { ran = append(ran, "b"); return nil }, true, "b"}, + } + failedTask, err := doTasksE(tasks) + if failedTask != "" || err != nil { + t.Errorf("doTasksE() = (%q, %v), want (\"\", nil) since the failing task did not abort", failedTask, err) + } + if got := strings.Join(ran, ","); got != "a,b" { + t.Errorf("ran tasks %q, want the run to continue past the non-abort failure", got) + } +} + +func TestDoTasksDelegatesToDoTasksE(t *testing.T) { + if !doTasks([]task{{func() error { return nil }, true, "ok"}}) { + t.Error("doTasks() = false, want true on success") + } + if doTasks([]task{{func() error { return errors.New("boom") }, true, "boom"}}) { + t.Error("doTasks() = true, want false when an abortOnError task fails") + } +} diff --git a/cmd/cli/service_windows.go b/cmd/cli/service_windows.go index fddb0ef8..a5a2ac29 100644 --- a/cmd/cli/service_windows.go +++ b/cmd/cli/service_windows.go @@ -1,23 +1,19 @@ package cli import ( + "errors" "os" - "reflect" "runtime" - "strconv" - "strings" "syscall" "time" "unsafe" - "github.com/microsoft/wmi/pkg/base/host" - "github.com/microsoft/wmi/pkg/base/instance" - "github.com/microsoft/wmi/pkg/base/query" - "github.com/microsoft/wmi/pkg/constant" "golang.org/x/sys/windows" + "golang.org/x/sys/windows/svc" "golang.org/x/sys/windows/svc/mgr" ) +// hasElevatedPrivilege checks if the current process has elevated privileges on Windows func hasElevatedPrivilege() (bool, error) { var sid *windows.SID if err := windows.AllocateAndInitializeSid( @@ -39,6 +35,65 @@ func hasElevatedPrivilege() (bool, error) { return token.IsMember(sid) } +// serviceLiveness is what could be established about the installed ctrld service. The +// three states are distinct because a caller that must not disturb a live service has to +// treat "could not tell" like "live", not like "stopped". +type serviceLiveness int + +const ( + // serviceLivenessUnknown means the question could not be answered: the SCM was + // unreachable, the caller lacked rights, or the query failed. + serviceLivenessUnknown serviceLiveness = iota + // serviceLivenessRunning means the service is running, starting, or paused - in every + // case a process that owns state. + serviceLivenessRunning + // serviceLivenessStopped means the service is installed and stopped, or not installed + // at all. Nothing of ctrld's is live. + serviceLivenessStopped +) + +// ctrldServiceLiveness reports what can be established about the installed ctrld service. +// +// Only serviceLivenessStopped is positive evidence that nothing is live. Every failure +// answers serviceLivenessUnknown rather than folding into "stopped": the SCM being +// unreachable says nothing about whether a service is running, and a caller that acts on +// that as absence would strip a live service's state. +// +// "Not installed" is deliberately stopped, not unknown: that is the answer, and it is +// exactly the host that needs stale state cleaned - an uninstall that left filters behind +// has no service left to protect. +func ctrldServiceLiveness() serviceLiveness { + m, err := mgr.Connect() + if err != nil { + return serviceLivenessUnknown + } + defer m.Disconnect() + + s, err := m.OpenService(ctrldServiceName) + if err != nil { + if errors.Is(err, windows.ERROR_SERVICE_DOES_NOT_EXIST) { + return serviceLivenessStopped + } + return serviceLivenessUnknown + } + defer s.Close() + + status, err := s.Query() + if err != nil { + return serviceLivenessUnknown + } + switch status.State { + case svc.Running, svc.StartPending, svc.ContinuePending, svc.PausePending, svc.Paused: + return serviceLivenessRunning + case svc.Stopped: + return serviceLivenessStopped + default: + // StopPending, and any state a later Windows adds: a process may still be + // holding its state, so this is no answer. + return serviceLivenessUnknown + } +} + // ConfigureWindowsServiceFailureActions checks if the given service // has the correct failure actions configured, and updates them if not. func ConfigureWindowsServiceFailureActions(serviceName string) error { @@ -72,15 +127,29 @@ func ConfigureWindowsServiceFailureActions(serviceName string) error { return err } - // Then proceed with existing actions, e.g. setting failure actions + // Recovery policy for a service that carries enforcement. + // + // ctrld's WFP session is dynamic, so Windows removes its filters when the process + // dies - a host with no ctrld is unfiltered rather than locked out. That makes the + // restart budget part of the enforcement story: three restarts five seconds apart + // with a two-minute reset window could be spent inside fifteen seconds, after which + // the service stays stopped and the host stays unfiltered until an operator acts. + // + // The delays back off instead, and the reset window is long enough that a burst + // cannot exhaust the budget faster than the backoff allows. A genuine crash loop + // still ends in a stopped service - that is the point of a bounded policy - but it + // takes minutes rather than seconds, and the third restart survives a transient + // failure that repeats. actions := []mgr.RecoveryAction{ - {Type: mgr.ServiceRestart, Delay: time.Second * 5}, // 5 seconds - {Type: mgr.ServiceRestart, Delay: time.Second * 5}, // 5 seconds - {Type: mgr.ServiceRestart, Delay: time.Second * 5}, // 5 seconds + {Type: mgr.ServiceRestart, Delay: time.Second * 5}, + {Type: mgr.ServiceRestart, Delay: time.Second * 30}, + {Type: mgr.ServiceRestart, Delay: time.Minute * 2}, } - // Set the recovery actions (3 restarts, reset period = 120). - err = s.SetRecoveryActions(actions, 120) + // Reset the failure count only after the service has stayed up longer than the whole + // backoff schedule, so repeated failures keep escalating instead of restarting the + // count from the first five-second delay. + err = s.SetRecoveryActions(actions, uint32((10 * time.Minute).Seconds())) if err != nil { return err } @@ -100,6 +169,7 @@ func ConfigureWindowsServiceFailureActions(serviceName string) error { return nil } +// openLogFile opens a log file with the specified mode on Windows func openLogFile(path string, mode int) (*os.File, error) { if len(path) == 0 { return nil, &os.PathError{Path: path, Op: "open", Err: syscall.ERROR_FILE_NOT_FOUND} @@ -151,77 +221,3 @@ func openLogFile(path string, mode int) (*os.File, error) { return os.NewFile(uintptr(handle), path), nil } - -const processEntrySize = uint32(unsafe.Sizeof(windows.ProcessEntry32{})) - -// hasLocalDnsServerRunning reports whether we are on Windows and having Dns server running. -func hasLocalDnsServerRunning() bool { - h, e := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0) - if e != nil { - return false - } - p := windows.ProcessEntry32{Size: processEntrySize} - for { - e := windows.Process32Next(h, &p) - if e != nil { - return false - } - if strings.ToLower(windows.UTF16ToString(p.ExeFile[:])) == "dns.exe" { - return true - } - } -} - -func isRunningOnDomainControllerWindows() (bool, int) { - whost := host.NewWmiLocalHost() - q := query.NewWmiQuery("Win32_ComputerSystem") - instances, err := instance.GetWmiInstancesFromHost(whost, string(constant.CimV2), q) - if err != nil { - mainLog.Load().Debug().Err(err).Msg("WMI query failed") - return false, 0 - } - if instances == nil { - mainLog.Load().Debug().Msg("WMI query returned nil instances") - return false, 0 - } - defer instances.Close() - - if len(instances) == 0 { - mainLog.Load().Debug().Msg("no rows returned from Win32_ComputerSystem") - return false, 0 - } - - val, err := instances[0].GetProperty("DomainRole") - if err != nil { - mainLog.Load().Debug().Err(err).Msg("failed to get DomainRole property") - return false, 0 - } - if val == nil { - mainLog.Load().Debug().Msg("DomainRole property is nil") - return false, 0 - } - - // Safely handle varied types: string or integer - var roleInt int - switch v := val.(type) { - case string: - // "4", "5", etc. - parsed, parseErr := strconv.Atoi(v) - if parseErr != nil { - mainLog.Load().Debug().Err(parseErr).Msgf("failed to parse DomainRole value %q", v) - return false, 0 - } - roleInt = parsed - case int8, int16, int32, int64: - roleInt = int(reflect.ValueOf(v).Int()) - case uint8, uint16, uint32, uint64: - roleInt = int(reflect.ValueOf(v).Uint()) - default: - mainLog.Load().Debug().Msgf("unexpected DomainRole type: %T value=%v", v, v) - return false, 0 - } - - // Check if role indicates a domain controller - isDC := roleInt == BackupDomainController || roleInt == PrimaryDomainController - return isDC, roleInt -} diff --git a/cmd/cli/service_windows_test.go b/cmd/cli/service_windows_test.go deleted file mode 100644 index 67c2725d..00000000 --- a/cmd/cli/service_windows_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package cli - -import ( - "testing" - "time" -) - -func Test_hasLocalDnsServerRunning(t *testing.T) { - start := time.Now() - hasDns := hasLocalDnsServerRunning() - t.Logf("Using Windows API takes: %d", time.Since(start).Milliseconds()) - - start = time.Now() - hasDnsPowershell := hasLocalDnsServerRunningPowershell() - t.Logf("Using Powershell takes: %d", time.Since(start).Milliseconds()) - - if hasDns != hasDnsPowershell { - t.Fatalf("result mismatch, want: %v, got: %v", hasDnsPowershell, hasDns) - } -} - -func hasLocalDnsServerRunningPowershell() bool { - _, err := powershell("Get-Process -Name DNS") - return err == nil -} diff --git a/cmd/cli/shutdown_test.go b/cmd/cli/shutdown_test.go new file mode 100644 index 00000000..b7356488 --- /dev/null +++ b/cmd/cli/shutdown_test.go @@ -0,0 +1,156 @@ +package cli + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func TestRunWaitStopsBeforeStartup(t *testing.T) { + p := &prog{ + waitCh: make(chan struct{}), + stopCh: make(chan struct{}), + runDone: make(chan struct{}), + } + p.logger.Store(mainLog.Load()) + require.NoError(t, p.Start(nil)) + close(p.stopCh) + select { + case <-p.runDone: + case <-time.After(time.Second): + t.Fatal("stop did not terminate workers waiting for startup") + } +} + +func TestRunDoesNotStartAfterStop(t *testing.T) { + for i := 0; i < 100; i++ { + p := &prog{waitCh: make(chan struct{}), stopCh: make(chan struct{})} + p.logger.Store(mainLog.Load()) + close(p.waitCh) + close(p.stopCh) + // cfg is deliberately nil: reaching component startup is a failure. + p.run(false, make(chan struct{})) + } +} + +func TestRunMobilePreflightCleanup(t *testing.T) { + const childEnv = "CTRLD_TEST_MOBILE_PREFLIGHT" + mode := os.Getenv(childEnv) + if mode == "" { + for _, mode := range []string{"stop", "failure"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunMobilePreflightCleanup$", "-test.v") + cmd.Env = append(os.Environ(), childEnv+"="+mode) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", out) + }) + } + return + } + + // Keep the real RunMobile -> run -> Start -> runWait path, but replace the + // platform cleanup and API boundaries before any host setup can happen. + isMobile = func() bool { return true } + networkUp = func() bool { return true } + cleanupCalls := 0 + cleanupStaleDNSInterceptStateFn = func() { cleanupCalls++ } + dir := t.TempDir() + configPath = filepath.Join(dir, "ctrld.toml") + require.NoError(t, os.WriteFile(configPath, []byte("[service]\n"), 0600)) + want := errors.New("preflight rejected") + for i := 0; i < 3; i++ { + stopCh := make(chan struct{}) + fetchCalls, exits := 0, 0 + processCDFlagsFn = func(ctx context.Context, _ *ctrld.Config) (*controld.ResolverConfig, error) { + fetchCalls++ + require.Eventually(t, startupWorkersRunning, time.Second, time.Millisecond) + if mode == "stop" { + close(stopCh) + <-ctx.Done() + return nil, ctx.Err() + } + return nil, want + } + callback := AppCallback{Exit: func(err string) { + exits++ + require.Equal(t, want.Error(), err) + }} + RunMobile(&AppConfig{CdUID: "testuid", HomeDir: dir, UpstreamProto: "doh", LogPath: filepath.Join(dir, "ctrld.log")}, &callback, stopCh) + require.Equal(t, 1, fetchCalls) + if mode == "failure" { + require.Equal(t, 1, exits) + require.False(t, stopRequested(stopCh), "cleanup must not close the controller-owned channel") + + } else { + require.Zero(t, exits) + } + require.False(t, startupWorkersRunning(), "RunMobile returned before its workers stopped") + if mode == "failure" { + close(stopCh) // A later Controller.Stop must remain safe. + } + } + require.Equal(t, 3, cleanupCalls, "the real stale intercept cleanup must never be reached") +} + +func startupWorkersRunning() bool { + stack := make([]byte, 1<<20) + n := runtime.Stack(stack, true) + return strings.Contains(string(stack[:n]), "(*prog).runWait(") || strings.Contains(string(stack[:n]), "(*prog).run(") +} + +func TestControlServerStopOnlyUnlinksOwnedListener(t *testing.T) { + // Keep the Unix socket path below macOS's limit, including its long TMPDIR. + dir, err := os.MkdirTemp("", "ctrld-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + path := filepath.Join(dir, "control.sock") + owner, err := newControlServer(path) + require.NoError(t, err) + require.NoError(t, owner.start()) + t.Cleanup(func() { _ = owner.stop() }) + unstarted, err := newControlServer(path) + require.NoError(t, err) + require.NoError(t, unstarted.stop()) + c, err := net.Dial("unix", path) + require.NoError(t, err, "stopping before startup must preserve someone else's socket") + require.NoError(t, c.SetDeadline(time.Now().Add(time.Second))) + _, err = io.WriteString(c, "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + require.NoError(t, err) + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.NoError(t, c.Close()) + require.NoError(t, owner.stop()) + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err), "owned socket must be unlinked on shutdown: %v", err) +} + +func TestShutdownClosesStartupLogConnection(t *testing.T) { + c, peer := net.Pipe() + defer peer.Close() + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{}), logConn: c} + p.logger.Store(mainLog.Load()) + require.NoError(t, p.shutdown()) + require.NoError(t, p.shutdown()) + _ = peer.SetWriteDeadline(time.Now().Add(time.Second)) + _, err := peer.Write([]byte("x")) + require.ErrorIs(t, err, io.ErrClosedPipe) +} diff --git a/cmd/cli/stop_control_server_test.go b/cmd/cli/stop_control_server_test.go new file mode 100644 index 00000000..01816ff6 --- /dev/null +++ b/cmd/cli/stop_control_server_test.go @@ -0,0 +1,150 @@ +package cli + +import ( + "context" + "net/http" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/kardianos/service" + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" +) + +// A stopping process must not answer /started for the service that replaces +// it. `ctrld upgrade` treats any reply as "the new binary is up", so a reply +// from the old process, while it waits for its components to stop, turned an +// upgrade to a never-ready build into "Upgrade successful" instead of a +// rollback (ctrld-qa upgrade-readiness-rollback on v1.0). +func TestStopClosesControlServerBeforeRunFinishes(t *testing.T) { + // A short path, as the other control server tests use: t.TempDir() is + // longer than a Unix socket path may be. + f, err := os.CreateTemp("", "") + require.NoError(t, err) + sock := f.Name() + f.Close() + t.Cleanup(func() { os.Remove(sock) }) + cs, err := newControlServer(sock) + require.NoError(t, err) + cs.register(startedPath, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + require.NoError(t, cs.start()) + + p := &prog{ + cfg: &ctrld.Config{}, + cs: cs, + waitCh: make(chan struct{}), stopCh: make(chan struct{}), runDone: make(chan struct{}), + runAbortCh: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), + pinCodeValidCh: make(chan struct{}, 1), + } + p.logger.Store(mainLog.Load()) + p.pinCodeValidCh <- struct{}{} // accept the stop even if a deactivation pin is configured + + cc := newControlClient(sock) + resp, err := cc.post(startedPath, nil) + require.NoError(t, err, "the running service answers") + resp.Body.Close() + + require.NoError(t, p.Stop(nil)) + // runDone is still open: the run has not finished, as when a component is + // slow to stop. The socket must already be closed. + _, err = cc.post(startedPath, nil) + require.Error(t, err, "a stopped service answered /started") + + p.stopControlServer() // the later releaseResources call is harmless +} + +// startTestControlServer starts a control server on a short socket path in a +// directory of its own, as the socket client expects, answering /started with +// whatever started returns. +func startTestControlServer(t *testing.T, started func() int) (dir, sock string, cs *controlServer) { + t.Helper() + // A short path, as the other control server tests use: t.TempDir() is + // longer than a Unix socket path may be. + dir, err := os.MkdirTemp("", "cs") + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(dir) }) + sock = filepath.Join(dir, ctrldControlUnixSock) + cs, err = newControlServer(sock) + require.NoError(t, err) + cs.register(startedPath, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(started()) })) + cs.register(listClientsPath, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + require.NoError(t, cs.start()) + t.Cleanup(func() { _ = cs.stop() }) + // start serves on a goroutine, and the server tracks the listener only + // once that runs: a Shutdown before then leaves the listener, and its + // unlink, to Serve's own deferred close. Wait until it answers, as a + // service that has been running does. A path other than /started keeps + // the started func's call count to the test's own requests. + require.Eventually(t, func() bool { + resp, err := newControlClient(sock).post(listClientsPath, nil) + if err != nil { + return false + } + resp.Body.Close() + return true + }, 5*time.Second, 10*time.Millisecond, "control server did not start serving") + return dir, sock, cs +} + +// Stop unlinks the socket path as it closes the socket, before the successor +// starts: unlinking it later removed the path the successor had bound. +func TestStopRemovesControlSocketPath(t *testing.T) { + _, sock, cs := startTestControlServer(t, func() int { return http.StatusOK }) + p := &prog{ + cfg: &ctrld.Config{}, + cs: cs, + waitCh: make(chan struct{}), stopCh: make(chan struct{}), runDone: make(chan struct{}), + runAbortCh: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), + pinCodeValidCh: make(chan struct{}, 1), + } + p.logger.Store(mainLog.Load()) + p.pinCodeValidCh <- struct{}{} + + require.NoError(t, p.Stop(nil)) + _, err := os.Stat(sock) + require.True(t, os.IsNotExist(err), "socket path still present after Stop: %v", err) +} + +// A stop path that never calls Stop still closes the control server. +func TestReleaseResourcesClosesControlServer(t *testing.T) { + _, sock, cs := startTestControlServer(t, func() int { return http.StatusOK }) + p := &prog{cfg: &ctrld.Config{}, cs: cs} + p.logger.Store(mainLog.Load()) + + p.releaseResources() + _, err := newControlClient(sock).post(startedPath, nil) + require.Error(t, err, "the control server answered after releaseResources") +} + +// `ctrld upgrade` must not count a successor that started but never became +// ready: it answers /started with 408, and only a 200 means ready. The plain +// socket client keeps accepting any answer, because the deactivation pin check +// must reach a running service whether or not it is ready. +func TestReadySocketControlClientRequiresReady(t *testing.T) { + running := &fakeService{statuses: []service.Status{service.StatusRunning}} + + t.Run("never ready", func(t *testing.T) { + dir, _, _ := startTestControlServer(t, func() int { return http.StatusRequestTimeout }) + require.Nil(t, newReadySocketControlClientWithTimeout(context.Background(), running, dir, 2*time.Second), + "a service answering 408 counted as ready") + require.NotNil(t, newSocketControlClientWithTimeout(context.Background(), running, dir, 2*time.Second), + "the plain client must still reach a running service that is not ready") + }) + + t.Run("becomes ready", func(t *testing.T) { + var calls atomic.Int32 + dir, _, _ := startTestControlServer(t, func() int { + if calls.Add(1) == 1 { + return http.StatusRequestTimeout + } + return http.StatusOK + }) + require.NotNil(t, dialSocketControlServer(context.Background(), running, dir, 10*time.Second, true), + "the poll gave up on a service that became ready") + require.GreaterOrEqual(t, calls.Load(), int32(2), "the poll stopped at the 408") + }) +} diff --git a/cmd/cli/suspend_detector.go b/cmd/cli/suspend_detector.go new file mode 100644 index 00000000..b3e3f815 --- /dev/null +++ b/cmd/cli/suspend_detector.go @@ -0,0 +1,109 @@ +package cli + +import "time" + +// Suspend/resume detection without an OS notification API. +// +// A host sleep suspends the whole process: timers stop advancing and resume where they +// left off, so a tick scheduled two seconds out can arrive a minute later. That +// overshoot is the only in-process evidence that the host slept, and nothing else tells +// the DNS-intercept repair paths that the state they proved before sleep - rule text, +// resolver list, upstream connections - may no longer hold. + +const ( + // suspendProbeInterval is how often the detector measures elapsed wall-clock time. + // Short enough that a resume is noticed within a couple of seconds, which is what + // the post-wake continuity contract measures from. + suspendProbeInterval = 2 * time.Second + + // suspendGapThreshold is the elapsed wall clock over one interval that counts as a + // suspend. A pfctl storm or a loaded host can stretch a tick, so this sits at + // several times the interval; a real sleep overshoots by far more. Erring low is + // cheap on purpose: a false positive costs one probe, and a stall this long is + // worth a probe either way. + suspendGapThreshold = 8 * time.Second +) + +// wallNanoNow reads the wall clock in nanoseconds. +// +// The detector works in wall-clock nanoseconds rather than time.Time on purpose. +// time.Time.Sub prefers the monotonic reading whenever both operands carry one, and +// Darwin's monotonic clock stops while the host is asleep - so a monotonic difference +// across a suspend is just the tick interval and hides the very gap this looks for. +// Taking int64 wall nanos makes that mistake unrepresentable rather than a comment. +func wallNanoNow() int64 { return time.Now().UnixNano() } + +// suspendDetector turns observations of a fixed-interval tick into suspend/resume +// events. It measures elapsed time rather than counting ticks, because a suspended +// process gets one late tick, not the ticks it missed. +type suspendDetector struct { + threshold time.Duration + wallNano func() int64 + lastNano int64 +} + +// newSuspendDetector returns a detector that reports gaps of at least threshold. +// wallNano is a seam for tests; nil means the real wall clock. +func newSuspendDetector(threshold time.Duration, wallNano func() int64) *suspendDetector { + if wallNano == nil { + wallNano = wallNanoNow + } + return &suspendDetector{threshold: threshold, wallNano: wallNano, lastNano: wallNano()} +} + +// observe records a tick and reports the elapsed time when it reaches the threshold, +// which means the process was not running for most of that span. +func (d *suspendDetector) observe() (time.Duration, bool) { + current := d.wallNano() + gap := time.Duration(current - d.lastNano) + d.lastNano = current + if gap < d.threshold { + return 0, false + } + return gap, true +} + +// reset re-baselines the detector to the current wall clock. Time that has already +// passed is deliberately discarded rather than measured: use it after doing work +// outside the sampling loop, which is not a suspend even though it looks like one. +func (d *suspendDetector) reset() { d.lastNano = d.wallNano() } + +// watchForResume calls onResume once per detected suspend/resume gap, until stopCh +// closes or tick is closed. tick is a parameter so tests can drive it directly. +// +// onResume runs on this goroutine, so recovery is serialized against detection - one +// resume cannot start a second recovery on top of the first. The cost is that no +// sampling happens while it runs, which the re-baseline below accounts for. +func watchForResume(stopCh <-chan struct{}, tick <-chan time.Time, d *suspendDetector, onResume func(gap time.Duration)) { + for { + select { + case <-stopCh: + return + case _, ok := <-tick: + if !ok { + return + } + if gap, resumed := d.observe(); resumed { + onResume(gap) + // Re-baseline: the recovery just ran, and its own runtime exceeds the + // threshold on its own (the post-wake probe schedule waits for seconds + // before a single probe timeout or reload). The ticker keeps firing and + // buffers a tick while it works, so without this the next observe() + // measures the callback and reports it as a fresh suspend - which starts + // another recovery, which is again long enough to look like a suspend. + // Under a persistent pf failure that is an unbounded reload loop, and + // each pass flushes pf state and kills in-flight DoH connections. + d.reset() + } + } + } +} + +// runSuspendWatcher watches for suspend/resume against a real ticker. +// +//lint:ignore U1000 used on darwin (post-wake interception verification) +func runSuspendWatcher(stopCh <-chan struct{}, onResume func(gap time.Duration)) { + ticker := time.NewTicker(suspendProbeInterval) + defer ticker.Stop() + watchForResume(stopCh, ticker.C, newSuspendDetector(suspendGapThreshold, nil), onResume) +} diff --git a/cmd/cli/suspend_detector_test.go b/cmd/cli/suspend_detector_test.go new file mode 100644 index 00000000..4f31a31c --- /dev/null +++ b/cmd/cli/suspend_detector_test.go @@ -0,0 +1,208 @@ +package cli + +import ( + "testing" + "time" +) + +// fakeWallClock returns the queued offsets from a fixed base, then repeats the last one. +type fakeWallClock struct { + nanos []int64 + calls int +} + +func (c *fakeWallClock) now() int64 { + if c.calls < len(c.nanos) { + n := c.nanos[c.calls] + c.calls++ + return n + } + return c.nanos[len(c.nanos)-1] +} + +// wallClockAt builds a clock whose readings are the given offsets from an arbitrary +// epoch. The first reading is consumed by newSuspendDetector; the rest are ticks. +func wallClockAt(offsets ...time.Duration) *fakeWallClock { + const base = int64(1787081656_000000000) // 2026-08-18T14:34:16-05:00 + nanos := make([]int64, 0, len(offsets)) + for _, off := range offsets { + nanos = append(nanos, base+int64(off)) + } + return &fakeWallClock{nanos: nanos} +} + +func TestSuspendDetectorIgnoresOrdinaryTicks(t *testing.T) { + // Construction, then three ticks two seconds apart, one stretched to 3s by a + // loaded host. None of these is a suspend. + clock := wallClockAt(0, 2*time.Second, 4*time.Second, 7*time.Second) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + for i := 0; i < 3; i++ { + if gap, resumed := d.observe(); resumed { + t.Errorf("tick %d reported a suspend of %s; ordinary jitter must not count", i, gap) + } + } +} + +// TestSuspendDetectorReportsTheGapOnce replays the captured incident: the last pre-sleep +// sample at 14:34:47 and the first post-resume one at 14:35:05. +func TestSuspendDetectorReportsTheGapOnce(t *testing.T) { + clock := wallClockAt( + 31*time.Second, // construction, 14:34:47 + 33*time.Second, // ordinary tick + 49*time.Second, // 14:35:05 - a 16s gap over a 2s interval + 51*time.Second, // ordinary again + ) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + if _, resumed := d.observe(); resumed { + t.Fatal("the tick before the suspend reported a resume") + } + gap, resumed := d.observe() + if !resumed { + t.Fatal("a 16s gap over a 2s interval was not reported as a suspend") + } + if gap != 16*time.Second { + t.Errorf("gap = %s, want 16s: the reported gap must be the measured wall clock", gap) + } + if _, resumed := d.observe(); resumed { + t.Error("the resume was reported twice; the baseline must advance to the late tick") + } +} + +// TestSuspendDetectorThresholdCoversShortNaps guards the lower bound. A nap only a few +// seconds longer than the interval still breaks pf translation, so the threshold must not +// drift up to a value that only catches long sleeps. +func TestSuspendDetectorThresholdCoversShortNaps(t *testing.T) { + if suspendGapThreshold > 10*time.Second { + t.Errorf("suspendGapThreshold = %s; a short nap would go unnoticed", suspendGapThreshold) + } + if suspendGapThreshold <= 2*suspendProbeInterval { + t.Errorf("suspendGapThreshold = %s is too close to the %s interval; ordinary jitter would fire it", + suspendGapThreshold, suspendProbeInterval) + } + + clock := wallClockAt(0, suspendGapThreshold) + d := newSuspendDetector(suspendGapThreshold, clock.now) + if _, resumed := d.observe(); !resumed { + t.Error("a gap exactly at the threshold was not reported") + } +} + +func TestSuspendDetectorIgnoresBackwardClockSteps(t *testing.T) { + // An NTP step backwards must not read as a resume, and must not leave the baseline + // in the future either: the next ordinary tick still has to look ordinary. + clock := wallClockAt(0, -30*time.Second, -28*time.Second) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + if gap, resumed := d.observe(); resumed { + t.Errorf("a backward clock step reported a suspend of %s", gap) + } + if gap, resumed := d.observe(); resumed { + t.Errorf("the tick after a backward step reported a suspend of %s", gap) + } +} + +func TestWatchForResumeCallsBackPerGap(t *testing.T) { + clock := wallClockAt( + 0, + 2*time.Second, // ordinary + 40*time.Second, // resume: 38s + 42*time.Second, // ordinary + 99*time.Second, // second resume: 57s + ) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + tick := make(chan time.Time, 4) + for i := 0; i < 4; i++ { + tick <- time.Time{} + } + close(tick) + + var gaps []time.Duration + watchForResume(nil, tick, d, func(gap time.Duration) { gaps = append(gaps, gap) }) + + if len(gaps) != 2 { + t.Fatalf("onResume called %d times (%v), want 2", len(gaps), gaps) + } + if gaps[0] != 38*time.Second || gaps[1] != 57*time.Second { + t.Errorf("gaps = %v, want [38s 57s]", gaps) + } +} + +// advancingClock is a wall clock the test moves by hand, so a callback's runtime can be +// simulated without sleeping for it. +type advancingClock struct{ nano int64 } + +func (c *advancingClock) now() int64 { return c.nano } +func (c *advancingClock) advance(d time.Duration) { c.nano += int64(d) } + +// TestWatchForResumeDoesNotRetriggerOnItsOwnCallback pins the loop this closes. onResume +// runs on the watcher goroutine, and the real callback - verifyInterceptAfterWake - takes +// longer than the gap threshold by design: its configured waits alone total more than the +// threshold, before any probe timeout or forced reload. The ticker buffers a tick while it +// works, so a detector that keeps its pre-callback baseline measures the callback itself as +// the next gap and starts another recovery, with another repair budget, indefinitely under +// a persistent pf failure. +func TestWatchForResumeDoesNotRetriggerOnItsOwnCallback(t *testing.T) { + clock := &advancingClock{} + d := newSuspendDetector(suspendGapThreshold, clock.now) + + // Capacity 1, like time.Ticker's channel: exactly one tick can queue up behind a + // callback in production. + tick := make(chan time.Time, 1) + calls := 0 + done := make(chan struct{}) + go func() { + defer close(done) + watchForResume(nil, tick, d, func(time.Duration) { + calls++ + if calls > 1 { + // A retrigger - the failure this test reports. There is nothing left to + // simulate, and tick is already closed. + return + } + // The recovery takes longer than the threshold, and a tick lands while it + // runs. Closing here ends the watcher after it drains that queued tick, + // which is the observation under test. + clock.advance(2 * suspendGapThreshold) + tick <- time.Time{} + close(tick) + }) + }() + + clock.advance(2 * suspendGapThreshold) // the genuine suspend + tick <- time.Time{} + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watchForResume did not finish; the queued tick was never consumed") + } + + if calls != 1 { + t.Errorf("onResume called %d times, want 1: the callback's own runtime was measured as a second suspend", calls) + } +} + +func TestWatchForResumeStopsOnStopCh(t *testing.T) { + stopCh := make(chan struct{}) + tick := make(chan time.Time) // never fires + clock := wallClockAt(0, time.Hour) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + done := make(chan struct{}) + go func() { + watchForResume(stopCh, tick, d, func(time.Duration) { + t.Error("onResume ran without a tick") + }) + close(done) + }() + + close(stopCh) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watchForResume did not return once stopCh was closed") + } +} diff --git a/cmd/cli/tailscale_localapi.go b/cmd/cli/tailscale_localapi.go new file mode 100644 index 00000000..3f3d3ab9 --- /dev/null +++ b/cmd/cli/tailscale_localapi.go @@ -0,0 +1,155 @@ +package cli + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "regexp" + "strconv" + "time" +) + +// tailscaleLocalAPITimeout bounds one LocalAPI read. Discovery runs on network +// change paths, so it must never stall them. +const tailscaleLocalAPITimeout = 2 * time.Second + +// tailscaleLocalAPIHost is the Host header Tailscale's LocalAPI requires. +const tailscaleLocalAPIHost = "local-tailscaled.sock" + +// tailscaleDNSConfigPath returns the network's DNS config, including Split DNS +// and App Connector routes. Tailscale added it in v1.98.0. +const tailscaleDNSConfigPath = "/localapi/v0/dns-config" + +// maxTailscaleDNSConfigBytes bounds the response body ctrld reads. +const maxTailscaleDNSConfigBytes = 4 << 20 + +var errTailscaleLocalAPIUnavailable = errors.New("tailscale LocalAPI not found") + +// tailscaleLocalAPIEndpoint is one way to reach a running Tailscale LocalAPI. +type tailscaleLocalAPIEndpoint struct { + name string + dial func(ctx context.Context) (net.Conn, error) + token string // HTTP Basic-Auth password; empty for the tailscaled socket +} + +// tailscaleAppStoreProc is the exact command name of the Mac App Store +// Tailscale network extension. +const tailscaleAppStoreProc = "IPNExtension" + +// sameUserProofPath matches the sameuserproof file of the Mac App Store +// Tailscale extension: /Library/Group Containers/.tailscale.ipn.macos/ +// sameuserproof--. Paths anywhere else are not accepted, because +// any local user can create a file with that name under /tmp. +var sameUserProofPath = regexp.MustCompile(`^/Users/[^/]+/Library/Group Containers/[^/]+\.tailscale\.ipn\.macos/sameuserproof-([0-9]+)-([^/]+)$`) + +// macOSSameUserProof is one App Store LocalAPI candidate found by lsof. +type macOSSameUserProof struct { + pid int + port int + token string +} + +// parseMacOSSameUserProofs extracts App Store LocalAPI candidates from +// `lsof -F pcn` output. A candidate needs a process whose command is exactly +// IPNExtension (lsof -c matches a prefix only) holding open a sameuserproof +// file in a user's Tailscale group container. The caller still has to verify +// the process code signature, because a command name is not proof. +func parseMacOSSameUserProofs(out []byte) []macOSSameUserProof { + var proofs []macOSSameUserProof + pid, command := 0, "" + sc := bufio.NewScanner(bytes.NewReader(out)) + for sc.Scan() { + line := sc.Text() + if line == "" { + continue + } + value := line[1:] + switch line[0] { + case 'p': + pid, command = 0, "" + if p, err := strconv.Atoi(value); err == nil && p > 0 { + pid = p + } + case 'c': + command = value + case 'n': + if pid == 0 || command != tailscaleAppStoreProc { + continue + } + m := sameUserProofPath.FindStringSubmatch(value) + if m == nil { + continue + } + port, err := strconv.Atoi(m[1]) + if err != nil || port <= 0 || port > 65535 { + continue + } + proofs = append(proofs, macOSSameUserProof{pid: pid, port: port, token: m[2]}) + } + } + return proofs +} + +// fetchTailscaleDNSConfig reads the dns-config body from one endpoint. +func fetchTailscaleDNSConfig(ctx context.Context, ep tailscaleLocalAPIEndpoint) ([]byte, error) { + tr := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return ep.dial(ctx) }, + DisableKeepAlives: true, + Proxy: nil, + } + defer tr.CloseIdleConnections() + client := &http.Client{Transport: tr} + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+tailscaleLocalAPIHost+tailscaleDNSConfigPath, nil) + if err != nil { + return nil, err + } + if ep.token != "" { + req.SetBasicAuth("", ep.token) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%s: %w", ep.name, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, maxTailscaleDNSConfigBytes+1)) + if err != nil { + return nil, fmt.Errorf("%s: %w", ep.name, err) + } + if resp.StatusCode != http.StatusOK { + // Older Tailscale has no dns-config endpoint and answers 404. + return nil, fmt.Errorf("%s: dns-config returned HTTP %d", ep.name, resp.StatusCode) + } + if len(body) > maxTailscaleDNSConfigBytes { + return nil, fmt.Errorf("%s: dns-config response exceeds %d bytes", ep.name, maxTailscaleDNSConfigBytes) + } + return body, nil +} + +// discoverTailscaleRoutesFrom tries each endpoint in order and returns the +// routes of the first that answers. The error never contains the token. +func discoverTailscaleRoutesFrom(ctx context.Context, endpoints []tailscaleLocalAPIEndpoint) ([]tailscaleRoute, error) { + if len(endpoints) == 0 { + return nil, errTailscaleLocalAPIUnavailable + } + var errs []error + for _, ep := range endpoints { + body, err := fetchTailscaleDNSConfig(ctx, ep) + if err != nil { + errs = append(errs, err) + continue + } + routes, err := parseTailscaleRoutes(body) + if err != nil { + errs = append(errs, fmt.Errorf("%s: decode dns-config: %w", ep.name, err)) + continue + } + return routes, nil + } + return nil, errors.Join(errs...) +} diff --git a/cmd/cli/tailscale_localapi_darwin.go b/cmd/cli/tailscale_localapi_darwin.go new file mode 100644 index 00000000..50216977 --- /dev/null +++ b/cmd/cli/tailscale_localapi_darwin.go @@ -0,0 +1,115 @@ +//go:build darwin + +package cli + +import ( + "context" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" +) + +// Locations used by the three macOS Tailscale variants. These match +// tailscale.com/safesocket and tailscale.com/paths. +const ( + tailscaleMacsysDir = "/Library/Tailscale" + tailscaledUnixSocket = "/var/run/tailscaled.socket" +) + +// tailscaleCodeRequirement accepts only the Mac App Store Tailscale network +// extension. Apple re-signs App Store code with its own leaf certificate +// (marker OID 1.2.840.113635.100.6.1.9) and signs a bundle identifier only for +// the developer that owns it; other builds carry Tailscale's team identifier +// W5364U7YZB, which Tailscale publishes for its macOS extension policy. +const tailscaleCodeRequirement = `anchor apple generic and identifier "io.tailscale.ipn.macos.network-extension" and ` + + `(certificate leaf[field.1.2.840.113635.100.6.1.9] or certificate leaf[subject.OU] = "W5364U7YZB")` + +// discoverTailscaleRoutesFn reads Tailscale DNS routes; tests replace it. +// +// discoverTailscaleRoutes reads Tailscale's DNS routes through its LocalAPI. +// ctrld runs as root, so it can read the credentials of each variant. The +// root-owned credentials come first: the Standalone system extension +// (/Library/Tailscale) and open source tailscaled (unix socket). The Mac App +// Store extension (sameuserproof file found with lsof) comes last, and only +// after its process passes a Tailscale code signature check, because any +// local user can start a process with that name. +var discoverTailscaleRoutesFn = discoverTailscaleRoutes + +// listAppStoreProofsFn lists the open files of IPNExtension processes; tests +// replace it. "+c 0" prints the full command name, so the parser can require +// an exact match; -c alone matches any command that starts with the name. +var listAppStoreProofsFn = func(ctx context.Context) ([]byte, error) { + return exec.CommandContext(ctx, "lsof", "-n", "-a", "+c", "0", "-c", tailscaleAppStoreProc, "-F", "pcn").Output() +} + +// verifyTailscaleProcessFn checks the code signature of a running process; +// tests replace it. +var verifyTailscaleProcessFn = verifyTailscaleProcess + +func discoverTailscaleRoutes(ctx context.Context) ([]tailscaleRoute, error) { + return discoverTailscaleRoutesFrom(ctx, tailscaleLocalAPIEndpoints(ctx)) +} + +func tailscaleLocalAPIEndpoints(ctx context.Context) []tailscaleLocalAPIEndpoint { + var eps []tailscaleLocalAPIEndpoint + + if portStr, err := os.Readlink(filepath.Join(tailscaleMacsysDir, "ipnport")); err == nil { + if port, err := strconv.Atoi(portStr); err == nil && port > 0 && port <= 65535 { + if b, err := os.ReadFile(filepath.Join(tailscaleMacsysDir, "sameuserproof-"+portStr)); err == nil { + if token := strings.TrimSpace(string(b)); token != "" { + eps = append(eps, tailscaleLocalAPIEndpoint{name: "standalone", dial: tcpLocalAPIDialer(port), token: token}) + } + } + } + } + + if fi, err := os.Stat(tailscaledUnixSocket); err == nil && fi.Mode()&os.ModeSocket != 0 { + eps = append(eps, tailscaleLocalAPIEndpoint{ + name: "tailscaled", + dial: func(ctx context.Context) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", tailscaledUnixSocket) + }, + }) + } + + return append(eps, appStoreLocalAPIEndpoints(ctx)...) +} + +// appStoreLocalAPIEndpoints returns the App Store LocalAPI endpoint of the +// first IPNExtension process that is signed by Tailscale. +func appStoreLocalAPIEndpoints(ctx context.Context) []tailscaleLocalAPIEndpoint { + out, err := listAppStoreProofsFn(ctx) + if err != nil { + return nil + } + verified := make(map[int]bool) + for _, proof := range parseMacOSSameUserProofs(out) { + ok, seen := verified[proof.pid] + if !seen { + ok = verifyTailscaleProcessFn(ctx, proof.pid) + verified[proof.pid] = ok + } + if ok { + return []tailscaleLocalAPIEndpoint{{name: "app-store", dial: tcpLocalAPIDialer(proof.port), token: proof.token}} + } + } + return nil +} + +// verifyTailscaleProcess reports whether the running process pid is signed by +// Tailscale. codesign validates a running process dynamically when given a +// pid instead of a path. +func verifyTailscaleProcess(ctx context.Context, pid int) bool { + return exec.CommandContext(ctx, "codesign", "--verify", "-R="+tailscaleCodeRequirement, strconv.Itoa(pid)).Run() == nil +} + +func tcpLocalAPIDialer(port int) func(ctx context.Context) (net.Conn, error) { + return func(ctx context.Context) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + } +} diff --git a/cmd/cli/tailscale_localapi_darwin_test.go b/cmd/cli/tailscale_localapi_darwin_test.go new file mode 100644 index 00000000..a7a5dd49 --- /dev/null +++ b/cmd/cli/tailscale_localapi_darwin_test.go @@ -0,0 +1,37 @@ +//go:build darwin + +package cli + +import ( + "context" + "testing" +) + +func TestAppStoreLocalAPIEndpointsRequireTailscaleSignature(t *testing.T) { + oldList, oldVerify := listAppStoreProofsFn, verifyTailscaleProcessFn + t.Cleanup(func() { listAppStoreProofsFn, verifyTailscaleProcessFn = oldList, oldVerify }) + + const proof = "/Users/u/Library/Group Containers/W5364U7YZB.io.tailscale.ipn.macos/sameuserproof-" + listAppStoreProofsFn = func(context.Context) ([]byte, error) { + return []byte("p100\ncIPNExtension\nn" + proof + "8080-spoof\n" + + "p200\ncIPNExtension\nn" + proof + "49152-real\n"), nil + } + var checked []int + verifyTailscaleProcessFn = func(_ context.Context, pid int) bool { + checked = append(checked, pid) + return pid == 200 + } + + eps := appStoreLocalAPIEndpoints(context.Background()) + if len(eps) != 1 || eps[0].name != "app-store" || eps[0].token != "real" { + t.Fatalf("endpoints = %+v, want only the signed process", eps) + } + if len(checked) != 2 || checked[0] != 100 || checked[1] != 200 { + t.Fatalf("verified pids = %v, want [100 200]", checked) + } + + verifyTailscaleProcessFn = func(context.Context, int) bool { return false } + if eps := appStoreLocalAPIEndpoints(context.Background()); len(eps) != 0 { + t.Fatalf("endpoints = %+v for unsigned processes, want none", eps) + } +} diff --git a/cmd/cli/tailscale_localapi_others.go b/cmd/cli/tailscale_localapi_others.go new file mode 100644 index 00000000..a528f3eb --- /dev/null +++ b/cmd/cli/tailscale_localapi_others.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package cli + +import "context" + +// discoverTailscaleRoutesFn is nil off macOS: other platforms publish +// Tailscale split domains through the OS (NRPT, systemd-resolved). +var discoverTailscaleRoutesFn func(context.Context) ([]tailscaleRoute, error) diff --git a/cmd/cli/tailscale_routes.go b/cmd/cli/tailscale_routes.go new file mode 100644 index 00000000..02288c82 --- /dev/null +++ b/cmd/cli/tailscale_routes.go @@ -0,0 +1,336 @@ +package cli + +import ( + "context" + "encoding/json" + "net" + "net/netip" + "net/url" + "slices" + "strings" + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/tsaddr" +) + +// maxTailscaleRoutes bounds the routes taken from one Tailscale DNS config so +// a very large App Connector set cannot grow the route table without limit. +const maxTailscaleRoutes = 10000 + +// tailscaleRouteRetryDelays are the waits before each retry after a failed +// LocalAPI read while quad-100 is still discovered. Retries stop after the +// last one, so a steady network costs nothing. +var tailscaleRouteRetryDelays = []time.Duration{5 * time.Second, 15 * time.Second, 60 * time.Second} + +// tailscaleDNSConfig is the subset of Tailscale's LocalAPI dns-config response +// (tailcfg.DNSConfig) that ctrld reads. Routes maps a DNS suffix to the +// resolvers Tailscale uses for it: Split DNS entries and App Connector domains. +type tailscaleDNSConfig struct { + Routes map[string][]struct { + Addr string `json:"Addr"` + } `json:"Routes"` +} + +// tailscaleRoute is one suffix that quad-100 resolves. +type tailscaleRoute struct { + Domain string + // NeedsPassthrough is set when quad-100 forwards the suffix as plain DNS + // on port 53 to a Tailscale IPv4 address. pf passes that traffic only + // through the VPN DNS interface passthrough, which exists in non-exit mode + // only. Everywhere else it would be redirected back to ctrld. + NeedsPassthrough bool +} + +// tailscaleResolverPath tells how quad-100 reaches one route resolver, as seen +// by ctrld's pf rules. +type tailscaleResolverPath int + +const ( + // tailscaleResolverLoopFree: not plain DNS on port 53 (DoH over the + // tailnet, peerAPI), or no upstream at all. pf never redirects it. + tailscaleResolverLoopFree tailscaleResolverPath = iota + // tailscaleResolverViaPassthrough: plain DNS on port 53 to a Tailscale + // IPv4 address, reached on the tunnel interface. + tailscaleResolverViaPassthrough + // tailscaleResolverLoops: plain DNS on port 53 to an address the + // passthrough does not cover (public, LAN, IPv6), or an unknown form. + tailscaleResolverLoops +) + +// classifyTailscaleResolver classifies one dnstype.Resolver Addr: a plain IP, +// an IP:port, or an http(s) URL. Unknown forms are treated as looping, so the +// suffix keeps going to Control D as it did before Tailscale routes existed. +func classifyTailscaleResolver(addr string) tailscaleResolverPath { + addr = strings.TrimSpace(addr) + if strings.HasPrefix(addr, "http://") || strings.HasPrefix(addr, "https://") { + u, err := url.Parse(addr) + if err != nil || u.Hostname() == "" { + return tailscaleResolverLoops + } + if u.Port() != "53" { + return tailscaleResolverLoopFree + } + ip, err := netip.ParseAddr(u.Hostname()) + if err != nil { + return tailscaleResolverLoops + } + return classifyTailscalePort53(ip) + } + if ip, err := netip.ParseAddr(addr); err == nil { + return classifyTailscalePort53(ip) + } + if ipp, err := netip.ParseAddrPort(addr); err == nil { + if ipp.Port() != 53 { + return tailscaleResolverLoopFree + } + return classifyTailscalePort53(ipp.Addr()) + } + return tailscaleResolverLoops +} + +// classifyTailscalePort53 classifies a plain port-53 resolver. The pf +// passthrough is IPv4-only and covers only the tunnel interface, which carries +// the Tailscale CGNAT range in non-exit mode. +func classifyTailscalePort53(ip netip.Addr) tailscaleResolverPath { + ip = ip.Unmap() + if ip.Is4() && tsaddr.CGNATRange().Contains(ip) { + return tailscaleResolverViaPassthrough + } + return tailscaleResolverLoops +} + +// parseTailscaleRoutes returns the sorted, normalized routes of a dns-config +// response. It drops the root, IP literals and names that cannot be DNS +// suffixes, and suffixes with a resolver that quad-100 can reach only through +// pf's port-53 redirect. Those keep going to Control D rather than loop. +func parseTailscaleRoutes(body []byte) ([]tailscaleRoute, error) { + var cfg tailscaleDNSConfig + if err := json.Unmarshal(body, &cfg); err != nil { + return nil, err + } + byDomain := make(map[string]tailscaleRoute, len(cfg.Routes)) + looping := 0 + for suffix, resolvers := range cfg.Routes { + domain := normalizeVPNDomain(strings.TrimPrefix(strings.TrimSpace(suffix), "*.")) + if !validTailscaleRouteDomain(domain) { + continue + } + route := tailscaleRoute{Domain: domain} + loops := false + for _, resolver := range resolvers { + switch classifyTailscaleResolver(resolver.Addr) { + case tailscaleResolverViaPassthrough: + route.NeedsPassthrough = true + case tailscaleResolverLoops: + loops = true + } + } + if loops { + looping++ + continue + } + if prev, ok := byDomain[domain]; ok { + route.NeedsPassthrough = route.NeedsPassthrough || prev.NeedsPassthrough + } + byDomain[domain] = route + } + routes := make([]tailscaleRoute, 0, len(byDomain)) + for _, route := range byDomain { + routes = append(routes, route) + } + slices.SortFunc(routes, func(a, b tailscaleRoute) int { return strings.Compare(a.Domain, b.Domain) }) + if looping > 0 { + mainLog.Load().Debug().Msgf("Tailscale DNS config: skipped %d routes whose port-53 resolvers pf would redirect back to ctrld", looping) + } + if len(routes) > maxTailscaleRoutes { + mainLog.Load().Warn().Msgf("Tailscale DNS config has %d routes; keeping the first %d", len(routes), maxTailscaleRoutes) + routes = routes[:maxTailscaleRoutes] + } + return routes, nil +} + +func validTailscaleRouteDomain(domain string) bool { + if domain == "" || net.ParseIP(domain) != nil { + return false + } + return !strings.ContainsAny(domain, " /:*\t\r\n") +} + +// isTailscaleServiceIP reports whether server is Tailscale's DNS service +// address (quad-100) in either address family. +func isTailscaleServiceIP(server string) bool { + ip := net.ParseIP(server) + if ip == nil { + return false + } + return ip.Equal(net.IP(tsaddr.TailscaleServiceIP().AsSlice())) || + ip.Equal(net.IP(tsaddr.TailscaleServiceIPv6().AsSlice())) +} + +func hasTailscaleResolver(configs []ctrld.VPNDNSConfig) bool { + for _, config := range configs { + if slices.ContainsFunc(config.Servers, isTailscaleServiceIP) { + return true + } + } + return false +} + +// withTailscaleRouteDomains returns configs with the route domains appended to +// every config whose servers include Tailscale's service IP. Other configs are +// returned unchanged, so non-Tailscale resolvers never gain these routes. +// Routes that need the pf passthrough are added only to a config that gets +// one: a named interface in non-exit mode. It also returns the set of domains +// that were added. +func withTailscaleRouteDomains(configs []ctrld.VPNDNSConfig, routes []tailscaleRoute) ([]ctrld.VPNDNSConfig, map[string]bool) { + if len(routes) == 0 { + return configs, nil + } + added := make(map[string]bool) + out := make([]ctrld.VPNDNSConfig, len(configs)) + for i, config := range configs { + out[i] = config + if !slices.ContainsFunc(config.Servers, isTailscaleServiceIP) { + continue + } + hasPassthrough := config.InterfaceName != "" && !config.IsExitMode + merged := append([]string{}, config.Domains...) + for _, route := range routes { + if route.NeedsPassthrough && !hasPassthrough { + continue + } + if !slices.Contains(merged, route.Domain) { + merged = append(merged, route.Domain) + added[route.Domain] = true + } + } + out[i].Domains = merged + } + return out, added +} + +// tailscaleRouteState keeps the last good Tailscale route set and the bounded +// retry after a failed read. +type tailscaleRouteState struct { + // last is the last successful read. It is guarded by the manager's + // discoveryMu, which every refresh holds. + last []tailscaleRoute + + mu sync.Mutex + timer *time.Timer + next int // index into tailscaleRouteRetryDelays of the next retry + retrying bool // a retry timer fired and its refresh has not read yet + closed bool +} + +// takeRetry reports whether the refresh now reading was started by a retry. +func (s *tailscaleRouteState) takeRetry() bool { + s.mu.Lock() + defer s.mu.Unlock() + retrying := s.retrying + s.retrying = false + return retrying +} + +// scheduleRetry arms the next retry after a failed read. A failure outside a +// retry restarts the sequence unless a retry is already pending. +func (s *tailscaleRouteState) scheduleRetry(fromRetry bool, refresh func()) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return + } + if !fromRetry { + if s.timer != nil { + return + } + s.next = 0 + } + if s.next >= len(tailscaleRouteRetryDelays) { + return + } + delay := tailscaleRouteRetryDelays[s.next] + s.next++ + s.timer = time.AfterFunc(delay, func() { + s.mu.Lock() + if s.closed { + s.mu.Unlock() + return + } + s.timer = nil + s.retrying = true + s.mu.Unlock() + refresh() + }) +} + +// stopRetry cancels a pending retry and resets the sequence. +func (s *tailscaleRouteState) stopRetry() { + s.mu.Lock() + defer s.mu.Unlock() + if s.timer != nil { + s.timer.Stop() + s.timer = nil + } + s.next = 0 + s.retrying = false +} + +// close stops retries for good. +func (s *tailscaleRouteState) close() { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + if s.timer != nil { + s.timer.Stop() + s.timer = nil + } +} + +// tailscaleRoutesFor reads the Tailscale DNS routes when a discovered resolver +// is Tailscale's. After a failed read it keeps the last good set and schedules +// a bounded retry; without a Tailscale resolver it forgets the set. +// The caller must hold m.discoveryMu. +func (m *vpnDNSManager) tailscaleRoutesFor(ctx context.Context, configs []ctrld.VPNDNSConfig) []tailscaleRoute { + fromRetry := m.tailscale.takeRetry() + if m.discoverTailscaleRoutes == nil || !hasTailscaleResolver(configs) { + m.tailscale.last = nil + m.tailscale.stopRetry() + return nil + } + readCtx, cancel := context.WithTimeout(ctx, tailscaleLocalAPITimeout) + defer cancel() + routes, err := m.discoverTailscaleRoutes(readCtx) + if err != nil { + ctrld.Log(ctx, ctrld.LoggerFromCtx(ctx).Debug().Err(err), "Tailscale DNS routes unavailable; keeping %d last known routes and retrying", len(m.tailscale.last)) + retry := m.retryRefresh + if retry == nil { + retry = func() { m.Refresh(m.backgroundCtx(), true) } + } + m.tailscale.scheduleRetry(fromRetry, retry) + return m.tailscale.last + } + m.tailscale.last = routes + m.tailscale.stopRetry() + return routes +} + +// backgroundCtx returns a context carrying the manager's logger for refreshes +// that start outside a caller's context, such as a Tailscale route retry. +func (m *vpnDNSManager) backgroundCtx() context.Context { + logger := mainLog.Load() + if m.logger != nil && m.logger.Load() != nil { + logger = m.logger.Load() + } + return ctrld.LoggerCtx(context.Background(), logger) +} + +// Close stops background work of the manager. It is safe on a nil manager. +func (m *vpnDNSManager) Close() { + if m == nil { + return + } + m.tailscale.close() +} diff --git a/cmd/cli/tailscale_routes_test.go b/cmd/cli/tailscale_routes_test.go new file mode 100644 index 00000000..a9c34f03 --- /dev/null +++ b/cmd/cli/tailscale_routes_test.go @@ -0,0 +1,451 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// tailscaleDNSConfigFixture is a reduced LocalAPI dns-config response with +// Split DNS zones, App Connector routes, a connector IP literal and the root. +// tail.example is a Split DNS zone on a tailnet resolver, which pf passes only +// in non-exit mode. lan.example and pub.example use resolvers off the tunnel, +// and mixed.example has one of each; pf would redirect those back to ctrld. +const tailscaleDNSConfigFixture = `{ + "Resolvers": [], + "Routes": { + "corp.example.": [], + "tail.example": [{"Addr": "100.64.0.2"}], + "lan.example": [{"Addr": "192.168.1.2"}], + "pub.example": [{"Addr": "8.8.8.8"}], + "v6.example": [{"Addr": "fd7a:115c:a1e0::2"}], + "mixed.example": [{"Addr": "100.64.0.2"}, {"Addr": "10.0.0.2"}], + "doh.example": [{"Addr": "https://dns.example/dns-query"}], + "backend.example.de": [{"Addr": "http://100.95.61.111:34024/dns-query"}], + "cd.backend.example.co.jp": [{"Addr": "http://100.94.53.33:34880/dns-query"}], + "Queue.RMQ4.Example.com": [{"Addr": "http://100.94.53.33:34880/dns-query"}], + "*.wild.example": [{"Addr": "http://100.1.2.3:1/dns-query"}], + "46.30.138.82": [{"Addr": "http://100.123.219.98:49667/dns-query"}], + "2001:db8::1": [{"Addr": "http://100.123.219.98:49667/dns-query"}], + ".": [{"Addr": "1.1.1.1"}], + "": [] + }, + "Domains": ["tailnet.ts.net"] +}` + +func TestParseTailscaleRoutes(t *testing.T) { + got, err := parseTailscaleRoutes([]byte(tailscaleDNSConfigFixture)) + if err != nil { + t.Fatal(err) + } + want := []tailscaleRoute{ + {Domain: "backend.example.de"}, + {Domain: "cd.backend.example.co.jp"}, + {Domain: "corp.example"}, + {Domain: "doh.example"}, + {Domain: "queue.rmq4.example.com"}, + {Domain: "tail.example", NeedsPassthrough: true}, + {Domain: "wild.example"}, + } + if !slices.Equal(got, want) { + t.Fatalf("parseTailscaleRoutes() = %v, want %v", got, want) + } + + if _, err := parseTailscaleRoutes([]byte("not json")); err == nil { + t.Fatal("expected an error for an undecodable body") + } +} + +func TestClassifyTailscaleResolver(t *testing.T) { + for addr, want := range map[string]tailscaleResolverPath{ + "http://100.95.61.111:34024/dns-query": tailscaleResolverLoopFree, + "https://dns.example/dns-query": tailscaleResolverLoopFree, + "100.64.0.2:5353": tailscaleResolverLoopFree, + "100.64.0.2": tailscaleResolverViaPassthrough, + "100.64.0.2:53": tailscaleResolverViaPassthrough, + "http://100.64.0.2:53/dns-query": tailscaleResolverViaPassthrough, + "10.0.0.2": tailscaleResolverLoops, + "8.8.8.8": tailscaleResolverLoops, + "fd7a:115c:a1e0::2": tailscaleResolverLoops, + "[2001:db8::1]:53": tailscaleResolverLoops, + "http://dns.example:53/": tailscaleResolverLoops, + "tls://dns.example": tailscaleResolverLoops, + "": tailscaleResolverLoops, + } { + if got := classifyTailscaleResolver(addr); got != want { + t.Errorf("classifyTailscaleResolver(%q) = %d, want %d", addr, got, want) + } + } +} + +func TestParseTailscaleRoutesCapsAndLogsTruncation(t *testing.T) { + var b strings.Builder + b.WriteString(`{"Routes":{`) + for i := 0; i < maxTailscaleRoutes+5; i++ { + if i > 0 { + b.WriteString(",") + } + fmt.Fprintf(&b, `"r%05d.example":[]`, i) + } + b.WriteString(`}}`) + logs := captureMainLog(t) + + got, err := parseTailscaleRoutes([]byte(b.String())) + if err != nil { + t.Fatal(err) + } + if len(got) != maxTailscaleRoutes { + t.Fatalf("kept %d routes, want %d", len(got), maxTailscaleRoutes) + } + if want := fmt.Sprintf("has %d routes; keeping the first %d", maxTailscaleRoutes+5, maxTailscaleRoutes); !strings.Contains(logs.String(), want) { + t.Fatalf("log does not report the truncation %q: %s", want, logs.String()) + } +} + +func TestWithTailscaleRouteDomainsOnlyExtendsTailscaleResolver(t *testing.T) { + configs := []ctrld.VPNDNSConfig{ + {InterfaceName: "utun4", Servers: []string{"100.100.100.100", "fd7a:115c:a1e0::53"}, Domains: []string{"tailnet.ts.net", "corp.example"}}, + {InterfaceName: "en0", Servers: []string{"192.168.1.1"}, Domains: []string{"lan"}}, + } + got, added := withTailscaleRouteDomains(configs, []tailscaleRoute{{Domain: "corp.example"}, {Domain: "backend.example.de"}}) + + if !added["backend.example.de"] || added["corp.example"] { + t.Fatalf("added = %v, want only backend.example.de", added) + } + if want := []string{"tailnet.ts.net", "corp.example", "backend.example.de"}; !slices.Equal(got[0].Domains, want) { + t.Fatalf("tailscale domains = %v, want %v", got[0].Domains, want) + } + if want := []string{"lan"}; !slices.Equal(got[1].Domains, want) { + t.Fatalf("non-tailscale domains changed: %v", got[1].Domains) + } + if want := []string{"tailnet.ts.net", "corp.example"}; !slices.Equal(configs[0].Domains, want) { + t.Fatalf("input configs were mutated: %v", configs[0].Domains) + } +} + +func TestVPNDNSRefreshRoutesTailscaleRoutesToQuad100(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{ + {InterfaceName: "utun4", Servers: []string{"100.100.100.100"}, Domains: []string{"tailnet.ts.net", "lan"}}, + {InterfaceName: "en0", Servers: []string{"192.168.1.248"}, Domains: []string{"lan"}}, + } + } + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + return parseTailscaleRoutes([]byte(tailscaleDNSConfigFixture)) + } + + m.Refresh(context.Background(), true) + + for _, name := range []string{"host.corp.example.", "backend.example.de.", "x.cd.backend.example.co.jp.", "host.tail.example."} { + if got := m.UpstreamForDomain(name); !slices.Equal(got, []string{"100.100.100.100"}) { + t.Errorf("UpstreamForDomain(%q) = %v, want quad-100", name, got) + } + } + for _, name := range []string{"example.de.", "other.example.com.", "46.30.138.82.", "host.lan.example.", "host.pub.example.", "host.v6.example.", "host.mixed.example."} { + if got := m.UpstreamForDomain(name); got != nil { + t.Errorf("UpstreamForDomain(%q) = %v, want no route", name, got) + } + } + for _, ex := range m.CurrentExemptions() { + if ex.Server != "100.100.100.100" && ex.Server != "192.168.1.248" { + t.Errorf("unexpected exemption %+v", ex) + } + } +} + +func TestVPNDNSRefreshSkipsTailscaleLookupWithoutQuad100(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{InterfaceName: "utun5", Servers: []string{"10.8.0.1"}, Domains: []string{"corp.example"}}} + } + var calls atomic.Int32 + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + calls.Add(1) + return []tailscaleRoute{{Domain: "should.not.appear"}}, nil + } + + m.Refresh(context.Background(), true) + + if calls.Load() != 0 { + t.Fatalf("Tailscale LocalAPI queried %d times without a Tailscale resolver", calls.Load()) + } + if got := m.UpstreamForDomain("x.should.not.appear."); got != nil { + t.Fatalf("unexpected route %v", got) + } +} + +func TestVPNDNSRefreshKeepsOSRoutesWhenTailscaleUnavailable(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{InterfaceName: "utun4", Servers: []string{"100.100.100.100"}, Domains: []string{"tailnet.ts.net"}}} + } + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + return nil, errTailscaleLocalAPIUnavailable + } + stubTailscaleRetryDelays(t, time.Hour) + defer m.Close() + + m.Refresh(context.Background(), true) + + if got := m.Routes(); len(got) != 1 || !slices.Equal(got["tailnet.ts.net"], []string{"100.100.100.100"}) { + t.Fatalf("routes = %v, want only the OS-discovered tailnet route", got) + } +} + +func TestVPNDNSRefreshPicksUpTailscaleRouteEdits(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{InterfaceName: "utun4", Servers: []string{"100.100.100.100"}, Domains: []string{"tailnet.ts.net"}}} + } + routes := []tailscaleRoute{{Domain: "corp.example"}} + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + return slices.Clone(routes), nil + } + + m.Refresh(context.Background(), true) + // A Tailscale reconnect is a tunnel change, which runs a refresh. + routes = []tailscaleRoute{{Domain: "new.connector.example"}} + m.Refresh(context.Background(), true) + + if got := m.UpstreamForDomain("api.new.connector.example."); !slices.Equal(got, []string{"100.100.100.100"}) { + t.Fatalf("new connector route = %v, want quad-100", got) + } + if got := m.UpstreamForDomain("host.corp.example."); got != nil { + t.Fatalf("removed route still present: %v", got) + } +} + +func TestParseMacOSSameUserProofs(t *testing.T) { + const good = "/Users/u/Library/Group Containers/W5364U7YZB.io.tailscale.ipn.macos/sameuserproof-49152-abcdef0123" + out := []byte("p123\ncIPNExtension\nn/private/var/folders/x\nn" + good + "\n" + + // A process whose name only starts with IPNExtension: lsof -c matches it. + "p200\ncIPNExtensionEvil\nn" + good + "\n" + + // The right name, but the proof file is outside a group container. + "p300\ncIPNExtension\nn/tmp/x.tailscale.ipn.macos/sameuserproof-8080-spoof\n" + + "n/Users/u/Library/Group Containers/io.tailscale.ipn.macos/sameuserproof-notaport-tok\n" + + "n/Users/u/Library/Group Containers/io.tailscale.ipn.macos/sameuserproof-49152-\n" + + "n/Users/u/Library/Group Containers/io.tailscale.ipn.macos/nested/sameuserproof-1-tok\n") + got := parseMacOSSameUserProofs(out) + want := []macOSSameUserProof{{pid: 123, port: 49152, token: "abcdef0123"}} + if !slices.Equal(got, want) { + t.Fatalf("parseMacOSSameUserProofs() = %+v, want %+v", got, want) + } +} + +func TestVPNDNSRefreshSkipsPassthroughRoutesInExitMode(t *testing.T) { + for _, tc := range []struct { + name string + exitMode bool + wantTail bool + }{ + {name: "split", exitMode: false, wantTail: true}, + {name: "exit node", exitMode: true, wantTail: false}, + } { + t.Run(tc.name, func(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{InterfaceName: "utun4", Servers: []string{"100.100.100.100"}, Domains: []string{"tailnet.ts.net"}, IsExitMode: tc.exitMode}} + } + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + return parseTailscaleRoutes([]byte(tailscaleDNSConfigFixture)) + } + + m.Refresh(context.Background(), true) + + // A peerAPI DoH connector never crosses pf's port-53 redirect. + if got := m.UpstreamForDomain("api.backend.example.de."); !slices.Equal(got, []string{"100.100.100.100"}) { + t.Fatalf("connector route = %v, want quad-100", got) + } + got := m.UpstreamForDomain("host.tail.example.") + if tc.wantTail && !slices.Equal(got, []string{"100.100.100.100"}) { + t.Fatalf("tailnet Split DNS route = %v, want quad-100", got) + } + if !tc.wantTail && got != nil { + t.Fatalf("tailnet Split DNS route = %v in exit mode, want none (pf has no passthrough)", got) + } + }) + } +} + +func TestVPNDNSRefreshKeepsLastTailscaleRoutesAndRetries(t *testing.T) { + stubTailscaleRetryDelays(t, 10*time.Millisecond, 10*time.Millisecond) + m := newVPNDNSManager(&mainLog, nil) + defer m.Close() + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{InterfaceName: "utun4", Servers: []string{"100.100.100.100"}, Domains: []string{"tailnet.ts.net"}}} + } + var mu sync.Mutex + results := []error{nil, errors.New("503 no netmap"), nil} + served := []tailscaleRoute{{Domain: "old.example"}} + var calls atomic.Int32 + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + mu.Lock() + defer mu.Unlock() + i := int(calls.Add(1)) - 1 + if i < len(results) && results[i] != nil { + return nil, results[i] + } + return slices.Clone(served), nil + } + + m.Refresh(context.Background(), true) + mu.Lock() + served = []tailscaleRoute{{Domain: "new.example"}} + mu.Unlock() + // This refresh fails: the last good routes stay. + m.Refresh(context.Background(), true) + if got := m.UpstreamForDomain("host.old.example."); !slices.Equal(got, []string{"100.100.100.100"}) { + t.Fatalf("last good route = %v after a failed read, want quad-100", got) + } + + // The retry reads again and replaces the set. + deadline := time.Now().Add(5 * time.Second) + for m.UpstreamForDomain("host.new.example.") == nil { + if time.Now().After(deadline) { + t.Fatalf("retry did not refresh the routes; %d LocalAPI calls", calls.Load()) + } + time.Sleep(5 * time.Millisecond) + } + if got := m.UpstreamForDomain("host.old.example."); got != nil { + t.Fatalf("replaced route still present: %v", got) + } + // A successful read stops the retries. + time.Sleep(50 * time.Millisecond) + if got := calls.Load(); got != 3 { + t.Fatalf("LocalAPI calls = %d, want 3 (no retry after success)", got) + } +} + +func TestVPNDNSTailscaleRetriesAreBounded(t *testing.T) { + stubTailscaleRetryDelays(t, time.Millisecond, time.Millisecond, time.Millisecond) + m := newVPNDNSManager(&mainLog, nil) + defer m.Close() + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{InterfaceName: "utun4", Servers: []string{"100.100.100.100"}}} + } + var calls atomic.Int32 + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + calls.Add(1) + return nil, errTailscaleLocalAPIUnavailable + } + + m.Refresh(context.Background(), true) + time.Sleep(200 * time.Millisecond) + + // One read on the refresh, then one per retry delay, then nothing. + if got := calls.Load(); got != 4 { + t.Fatalf("LocalAPI calls = %d, want 4", got) + } +} + +func TestVPNDNSTailscaleRoutesDroppedWithoutQuad100(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + withTS := true + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + if withTS { + return []ctrld.VPNDNSConfig{{InterfaceName: "utun4", Servers: []string{"100.100.100.100"}}} + } + return nil + } + m.discoverTailscaleRoutes = func(context.Context) ([]tailscaleRoute, error) { + return []tailscaleRoute{{Domain: "old.example"}}, nil + } + m.Refresh(context.Background(), true) + withTS = false + m.Refresh(context.Background(), false) + if m.tailscale.last != nil { + t.Fatalf("last Tailscale routes kept without quad-100: %v", m.tailscale.last) + } +} + +func TestRetryVPNDNSRefreshUsesNetworkActivityGate(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + var calls atomic.Int32 + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + calls.Add(1) + return nil + } + p := &prog{vpnDNS: m} + retry := p.retryVPNDNSRefresh(m) + + retry() + if got := calls.Load(); got != 1 { + t.Fatalf("refreshes = %d for the live manager, want 1", got) + } + + p.vpnDNS = newVPNDNSManager(&mainLog, nil) + retry() + if got := calls.Load(); got != 1 { + t.Fatalf("retry of a replaced manager refreshed it (%d refreshes)", got) + } + + p.vpnDNS = m + p.closeNetMonitor() + retry() + if got := calls.Load(); got != 1 { + t.Fatalf("retry refreshed after shutdown (%d refreshes)", got) + } +} + +func stubTailscaleRetryDelays(t *testing.T, delays ...time.Duration) { + t.Helper() + old := tailscaleRouteRetryDelays + tailscaleRouteRetryDelays = delays + t.Cleanup(func() { tailscaleRouteRetryDelays = old }) +} + +func TestDiscoverTailscaleRoutesFromLocalAPI(t *testing.T) { + const token = "s3cret-token" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != tailscaleDNSConfigPath || r.Host != tailscaleLocalAPIHost { + http.NotFound(w, r) + return + } + if _, pass, ok := r.BasicAuth(); !ok || pass != token { + http.Error(w, "auth required", http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte(tailscaleDNSConfigFixture)) + })) + defer srv.Close() + dial := func(ctx context.Context) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", srv.Listener.Addr().String()) + } + + got, err := discoverTailscaleRoutesFrom(context.Background(), []tailscaleLocalAPIEndpoint{ + {name: "stale", dial: func(context.Context) (net.Conn, error) { return nil, errors.New("connection refused") }}, + {name: "wrong-token", dial: dial, token: "nope"}, + {name: "good", dial: dial, token: token}, + }) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(got, tailscaleRoute{Domain: "corp.example"}) || !slices.Contains(got, tailscaleRoute{Domain: "backend.example.de"}) { + t.Fatalf("routes = %v", got) + } + + _, err = discoverTailscaleRoutesFrom(context.Background(), []tailscaleLocalAPIEndpoint{{name: "wrong-token", dial: dial, token: "nope"}}) + if err == nil { + t.Fatal("expected an error for a rejected token") + } + if strings.Contains(err.Error(), "nope") || strings.Contains(err.Error(), token) { + t.Fatalf("error leaks the token: %v", err) + } + + if _, err := discoverTailscaleRoutesFrom(context.Background(), nil); !errors.Is(err, errTailscaleLocalAPIUnavailable) { + t.Fatalf("err = %v, want errTailscaleLocalAPIUnavailable", err) + } +} diff --git a/cmd/cli/tunnel_owner.go b/cmd/cli/tunnel_owner.go new file mode 100644 index 00000000..b57b0852 --- /dev/null +++ b/cmd/cli/tunnel_owner.go @@ -0,0 +1,35 @@ +package cli + +import ( + "bufio" + "io" + "strings" +) + +// agentDescriptionKey starts the quoted product name on an ifconfig agent line. +const agentDescriptionKey = `desc:"` + +// parseIfconfigAgent returns the owner of a tunnel from "ifconfig -v utunN" +// output. The owner is the description of the first agent line, and a tunnel that +// no network extension owns has no agent line, so the result is "". +func parseIfconfigAgent(r io.Reader) string { + if r == nil { + return "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "agent ") { + continue + } + _, quoted, ok := strings.Cut(line, agentDescriptionKey) + if !ok { + continue + } + if description, _, ok := strings.Cut(quoted, `"`); ok && description != "" { + return description + } + } + // An owner that the read did not reach is not an owner that does not exist. + return "" +} diff --git a/cmd/cli/tunnel_owner_test.go b/cmd/cli/tunnel_owner_test.go new file mode 100644 index 00000000..76ba6235 --- /dev/null +++ b/cmd/cli/tunnel_owner_test.go @@ -0,0 +1,83 @@ +package cli + +import ( + "strings" + "testing" +) + +// Captured from "ifconfig -v utunN" on macOS. A tunnel that a network extension +// owns carries an agent line with the product name in desc. +const ifconfigWindscribeTunnel = `utun4: flags=8051 mtu 1420 + index 22 + eflags=1002080 + options=6460 + inet 100.96.0.2 --> 100.96.0.2 netmask 0xffffffff + nd6 options=201 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Windscribe VPN" + agent domain:Multipath type:Path Source flags:0x3 desc:"Multipath path manager" + netif: 6d1e1b2a-4c55-4f0b-9f0a-1d2e3f4a5b6c + scheduler: FQ_CODEL + link quality: 100 (good) +` + +const ifconfigTailscaleTunnel = `utun6: flags=8051 mtu 1280 + index 24 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Tailscale Tunnel" + nd6 options=201 +` + +const ifconfigSystemTunnel = `utun0: flags=8051 mtu 1500 + index 16 + eflags=1002080 + inet6 fe80::e1a2:3b4c:5d6e:7f80%utun0 prefixlen 64 scopeid 0x10 + nd6 options=201 + scheduler: FQ_CODEL +` + +func Test_parseIfconfigAgent(t *testing.T) { + tests := []struct { + name string + output string + want string + }{ + { + name: "a VPN network extension owns the tunnel", + output: ifconfigWindscribeTunnel, + want: "Windscribe VPN", + }, + { + name: "a mesh client owns the tunnel", + output: ifconfigTailscaleTunnel, + want: "Tailscale Tunnel", + }, + { + name: "no agent line", + output: ifconfigSystemTunnel, + }, + { + name: "no output", + output: "", + }, + { + name: "an agent line without a description", + output: "\tagent domain:Multipath type:Path Source flags:0x3\n\tagent domain:NetworkExtension type:NetworkExtension flags:0xf desc:\"Windscribe VPN\"\n", + want: "Windscribe VPN", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := parseIfconfigAgent(strings.NewReader(tc.output)); got != tc.want { + t.Fatalf("parseIfconfigAgent() = %q, want %q", got, tc.want) + } + }) + } +} + +// Test_parseIfconfigAgentRejectsAnOversizedLine covers a truncated read. An +// owner that the reader did not reach is not an owner that does not exist. +func Test_parseIfconfigAgentRejectsAnOversizedLine(t *testing.T) { + output := strings.Repeat("a", 70<<10) + "\n" + ifconfigWindscribeTunnel + if got := parseIfconfigAgent(strings.NewReader(output)); got != "" { + t.Fatalf("tunnel owner of a truncated read = %q, want an empty owner", got) + } +} diff --git a/cmd/cli/uninstall_invalid_test.go b/cmd/cli/uninstall_invalid_test.go new file mode 100644 index 00000000..664e97b2 --- /dev/null +++ b/cmd/cli/uninstall_invalid_test.go @@ -0,0 +1,62 @@ +package cli + +import "testing" + +// Test_ensureRunningIfaceForInvalidUninstall is a regression test for issue-556: +// after a reboot, the invalid-device self-uninstall path could run before the +// running interface was known. Because resetDNS (via resetDNSForRunningIface) +// silently skips DNS restoration when p.runningIface is empty, the OS was left +// pointed at ctrld's local listener with no internet after the service was +// removed. ensureRunningIfaceForInvalidUninstall must populate p.runningIface +// before resetDNS runs. +func Test_ensureRunningIfaceForInvalidUninstall(t *testing.T) { + // preRun mutates the package-level iface global; restore it after the test. + origIface := iface + t.Cleanup(func() { iface = origIface }) + + // newService needs the package service config; it is safe to build here + // because ensureRunningIfaceForInvalidUninstall only queries the (absent) + // control socket via runningIface, which returns nil when ctrld is not + // running, and performs no DNS or service mutation. + sc := NewServiceCommand() + s, err := sc.newService(&prog{}, sc.createServiceConfig()) + if err != nil { + t.Fatalf("newService: %v", err) + } + + t.Run("iface flag already resolved but not yet copied", func(t *testing.T) { + iface = "eth-test" + p := &prog{} + + // Precondition mirrors the buggy post-reboot state: an empty running + // interface would make resetDNS skip restoration entirely. + if p.runningIface != "" { + t.Fatalf("precondition: runningIface = %q, want empty", p.runningIface) + } + + ensureRunningIfaceForInvalidUninstall(p, s) + + if p.runningIface == "" { + t.Fatal("runningIface still empty after prepare: resetDNS would skip DNS " + + "restoration and leave the OS pointed at ctrld's local listener") + } + if p.runningIface != "eth-test" { + t.Fatalf("runningIface = %q, want the resolved iface %q", p.runningIface, "eth-test") + } + }) + + t.Run("iface unset falls back to auto-detected interface", func(t *testing.T) { + iface = "" + p := &prog{} + + ensureRunningIfaceForInvalidUninstall(p, s) + + // With iface unset the prep resolves "auto" to the default interface + // (defaultIfaceName never returns empty on the supported platforms), so + // resetDNS has a concrete interface to restore. + if p.runningIface == "" { + t.Fatal("runningIface still empty after prepare with iface unset: " + + "resetDNS would skip DNS restoration") + } + }) +} diff --git a/cmd/cli/upstream_monitor.go b/cmd/cli/upstream_monitor.go index 6e19e38a..f660ed2a 100644 --- a/cmd/cli/upstream_monitor.go +++ b/cmd/cli/upstream_monitor.go @@ -1,7 +1,9 @@ package cli import ( + "strings" "sync" + "sync/atomic" "time" "github.com/Control-D-Inc/ctrld" @@ -12,31 +14,86 @@ const ( maxFailureRequest = 50 // checkUpstreamBackoffSleep is the time interval between each upstream checks. checkUpstreamBackoffSleep = 2 * time.Second + // checkUpstreamUnreachableBackoffMax caps the recovery retry interval for an + // endpoint that keeps failing with a network-unreachable error. It bounds + // the backoff so an unroutable endpoint is still re-probed periodically and + // recovers once the route returns. + checkUpstreamUnreachableBackoffMax = 60 * time.Second + // upstreamDownDelay is the time that the failures of an upstream must last + // before the monitor marks it down. + upstreamDownDelay = 10 * time.Second + // upstreamRecoveredDelay is the time that a reset holds back the failure + // counts of an upstream. + upstreamRecoveredDelay = 1 * time.Second ) +// unreachableRecoveryBackoff returns the retry interval for the given streak of +// consecutive network-unreachable failures. It starts at checkUpstreamBackoffSleep +// and doubles each attempt, capped at checkUpstreamUnreachableBackoffMax. +func unreachableRecoveryBackoff(streak int) time.Duration { + d := checkUpstreamBackoffSleep + for i := 1; i < streak; i++ { + d *= 2 + if d >= checkUpstreamUnreachableBackoffMax { + return checkUpstreamUnreachableBackoffMax + } + } + return d +} + +// upstreamDownState remembers when an upstream went down and how many +// failures put it there, so the up event can report the outage. +type upstreamDownState struct { + since time.Time + failures uint64 +} + // upstreamMonitor performs monitoring upstreams health. type upstreamMonitor struct { - cfg *ctrld.Config + cfg *ctrld.Config + logger atomic.Pointer[ctrld.Logger] - mu sync.RWMutex + mu sync.RWMutex + // down holds one entry for each upstream that is down now. + down map[string]upstreamDownState checking map[string]bool - down map[string]bool failureReq map[string]uint64 recovered map[string]bool // failureTimerActive tracks if a timer is already running for a given upstream. failureTimerActive map[string]bool + // retired is set when a reload replaced this monitor. A timer that was + // armed before the reload must not open an outage that nothing closes. + retired bool + + // now is a seam, so a test can set the reported down time. + now func() time.Time + // after arms a one-shot timer, so a test fires the check itself instead of + // waiting for it. nil means time.AfterFunc. + after func(time.Duration, func()) +} + +// afterFunc returns the timer seam. A real timer drops its handle, because +// nothing stops these checks. +func (um *upstreamMonitor) afterFunc() func(time.Duration, func()) { + if um.after != nil { + return um.after + } + return func(d time.Duration, fn func()) { time.AfterFunc(d, fn) } } -func newUpstreamMonitor(cfg *ctrld.Config) *upstreamMonitor { +// newUpstreamMonitor creates a new upstream monitor instance +func newUpstreamMonitor(cfg *ctrld.Config, logger *ctrld.Logger) *upstreamMonitor { um := &upstreamMonitor{ cfg: cfg, checking: make(map[string]bool), - down: make(map[string]bool), + down: make(map[string]upstreamDownState), failureReq: make(map[string]uint64), recovered: make(map[string]bool), failureTimerActive: make(map[string]bool), + now: time.Now, } + um.logger.Store(logger) for n := range cfg.Upstream { upstream := upstreamPrefix + n um.reset(upstream) @@ -52,8 +109,11 @@ func (um *upstreamMonitor) increaseFailureCount(upstream string) { um.mu.Lock() defer um.mu.Unlock() + if um.retired { + return + } if um.recovered[upstream] { - mainLog.Load().Debug().Msgf("upstream %q is recovered, skipping failure count increase", upstream) + um.logger.Load().Debug().Msgf("Upstream %q is recovered, skipping failure count increase", upstream) return } @@ -61,31 +121,166 @@ func (um *upstreamMonitor) increaseFailureCount(upstream string) { failedCount := um.failureReq[upstream] // Log the updated failure count. - mainLog.Load().Debug().Msgf("upstream %q failure count updated to %d", upstream, failedCount) + um.logger.Load().Debug().Msgf("Upstream %q failure count updated to %d", upstream, failedCount) - // If this is the first failure and no timer is running, start a 10-second timer. + // If this is the first failure and no timer is running, start the timer. if failedCount == 1 && !um.failureTimerActive[upstream] { um.failureTimerActive[upstream] = true - go func(upstream string) { - time.Sleep(10 * time.Second) - um.mu.Lock() - defer um.mu.Unlock() - // If no success occurred during the 10-second window (i.e. counter remains > 0) - // and the upstream is not in a recovered state, mark it as down. - if um.failureReq[upstream] > 0 && !um.recovered[upstream] { - um.down[upstream] = true - mainLog.Load().Warn().Msgf("upstream %q marked as down after 10 seconds (failure count: %d)", upstream, um.failureReq[upstream]) - } - // Reset the timer flag so that a new timer can be spawned if needed. - um.failureTimerActive[upstream] = false - }(upstream) + um.afterFunc()(upstreamDownDelay, func() { um.markDownAfterDelay(upstream) }) } // If the failure count quickly reaches the threshold, mark the upstream as down immediately. if failedCount >= maxFailureRequest { - um.down[upstream] = true - mainLog.Load().Warn().Msgf("upstream %q marked as down immediately (failure count: %d)", upstream, failedCount) + um.markDown(upstream, failedCount, "immediate") + } +} + +// markDownAfterDelay marks an upstream down when its failures did not clear +// while the timer ran. A success during that time leaves the count at zero. +func (um *upstreamMonitor) markDownAfterDelay(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + if um.retired { + return + } + if um.failureReq[upstream] > 0 && !um.recovered[upstream] { + um.markDown(upstream, um.failureReq[upstream], "timer") + } + // The next failure of this upstream can arm a new timer. + um.failureTimerActive[upstream] = false +} + +// markDown marks an upstream as down and logs the transition once. The caller holds the lock. +func (um *upstreamMonitor) markDown(upstream string, failures uint64, reason string) { + if um.retired { + return + } + if _, down := um.down[upstream]; down { + return + } + um.down[upstream] = upstreamDownState{since: um.now(), failures: failures} + journal(um.logger.Load().Warn()). + Str("upstream", journalUpstreamName(upstream)). + Str("state", "down"). + Uint64("failure_count", failures). + Str("reason", reason). + Msg("Upstream state changed") +} + +// markUp marks an upstream as up and logs the transition once. The caller holds the lock. +func (um *upstreamMonitor) markUp(upstream, reason string) { + state, down := um.down[upstream] + if !down { + return + } + delete(um.down, upstream) + journal(um.logger.Load().Info()). + Str("upstream", journalUpstreamName(upstream)). + Str("state", "up"). + Str("reason", reason). + Int64("down_for_ms", um.now().Sub(state.since).Milliseconds()). + Uint64("failure_count", state.failures). + Msg("Upstream state changed") +} + +// journalUpstreamName bounds the upstream name that a journal event carries. +// ctrld generates upstream.os, upstream., and upstream.internal_; every +// other key is operator text that can hold a token, so it reads as +// upstream.custom. An empty name holds no operator text and stays empty, +// because a line that names no upstream must not read as a custom one. +func journalUpstreamName(upstream string) string { + if upstream == "" || upstream == upstreamOS || generatedUpstreamKey(strings.TrimPrefix(upstream, upstreamPrefix)) { + return upstream + } + return upstreamPrefix + "custom" +} + +// journalUpstreamNames bounds every name of a list, for a line that names +// several upstreams. +func journalUpstreamNames(upstreams []string) []string { + names := make([]string, 0, len(upstreams)) + for _, upstream := range upstreams { + names = append(names, journalUpstreamName(upstream)) + } + return names +} + +// generatedUpstreamKey reports whether key is a number, or internal_ and a +// number, the two shapes that ctrld writes itself. +func generatedUpstreamKey(key string) bool { + digits := strings.TrimPrefix(key, internalDomainUpstreamPrefix) + if digits == "" { + return false + } + for _, r := range digits { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// retire closes the outage of every upstream that is still down and stops +// every later state change. A reload builds a new monitor, so the journal ends +// the outages of the old one, and the old timers must not open new ones. +func (um *upstreamMonitor) retire() { + um.mu.Lock() + defer um.mu.Unlock() + for upstream := range um.down { + um.markUp(upstream, "reload") } + um.retired = true + clear(um.failureReq) + clear(um.failureTimerActive) +} + +// noteSuccess records an answer from an upstream. It does not set the recovered +// flag, because that flag stops failure counts for one second and an answer +// must not hide the failures that follow it. +func (um *upstreamMonitor) noteSuccess(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + + um.failureReq[upstream] = 0 + um.markUp(upstream, "recovered") +} + +// countDown reports how many upstreams are down at this time. +func (um *upstreamMonitor) countDown() int { + um.mu.RLock() + defer um.mu.RUnlock() + return len(um.down) +} + +// countDownExcept reports how many upstreams are down at this time, without +// the ones that skip accepts. The grade of the query path leaves out the +// resolvers that only an endpoint on the organization network can reach. +func (um *upstreamMonitor) countDownExcept(skip func(string) bool) int { + if skip == nil { + return um.countDown() + } + um.mu.RLock() + defer um.mu.RUnlock() + count := 0 + for upstream := range um.down { + if skip(upstream) { + continue + } + count++ + } + return count +} + +// downFor reports how long an upstream has been down. An upstream that is up +// gives zero, so a caller logs the field without a branch. +func (um *upstreamMonitor) downFor(upstream string) time.Duration { + um.mu.RLock() + defer um.mu.RUnlock() + state, down := um.down[upstream] + if !down { + return 0 + } + return um.now().Sub(state.since) } // isDown reports whether the given upstream is being marked as down. @@ -93,23 +288,26 @@ func (um *upstreamMonitor) isDown(upstream string) bool { um.mu.Lock() defer um.mu.Unlock() - return um.down[upstream] + _, down := um.down[upstream] + return down } // reset marks an upstream as up and set failed queries counter to zero. func (um *upstreamMonitor) reset(upstream string) { um.mu.Lock() um.failureReq[upstream] = 0 - um.down[upstream] = false + um.markUp(upstream, "recovered") um.recovered[upstream] = true um.mu.Unlock() - go func() { - // debounce the recovery to avoid incrementing failure counts already in flight - time.Sleep(1 * time.Second) - um.mu.Lock() - um.recovered[upstream] = false - um.mu.Unlock() - }() + // The flag holds back the failure counts that are already in flight. + um.afterFunc()(upstreamRecoveredDelay, func() { um.clearRecovered(upstream) }) +} + +// clearRecovered lets the failures of an upstream count again. +func (um *upstreamMonitor) clearRecovered(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + um.recovered[upstream] = false } // countHealthy returns the number of upstreams in the provided map that are considered healthy. @@ -117,7 +315,7 @@ func (um *upstreamMonitor) countHealthy(upstreams []string) int { var count int um.mu.RLock() for _, upstream := range upstreams { - if !um.down[upstream] { + if _, down := um.down[upstream]; !down { count++ } } diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go new file mode 100644 index 00000000..9a4d3d08 --- /dev/null +++ b/cmd/cli/upstream_monitor_test.go @@ -0,0 +1,291 @@ +package cli + +import ( + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// upstreamStateMessage is the message of every upstream transition event. +const upstreamStateMessage = "Upstream state changed" + +// downMonitor returns a monitor on a clock the test owns, with the upstream down. +func downMonitor(t *testing.T, upstream string) (*upstreamMonitor, *syncBuffer, *time.Time) { + t.Helper() + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + now := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + um.now = func() time.Time { return now } + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 1 { + t.Fatalf("state events after the failures: got %d, want 1", got) + } + return um, logs, &now +} + +func Test_upstreamMonitorLogsOneDownEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + + for i := 0; i < 633; i++ { + um.increaseFailureCount(upstream) + } + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events: got %d, want 1", len(events)) + } + wantField(t, events[0], "level", "warn") + wantField(t, events[0], "journal", true) + wantField(t, events[0], "upstream", upstream) + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "immediate") + wantField(t, events[0], "failure_count", float64(maxFailureRequest)) +} + +func Test_upstreamMonitorTimerLogsDownEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + um.increaseFailureCount(upstream) + if len(armed) != 1 { + t.Fatalf("armed timers: got %d, want 1", len(armed)) + } + armed[0]() + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events: got %d, want 1", len(events)) + } + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "timer") + wantField(t, events[0], "failure_count", float64(1)) +} + +func Test_upstreamMonitorResetLogsOneUpEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(1500 * time.Millisecond) + um.reset(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "level", "info") + wantField(t, up, "journal", true) + wantField(t, up, "upstream", upstream) + wantField(t, up, "state", "up") + wantField(t, up, "reason", "recovered") + wantField(t, up, "down_for_ms", float64(1500)) + wantField(t, up, "failure_count", float64(maxFailureRequest)) +} + +func Test_upstreamMonitorRetireLogsTheOpenOutage(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(3 * time.Second) + um.retire() + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "upstream", upstream) + wantField(t, up, "state", "up") + wantField(t, up, "reason", "reload") + wantField(t, up, "down_for_ms", float64(3000)) + + um.retire() + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 { + t.Fatalf("state events after the second retire: got %d, want 2", got) + } +} + +func Test_upstreamMonitorNoteSuccessLogsOneUpEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(2 * time.Second) + um.noteSuccess(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "state", "up") + wantField(t, up, "reason", "recovered") + wantField(t, up, "down_for_ms", float64(2000)) + wantField(t, up, "failure_count", float64(maxFailureRequest)) + if got := um.failureReq[upstream]; got != 0 { + t.Fatalf("failure count after the answer: got %d, want 0", got) + } +} + +func Test_upstreamMonitorNoteSuccessOnUpUpstreamIsQuiet(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + + um.noteSuccess(upstream) + um.noteSuccess(upstream) + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 { + t.Fatalf("state events: got %d, want 0", got) + } +} + +// Test_upstreamMonitorRetiredTimerStaysSilent fires the down timer that a +// failure armed before the reload retired the monitor. A retired monitor +// owns no outage, so the timer must add no event. +func Test_upstreamMonitorRetiredTimerStaysSilent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + um.increaseFailureCount(upstream) + um.retire() + for _, fire := range armed { + fire() + } + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 { + t.Fatalf("state events from the retired monitor: got %d, want 0: %s", got, logs.String()) + } + if um.isDown(upstream) { + t.Fatal("the retired monitor marked the upstream down") + } +} + +// Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent retires a monitor +// that already reported an outage. The armed timer and a late failure must +// add no event after the closing up event. +func Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + um.retire() + before := jsonLogEvents(t, logs, upstreamStateMessage) + if len(before) != 2 { + t.Fatalf("state events after the retire: got %d, want down and up", len(before)) + } + for _, fire := range armed { + fire() + } + um.increaseFailureCount(upstream) + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 { + t.Fatalf("state events after the retired timer fired: got %d, want 2: %s", got, logs.String()) + } + if um.isDown(upstream) { + t.Fatal("the retired monitor marked the upstream down again") + } +} + +// Test_upstreamMonitorKeepsACustomNameOutOfTheJournal marks an upstream with +// an operator-chosen key down. The key can hold a token, so the event names +// the upstream by a bounded name. +func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) { + const upstream = upstreamPrefix + "dns.controld.com/org-v1-SECRET0123" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + um.noteSuccess(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want down and up", len(events)) + } + for _, event := range events { + wantField(t, event, "upstream", upstreamPrefix+"custom") + } + for _, generated := range []string{upstreamOS, upstreamPrefix + "0", upstreamPrefix + "internal_3"} { + if got := journalUpstreamName(generated); got != generated { + t.Fatalf("journalUpstreamName(%q) = %q, want the name itself", generated, got) + } + } +} + +func Test_upstreamMonitorCountDown(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + if got := um.countDown(); got != 0 { + t.Fatalf("countDown on a new monitor = %d, want 0", got) + } + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+"1", 3, "immediate") + um.mu.Unlock() + if got := um.countDown(); got != 2 { + t.Fatalf("countDown with two down upstreams = %d, want 2", got) + } +} + +func Test_upstreamMonitorDownFor(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, _, now := downMonitor(t, upstream) + + *now = now.Add(2500 * time.Millisecond) + + if got := um.downFor(upstream); got != 2500*time.Millisecond { + t.Fatalf("downFor a down upstream = %s, want 2.5s", got) + } + if got := um.downFor(upstreamPrefix + "9"); got != 0 { + t.Fatalf("downFor an unknown upstream = %s, want 0", got) + } + + um.reset(upstream) + + if got := um.downFor(upstream); got != 0 { + t.Fatalf("downFor an upstream that is up = %s, want 0", got) + } +} + +func Test_upstreamMonitorCountDownExcept(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}, mainLog.Load()) + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"1", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"2", 3, "immediate") + um.mu.Unlock() + + if got := um.countDown(); got != 3 { + t.Fatalf("countDown = %d, want 3", got) + } + p := internalDomainUpstreamsProg(map[string]string{ + "0": "", + internalDomainUpstreamPrefix + "1": internalDomainUpstreamFallback, + internalDomainUpstreamPrefix + "2": internalDomainUpstreamOnly, + }) + if got := um.countDownExcept(p.isInternalDomainUpstream); got != 1 { + t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got) + } + if got := um.countDownExcept(nil); got != 3 { + t.Fatalf("countDownExcept(nil) = %d, want 3", got) + } +} diff --git a/cmd/cli/upstream_probe_journal_test.go b/cmd/cli/upstream_probe_journal_test.go new file mode 100644 index 00000000..c1bbfe88 --- /dev/null +++ b/cmd/cli/upstream_probe_journal_test.go @@ -0,0 +1,167 @@ +package cli + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// probeSecret is the operator text that a custom upstream key, name, and +// endpoint hold in these tests. +const probeSecret = "org-v1-SECRET0123" + +// customProbeUpstream builds an upstream whose key, name, and endpoint all +// hold operator text. +func customProbeUpstream() (string, *ctrld.UpstreamConfig) { + return upstreamPrefix + probeSecret, &ctrld.UpstreamConfig{ + Name: probeSecret, + Endpoint: "https://dns.example.com/" + probeSecret, + } +} + +// retainedProbeLines returns the captured lines that the journal keeps. When +// messages are given, only the lines that start with one of them come back. +// captureDebugMainLog swaps the process-wide mainLog, so any goroutine of +// another test can add a warn or error line to the same buffer; a caller that +// asserts on field values names the lines its own probe wrote. +func retainedProbeLines(t *testing.T, logs *syncBuffer, messages ...string) []map[string]any { + t.Helper() + var retained []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + switch event["level"] { + case "error", "warn": + default: + continue + } + if len(messages) == 0 { + retained = append(retained, event) + continue + } + message, _ := event["message"].(string) + for _, want := range messages { + if strings.HasPrefix(message, want) { + retained = append(retained, event) + break + } + } + } + return retained +} + +// wantNoProbeSecret fails when a retained line carries the operator text. +func wantNoProbeSecret(t *testing.T, logs *syncBuffer) { + t.Helper() + for _, event := range retainedProbeLines(t, logs) { + for field, value := range event { + text, ok := value.(string) + if ok && strings.Contains(text, probeSecret) { + t.Fatalf("retained field %q holds the operator text: %s", field, text) + } + } + } +} + +// Test_upstreamFailureLogKeepsACustomKeyOutOfTheJournal drives one recovery +// check failure against an operator-chosen upstream key. The journal keeps +// error lines, so the key, the name, and the endpoint must stay off the line. +func Test_upstreamFailureLogKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + upstream, uc := customProbeUpstream() + + (&upstreamFailureLog{}).report(probeTestProg(), upstream, uc, errors.New("probe failed"), time.Second) + + retained := retainedProbeLines(t, logs, "Upstream check failed after ") + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) + if !strings.Contains(logs.String(), probeSecret) { + t.Fatal("the operator text must stay at debug level for diagnosis") + } +} + +// Test_checkUpstreamOnceKeepsACustomKeyOutOfTheJournal drives the resolver +// creation failure of a recovery check. It logs at error level, so the journal +// keeps it. +func Test_checkUpstreamOnceKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + upstream, uc := customProbeUpstream() + // An empty type has no resolver, so the check fails before it sends. + uc.Type = "" + p := &prog{} + p.logger.Store(mainLog.Load()) + + if err := p.checkUpstreamOnce(upstream, uc, &upstreamFailureLog{}); err == nil { + t.Fatal("expected the resolver creation to fail") + } + + retained := retainedProbeLines(t, logs, "Failed to create resolver") + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) +} + +// Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal drives the loop checker +// against an operator-chosen upstream key. The loop checker writes at warn +// level, so the journal keeps its line too. +func Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + uc := &ctrld.UpstreamConfig{ + Name: probeSecret, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 200, + } + uc.Init(context.Background()) + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{probeSecret: uc}} + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg, mainLog.Load()), loop: make(map[string]bool)} + p.logger.Store(mainLog.Load()) + + p.checkDnsLoop() + + retained := retainedProbeLines(t, logs, "Could not perform loop check", "Could not send DNS loop check query") + if len(retained) == 0 { + t.Fatal("the loop check failure must stay visible above debug") + } + for _, event := range retained { + wantField(t, event, "upstream", upstreamPrefix+"custom") + } + wantNoProbeSecret(t, logs) +} + +// Test_logUpstreamPingFailuresKeepsACustomKeyOutOfTheJournal drives the report +// that follows a failed self-check. It writes at error level, so the journal +// keeps it and the operator text must stay off the line. +func Test_logUpstreamPingFailuresKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + _, uc := customProbeUpstream() + origPing := upstreamPingFn + t.Cleanup(func() { upstreamPingFn = origPing }) + upstreamPingFn = func(*ctrld.UpstreamConfig, context.Context) error { return errors.New("ping failed") } + + logUpstreamPingFailures(context.Background(), map[string]*ctrld.UpstreamConfig{probeSecret: uc}) + + retained := retainedProbeLines(t, logs, "Failed to connect to the upstream") + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) + if !strings.Contains(logs.String(), probeSecret) { + t.Fatal("the operator text must stay at debug level for diagnosis") + } +} + +// probeTestProg returns a prog that logs through the captured main logger. +func probeTestProg() *prog { + p := &prog{} + p.logger.Store(mainLog.Load()) + return p +} diff --git a/cmd/cli/upstream_retirement_test.go b/cmd/cli/upstream_retirement_test.go new file mode 100644 index 00000000..f6a712a1 --- /dev/null +++ b/cmd/cli/upstream_retirement_test.go @@ -0,0 +1,101 @@ +package cli + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" +) + +func TestCloseReplacedUpstreamsPreservesRetainedAndNew(t *testing.T) { + certServer := httptest.NewTLSServer(nil) + certificate := certServer.TLS.Certificates[0] + roots := x509.NewCertPool() + roots.AddCert(certServer.Certificate()) + certServer.Close() + + type upstream struct { + config *ctrld.UpstreamConfig + entered <-chan struct{} + release func() + } + newUpstream := func() upstream { + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{certificate}}) + require.NoError(t, err) + entered, release := make(chan struct{}), make(chan struct{}) + var enteredOnce, releaseOnce sync.Once + s := &dns.Server{Listener: ln, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { + enteredOnce.Do(func() { close(entered) }) + <-release + answer := new(dns.Msg) + answer.SetReply(m) + _ = w.WriteMsg(answer) + })} + done := make(chan error, 1) + go func() { done <- s.ActivateAndServe() }() + uc := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1"} + uc.Init(context.Background()) + uc.SetCertPool(roots) + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(func() { + unblock() + uc.CloseTransports() + _ = s.Shutdown() + _ = ln.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Error("test DNS server did not stop") + } + }) + return upstream{uc, entered, unblock} + } + removed, retained, replacement := newUpstream(), newUpstream(), newUpstream() + query := func(u upstream) <-chan error { + result := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + r, err := ctrld.NewResolver(ctx, u.config) + if err == nil { + q := new(dns.Msg) + q.SetQuestion("retirement.example.", dns.TypeA) + _, err = r.Resolve(ctx, q) + } + result <- err + }() + return result + } + removedResult, retainedResult := query(removed), query(retained) + for _, ch := range []<-chan struct{}{removed.entered, retained.entered} { + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("query did not reach local upstream") + } + } + closeReplacedUpstreams(map[string]*ctrld.UpstreamConfig{ + "0": removed.config, "1": retained.config, "nil": nil, + }, map[string]*ctrld.UpstreamConfig{ + "0": replacement.config, "renamed": retained.config, + }) + select { + case err := <-removedResult: + require.ErrorIs(t, err, net.ErrClosed) + case <-time.After(time.Second): + t.Fatal("removed upstream's active query was not closed") + } + retained.release() + require.NoError(t, <-retainedResult, "retained pointer must remain usable under a different key") + replacement.release() + require.NoError(t, <-query(replacement), "new upstream must not be retired with the old key") +} diff --git a/cmd/cli/vpn_dns.go b/cmd/cli/vpn_dns.go new file mode 100644 index 00000000..d383fe00 --- /dev/null +++ b/cmd/cli/vpn_dns.go @@ -0,0 +1,531 @@ +package cli + +import ( + "context" + "fmt" + "net" + "runtime" + "strings" + "sync" + "sync/atomic" + + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld" +) + +var vpnDNSSettlingEnabled = runtime.GOOS == "windows" + +// vpnDNSExemption represents a VPN DNS server that needs pf/WFP exemption, +// including the interface it was discovered on. The interface is used on macOS +// to create interface-scoped pf exemptions that allow the VPN's local DNS +// handler (e.g., Tailscale's MagicDNS Network Extension) to receive queries +// from all processes — not just ctrld. Without the interface scope, VPN DNS +// handlers that operate at the packet level (Network Extensions) never see +// the queries because pf intercepts them first. +type vpnDNSExemption struct { + Server string // DNS server IP (e.g., "100.100.100.100") + Interface string // Interface name from scutil (e.g., "utun11"), may be empty + IsExitMode bool // True if this VPN is in exit/full-tunnel mode (all traffic routed through VPN) +} + +// vpnDNSExemptFunc is called when VPN DNS servers change, to update +// the intercept layer (WFP/pf) to permit VPN DNS traffic. +// On macOS, exemptions are interface-scoped to allow VPN local DNS handlers +// (e.g., Tailscale MagicDNS) to receive queries from all processes. +type vpnDNSExemptFunc func(exemptions []vpnDNSExemption) error + +// vpnDNSManager tracks active VPN DNS configurations and provides +// domain-to-upstream routing for VPN split DNS. +type vpnDNSManager struct { + mu sync.RWMutex + configs []ctrld.VPNDNSConfig + // Map of domain suffix → DNS servers for fast lookup + routes map[string][]string + logger *atomic.Pointer[ctrld.Logger] + // DNS servers from VPN interfaces that have no domain/suffix config. + // These are NOT added to the global OS resolver. They're only used + // as additional nameservers for queries that match split-DNS rules + // (from ctrld config, AD domain, or VPN suffix config). + domainlessServers []string + // appliedExemptions advances only after the platform PF/WFP callback succeeds. + // Keeping it separate from discovered configs makes failed rule updates retryable. + appliedExemptions []vpnDNSExemption + // retainedAfterEmptyDiscovery means Windows reported an empty VPN DNS + // snapshot once while previous VPN DNS state existed. We keep that last-known + // state for one guarded refresh cycle because Windows can briefly report an + // intermediate empty adapter/DNS state after sleep/wake or reconnect. + retainedAfterEmptyDiscovery bool + // discoverVPNDNS is injected for tests so Refresh does not depend on the + // runner host's real VPN/virtual adapter state. + discoverVPNDNS func(context.Context) []ctrld.VPNDNSConfig + // discoverTailscaleRoutes reads the Split DNS and App Connector suffixes + // that Tailscale resolves internally and does not publish to the OS. Nil + // where the OS already exposes them. + discoverTailscaleRoutes func(context.Context) ([]tailscaleRoute, error) + // tailscale keeps the last good Tailscale route set and the bounded retry + // after a failed LocalAPI read. + tailscale tailscaleRouteState + // retryRefresh runs the refresh of a Tailscale route retry. The program + // sets it to admit the refresh through its network activity gate, so a + // retry never runs during shutdown. Nil means m.Refresh(true). + retryRefresh func() + // refreshStateMu keeps noisy network-change storms from running overlapping + // full VPN DNS refreshes and retains one trailing refresh when an event arrives + // during discovery so the newest OS state is not lost. + refreshStateMu sync.Mutex + refreshRunning bool + refreshPending bool + discoveryMu sync.Mutex + // repeats holds the last refresh summary. The manager has no prog, and a + // network change storm runs one refresh per delta. + repeats repeatLogger + // Called when VPN DNS server list changes, to update intercept exemptions. + onServersChanged vpnDNSExemptFunc +} + +// vpnDNSRefreshKey names the repeat counter of the refresh summary. +const vpnDNSRefreshKey = "vpn_dns_refresh" + +// newVPNDNSManager creates a new manager. Only call when dnsIntercept is active. +// exemptFunc is called whenever VPN DNS servers are discovered/changed, to update +// the OS-level intercept rules to permit ctrld's outbound queries to those IPs. +func newVPNDNSManager(logger *atomic.Pointer[ctrld.Logger], exemptFunc vpnDNSExemptFunc) *vpnDNSManager { + return &vpnDNSManager{ + routes: make(map[string][]string), + logger: logger, + discoverVPNDNS: ctrld.DiscoverVPNDNS, + discoverTailscaleRoutes: discoverTailscaleRoutesFn, + onServersChanged: exemptFunc, + } +} + +// Refresh re-discovers VPN DNS configs from the OS. +// Called on network change events. Overlapping calls are coalesced into one +// trailing refresh so a newer OS snapshot is never silently discarded. +func (m *vpnDNSManager) Refresh(ctx context.Context, guardAgainstNoNameservers ...bool) { + logger := ctrld.LoggerFromCtx(ctx) + guardedRefresh := len(guardAgainstNoNameservers) > 0 && guardAgainstNoNameservers[0] + m.refreshStateMu.Lock() + if m.refreshRunning { + m.refreshPending = true + m.refreshStateMu.Unlock() + ctrld.Log(ctx, logger.Debug(), "VPN DNS refresh already running, coalescing trailing refresh") + return + } + m.refreshRunning = true + m.refreshStateMu.Unlock() + + for { + m.refreshOnce(ctx, guardedRefresh) + + m.refreshStateMu.Lock() + if m.refreshPending { + m.refreshPending = false + m.refreshStateMu.Unlock() + guardedRefresh = true + continue + } + m.refreshRunning = false + m.refreshStateMu.Unlock() + return + } +} + +func (m *vpnDNSManager) refreshOnce(ctx context.Context, guardAgainstNoNameservers bool) { + logger := ctrld.LoggerFromCtx(ctx) + m.discoveryMu.Lock() + defer m.discoveryMu.Unlock() + + ctrld.Log(ctx, logger.Debug(), "Refreshing VPN DNS configurations") + discoverVPNDNS := m.discoverVPNDNS + if discoverVPNDNS == nil { + discoverVPNDNS = ctrld.DiscoverVPNDNS + } + configs := discoverVPNDNS(ctx) + + // Detect exit mode: if the default route goes through a VPN DNS interface, + // the VPN is routing ALL traffic (exit node / full tunnel). This is more + // reliable than scutil flag parsing because the routing table is the ground + // truth for traffic flow. + if dri, err := netmon.DefaultRouteInterface(); err == nil && dri != "" { + for i := range configs { + if configs[i].InterfaceName == dri { + if !configs[i].IsExitMode { + ctrld.Log(ctx, logger.Info(), "VPN DNS on %s: default route interface match — EXIT MODE (route-based detection)", dri) + } + configs[i].IsExitMode = true + } + } + } + // Tailscale routes go in after exit mode is known: routes that need the + // pf passthrough are skipped in exit mode. + configs, tailscaleDomains := withTailscaleRouteDomains(configs, m.tailscaleRoutesFor(ctx, configs)) + if len(tailscaleDomains) > 0 { + ctrld.Log(ctx, logger.Debug(), "Added %d Tailscale DNS routes -> Tailscale resolver", len(tailscaleDomains)) + } + + m.mu.Lock() + defer m.mu.Unlock() + + if vpnDNSSettlingEnabled && len(configs) == 0 && guardAgainstNoNameservers && m.hasVPNDNSStateLocked() { + if !m.retainedAfterEmptyDiscovery { + exemptions := m.currentExemptionsLocked() + m.retainedAfterEmptyDiscovery = true + ctrld.Log(ctx, logger.Debug(), + "VPN DNS discovery empty; retaining last-known VPN DNS state for one guarded refresh (%d domainless servers, %d exemptions)", + len(m.domainlessServers), len(exemptions)) + if m.onServersChanged != nil { + if err := m.onServersChanged(exemptions); err != nil { + ctrld.Log(ctx, logger.Error().Err(err), "Failed to re-apply retained VPN DNS exemptions") + } else { + m.appliedExemptions = append([]vpnDNSExemption(nil), exemptions...) + } + } + return + } + ctrld.Log(ctx, logger.Debug(), + "VPN DNS discovery still empty on next guarded refresh; clearing retained VPN DNS state (%d domainless servers)", + len(m.domainlessServers)) + } + + // Any discovery path that does not return with retained state clears the + // settling marker: non-empty discovery replaces old servers immediately, and + // an unguarded/second empty discovery clears stale state below. + m.retainedAfterEmptyDiscovery = false + m.configs = configs + m.routes = make(map[string][]string) + + // Build domain -> DNS servers mapping + for _, config := range configs { + ctrld.Log(ctx, logger.Debug(), "Processing VPN interface %s with %d domains and %d servers", + config.InterfaceName, len(config.Domains), len(config.Servers)) + + for _, domain := range config.Domains { + domain = normalizeVPNDomain(domain) + + if domain != "" { + m.routes[domain] = append([]string{}, config.Servers...) + // Tailscale routes can number in the thousands; one summary + // line above covers them. + if !tailscaleDomains[domain] { + ctrld.Log(ctx, logger.Debug(), "Added VPN DNS route: %s -> %v", domain, config.Servers) + } + } + } + } + + // Collect unique VPN DNS exemptions (server + interface) for pf/WFP rules. + // We track server+interface pairs because the same server IP on different + // interfaces needs separate exemptions (interface-scoped on macOS). + type exemptionKey struct{ server, iface string } + seen := make(map[exemptionKey]bool) + var exemptions []vpnDNSExemption + for _, config := range configs { + for _, server := range config.Servers { + key := exemptionKey{server, config.InterfaceName} + if !seen[key] { + seen[key] = true + exemptions = append(exemptions, vpnDNSExemption{ + Server: server, + Interface: config.InterfaceName, + IsExitMode: config.IsExitMode, + }) + } + } + } + + // Collect domain-less VPN DNS servers. These are NOT added to the global + // OS resolver (that would pollute captive portal / DHCP flows). Instead, + // they're stored separately and only used for queries that match existing + // split-DNS rules (from ctrld config, AD domain, or VPN suffix config). + var domainlessServers []string + seenDomainless := make(map[string]bool) + for _, config := range configs { + if len(config.Domains) == 0 && len(config.Servers) > 0 { + ctrld.Log(ctx, logger.Debug(), "VPN interface %s has DNS servers but no domains, storing as split-rule fallback: %v", + config.InterfaceName, config.Servers) + for _, server := range config.Servers { + if !seenDomainless[server] { + seenDomainless[server] = true + domainlessServers = append(domainlessServers, server) + } + } + } + } + m.domainlessServers = domainlessServers + + summary := fmt.Sprintf("%d configs, %d routes, %d domainless servers, %d unique exemptions", + len(m.configs), len(m.routes), len(m.domainlessServers), len(exemptions)) + if changed, repeats := m.repeats.changed(vpnDNSRefreshKey, summary); changed { + ctrld.Log(ctx, logger.Debug().Uint64("repeats", repeats), "VPN DNS refresh completed: %s", summary) + } + + // Update intercept rules only when desired exemptions differ from the last + // successfully applied set. Failed PF/WFP callbacks remain retryable on the + // next refresh even when discovery returns the same VPN DNS state. + m.updateInterceptExemptionsIfChanged(ctx, logger, exemptions, "VPN DNS") +} + +func (m *vpnDNSManager) updateInterceptExemptionsIfChanged(ctx context.Context, logger *ctrld.Logger, desired []vpnDNSExemption, reason string) { + if m.onServersChanged == nil { + return + } + if vpnDNSExemptionsEqual(m.appliedExemptions, desired) { + ctrld.Log(ctx, logger.Debug(), "VPN DNS exemptions unchanged after %s refresh; skipping intercept rule update", reason) + return + } + if err := m.onServersChanged(desired); err != nil { + ctrld.Log(ctx, logger.Error().Err(err), "Failed to update intercept exemptions for VPN DNS servers") + return + } + m.appliedExemptions = append([]vpnDNSExemption(nil), desired...) +} + +// RefreshRoutesOnly re-discovers VPN DNS configs and updates ctrld's +// in-memory split-DNS routes. It applies intercept exemptions only when that set +// changes, while holding the shared discovery lane so a concurrent full refresh +// cannot commit a newer snapshot and then be overwritten by this one. +func (m *vpnDNSManager) RefreshRoutesOnly() (routes, domainlessServers, exemptions int) { + logger := mainLog.Load() + if m.logger != nil && m.logger.Load() != nil { + logger = m.logger.Load() + } + ctx := ctrld.LoggerCtx(context.Background(), logger) + + m.discoveryMu.Lock() + defer m.discoveryMu.Unlock() + + ctrld.Log(ctx, logger.Debug(), "Refreshing VPN DNS route state only") + discoverVPNDNS := m.discoverVPNDNS + if discoverVPNDNS == nil { + discoverVPNDNS = ctrld.DiscoverVPNDNS + } + configs := discoverVPNDNS(ctx) + + if dri, err := netmon.DefaultRouteInterface(); err == nil && dri != "" { + for i := range configs { + if configs[i].InterfaceName == dri { + configs[i].IsExitMode = true + } + } + } + configs, _ = withTailscaleRouteDomains(configs, m.tailscaleRoutesFor(ctx, configs)) + + m.mu.Lock() + defer m.mu.Unlock() + + m.retainedAfterEmptyDiscovery = false + m.configs = configs + m.routes = make(map[string][]string) + + for _, config := range configs { + for _, domain := range config.Domains { + domain = normalizeVPNDomain(domain) + if domain != "" { + m.routes[domain] = append([]string{}, config.Servers...) + } + } + } + + var domainless []string + seenDomainless := make(map[string]bool) + for _, config := range configs { + if len(config.Domains) == 0 && len(config.Servers) > 0 { + for _, server := range config.Servers { + if !seenDomainless[server] { + seenDomainless[server] = true + domainless = append(domainless, server) + } + } + } + } + m.domainlessServers = domainless + currentExemptions := m.currentExemptionsLocked() + + ctrld.Log(ctx, logger.Debug(), "VPN DNS route-only refresh completed: %d configs, %d routes, %d domainless servers, %d exemptions", + len(m.configs), len(m.routes), len(m.domainlessServers), len(currentExemptions)) + m.updateInterceptExemptionsIfChanged(ctx, logger, currentExemptions, "route-only VPN DNS") + return len(m.routes), len(m.domainlessServers), len(currentExemptions) +} + +func (m *vpnDNSManager) markInterceptExemptionsApplied(applied []vpnDNSExemption) { + m.mu.Lock() + defer m.mu.Unlock() + if vpnDNSExemptionsEqual(m.currentExemptionsLocked(), applied) { + m.appliedExemptions = append([]vpnDNSExemption(nil), applied...) + } +} + +func (m *vpnDNSManager) interceptExemptionsPending() bool { + m.mu.RLock() + defer m.mu.RUnlock() + return !vpnDNSExemptionsEqual(m.appliedExemptions, m.currentExemptionsLocked()) +} + +func (m *vpnDNSManager) hasVPNDNSStateLocked() bool { + return len(m.configs) > 0 || len(m.routes) > 0 || len(m.domainlessServers) > 0 +} + +func (m *vpnDNSManager) currentExemptionsLocked() []vpnDNSExemption { + type key struct{ server, iface string } + seen := make(map[key]bool) + var exemptions []vpnDNSExemption + for _, config := range m.configs { + for _, server := range config.Servers { + k := key{server, config.InterfaceName} + if seen[k] { + continue + } + seen[k] = true + exemptions = append(exemptions, vpnDNSExemption{ + Server: server, + Interface: config.InterfaceName, + IsExitMode: config.IsExitMode, + }) + } + } + return exemptions +} + +// ShouldFailClosedAfterVPNDNSTransportFailure reports whether split-rule +// queries should fail closed instead of falling back to OS/public DNS after +// every candidate VPN DNS server failed before returning a DNS packet. This is +// Windows-only and only active while serving retained VPN DNS state from a +// guarded empty discovery, which is the short window where Windows can report +// VPN DNS before routes to those servers are usable after wake/reconnect. +func (m *vpnDNSManager) ShouldFailClosedAfterVPNDNSTransportFailure(domain string, servers []string) bool { + m.mu.RLock() + defer m.mu.RUnlock() + + if !vpnDNSSettlingEnabled || len(servers) == 0 || !m.retainedAfterEmptyDiscovery || !m.hasVPNDNSStateLocked() { + return false + } + + logger := m.logger.Load() + if logger != nil { + logger.Debug().Msgf( + "VPN DNS transport failed for %s while retained VPN DNS state is active; suppressing OS fallback for this query (servers=%v)", + domain, servers) + } + return true +} + +// VPNDNSReachable records that a VPN DNS server returned a DNS response. The +// response may be negative (NXDOMAIN/SERVFAIL); the important signal is that +// the VPN DNS transport is reachable again. +func (m *vpnDNSManager) VPNDNSReachable() { + m.mu.Lock() + defer m.mu.Unlock() + if m.retainedAfterEmptyDiscovery { + logger := m.logger.Load() + if logger != nil { + logger.Debug().Msg("VPN DNS transport recovered; clearing retained-empty-discovery state") + } + } + m.retainedAfterEmptyDiscovery = false +} + +// normalizeVPNDomain accepts OS suffix and Linux routing-domain spellings. +// The root alone is not a split-DNS route (including the Linux ~. catch-all). +func normalizeVPNDomain(domain string) string { + domain = strings.TrimPrefix(domain, "~") + domain = strings.TrimPrefix(domain, ".") + return strings.ToLower(strings.TrimSuffix(domain, ".")) +} + +// UpstreamForDomain checks if the domain matches any VPN search domain. +// Returns VPN DNS servers if matched, nil otherwise. +// Uses suffix matching: "foo.provisur.local" matches "provisur.local" +func (m *vpnDNSManager) UpstreamForDomain(domain string) []string { + if domain == "" { + return nil + } + + m.mu.RLock() + defer m.mu.RUnlock() + + // Normalize domain (remove trailing dot, convert to lowercase) + domain = strings.TrimSuffix(domain, ".") + domain = strings.ToLower(domain) + + // Walk label boundaries from most to least specific. Map iteration order + // must never choose a parent VPN's resolver over a matching child zone. + for domain != "" { + if servers, ok := m.routes[domain]; ok { + return append([]string{}, servers...) + } + i := strings.IndexByte(domain, '.') + if i < 0 { + break + } + domain = domain[i+1:] + } + + return nil +} + +// DomainlessServers returns VPN DNS servers that have no associated domains. +// These should only be used for queries matching split-DNS rules, not for +// general OS resolver queries (to avoid polluting captive portal / DHCP flows). +func (m *vpnDNSManager) DomainlessServers() []string { + m.mu.RLock() + defer m.mu.RUnlock() + return append([]string{}, m.domainlessServers...) +} + +// CurrentServers returns the current set of unique VPN DNS server IPs. +// Used by pf anchor rebuild to include VPN DNS exemptions without a full Refresh(). +func (m *vpnDNSManager) CurrentServers() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + seen := make(map[string]bool) + var servers []string + for _, ss := range m.routes { + for _, s := range ss { + if !seen[s] { + seen[s] = true + servers = append(servers, s) + } + } + } + return servers +} + +// CurrentExemptions returns VPN DNS server + interface pairs for pf exemption rules. +// Used by pf anchor rebuild paths that need interface-scoped exemptions. +func (m *vpnDNSManager) CurrentExemptions() []vpnDNSExemption { + m.mu.RLock() + defer m.mu.RUnlock() + return m.currentExemptionsLocked() +} + +// Routes returns a copy of the current VPN DNS routes for debugging. +func (m *vpnDNSManager) Routes() map[string][]string { + m.mu.RLock() + defer m.mu.RUnlock() + + routes := make(map[string][]string) + for domain, servers := range m.routes { + routes[domain] = append([]string{}, servers...) + } + return routes +} + +// upstreamConfigFor creates a legacy upstream configuration for the given VPN DNS server. +func (m *vpnDNSManager) upstreamConfigFor(server string) *ctrld.UpstreamConfig { + // Use net.JoinHostPort to correctly handle both IPv4 and IPv6 addresses. + // Previously, the strings.Contains(":") check would skip appending ":53" + // for IPv6 addresses (they contain colons), leaving a bare address like + // "2a0d:6fc0:9b0:3600::1" which net.Dial rejects with "too many colons". + // net.JoinHostPort produces "[2a0d:6fc0:9b0:3600::1]:53" as required. + endpoint := net.JoinHostPort(server, "53") + + return &ctrld.UpstreamConfig{ + Name: "VPN DNS", + Type: ctrld.ResolverTypeLegacy, + Endpoint: endpoint, + Timeout: 2000, // 2 second timeout for VPN DNS queries + } +} diff --git a/cmd/cli/vpn_dns_convergence_test.go b/cmd/cli/vpn_dns_convergence_test.go new file mode 100644 index 00000000..7ef2a92a --- /dev/null +++ b/cmd/cli/vpn_dns_convergence_test.go @@ -0,0 +1,135 @@ +package cli + +import ( + "context" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +func TestVPNDNSLongestNormalizedSuffixAcrossRefreshes(t *testing.T) { + for _, routeOnly := range []bool{false, true} { + t.Run(map[bool]string{false: "full", true: "route-only"}[routeOnly], func(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{ + {Domains: []string{".EXAMPLE.com."}, Servers: []string{"10.0.0.1"}}, + {Domains: []string{"~Child.Example.COM."}, Servers: []string{"10.0.0.2", "10.0.0.3"}}, + {Domains: []string{".", "~."}, Servers: []string{"10.0.0.4"}}, + } + } + if routeOnly { + m.RefreshRoutesOnly() + } else { + m.Refresh(context.Background(), true) + } + for query, want := range map[string][]string{ + "example.com.": {"10.0.0.1"}, "other.example.com": {"10.0.0.1"}, + "child.example.com.": {"10.0.0.2", "10.0.0.3"}, + "HOST.Deep.Child.Example.COM.": {"10.0.0.2", "10.0.0.3"}, + "notexample.com.": nil, "example.com.other.": nil, "public.test.": nil, + } { + for i := 0; i < 1000; i++ { + if got := m.UpstreamForDomain(query); !reflect.DeepEqual(got, want) { + t.Fatalf("%s: got %v want %v", query, got, want) + } + } + } + got := m.UpstreamForDomain("child.example.com") + got[0] = "mutated" + if m.UpstreamForDomain("child.example.com")[0] != "10.0.0.2" { + t.Fatal("lookup leaked mutable slice") + } + }) + } +} + +// Drive the real poller diff -> journal callback -> manager discovery path. +// Both OS boundaries are fixtures, so this test cannot touch host DNS or PF. +func TestVPNDNSJournalLatePublicationConverges(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + var discoveryCalls, updates int + var snapshot []ctrld.VPNDNSConfig + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { discoveryCalls++; return snapshot } + m.onServersChanged = func([]vpnDNSExemption) error { updates++; return nil } + p := &prog{vpnDNS: m, csSetDnsDone: make(chan struct{})} + p.logger.Store(mainLog.Load()) + close(p.csSetDnsDone) + var table []dnsResolverEntry + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return table, nil }) + emit := func() { p.logDNSConfigChanges(poller.pollOnce(time.Now())) } + emit() // empty startup baseline + for _, server := range []string{"10.0.0.1", "10.0.0.2"} { + snapshot = []ctrld.VPNDNSConfig{{InterfaceName: "utun-fixture", Domains: []string{"corp.example."}, Servers: []string{server}}} + table = []dnsResolverEntry{{Interface: "utun-fixture", Nameservers: []string{server}}} + emit() + if got := m.UpstreamForDomain("host.corp.example."); len(got) != 1 || got[0] != server { + t.Fatalf("late DNS-only change not published: %v", got) + } + for i := 0; i < 100; i++ { + emit() + } + } + if discoveryCalls != 2 || updates != 2 { + t.Fatalf("unchanged tables caused refresh storm: discoveries=%d updates=%d", discoveryCalls, updates) + } + // Removal is a DNS-only delta too; keep Windows' one-empty-snapshot guard. + snapshot = nil + table = nil + emit() + if vpnDNSSettlingEnabled { // Preserve Windows' one-empty-snapshot guard. + m.Refresh(context.Background(), true) + } + if got := m.UpstreamForDomain("host.corp.example"); len(got) != 0 { + t.Fatalf("removed DNS retained: %v", got) + } + p.closeNetMonitor() +} + +func TestVPNDNSJournalShutdownDrainsAndRejectsRefresh(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + entered, release := make(chan struct{}), make(chan struct{}) + var calls, updates atomic.Int32 + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + calls.Add(1) + close(entered) + <-release + return []ctrld.VPNDNSConfig{{Domains: []string{"corp.example"}, Servers: []string{"10.0.0.1"}}} + } + m.onServersChanged = func([]vpnDNSExemption) error { updates.Add(1); return nil } + p := &prog{vpnDNS: m, csSetDnsDone: make(chan struct{})} + p.logger.Store(mainLog.Load()) + entries := []dnsResolverEntry{{Action: resolverActionChanged}} + p.logDNSConfigChanges(entries) + if calls.Load() != 0 { + t.Fatal("discovery ran before DNS initialization") + } + close(p.csSetDnsDone) + callbackDone := make(chan struct{}) + go func() { defer close(callbackDone); p.logDNSConfigChanges(entries) }() + <-entered + stopDone := make(chan struct{}) + go func() { p.closeNetMonitor(); close(stopDone) }() + <-p.networkActivityDone() + select { + case <-stopDone: + t.Fatal("shutdown did not drain in-flight refresh") + default: + } + close(release) + <-callbackDone + <-stopDone + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { defer wg.Done(); p.logDNSConfigChanges(entries) }() + } + wg.Wait() + if calls.Load() != 1 || updates.Load() != 1 { + t.Fatalf("post-stop mutation: discovery=%d updates=%d", calls.Load(), updates.Load()) + } +} diff --git a/cmd/cli/vpn_dns_test.go b/cmd/cli/vpn_dns_test.go new file mode 100644 index 00000000..f773c926 --- /dev/null +++ b/cmd/cli/vpn_dns_test.go @@ -0,0 +1,351 @@ +package cli + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +func withVPNDNSSettlingEnabled(t *testing.T) { + t.Helper() + old := vpnDNSSettlingEnabled + vpnDNSSettlingEnabled = true + t.Cleanup(func() { vpnDNSSettlingEnabled = old }) +} + +func TestVPNDNSRefreshCoalescesConcurrentTrailingRefresh(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + started := make(chan struct{}) + release := make(chan struct{}) + done := make(chan struct{}) + var once sync.Once + var calls atomic.Int32 + + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + call := calls.Add(1) + once.Do(func() { close(started) }) + <-release + if call == 2 { + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-latest", + Servers: []string{"10.0.0.2"}, + Domains: []string{"latest.internal"}, + }} + } + return nil + } + + go func() { + defer close(done) + m.Refresh(context.Background(), true) + }() + + <-started + m.Refresh(context.Background(), true) + close(release) + <-done + + if calls.Load() != 2 { + t.Fatalf("expected one active and one trailing discovery call, got %d", calls.Load()) + } + if got := m.Routes()["latest.internal"]; len(got) != 1 || got[0] != "10.0.0.2" { + t.Fatalf("trailing refresh did not publish latest OS snapshot: %v", got) + } +} + +func TestVPNDNSRefreshRetainsStateForOneGuardedEmptyDiscovery(t *testing.T) { + withVPNDNSSettlingEnabled(t) + var gotExemptions []vpnDNSExemption + m := newVPNDNSManager(&mainLog, func(exemptions []vpnDNSExemption) error { + gotExemptions = exemptions + return nil + }) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { return nil } + m.configs = []ctrld.VPNDNSConfig{{ + InterfaceName: "Ethernet 6", + Servers: []string{"10.25.37.21", "10.25.37.22"}, + }} + m.domainlessServers = []string{"10.25.37.21", "10.25.37.22"} + + m.Refresh(context.Background(), true) + + if got := m.DomainlessServers(); len(got) != 2 { + t.Fatalf("expected retained domainless servers, got %v", got) + } + if len(gotExemptions) != 2 { + t.Fatalf("expected retained exemptions to be re-applied, got %v", gotExemptions) + } + if !m.retainedAfterEmptyDiscovery { + t.Fatal("expected empty discovery retention to be marked") + } +} + +func TestVPNDNSRefreshClearsOnSecondGuardedEmptyDiscovery(t *testing.T) { + withVPNDNSSettlingEnabled(t) + var gotExemptions []vpnDNSExemption + updates := 0 + m := newVPNDNSManager(&mainLog, func(exemptions []vpnDNSExemption) error { + updates++ + gotExemptions = exemptions + return nil + }) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { return nil } + m.configs = []ctrld.VPNDNSConfig{{ + InterfaceName: "Ethernet 6", + Servers: []string{"10.25.37.21"}, + }} + m.domainlessServers = []string{"10.25.37.21"} + m.appliedExemptions = []vpnDNSExemption{{Server: "10.25.37.21", Interface: "Ethernet 6"}} + m.retainedAfterEmptyDiscovery = true + + m.Refresh(context.Background(), true) + + if got := m.DomainlessServers(); len(got) != 0 { + t.Fatalf("expected domainless servers to be cleared on second empty discovery, got %v", got) + } + if updates != 1 || len(gotExemptions) != 0 { + t.Fatalf("expected one empty exemption update after clearing stale state, calls=%d exemptions=%v", updates, gotExemptions) + } + if m.retainedAfterEmptyDiscovery { + t.Fatal("expected retained empty-discovery marker to be cleared with stale state") + } +} + +func TestVPNDNSRefreshSkipsUnchangedInterceptExemptions(t *testing.T) { + var updates [][]vpnDNSExemption + m := newVPNDNSManager(&mainLog, func(exemptions []vpnDNSExemption) error { + updates = append(updates, append([]vpnDNSExemption{}, exemptions...)) + return nil + }) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-test", + Servers: []string{"10.102.26.10"}, + Domains: []string{"example.internal"}, + }} + } + + m.Refresh(context.Background(), true) + m.Refresh(context.Background(), true) + + if len(updates) != 1 { + t.Fatalf("expected exactly one intercept exemption update for unchanged VPN DNS state, got %d", len(updates)) + } + if len(updates[0]) != 1 || updates[0][0].Server != "10.102.26.10" || updates[0][0].Interface != "utun-test" { + t.Fatalf("unexpected exemption update: %+v", updates[0]) + } +} + +func TestVPNDNSRefreshRetriesFailedInterceptExemptionUpdate(t *testing.T) { + attempts := 0 + m := newVPNDNSManager(&mainLog, func([]vpnDNSExemption) error { + attempts++ + if attempts == 1 { + return errors.New("pf update failed") + } + return nil + }) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-test", + Servers: []string{"10.102.26.10"}, + Domains: []string{"internal.test"}, + }} + } + + m.Refresh(context.Background(), true) + if !m.interceptExemptionsPending() { + t.Fatal("failed intercept exemption update was not retained for retry") + } + m.Refresh(context.Background(), true) + if m.interceptExemptionsPending() { + t.Fatal("successful intercept exemption retry did not advance applied state") + } + m.Refresh(context.Background(), true) + + if attempts != 2 { + t.Fatalf("intercept exemption update attempts = %d, want failed attempt plus one retry", attempts) + } + if len(m.appliedExemptions) != 1 || m.appliedExemptions[0].Server != "10.102.26.10" { + t.Fatalf("applied exemptions = %+v, want successful retry state", m.appliedExemptions) + } +} + +func TestVPNDNSMarkAppliedExemptionsRejectsStaleSnapshot(t *testing.T) { + m := newVPNDNSManager(&mainLog, nil) + m.configs = []ctrld.VPNDNSConfig{{InterfaceName: "utun-new", Servers: []string{"10.0.0.2"}}} + + m.markInterceptExemptionsApplied([]vpnDNSExemption{{Server: "10.0.0.1", Interface: "utun-old"}}) + if !m.interceptExemptionsPending() { + t.Fatal("stale PF snapshot incorrectly advanced applied exemptions") + } + + m.markInterceptExemptionsApplied([]vpnDNSExemption{{Server: "10.0.0.2", Interface: "utun-new"}}) + if m.interceptExemptionsPending() { + t.Fatal("current PF snapshot did not advance applied exemptions") + } +} + +func TestVPNDNSTransportFailureSuppressesFallbackOnlyWhileRetainingState(t *testing.T) { + withVPNDNSSettlingEnabled(t) + m := newVPNDNSManager(&mainLog, nil) + m.domainlessServers = []string{"10.25.37.21"} + + if m.ShouldFailClosedAfterVPNDNSTransportFailure("splunk.aws.arena.net.", []string{"10.25.37.21"}) { + t.Fatal("did not expect transport failure to suppress OS fallback outside retained empty-discovery state") + } + + m.retainedAfterEmptyDiscovery = true + if !m.ShouldFailClosedAfterVPNDNSTransportFailure("splunk.aws.arena.net.", []string{"10.25.37.21"}) { + t.Fatal("expected transport failure to suppress OS fallback while retained state is active") + } + + m.VPNDNSReachable() + if m.retainedAfterEmptyDiscovery { + t.Fatal("expected reachable DNS response to clear retained empty-discovery state") + } +} + +func TestVPNDNSFullAndRouteOnlyDiscoveryAreSerialized(t *testing.T) { + var updateMu sync.Mutex + var exemptionUpdates []string + m := newVPNDNSManager(&mainLog, func(exemptions []vpnDNSExemption) error { + updateMu.Lock() + defer updateMu.Unlock() + if len(exemptions) == 0 { + exemptionUpdates = append(exemptionUpdates, "") + } else { + exemptionUpdates = append(exemptionUpdates, exemptions[0].Server) + } + return nil + }) + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondStarted := make(chan struct{}) + var calls atomic.Int32 + + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + switch calls.Add(1) { + case 1: + close(firstStarted) + <-releaseFirst + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-old", + Servers: []string{"10.0.0.1"}, + Domains: []string{"old.internal"}, + }} + case 2: + close(secondStarted) + return []ctrld.VPNDNSConfig{{ + InterfaceName: "utun-new", + Servers: []string{"10.0.0.2"}, + Domains: []string{"new.internal"}, + }} + default: + t.Fatalf("unexpected discovery call %d", calls.Load()) + return nil + } + } + + routesDone := make(chan struct{}) + go func() { + defer close(routesDone) + m.RefreshRoutesOnly() + }() + <-firstStarted + + fullDone := make(chan struct{}) + go func() { + defer close(fullDone) + m.Refresh(context.Background(), false) + }() + + select { + case <-secondStarted: + t.Fatal("full and route-only VPN DNS discovery overlapped") + case <-time.After(50 * time.Millisecond): + } + close(releaseFirst) + + select { + case <-routesDone: + case <-time.After(time.Second): + t.Fatal("route-only refresh did not finish") + } + select { + case <-fullDone: + case <-time.After(time.Second): + t.Fatal("full refresh did not finish") + } + + routes := m.Routes() + if _, ok := routes["old.internal"]; ok { + t.Fatalf("older route-only snapshot overwrote newer full refresh: %v", routes) + } + if got := routes["new.internal"]; len(got) != 1 || got[0] != "10.0.0.2" { + t.Fatalf("final VPN DNS routes = %v, want new.internal -> 10.0.0.2", routes) + } + updateMu.Lock() + defer updateMu.Unlock() + if len(exemptionUpdates) != 2 || exemptionUpdates[0] != "10.0.0.1" || exemptionUpdates[1] != "10.0.0.2" { + t.Fatalf("serialized exemption updates = %v, want old then new", exemptionUpdates) + } +} + +// eventsWithPrefix returns the captured events whose message starts with the +// prefix. Several lines of this module carry a value in the message, so no +// constant matches the whole of it. +func eventsWithPrefix(t *testing.T, logs *syncBuffer, prefix string) []map[string]any { + t.Helper() + var matched []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + matched = append(matched, event) + } + } + return matched +} + +// TestVPNDNSRefreshSummaryLogsOnChangeOnly covers the refresh summary. A network +// change storm runs the refresh every few seconds, and the same summary in every +// one of those lines buries the refresh that changed something. +func TestVPNDNSRefreshSummaryLogsOnChangeOnly(t *testing.T) { + logs := captureDebugMainLog(t) + m := newVPNDNSManager(&mainLog, nil) + configs := []ctrld.VPNDNSConfig{{ + InterfaceName: "utun4", + Servers: []string{"10.0.0.1"}, + Domains: []string{"corp.internal"}, + }} + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { return configs } + + // Refresh logs through the logger of its context. + ctx := ctrld.LoggerCtx(context.Background(), mainLog.Load()) + m.Refresh(ctx, true) + m.Refresh(ctx, true) + + summaries := eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 1 { + t.Fatalf("got %d refresh summaries for two identical results, want 1", len(summaries)) + } + wantField(t, summaries[0], "repeats", float64(0)) + + configs = append(configs, ctrld.VPNDNSConfig{ + InterfaceName: "utun6", + Servers: []string{"10.0.0.2"}, + }) + m.Refresh(ctx, true) + + summaries = eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 2 { + t.Fatalf("got %d refresh summaries after the result changed, want 2", len(summaries)) + } + wantField(t, summaries[1], "repeats", float64(1)) +} diff --git a/cmd/ctrld_library/main.go b/cmd/ctrld_library/main.go index b2e643db..9ae8d79c 100644 --- a/cmd/ctrld_library/main.go +++ b/cmd/ctrld_library/main.go @@ -1,18 +1,32 @@ package ctrld_library import ( + "sync" + "time" + "github.com/Control-D-Inc/ctrld/cmd/cli" ) -// Controller holds global state +// stopTimeout bounds how long Stop waits for ctrld to release its resources. +const stopTimeout = 15 * time.Second + +var ( + runMobile = cli.RunMobile + checkDeactivationPin = cli.CheckDeactivationPin +) + +// Controller holds global state. Start, Stop and IsRunning are safe for +// concurrent use. AppCallback and Config must not be accessed concurrently. type Controller struct { + mu sync.Mutex + pinMu sync.Mutex // Serializes PIN checks, which use CLI global state. stopCh chan struct{} + doneCh chan struct{} AppCallback AppCallback Config cli.AppConfig } // NewController provides reference to global state to be managed by android vpn service and iOS network extension. -// reference is not safe for concurrent use. func NewController(appCallback AppCallback) *Controller { return &Controller{AppCallback: appCallback} } @@ -26,26 +40,39 @@ type AppCallback interface { } // Start configures utility with config.toml from provided directory. -// This function will block until Stop is called +// This function blocks until the run exits, including teardown after Stop. +// A call made while a run is active or stopping returns immediately. // Check port availability prior to calling it. func (c *Controller) Start(CdUID string, ProvisionID string, CustomHostname string, HomeDir string, UpstreamProto string, logLevel int, logPath string) { - if c.stopCh == nil { - c.stopCh = make(chan struct{}) - c.Config = cli.AppConfig{ - CdUID: CdUID, - ProvisionID: ProvisionID, - CustomHostname: CustomHostname, - HomeDir: HomeDir, - UpstreamProto: UpstreamProto, - Verbose: logLevel, - LogPath: logPath, - } - appCallback := mapCallback(c.AppCallback) - cli.RunMobile(&c.Config, &appCallback, c.stopCh) + c.mu.Lock() + if c.stopCh != nil { + c.mu.Unlock() + return + } + stopCh, doneCh := make(chan struct{}), make(chan struct{}) + c.stopCh, c.doneCh = stopCh, doneCh + c.Config = cli.AppConfig{ + CdUID: CdUID, + ProvisionID: ProvisionID, + CustomHostname: CustomHostname, + HomeDir: HomeDir, + UpstreamProto: UpstreamProto, + Verbose: logLevel, + LogPath: logPath, } + appCallback := mapCallback(c.AppCallback) + c.mu.Unlock() + defer func() { + c.mu.Lock() + defer c.mu.Unlock() + // Only the completed run releases the guard, even if Stop timed out. + close(doneCh) + c.stopCh, c.doneCh = nil, nil + }() + runMobile(&c.Config, &appCallback, stopCh) } -// As workaround to avoid circular dependency between cli and ctrld_library module +// mapCallback maps the AppCallback interface to cli.AppCallback to avoid circular dependency func mapCallback(callback AppCallback) cli.AppCallback { return cli.AppCallback{ HostName: func() string { @@ -63,20 +90,49 @@ func mapCallback(callback AppCallback) cli.AppCallback { } } +// Stop requests shutdown after validating pin, unless restart bypasses the check. +// A zero result means the stop request was accepted, not that teardown finished. +// After cancellation it waits up to 15 seconds. Callers must check IsRunning +// after Stop: while it remains true, teardown still owns the session and Start +// will be rejected. PIN failure codes are returned unchanged. func (c *Controller) Stop(restart bool, pin int64) int { - var errorCode = 0 + // Keep this request bound to its original run while PIN validation blocks. + c.mu.Lock() + stopCh, doneCh := c.stopCh, c.doneCh + c.mu.Unlock() + // Force disconnect without checking pin. // In iOS restart is required if vpn detects no connectivity after network change. if !restart { - errorCode = cli.CheckDeactivationPin(pin, c.stopCh) + c.pinMu.Lock() + errorCode := checkDeactivationPin(pin, stopCh) + c.pinMu.Unlock() + if errorCode != 0 { + return errorCode + } } - if errorCode == 0 && c.stopCh != nil { - close(c.stopCh) - c.stopCh = nil + if stopCh != nil { + c.mu.Lock() + select { + case <-stopCh: + default: + close(stopCh) + } + c.mu.Unlock() + + timer := time.NewTimer(stopTimeout) + defer timer.Stop() + select { + case <-doneCh: + case <-timer.C: + } } - return errorCode + return 0 } +// IsRunning remains true until the run has finished releasing its resources. func (c *Controller) IsRunning() bool { + c.mu.Lock() + defer c.mu.Unlock() return c.stopCh != nil } diff --git a/cmd/ctrld_library/main_test.go b/cmd/ctrld_library/main_test.go new file mode 100644 index 00000000..3ccb5331 --- /dev/null +++ b/cmd/ctrld_library/main_test.go @@ -0,0 +1,275 @@ +package ctrld_library + +import ( + "runtime" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/Control-D-Inc/ctrld/cmd/cli" +) + +// Tests replace the first host-touching calls, not the Controller methods. +// They must not run in parallel because these seams are package globals. +func stubMobile(t *testing.T, run func(*cli.AppConfig, *cli.AppCallback, chan struct{}), check func(int64, chan struct{}) int) { + t.Helper() + oldRun, oldCheck := runMobile, checkDeactivationPin + runMobile, checkDeactivationPin = run, check + t.Cleanup(func() { runMobile, checkDeactivationPin = oldRun, oldCheck }) +} + +func startController(c *Controller) <-chan struct{} { + done := make(chan struct{}) + go func() { + c.Start("uid", "provision", "hostname", "home", "doh", 2, "log") + close(done) + }() + return done +} + +func TestControllerConcurrentLifecycle(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + var runs, active atomic.Int32 + var pinCalls int // Deliberately non-atomic: PIN checks must be serialized. + stubMobile(t, func(config *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + if active.Add(1) != 1 { + t.Error("overlapping mobile runs") + } + defer active.Add(-1) + runs.Add(1) + want := cli.AppConfig{CdUID: "uid", ProvisionID: "provision", CustomHostname: "hostname", HomeDir: "home", UpstreamProto: "doh", Verbose: 2, LogPath: "log"} + if *config != want { + t.Errorf("config = %+v, want %+v", *config, want) + } + <-stop + }, func(_ int64, _ chan struct{}) int { + pinCalls++ + return 0 + }) + + if c.IsRunning() || c.Stop(true, 0) != 0 { + t.Fatal("new controller must be stopped") + } + for cycle := 0; cycle < 25; cycle++ { + var starters sync.WaitGroup + for i := 0; i < 16; i++ { + starters.Add(1) + go func() { defer starters.Done(); <-startController(c) }() + } + synctest.Wait() + if !c.IsRunning() || runs.Load() != int32(cycle+1) { + t.Fatal("concurrent Starts did not claim exactly one run") + } + var stoppers sync.WaitGroup + for i := 0; i < 16; i++ { + stoppers.Add(1) + go func() { + defer stoppers.Done() + c.IsRunning() + if code := c.Stop(false, 1234); code != 0 { + t.Errorf("Stop = %d", code) + } + c.IsRunning() + }() + } + stoppers.Wait() + starters.Wait() + if c.IsRunning() || active.Load() != 0 { + t.Fatal("Stop returned before run completion") + } + } + if pinCalls != 25*16 { + t.Fatalf("PIN checks = %d, want %d", pinCalls, 25*16) + } + }) +} + +func TestControllerStopWaitsAndTimeoutKeepsGuard(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + release := make(chan struct{}) + var runs int + stubMobile(t, func(_ *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + runs++ + <-stop + <-release + }, func(int64, chan struct{}) int { t.Error("restart checked PIN"); return 1 }) + started := startController(c) + synctest.Wait() + stopped := make(chan struct{}) + before := time.Now() + go func() { + var stoppers sync.WaitGroup + for i := 0; i < 8; i++ { + stoppers.Add(1) + go func() { defer stoppers.Done(); c.Stop(true, 0) }() + } + stoppers.Wait() + close(stopped) + }() + synctest.Wait() + select { + case <-stopped: + t.Fatal("Stop did not wait for teardown") + default: + } + c.Start("", "", "", "", "", 0, "") + if !c.IsRunning() || runs != 1 { + t.Fatal("guard released during teardown") + } + <-stopped // synctest advances virtual time, not wall time. + if elapsed := time.Since(before); elapsed != stopTimeout { + t.Fatalf("Stop timeout = %v, want %v", elapsed, stopTimeout) + } + c.Start("", "", "", "", "", 0, "") + if !c.IsRunning() || runs != 1 { + t.Fatal("timeout released a live run's guard") + } + close(release) + <-started + if c.IsRunning() { + t.Fatal("completed teardown retained guard") + } + started = startController(c) + synctest.Wait() + c.Stop(true, 0) + <-started + if runs != 2 || c.IsRunning() { + t.Fatal("could not restart after timed-out teardown completed") + } + }) +} + +func TestControllerPinBehavior(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + var runStop chan struct{} + var pins []int64 + stubMobile(t, func(_ *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + runStop = stop + <-stop + }, func(pin int64, stop chan struct{}) int { + pins = append(pins, pin) + if stop != runStop { + t.Error("PIN check received wrong run") + } + if pin == 1234 { + return 0 + } + return 42 + }) + if code := c.Stop(false, 9); code != 42 { + t.Fatalf("idle Stop PIN result = %d, want 42", code) + } + started := startController(c) + synctest.Wait() + if code := c.Stop(false, 8); code != 42 || !c.IsRunning() { + t.Fatalf("invalid PIN Stop = %d, running = %v", code, c.IsRunning()) + } + select { + case <-runStop: + t.Fatal("invalid PIN stopped run") + default: + } + if code := c.Stop(false, 1234); code != 0 { + t.Fatalf("valid PIN Stop = %d", code) + } + <-started + started = startController(c) + synctest.Wait() + c.Stop(true, 9999) + <-started + if len(pins) != 3 || pins[0] != 9 || pins[1] != 8 || pins[2] != 1234 { + t.Fatalf("PIN calls = %v", pins) + } + }) +} + +func TestControllerSlowPinCannotStopReplacement(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + pinStarted, releasePin := make(chan struct{}), make(chan struct{}) + var currentStop chan struct{} + stubMobile(t, func(_ *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + currentStop = stop + <-stop + }, func(_ int64, stop chan struct{}) int { + if stop != currentStop { + t.Error("wrong PIN session") + } + close(pinStarted) + <-releasePin + return 0 + }) + first := startController(c) + synctest.Wait() + oldStop := make(chan struct{}) + go func() { c.Stop(false, 1234); close(oldStop) }() + <-pinStarted + c.Stop(true, 0) + <-first + second := startController(c) + synctest.Wait() + close(releasePin) + <-oldStop + if !c.IsRunning() { + t.Fatal("old Stop cleared replacement guard") + } + select { + case <-currentStop: + t.Fatal("old Stop cancelled replacement") + default: + } + c.Stop(true, 0) + <-second + }) +} + +func TestControllerMixedConcurrentCalls(t *testing.T) { + c := NewController(nil) + var active atomic.Int32 + stubMobile(t, func(config *cli.AppConfig, _ *cli.AppCallback, _ chan struct{}) { + if active.Add(1) != 1 { + t.Error("overlapping mobile runs") + } + defer active.Add(-1) + runtime.Gosched() + if config.CdUID != "uid" { + t.Error("active config changed") + } + }, func(int64, chan struct{}) int { return 0 }) + var workers sync.WaitGroup + for i := 0; i < 12; i++ { + workers.Add(1) + go func() { + defer workers.Done() + for j := 0; j < 100; j++ { + c.Start("uid", "", "", "", "", 0, "") + c.IsRunning() + c.Stop(j%2 == 0, 1234) + } + }() + } + workers.Wait() + if c.IsRunning() || active.Load() != 0 { + t.Fatal("concurrent calls left a run active") + } +} + +func TestControllerRunReturnReleasesGuard(t *testing.T) { + c := NewController(nil) + var runs int + stubMobile(t, func(*cli.AppConfig, *cli.AppCallback, chan struct{}) { runs++ }, func(int64, chan struct{}) int { return 0 }) + for i := 0; i < 2; i++ { + c.Start("", "", "", "", "", 0, "") + if c.IsRunning() { + t.Fatal("returned run retained guard") + } + } + if runs != 2 { + t.Fatalf("runs = %d, want 2", runs) + } +} diff --git a/config.go b/config.go index 73484d70..f894bdba 100644 --- a/config.go +++ b/config.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net" "net/http" "net/netip" @@ -27,7 +26,6 @@ import ( "github.com/go-playground/validator/v10" "github.com/miekg/dns" "github.com/spf13/viper" - "golang.org/x/net/http2" "golang.org/x/sync/singleflight" "tailscale.com/logtail/backoff" "tailscale.com/net/tsaddr" @@ -79,10 +77,20 @@ const ( controlDNetDomain = "controld.net" controlDDevDomain = "controld.dev" + // nextDNSDomain is the parent domain of the NextDNS DoH endpoints. Beside + // dns.nextdns.io, NextDNS serves alternative endpoints under it, such as + // ultralow.dns.nextdns.io and anycast.dns2.nextdns.io, which are the same + // service and take the same client info headers. + nextDNSDomain = "nextdns.io" + endpointPrefixHTTPS = "https://" endpointPrefixQUIC = "quic://" endpointPrefixH3 = "h3://" endpointPrefixSdns = "sdns://" + + rebootstrapNotStarted = 0 + rebootstrapStarted = 1 + rebootstrapInProgress = 2 ) var ( @@ -114,6 +122,10 @@ func SetConfigNameWithPath(v *viper.Viper, name, configPath string) { // InitConfig initializes default config values for given *viper.Viper instance. func InitConfig(v *viper.Viper, name string) { + ctx := context.Background() + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "Config initialization started") + v.SetDefault("listener", map[string]*ListenerConfig{ "0": { IP: "", @@ -152,6 +164,8 @@ func InitConfig(v *viper.Viper, name string) { Timeout: 3000, }, }) + + Log(ctx, logger.Debug(), "Config initialization completed") } // Config represents ctrld supported configuration. @@ -213,8 +227,13 @@ func (c *Config) FirstUpstream() *UpstreamConfig { // ServiceConfig specifies the general ctrld config. type ServiceConfig struct { + // AllowUnprivilegedLogSend is a macOS-only, administrator-controlled opt-in. + AllowUnprivilegedLogSend bool `mapstructure:"allow_unprivileged_log_send" toml:"allow_unprivileged_log_send,omitempty"` + LogLevel string `mapstructure:"log_level" toml:"log_level,omitempty"` LogPath string `mapstructure:"log_path" toml:"log_path,omitempty"` + LogMaxSizeMB int `mapstructure:"log_max_size_mb" toml:"log_max_size_mb,omitempty" validate:"omitempty,gte=1,lte=1024"` + LogMaxBackups *int `mapstructure:"log_max_backups" toml:"log_max_backups,omitempty" validate:"omitempty,gte=0,lte=64"` CacheEnable bool `mapstructure:"cache_enable" toml:"cache_enable,omitempty"` CacheSize int `mapstructure:"cache_size" toml:"cache_size,omitempty"` CacheTTLOverride int `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"` @@ -237,8 +256,33 @@ type ServiceConfig struct { RefetchTime *int `mapstructure:"refetch_time" toml:"refetch_time,omitempty"` ForceRefetchWaitTime *int `mapstructure:"force_refetch_wait_time" toml:"force_refetch_wait_time,omitempty"` LeakOnUpstreamFailure *bool `mapstructure:"leak_on_upstream_failure" toml:"leak_on_upstream_failure,omitempty"` - Daemon bool `mapstructure:"-" toml:"-"` - AllocateIP bool `mapstructure:"-" toml:"-"` + InterceptMode string `mapstructure:"intercept_mode" toml:"intercept_mode,omitempty" validate:"omitempty,oneof=off dns hard"` + NRPTRecoveryMaxAttempts *int `mapstructure:"nrpt_recovery_max_attempts" toml:"nrpt_recovery_max_attempts,omitempty" validate:"omitempty,gte=0"` + NRPTRecoveryCooldown *time.Duration `mapstructure:"nrpt_recovery_cooldown" toml:"nrpt_recovery_cooldown,omitempty"` + // FirewallMode controls the DNS-resolved IP allowlist. When "on", only IPs + // that were successfully resolved by ctrld are allowed for outbound connections. + // This closes the "DNS gap" where apps bypass DNS policy using hardcoded IPs. + // Requires intercept mode to be active for enforcement on desktop platforms. + // On mobile, the netstack layer uses the allowlist directly. + FirewallMode string `mapstructure:"firewall_mode" toml:"firewall_mode,omitempty" validate:"omitempty,oneof=off on"` + // FirewallForwardedSources lists VM/container source subnets (CIDR) whose + // forwarded/NATed DNS is redirected through ctrld under Firewall Mode, so + // guest resolutions are policy-enforced and populate the allowlist, and guest + // egress to allowed destinations is permitted without an interface-wide + // bypass. These augment auto-detection of VM networks and are the supported way + // to trust a stack that auto-detection cannot prove ownership of. Empty (the + // default) preserves prior behavior. macOS only; see + // buildPFForwardedSourceRules. + // + // Deliberately not validated with `cidr`: entries are checked at use time and a + // bad one is dropped with a warning while the rest of the set still applies + // (see firewallForwardedSources). A hard validator here would make one typo in + // an MDM-pushed subnet fatal at startup - validateConfig exits the process - + // taking down DNS service for the whole host over a line that only ever + // widened a firewall allowance. + FirewallForwardedSources []string `mapstructure:"firewall_forwarded_sources" toml:"firewall_forwarded_sources,omitempty"` + Daemon bool `mapstructure:"-" toml:"-"` + AllocateIP bool `mapstructure:"-" toml:"-"` } // NetworkConfig specifies configuration for networks where ctrld will handle requests. @@ -255,8 +299,13 @@ type UpstreamConfig struct { Endpoint string `mapstructure:"endpoint" toml:"endpoint,omitempty"` BootstrapIP string `mapstructure:"bootstrap_ip" toml:"bootstrap_ip,omitempty"` Domain string `mapstructure:"-" toml:"-"` - IPStack string `mapstructure:"ip_stack" toml:"ip_stack,omitempty" validate:"ipstack"` - Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` + // InternalDomain is set only by ctrld, on the upstreams it generates for + // organization Internal Domains, to the mode they serve. It is never read + // from or written to a configuration file, so an upstream a configuration + // defines is never mistaken for a generated one, whatever it is named. + InternalDomain string `mapstructure:"-" toml:"-"` + IPStack string `mapstructure:"ip_stack" toml:"ip_stack,omitempty" validate:"ipstack"` + Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` // The caller should not access this field directly. // Use UpstreamSendClientInfo instead. SendClientInfo *bool `mapstructure:"send_client_info" toml:"send_client_info,omitempty"` @@ -265,7 +314,7 @@ type UpstreamConfig struct { Discoverable *bool `mapstructure:"discoverable" toml:"discoverable"` g singleflight.Group - rebootstrap atomic.Bool + rebootstrap atomic.Int64 bootstrapIPs []string bootstrapIPs4 []string bootstrapIPs6 []string @@ -276,10 +325,23 @@ type UpstreamConfig struct { http3RoundTripper http.RoundTripper http3RoundTripper4 http.RoundTripper http3RoundTripper6 http.RoundTripper + doqConnPool *doqConnPool + doqConnPool4 *doqConnPool + doqConnPool6 *doqConnPool + dotClientPool *dotConnPool + dotClientPool4 *dotConnPool + dotClientPool6 *dotConnPool certPool *x509.CertPool u *url.URL fallbackOnce sync.Once uid string + + // bootstrapIncomplete is set while the bootstrap addresses lack a family + // the ip_stack needs; the next re-bootstrap looks them up again. + bootstrapIncomplete atomic.Bool + // directFallback is set once FallbackToDirectIP has fired: DoH and DoH3 + // then also try the Control D direct IPs when the bootstrap addresses fail. + directFallback atomic.Bool } // ListenerConfig specifies the networks configuration that ctrld will run on. @@ -309,14 +371,20 @@ func (lc *ListenerConfig) IsDirectDnsListener() bool { } } +// MatchingConfig defines the configuration for rule matching behavior +type MatchingConfig struct { + Order []string `mapstructure:"order" toml:"order,omitempty"` +} + // ListenerPolicyConfig specifies the policy rules for ctrld to filter incoming requests. type ListenerPolicyConfig struct { - Name string `mapstructure:"name" toml:"name,omitempty"` - Networks []Rule `mapstructure:"networks" toml:"networks,omitempty,inline,multiline" validate:"dive,len=1"` - Rules []Rule `mapstructure:"rules" toml:"rules,omitempty,inline,multiline" validate:"dive,len=1"` - Macs []Rule `mapstructure:"macs" toml:"macs,omitempty,inline,multiline" validate:"dive,len=1"` - FailoverRcodes []string `mapstructure:"failover_rcodes" toml:"failover_rcodes,omitempty" validate:"dive,dnsrcode"` - FailoverRcodeNumbers []int `mapstructure:"-" toml:"-"` + Name string `mapstructure:"name" toml:"name,omitempty"` + Networks []Rule `mapstructure:"networks" toml:"networks,omitempty,inline,multiline" validate:"dive,len=1"` + Rules []Rule `mapstructure:"rules" toml:"rules,omitempty,inline,multiline" validate:"dive,len=1"` + Macs []Rule `mapstructure:"macs" toml:"macs,omitempty,inline,multiline" validate:"dive,len=1"` + FailoverRcodes []string `mapstructure:"failover_rcodes" toml:"failover_rcodes,omitempty" validate:"dive,dnsrcode"` + FailoverRcodeNumbers []int `mapstructure:"-" toml:"-"` + Matching *MatchingConfig `mapstructure:"-" toml:"-"` } // Rule is a map from source to list of upstreams. @@ -325,12 +393,13 @@ type ListenerPolicyConfig struct { type Rule map[string][]string // Init initialized necessary values for an UpstreamConfig. -func (uc *UpstreamConfig) Init() { +func (uc *UpstreamConfig) Init(ctx context.Context) { + logger := LoggerFromCtx(ctx) if err := uc.initDnsStamps(); err != nil { - ProxyLogger.Load().Fatal().Err(err).Msg("invalid DNS Stamps") + logger.Fatal().Err(err).Msg("Invalid dns stamps") } uc.initDoHScheme() - uc.uid = upstreamUID() + uc.uid = upstreamUID(ctx) if u, err := url.Parse(uc.Endpoint); err == nil { uc.Domain = u.Hostname() switch uc.Type { @@ -350,6 +419,9 @@ func (uc *UpstreamConfig) Init() { } } if uc.IPStack == "" { + // Set default IP stack based on upstream type + // Control-D upstreams use split stack for better IPv4/IPv6 handling, + // while other upstreams use both stacks for maximum compatibility if uc.IsControlD() { uc.IPStack = IpStackSplit } else { @@ -411,7 +483,7 @@ func (uc *UpstreamConfig) IsDiscoverable() bool { return *uc.Discoverable } switch uc.Type { - case ResolverTypeOS, ResolverTypeLegacy, ResolverTypePrivate, ResolverTypeLocal: + case ResolverTypeOS, ResolverTypeLegacy, ResolverTypePrivate: if ip, err := netip.ParseAddr(uc.Domain); err == nil { return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || tsaddr.CGNATRange().Contains(ip) } @@ -434,117 +506,333 @@ func (uc *UpstreamConfig) UID() string { return uc.uid } -// SetupBootstrapIP manually find all available IPs of the upstream. -// The first usable IP will be used as bootstrap IP of the upstream. +// bootstrapLookupFn resolves domain for bootstrapping. It is a variable so +// tests control the lookup result. +var bootstrapLookupFn = lookupIP + +// SetupBootstrapIP sets up bootstrap IPs for the upstream config. // The upstream domain will be looked up using following orders: // // - Current system DNS settings. // - Direct IPs table for ControlD upstreams. // - ControlD Bootstrap DNS 76.76.2.22 // -// The setup process will block until there's usable IPs found. -func (uc *UpstreamConfig) SetupBootstrapIP() { +// The later sources fill in what the earlier ones could not: the whole list, +// and also an address family the ip_stack needs that a lookup left out, such +// as an AAAA-only answer for a split-stack upstream, whose A queries need IPv4. +// +// The setup process will block until there's usable IPs found. A result that +// still lacks a needed family is used as it is, and the next re-bootstrap +// looks the missing family up again. +func (uc *UpstreamConfig) SetupBootstrapIP(ctx context.Context) { + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "Setting up bootstrap IPs for upstream: %s", uc.Name) + b := backoff.NewBackoff("setupBootstrapIP", func(format string, args ...any) {}, 10*time.Second) - isControlD := uc.IsControlD() - nss := initDefaultOsResolver() + nss := initDefaultOsResolver(ctx) for { - uc.bootstrapIPs = lookupIP(uc.Domain, uc.Timeout, nss) - // For ControlD upstream, the bootstrap IPs could not be RFC 1918 addresses, - // filtering them out here to prevent weird behavior. - if isControlD { - n := 0 - for _, ip := range uc.bootstrapIPs { - netIP := net.ParseIP(ip) - if netIP != nil && !netIP.IsPrivate() { - uc.bootstrapIPs[n] = ip - n++ - } - } - uc.bootstrapIPs = uc.bootstrapIPs[:n] - if len(uc.bootstrapIPs) == 0 { - uc.bootstrapIPs = bootstrapIPsFromControlDDomain(uc.Domain) - ProxyLogger.Load().Warn().Msgf("no record found for %q, lookup from direct IP table", uc.Domain) - } + if ips := uc.lookupBootstrapIPs(ctx, nss); len(ips) > 0 { + uc.setBootstrapIPs(ctx, ips) + break } - if len(uc.bootstrapIPs) == 0 { - ProxyLogger.Load().Warn().Msgf("no record found for %q, using bootstrap server: %s", uc.Domain, PremiumDNSBoostrapIP) - uc.bootstrapIPs = lookupIP(uc.Domain, uc.Timeout, []string{net.JoinHostPort(PremiumDNSBoostrapIP, "53")}) + logger.Warn().Msg("Could not resolve bootstrap ips, retrying...") + b.BackOff(context.Background(), errors.New("no bootstrap IPs")) + } + logger.Debug().Msgf("Bootstrap ips: %v", uc.bootstrapIPs) + Log(ctx, logger.Debug(), "Bootstrap IP setup completed for upstream: %s", uc.Name) +} +// lookupBootstrapIPs runs one round of the bootstrap lookup through nss and the +// fallback sources, and returns what it found, which may be incomplete for the +// ip_stack or empty. +func (uc *UpstreamConfig) lookupBootstrapIPs(ctx context.Context, nss []string) []string { + logger := LoggerFromCtx(ctx) + isControlD := uc.IsControlD() + Log(ctx, logger.Debug(), "Looking up bootstrap IPs for domain: %s", uc.Domain) + ips := bootstrapLookupFn(ctx, uc.Domain, uc.Timeout, nss) + // For ControlD upstream, the bootstrap IPs could not be RFC 1918 addresses, + // filtering them out here to prevent weird behavior. + if isControlD { + n := 0 + for _, ip := range ips { + netIP := net.ParseIP(ip) + if netIP != nil && !netIP.IsPrivate() { + ips[n] = ip + n++ + } } - if len(uc.bootstrapIPs) > 0 { - break + ips = ips[:n] + if !uc.bootstrapComplete(ctx, ips) { + logger.Warn().Msgf("Incomplete records for %q (%v), filling in from direct ip table", uc.Domain, ips) + ips = mergeMissingFamilies(ips, bootstrapIPsFromControlDDomain(uc.Domain)) } - ProxyLogger.Load().Warn().Msg("could not resolve bootstrap IPs, retrying...") - b.BackOff(context.Background(), errors.New("no bootstrap IPs")) } - for _, ip := range uc.bootstrapIPs { + if !uc.bootstrapComplete(ctx, ips) { + logger.Warn().Msgf("Incomplete records for %q (%v), using bootstrap server: %s", uc.Domain, ips, PremiumDNSBoostrapIP) + ips = mergeMissingFamilies(ips, bootstrapLookupFn(ctx, uc.Domain, uc.Timeout, []string{net.JoinHostPort(PremiumDNSBoostrapIP, "53")})) + } + return ips +} + +// setBootstrapIPs replaces the bootstrap addresses, and records whether they +// cover the families the ip_stack needs. +func (uc *UpstreamConfig) setBootstrapIPs(ctx context.Context, ips []string) { + ips4, ips6 := splitAddressFamilies(ips) + uc.bootstrapIPs, uc.bootstrapIPs4, uc.bootstrapIPs6 = ips, ips4, ips6 + complete := uc.bootstrapComplete(ctx, ips) + uc.bootstrapIncomplete.Store(!complete) + if !complete { + logger := LoggerFromCtx(ctx) + logger.Warn().Msgf("Bootstrap ips for %q lack an address family needed by ip_stack %q: %v; will look up again on re-bootstrap", uc.Domain, uc.IPStack, ips) + } +} + +// bootstrapComplete reports whether ips cover the address families the +// upstream's ip_stack needs: IPv4 for v4, IPv6 for v6, either for both, and +// for split IPv4 for A queries plus IPv6 for the others when the host has IPv6. +// An IPv4-only host never needs IPv6. +func (uc *UpstreamConfig) bootstrapComplete(ctx context.Context, ips []string) bool { + ips4, ips6 := splitAddressFamilies(ips) + switch uc.IPStack { + case IpStackV4: + return len(ips4) > 0 + case IpStackV6: + return len(ips6) > 0 + case IpStackSplit: + return len(ips4) > 0 && (len(ips6) > 0 || !HasIPv6(ctx)) + } + return len(ips) > 0 +} + +// splitAddressFamilies returns the IPv4 and the IPv6 addresses of ips. +func splitAddressFamilies(ips []string) (ips4, ips6 []string) { + for _, ip := range ips { if ctrldnet.IsIPv6(ip) { - uc.bootstrapIPs6 = append(uc.bootstrapIPs6, ip) + ips6 = append(ips6, ip) } else { - uc.bootstrapIPs4 = append(uc.bootstrapIPs4, ip) + ips4 = append(ips4, ip) } } - ProxyLogger.Load().Debug().Msgf("bootstrap IPs: %v", uc.bootstrapIPs) + return ips4, ips6 +} + +// mergeMissingFamilies adds to ips the addresses of extra whose family ips +// does not have at all. A family ips already has is kept as it is. +func mergeMissingFamilies(ips, extra []string) []string { + ips4, ips6 := splitAddressFamilies(ips) + extra4, extra6 := splitAddressFamilies(extra) + merged := append([]string{}, ips...) + if len(ips4) == 0 { + merged = append(merged, extra4...) + } + if len(ips6) == 0 { + merged = append(merged, extra6...) + } + return merged +} + +// refreshIncompleteBootstrap looks the bootstrap addresses up again, once, when +// they lack a family the ip_stack needs. Rebuilding the transports from the +// same incomplete lists would never recover; this picks up the missing family +// once DNS is reachable again. Families it still cannot find keep their current +// addresses. +func (uc *UpstreamConfig) refreshIncompleteBootstrap(ctx context.Context) { + if uc.BootstrapIP != "" || !uc.bootstrapIncomplete.Load() { + return + } + ips := uc.lookupBootstrapIPs(ctx, initDefaultOsResolver(ctx)) + if !uc.bootstrapComplete(ctx, ips) { + ips = mergeMissingFamilies(ips, uc.bootstrapIPs) + } + if len(ips) == 0 { + return + } + uc.setBootstrapIPs(ctx, ips) + logger := LoggerFromCtx(ctx) + logger.Debug().Msgf("Refreshed bootstrap ips for %q: %v", uc.Domain, ips) } // ReBootstrap re-setup the bootstrap IP and the transport. -func (uc *UpstreamConfig) ReBootstrap() { +func (uc *UpstreamConfig) ReBootstrap(ctx context.Context) { switch uc.Type { - case ResolverTypeDOH, ResolverTypeDOH3: + case ResolverTypeDOH, ResolverTypeDOH3, ResolverTypeDOQ, ResolverTypeDOT: default: return } _, _, _ = uc.g.Do("ReBootstrap", func() (any, error) { - if uc.rebootstrap.CompareAndSwap(false, true) { - ProxyLogger.Load().Debug().Msgf("re-bootstrapping upstream ip for %v", uc) + if uc.rebootstrap.CompareAndSwap(rebootstrapNotStarted, rebootstrapStarted) { + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "Re-bootstrapping upstream: %s", uc.Name) } return true, nil }) } -// SetupTransport initializes the network transport used to connect to upstream server. -// For now, only DoH upstream is supported. -func (uc *UpstreamConfig) SetupTransport() { +// ForceReBootstrap immediately creates a new transport (closing old idle +// connections first) without waiting for the lazy re-bootstrap mechanism. +// Used after pf state table flushes where existing TCP/QUIC connections +// are dead and we need fresh connections immediately. +func (uc *UpstreamConfig) ForceReBootstrap(ctx context.Context) { switch uc.Type { - case ResolverTypeDOH: - uc.setupDOHTransport() - case ResolverTypeDOH3: - uc.setupDOH3Transport() + case ResolverTypeDOH, ResolverTypeDOH3, ResolverTypeDOQ, ResolverTypeDOT: + default: + return + } + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "force re-bootstrapping upstream transport for %v", uc) + uc.closeTransports() + uc.refreshIncompleteBootstrap(ctx) + uc.SetupTransport(ctx) + // Clear any pending lazy re-bootstrap flag so ensureSetupTransport() + // doesn't redundantly recreate the transport we just built. + uc.rebootstrap.Store(rebootstrapNotStarted) +} + +// CloseTransports retires the upstream's transports, including active DoT and +// HTTP/3 connections. Callers must stop publishing this upstream before calling +// it and must not reuse it without calling SetupTransport. +func (uc *UpstreamConfig) CloseTransports() { + uc.retireTransports() +} + +// retireTransports permanently closes DoT pools and HTTP/3 transports before +// their owner discards them. Like closeTransports, it does not acquire an +// upstream lock: callers that serialize slot access must keep that serialization +// across retirement and replacement. +func (uc *UpstreamConfig) retireTransports() { + uc.closeTransports() + for _, p := range []*dotConnPool{uc.dotClientPool, uc.dotClientPool4, uc.dotClientPool6} { + if p != nil { + p.Close() + } + } + // Unlike CloseIdleConnections, Close also cancels pending dials and active + // HTTP/3 requests, and prevents late requests from reviving the transport. + for _, rt := range []http.RoundTripper{uc.http3RoundTripper, uc.http3RoundTripper4, uc.http3RoundTripper6} { + if c, ok := rt.(interface{ Close() error }); ok { + _ = c.Close() + } + } +} + +// closeTransports closes idle connections on all existing transports. +// It does not retire DoT pools or HTTP/3 transports: active work may continue +// and the resources remain usable. Use retireTransports before replacing them. +func (uc *UpstreamConfig) closeTransports() { + for _, t := range []*http.Transport{uc.transport, uc.transport4, uc.transport6} { + if t != nil { + t.CloseIdleConnections() + } + } + for _, p := range []*doqConnPool{uc.doqConnPool, uc.doqConnPool4, uc.doqConnPool6} { + if p != nil { + p.CloseIdleConnections() + } + } + for _, p := range []*dotConnPool{uc.dotClientPool, uc.dotClientPool4, uc.dotClientPool6} { + if p != nil { + p.CloseIdleConnections() + } + } + for _, rt := range []http.RoundTripper{uc.http3RoundTripper, uc.http3RoundTripper4, uc.http3RoundTripper6} { + if c, ok := rt.(interface{ CloseIdleConnections() }); ok { + c.CloseIdleConnections() + } } } -func (uc *UpstreamConfig) setupDOHTransport() { +// SetupTransport initializes the network transport used to connect to upstream servers. +// For now, DoH/DoH3/DoQ/DoT upstreams are supported. +func (uc *UpstreamConfig) SetupTransport(ctx context.Context) { + switch uc.Type { + case ResolverTypeDOH, ResolverTypeDOH3, ResolverTypeDOQ, ResolverTypeDOT: + default: + return + } + + // Retire old DoT/HTTP3 resources before replacing their slots. Idle-only + // cleanup would leave active queries and late work owning orphaned resources. + // DoH and DoQ retain their existing cleanup semantics. + uc.retireTransports() + + ips := uc.bootstrapIPs switch uc.IPStack { - case IpStackBoth, "": - uc.transport = uc.newDOHTransport(uc.bootstrapIPs) case IpStackV4: - uc.transport = uc.newDOHTransport(uc.bootstrapIPs4) + ips = uc.bootstrapIPs4 case IpStackV6: - uc.transport = uc.newDOHTransport(uc.bootstrapIPs6) - case IpStackSplit: - uc.transport4 = uc.newDOHTransport(uc.bootstrapIPs4) - if HasIPv6() { - uc.transport6 = uc.newDOHTransport(uc.bootstrapIPs6) + ips = uc.bootstrapIPs6 + } + uc.transport = uc.newDOHTransport(ctx, ips) + uc.http3RoundTripper = uc.newDOH3Transport(ctx, ips) + uc.doqConnPool = uc.newDOQConnPool(ctx, ips) + uc.dotClientPool = uc.newDOTClientPool(ctx, ips) + if uc.IPStack == IpStackSplit { + // Split picks the transport by query type, not a hard family limit: + // while a family is missing, its queries borrow the other family's + // addresses rather than fail on an empty transport until the + // re-bootstrap finds it (refreshIncompleteBootstrap). + ips4, ips6 := uc.bootstrapIPs4, uc.bootstrapIPs6 + if len(ips4) == 0 { + ips4 = ips6 + } + if len(ips6) == 0 { + ips6 = ips4 + } + uc.transport4 = uc.newDOHTransport(ctx, ips4) + uc.http3RoundTripper4 = uc.newDOH3Transport(ctx, ips4) + uc.doqConnPool4 = uc.newDOQConnPool(ctx, ips4) + uc.dotClientPool4 = uc.newDOTClientPool(ctx, ips4) + if HasIPv6(ctx) { + uc.transport6 = uc.newDOHTransport(ctx, ips6) + uc.http3RoundTripper6 = uc.newDOH3Transport(ctx, ips6) + uc.doqConnPool6 = uc.newDOQConnPool(ctx, ips6) + uc.dotClientPool6 = uc.newDOTClientPool(ctx, ips6) } else { uc.transport6 = uc.transport4 + uc.http3RoundTripper6 = uc.http3RoundTripper4 + uc.doqConnPool6 = uc.doqConnPool4 + uc.dotClientPool6 = uc.dotClientPool4 } - uc.transport = uc.newDOHTransport(uc.bootstrapIPs) } } -func (uc *UpstreamConfig) newDOHTransport(addrs []string) *http.Transport { +func (uc *UpstreamConfig) ensureSetupTransport(ctx context.Context) { + uc.transportOnce.Do(func() { + uc.SetupTransport(ctx) + }) + + if uc.rebootstrap.CompareAndSwap(rebootstrapStarted, rebootstrapInProgress) { + uc.refreshIncompleteBootstrap(ctx) + uc.SetupTransport(ctx) + uc.rebootstrap.Store(rebootstrapNotStarted) + } +} + +// HTTP/2 health check for DoH connections: after dohSendPingTimeout without a +// received frame, send a PING; close the connection if no reply arrives within +// dohPingTimeout. Variables only so tests can shorten them. +var ( + dohSendPingTimeout = 10 * time.Second + dohPingTimeout = 5 * time.Second +) + +func (uc *UpstreamConfig) newDOHTransport(ctx context.Context, addrs []string) *http.Transport { + if uc.Type != ResolverTypeDOH { + return nil + } transport := http.DefaultTransport.(*http.Transport).Clone() transport.MaxIdleConnsPerHost = 100 transport.TLSClientConfig = &tls.Config{ RootCAs: uc.certPool, ClientSessionCache: tls.NewLRUClientSessionCache(0), + MinVersion: tls.VersionTLS12, } // Prevent bad tcp connection hanging the requests for too long. // See: https://github.com/golang/go/issues/36026 - if t2, err := http2.ConfigureTransports(transport); err == nil { - t2.ReadIdleTimeout = 10 * time.Second - t2.PingTimeout = 5 * time.Second + transport.HTTP2 = &http.HTTP2Config{ + SendPingTimeout: dohSendPingTimeout, + PingTimeout: dohPingTimeout, } dialerTimeoutMs := 2000 @@ -552,26 +840,31 @@ func (uc *UpstreamConfig) newDOHTransport(addrs []string) *http.Transport { dialerTimeoutMs = uc.Timeout } dialerTimeout := time.Duration(dialerTimeoutMs) * time.Millisecond + logger := LoggerFromCtx(ctx) transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { _, port, _ := net.SplitHostPort(addr) if uc.BootstrapIP != "" { dialer := net.Dialer{Timeout: dialerTimeout, KeepAlive: dialerTimeout} addr := net.JoinHostPort(uc.BootstrapIP, port) - Log(ctx, ProxyLogger.Load().Debug(), "sending doh request to: %s", addr) + Log(ctx, logger.Debug(), "Sending doh request to: %s", addr) return dialer.DialContext(ctx, network, addr) } pd := &ctrldnet.ParallelDialer{} pd.Timeout = dialerTimeout pd.KeepAlive = dialerTimeout - dialAddrs := make([]string, len(addrs)) - for i := range addrs { - dialAddrs[i] = net.JoinHostPort(addrs[i], port) - } - conn, err := pd.DialContext(ctx, network, dialAddrs, ProxyLogger.Load()) + conn, err := pd.DialContext(ctx, network, joinHostPorts(addrs, port), logger.Logger) if err != nil { - return nil, err + direct := uc.directFallbackIPs() + if len(direct) == 0 { + return nil, err + } + Log(ctx, logger.Debug().Err(err), "Bootstrap addresses failed, dialing direct ips %v", direct) + var directErr error + if conn, directErr = pd.DialContext(ctx, network, joinHostPorts(direct, port), logger.Logger); directErr != nil { + return nil, errors.Join(err, directErr) + } } - Log(ctx, ProxyLogger.Load().Debug(), "sending doh request to: %s", conn.RemoteAddr()) + Log(ctx, logger.Debug(), "Sending doh request to: %s", conn.RemoteAddr()) return conn, nil } runtime.SetFinalizer(transport, func(transport *http.Transport) { @@ -581,21 +874,22 @@ func (uc *UpstreamConfig) newDOHTransport(addrs []string) *http.Transport { } // Ping warms up the connection to DoH/DoH3 upstream. -func (uc *UpstreamConfig) Ping() { - if err := uc.ping(); err != nil { - ProxyLogger.Load().Debug().Err(err).Msgf("upstream ping failed: %s", uc.Endpoint) - _ = uc.FallbackToDirectIP() +func (uc *UpstreamConfig) Ping(ctx context.Context) { + if err := uc.ping(ctx); err != nil { + logger := LoggerFromCtx(ctx) + logger.Debug().Err(err).Msgf("Upstream ping failed: %s", uc.Endpoint) + _ = uc.FallbackToDirectIP(ctx) } } // ErrorPing is like Ping, but return an error if any. -func (uc *UpstreamConfig) ErrorPing() error { - return uc.ping() +func (uc *UpstreamConfig) ErrorPing(ctx context.Context) error { + return uc.ping(ctx) } -func (uc *UpstreamConfig) ping() error { +func (uc *UpstreamConfig) ping(ctx context.Context) error { switch uc.Type { - case ResolverTypeDOH, ResolverTypeDOH3: + case ResolverTypeDOH, ResolverTypeDOH3, ResolverTypeDOQ: default: return nil } @@ -622,13 +916,21 @@ func (uc *UpstreamConfig) ping() error { for _, typ := range []uint16{dns.TypeA, dns.TypeAAAA} { switch uc.Type { case ResolverTypeDOH: - if err := ping(uc.dohTransport(typ)); err != nil { + if err := ping(uc.dohTransport(ctx, typ)); err != nil { return err } case ResolverTypeDOH3: - if err := ping(uc.doh3Transport(typ)); err != nil { + if err := ping(uc.doh3Transport(ctx, typ)); err != nil { return err } + case ResolverTypeDOQ: + // For DoQ, we just ensure transport is set up by calling doqTransport + // DoQ doesn't use HTTP, so we can't ping it the same way + _ = uc.doqTransport(ctx, typ) + case ResolverTypeDOT: + // For DoT, we just ensure transport is set up by calling dotTransport + // DoT doesn't use HTTP, so we can't ping it the same way + _ = uc.dotTransport(ctx, typ) } } @@ -651,6 +953,7 @@ func (uc *UpstreamConfig) IsControlD() bool { return false } +// isNextDNS reports whether this is a NextDNS upstream. func (uc *UpstreamConfig) isNextDNS() bool { domain := uc.Domain if domain == "" { @@ -658,53 +961,15 @@ func (uc *UpstreamConfig) isNextDNS() bool { domain = u.Hostname() } } - return domain == "dns.nextdns.io" -} - -func (uc *UpstreamConfig) dohTransport(dnsType uint16) http.RoundTripper { - uc.transportOnce.Do(func() { - uc.SetupTransport() - }) - if uc.rebootstrap.CompareAndSwap(true, false) { - uc.SetupTransport() - } - switch uc.IPStack { - case IpStackBoth, IpStackV4, IpStackV6: - return uc.transport - case IpStackSplit: - switch dnsType { - case dns.TypeA: - return uc.transport4 - default: - return uc.transport6 - } - } - return uc.transport + return dns.IsSubDomain(nextDNSDomain, domain) } -func (uc *UpstreamConfig) bootstrapIPForDNSType(dnsType uint16) string { - switch uc.IPStack { - case IpStackBoth: - return pick(uc.bootstrapIPs) - case IpStackV4: - return pick(uc.bootstrapIPs4) - case IpStackV6: - return pick(uc.bootstrapIPs6) - case IpStackSplit: - switch dnsType { - case dns.TypeA: - return pick(uc.bootstrapIPs4) - default: - if HasIPv6() { - return pick(uc.bootstrapIPs6) - } - return pick(uc.bootstrapIPs4) - } - } - return pick(uc.bootstrapIPs) +func (uc *UpstreamConfig) dohTransport(ctx context.Context, dnsType uint16) http.RoundTripper { + uc.ensureSetupTransport(ctx) + return transportByIpStack(uc.IPStack, dnsType, uc.transport, uc.transport4, uc.transport6) } -func (uc *UpstreamConfig) netForDNSType(dnsType uint16) (string, string) { +func (uc *UpstreamConfig) netForDNSType(ctx context.Context, dnsType uint16) (string, string) { switch uc.IPStack { case IpStackBoth: return "tcp-tls", "udp" @@ -717,7 +982,7 @@ func (uc *UpstreamConfig) netForDNSType(dnsType uint16) (string, string) { case dns.TypeA: return "tcp4-tls", "udp4" default: - if HasIPv6() { + if HasIPv6(ctx) { return "tcp6-tls", "udp6" } return "tcp4-tls", "udp4" @@ -797,8 +1062,43 @@ func (uc *UpstreamConfig) Context(ctx context.Context) (context.Context, context return context.WithCancel(ctx) } -// FallbackToDirectIP changes ControlD upstream endpoint to use direct IP instead of domain. -func (uc *UpstreamConfig) FallbackToDirectIP() bool { +// controlDDirectIPsFn returns the direct IPs of a Control D domain. It is a +// variable so tests can point the fallback at local listeners. +var controlDDirectIPsFn = bootstrapIPsFromControlDDomain + +// directFallbackIPs returns the Control D direct IPs that DoH and DoH3 try +// when the bootstrap addresses fail, once FallbackToDirectIP has fired, or nil. +// They are limited to the families the ip_stack dials: an IPv6-only upstream +// must not be sent to an IPv4 literal it cannot reach. +func (uc *UpstreamConfig) directFallbackIPs() []string { + if !uc.directFallback.Load() { + return nil + } + ips4, ips6 := splitAddressFamilies(controlDDirectIPsFn(uc.Domain)) + switch uc.IPStack { + case IpStackV4: + return ips4 + case IpStackV6: + return ips6 + } + return append(ips4, ips6...) +} + +// joinHostPorts returns ips joined with port. +func joinHostPorts(ips []string, port string) []string { + addrs := make([]string, len(ips)) + for i, ip := range ips { + addrs[i] = net.JoinHostPort(ip, port) + } + return addrs +} + +// FallbackToDirectIP lets a ControlD DoH/DoH3 upstream also dial its direct +// IPs, which DoH and DoH3 try only when the bootstrap addresses are empty or +// all fail. The bootstrap list keeps precedence, so a working or refreshed +// list is used again as soon as it can be, and nothing is pinned. The request +// URL keeps the domain, so TLS still verifies the certificate for it. +func (uc *UpstreamConfig) FallbackToDirectIP(ctx context.Context) bool { if !uc.IsControlD() { return false } @@ -808,17 +1108,13 @@ func (uc *UpstreamConfig) FallbackToDirectIP() bool { done := false uc.fallbackOnce.Do(func() { - var ip string - switch { - case dns.IsSubDomain(PremiumDnsDomain, uc.Domain): - ip = PremiumDNSBoostrapIP - case dns.IsSubDomain(FreeDnsDomain, uc.Domain): - ip = FreeDNSBoostrapIP - default: + ips := controlDDirectIPsFn(uc.Domain) + if len(ips) == 0 { return } - ProxyLogger.Load().Warn().Msgf("using direct IP for %q: %s", uc.Endpoint, ip) - uc.u.Host = ip + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Warn(), "Falling back to direct ips for %q when its bootstrap addresses fail: %v", uc.Endpoint, ips) + uc.directFallback.Store(true) done = true }) return done @@ -826,12 +1122,18 @@ func (uc *UpstreamConfig) FallbackToDirectIP() bool { // Init initialized necessary values for an ListenerConfig. func (lc *ListenerConfig) Init() { + logger := LoggerFromCtx(context.Background()) + Log(context.Background(), logger.Debug(), "Initializing listener config") + if lc.Policy != nil { lc.Policy.FailoverRcodeNumbers = make([]int, len(lc.Policy.FailoverRcodes)) for i, rcode := range lc.Policy.FailoverRcodes { lc.Policy.FailoverRcodeNumbers[i] = dnsrcode.FromString(rcode) } + Log(context.Background(), logger.Debug(), "Listener policy initialized with %d failover rcodes", len(lc.Policy.FailoverRcodes)) } + + Log(context.Background(), logger.Debug(), "Listener config initialization completed") } // ValidateConfig validates the given config. @@ -946,16 +1248,13 @@ func ResolverTypeFromEndpoint(endpoint string) string { return ResolverTypeDOT } -func pick(s []string) string { - return s[rand.Intn(len(s))] -} - // upstreamUID generates an unique identifier for an upstream. -func upstreamUID() string { +func upstreamUID(ctx context.Context) string { + logger := LoggerFromCtx(ctx) b := make([]byte, 4) for { if _, err := crand.Read(b); err != nil { - ProxyLogger.Load().Warn().Err(err).Msg("could not generate uid for upstream, retrying...") + logger.Warn().Err(err).Msg("Could not generate uid for upstream, retrying...") continue } return hex.EncodeToString(b) @@ -985,3 +1284,18 @@ func bootstrapIPsFromControlDDomain(domain string) []string { } return nil } + +func transportByIpStack[T any](ipStack string, dnsType uint16, transport, transport4, transport6 T) T { + switch ipStack { + case IpStackBoth, IpStackV4, IpStackV6: + return transport + case IpStackSplit: + switch dnsType { + case dns.TypeA: + return transport4 + default: + return transport6 + } + } + return transport +} diff --git a/config_bootstrap_test.go b/config_bootstrap_test.go new file mode 100644 index 00000000..65851e48 --- /dev/null +++ b/config_bootstrap_test.go @@ -0,0 +1,334 @@ +package ctrld + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "slices" + "strconv" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" +) + +// useBootstrapLookup replaces the bootstrap lookup. system answers lookups +// through the system nameservers, controlD lookups through the Control D +// bootstrap server; the test changes them through the returned pointers. +func useBootstrapLookup(t *testing.T, system, controlD []string) (*[]string, *[]string) { + t.Helper() + sys, cd := append([]string{}, system...), append([]string{}, controlD...) + old := bootstrapLookupFn + bootstrapLookupFn = func(_ context.Context, domain string, timeout int, nss []string) []string { + if slices.Equal(nss, []string{net.JoinHostPort(PremiumDNSBoostrapIP, "53")}) { + return append([]string{}, cd...) + } + return append([]string{}, sys...) + } + t.Cleanup(func() { bootstrapLookupFn = old }) + return &sys, &cd +} + +// useIPv6 pins whether the host has IPv6. HasIPv6 probes once per process, so +// the probe runs first and the pinned value replaces its result. +func useIPv6(t *testing.T, available bool) { + t.Helper() + old := HasIPv6(context.Background()) + SetIPv6Available(available) + t.Cleanup(func() { SetIPv6Available(old) }) +} + +func newBootstrapTestUpstream(typ, endpoint, ipStack string) *UpstreamConfig { + uc := &UpstreamConfig{Type: typ, Endpoint: endpoint, Timeout: 1000} + uc.Init(context.Background()) + uc.IPStack = ipStack + return uc +} + +// Bootstrap results must cover the address families the ip_stack needs (#624). +// A partial lookup is filled in from the Control D direct-IP table and the +// Control D bootstrap server; what is still missing marks the upstream for a +// new lookup instead of becoming a permanently empty transport. An IPv4-only +// host never needs IPv6. +func TestSetupBootstrapIPCompletesPerStack(t *testing.T) { + const ( + cdEndpoint = "https://dns.controld.com/abcdef" + otherEndpoint = "https://dns.example.com/dns-query" + v4, v6 = "192.0.2.10", "2001:db8::10" + ) + for _, tc := range []struct { + name string + endpoint string + ipStack string + ipv6 bool + system, cdBoot []string + want4, want6 []string + wantIncomplete bool + }{ + {name: "Control D split, AAAA only", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: true, + system: []string{"2606:1a40::99"}, want4: []string{PremiumDNSBoostrapIP}, want6: []string{"2606:1a40::99"}}, + {name: "Control D split, A only", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: true, + system: []string{"76.76.2.99"}, want4: []string{"76.76.2.99"}, want6: []string{PremiumDNSBoostrapIPv6}}, + {name: "Control D split, A only, IPv4-only host", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: false, + system: []string{"76.76.2.99"}, want4: []string{"76.76.2.99"}}, + {name: "Control D split, complete", endpoint: cdEndpoint, ipStack: IpStackSplit, ipv6: true, + system: []string{"2606:1a40::99", "76.76.2.99"}, want4: []string{"76.76.2.99"}, want6: []string{"2606:1a40::99"}}, + {name: "other v4, AAAA only", endpoint: otherEndpoint, ipStack: IpStackV4, ipv6: true, + system: []string{v6}, want6: []string{v6}, wantIncomplete: true}, + {name: "other v6, A only", endpoint: otherEndpoint, ipStack: IpStackV6, ipv6: true, + system: []string{v4}, want4: []string{v4}, wantIncomplete: true}, + {name: "other v4, complete", endpoint: otherEndpoint, ipStack: IpStackV4, ipv6: true, + system: []string{v4}, want4: []string{v4}}, + {name: "other both, AAAA only", endpoint: otherEndpoint, ipStack: IpStackBoth, ipv6: true, + system: []string{v6}, want6: []string{v6}}, + {name: "other split, AAAA only, bootstrap server has A", endpoint: otherEndpoint, ipStack: IpStackSplit, ipv6: true, + system: []string{v6}, cdBoot: []string{v4}, want4: []string{v4}, want6: []string{v6}}, + {name: "other split, AAAA only, nowhere has A", endpoint: otherEndpoint, ipStack: IpStackSplit, ipv6: true, + system: []string{v6}, want6: []string{v6}, wantIncomplete: true}, + } { + t.Run(tc.name, func(t *testing.T) { + useIPv6(t, tc.ipv6) + useBootstrapLookup(t, tc.system, tc.cdBoot) + uc := newBootstrapTestUpstream(ResolverTypeDOH, tc.endpoint, tc.ipStack) + uc.SetupBootstrapIP(context.Background()) + if !slices.Equal(uc.bootstrapIPs4, tc.want4) || !slices.Equal(uc.bootstrapIPs6, tc.want6) { + t.Fatalf("bootstrap IPs = v4 %v, v6 %v; want v4 %v, v6 %v", uc.bootstrapIPs4, uc.bootstrapIPs6, tc.want4, tc.want6) + } + if got := uc.bootstrapIncomplete.Load(); got != tc.wantIncomplete { + t.Fatalf("bootstrapIncomplete = %v, want %v", got, tc.wantIncomplete) + } + }) + } +} + +// After a partial lookup, the transport recovers once the missing family can +// be resolved again, without a restart: a re-bootstrap looks it up instead of +// rebuilding the transport from the same incomplete lists. Until then a split +// transport borrows the other family rather than staying empty. +func TestReBootstrapRefreshesMissingFamily(t *testing.T) { + const v4, v6 = "192.0.2.10", "2001:db8::10" + for _, tc := range []struct { + name string + ipStack string + wantInterimA4 []string // addresses A queries dial before recovery + }{ + {name: "v4", ipStack: IpStackV4, wantInterimA4: nil}, + {name: "split", ipStack: IpStackSplit, wantInterimA4: []string{v6}}, + } { + t.Run(tc.name, func(t *testing.T) { + useIPv6(t, true) + system, _ := useBootstrapLookup(t, []string{v6}, nil) + uc := newBootstrapTestUpstream(ResolverTypeDOT, "dot.example.com:853", tc.ipStack) + t.Cleanup(uc.CloseTransports) + uc.SetupBootstrapIP(context.Background()) + if !uc.bootstrapIncomplete.Load() { + t.Fatal("an AAAA-only result was accepted as complete") + } + if got := uc.dotTransport(context.Background(), dns.TypeA).addrs; !slices.Equal(got, tc.wantInterimA4) { + t.Fatalf("A-query addresses before recovery = %v, want %v", got, tc.wantInterimA4) + } + + // DNS is still broken: a re-bootstrap keeps what it has. + uc.ReBootstrap(context.Background()) + if got := uc.dotTransport(context.Background(), dns.TypeA).addrs; !slices.Equal(got, tc.wantInterimA4) { + t.Fatalf("A-query addresses while DNS is down = %v, want %v", got, tc.wantInterimA4) + } + + // DNS is back: the next re-bootstrap picks up the missing family. + *system = []string{v6, v4} + uc.ReBootstrap(context.Background()) + if got := uc.dotTransport(context.Background(), dns.TypeA).addrs; !slices.Equal(got, []string{v4}) { + t.Fatalf("A-query addresses after recovery = %v, want [%s]", got, v4) + } + if uc.bootstrapIncomplete.Load() { + t.Fatal("still incomplete after the missing family was found") + } + }) + } +} + +// A complete result is not looked up again on every re-bootstrap. +func TestReBootstrapKeepsCompleteResult(t *testing.T) { + useIPv6(t, true) + lookups := 0 + old := bootstrapLookupFn + bootstrapLookupFn = func(context.Context, string, int, []string) []string { + lookups++ + return []string{"192.0.2.10", "2001:db8::10"} + } + t.Cleanup(func() { bootstrapLookupFn = old }) + uc := newBootstrapTestUpstream(ResolverTypeDOT, "dot.example.com:853", IpStackSplit) + t.Cleanup(uc.CloseTransports) + uc.SetupBootstrapIP(context.Background()) + before := lookups + uc.ReBootstrap(context.Background()) + _ = uc.dotTransport(context.Background(), dns.TypeA) + if lookups != before { + t.Fatalf("a complete result was looked up again: %d lookups", lookups-before) + } +} + +// useDirectIPs replaces the Control D direct-IP table. +func useDirectIPs(t *testing.T, ips ...string) { + t.Helper() + old := controlDDirectIPsFn + controlDDirectIPsFn = func(string) []string { return append([]string{}, ips...) } + t.Cleanup(func() { controlDDirectIPsFn = old }) +} + +// listenLocal listens on ip:port, or on a free port when port is "0", and +// returns the port. It skips the test when ip is not available here. +func listenLocal(t *testing.T, ip, port string) string { + t.Helper() + ln, err := net.Listen("tcp", net.JoinHostPort(ip, port)) + if err != nil { + t.Skipf("cannot listen on %s: %v", ip, err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + } + }() + return strconv.Itoa(ln.Addr().(*net.TCPAddr).Port) +} + +func dialDOH(t *testing.T, transport *http.Transport, port string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + conn, err := transport.DialContext(ctx, "tcp", net.JoinHostPort("dns.controld.com", port)) + if err != nil { + return "", err + } + defer conn.Close() + host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) + return host, nil +} + +// The DoH direct-IP fallback adds the Control D direct IPs as a backup for +// when the bootstrap addresses are empty or all fail. It never replaces a +// working list, so it is not pinned until restart, and it only dials the +// families the ip_stack dials. +func TestFallbackToDirectIPIsABackup(t *testing.T) { + t.Run("v6 upstream keeps dialing its IPv6 list after the fallback", func(t *testing.T) { + port := listenLocal(t, "::1", "0") + useDirectIPs(t, "127.0.0.1") + uc := newBootstrapTestUpstream(ResolverTypeDOH, "https://dns.controld.com/abcdef", IpStackV6) + transport := uc.newDOHTransport(context.Background(), []string{"::1"}) + t.Cleanup(transport.CloseIdleConnections) + if !uc.FallbackToDirectIP(context.Background()) { + t.Fatal("FallbackToDirectIP did not fire for a Control D upstream") + } + if got, err := dialDOH(t, transport, port); err != nil || got != "::1" { + t.Fatalf("dialed %q (err %v), want the bootstrap address ::1", got, err) + } + if uc.u.Host != "dns.controld.com" { + t.Fatalf("request host = %q, want the domain kept for TLS", uc.u.Host) + } + }) + + t.Run("v6 upstream with no addresses only gets IPv6 direct IPs", func(t *testing.T) { + port := listenLocal(t, "::1", "0") + useDirectIPs(t, "127.0.0.1", "::1") + uc := newBootstrapTestUpstream(ResolverTypeDOH, "https://dns.controld.com/abcdef", IpStackV6) + transport := uc.newDOHTransport(context.Background(), nil) + t.Cleanup(transport.CloseIdleConnections) + uc.FallbackToDirectIP(context.Background()) + if got := uc.directFallbackIPs(); !slices.Equal(got, []string{"::1"}) { + t.Fatalf("direct IPs for a v6 upstream = %v, want only [::1]", got) + } + if got, err := dialDOH(t, transport, port); err != nil || got != "::1" { + t.Fatalf("dialed %q (err %v), want the IPv6 direct IP ::1", got, err) + } + }) + + t.Run("direct IPs only after the list fails, and the list wins again once reachable", func(t *testing.T) { + port := listenLocal(t, "127.0.0.1", "0") // the direct IP + useDirectIPs(t, "127.0.0.1") + uc := newBootstrapTestUpstream(ResolverTypeDOH, "https://dns.controld.com/abcdef", IpStackSplit) + transport := uc.newDOHTransport(context.Background(), []string{"::1"}) // nothing listens there yet + t.Cleanup(transport.CloseIdleConnections) + + if _, err := dialDOH(t, transport, port); err == nil { + t.Fatal("a failed list was retried on the direct IPs before the fallback fired") + } + uc.FallbackToDirectIP(context.Background()) + if got, err := dialDOH(t, transport, port); err != nil || got != "127.0.0.1" { + t.Fatalf("dialed %q (err %v), want the direct IP after the list failed", got, err) + } + + // The bootstrap address is reachable again: it is dialed, not the direct IP. + listenLocal(t, "::1", port) + if got, err := dialDOH(t, transport, port); err != nil || got != "::1" { + t.Fatalf("dialed %q (err %v), want the recovered bootstrap address ::1", got, err) + } + }) + t.Run("doh3 dials the direct IPs after the list fails", func(t *testing.T) { + cert := generateTestCertificate(t) + ln, err := quic.ListenAddr("127.0.0.1:0", &tls.Config{ + Certificates: []tls.Certificate{cert.tlsCert}, + NextProtos: []string{http3.NextProtoH3}, + MinVersion: tls.VersionTLS13, + }, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + if _, err := ln.Accept(context.Background()); err != nil { + return + } + } + }() + port := strconv.Itoa(ln.Addr().(*net.UDPAddr).Port) + useDirectIPs(t, "127.0.0.1") + uc := newBootstrapTestUpstream(ResolverTypeDOH3, "https://dns.controld.com/abcdef", IpStackV4) + rt := uc.newDOH3Transport(context.Background(), nil).(*http3.Transport) + t.Cleanup(func() { _ = rt.Close() }) + + pool := x509.NewCertPool() + pool.AddCert(cert.cert) + tlsCfg := &tls.Config{RootCAs: pool, ServerName: "localhost", NextProtos: []string{http3.NextProtoH3}, MinVersion: tls.VersionTLS13} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + addr := net.JoinHostPort("dns.controld.com", port) + + if _, err := rt.Dial(ctx, addr, tlsCfg, nil); err == nil { + t.Fatal("dial with no addresses succeeded before the fallback") + } + uc.FallbackToDirectIP(context.Background()) + conn, err := rt.Dial(ctx, addr, tlsCfg, nil) + if err != nil { + t.Fatalf("doh3 dial after the fallback: %v", err) + } + defer conn.CloseWithError(0, "") + if host, _, _ := net.SplitHostPort(conn.RemoteAddr().String()); host != "127.0.0.1" { + t.Fatalf("doh3 dialed %q, want the direct IP 127.0.0.1", host) + } + }) +} + +// DoT verifies the server as the upstream domain on every dial path, including +// the empty-address path that dials the endpoint: tls.Client fails every +// handshake without ServerName, and verification must stay on. +func TestDOTClientPoolServerName(t *testing.T) { + uc := newBootstrapTestUpstream(ResolverTypeDOT, "dot.example.com:853", IpStackBoth) + p := newDOTClientPool(context.Background(), uc, nil) + t.Cleanup(p.Close) + if p.tlsConfig.ServerName != "dot.example.com" { + t.Fatalf("ServerName = %q, want the upstream domain", p.tlsConfig.ServerName) + } + if p.tlsConfig.InsecureSkipVerify { + t.Fatal("certificate verification is disabled") + } +} diff --git a/config_forwarded_sources_test.go b/config_forwarded_sources_test.go new file mode 100644 index 00000000..436cc771 --- /dev/null +++ b/config_forwarded_sources_test.go @@ -0,0 +1,41 @@ +package ctrld_test + +import ( + "testing" + + "github.com/go-playground/validator/v10" + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// TestValidateConfig_FirewallForwardedSourcesLenient verifies a bad +// firewall_forwarded_sources entry does not fail config validation. +// +// This field is deliberately not validated with `cidr`: validateConfig failure exits +// the process, so a hard validator would turn one typo in an MDM-pushed subnet into a +// host-wide DNS outage. Bad entries are dropped with a warning at use time instead +// (see firewallForwardedSources on darwin), which is what the docs promise. +func TestValidateConfig_FirewallForwardedSourcesLenient(t *testing.T) { + tests := []struct { + name string + sources []string + }{ + {"malformed entry", []string{"not-a-cidr"}}, + {"missing prefix length", []string{"192.168.64.0"}}, + {"bad entry alongside good ones", []string{"192.168.64.0/24", "oops", "10.0.0.0/8"}}, + {"non-IPv4 entry", []string{"fd00::/64"}}, + {"empty string", []string{""}}, + {"valid entries", []string{"192.168.64.0/24"}}, + {"unset", nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := testhelper.SampleConfig(t) + cfg.Service.FirewallForwardedSources = tc.sources + require.NoError(t, ctrld.ValidateConfig(validator.New(), cfg), + "a bad forwarded-source entry must not stop ctrld from starting") + }) + } +} diff --git a/config_internal_test.go b/config_internal_test.go index b37e982f..b1f40424 100644 --- a/config_internal_test.go +++ b/config_internal_test.go @@ -1,7 +1,9 @@ package ctrld import ( + "context" "net/url" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -36,10 +38,10 @@ func TestUpstreamConfig_SetupBootstrapIP(t *testing.T) { t.Run(tc.name, func(t *testing.T) { // Enable parallel tests once https://github.com/microsoft/wmi/issues/165 fixed. // t.Parallel() - tc.uc.Init() - tc.uc.SetupBootstrapIP() + tc.uc.Init(context.Background()) + tc.uc.SetupBootstrapIP(context.Background()) if len(tc.uc.bootstrapIPs) == 0 { - t.Log(defaultNameservers()) + t.Log(defaultNameservers(context.Background())) t.Fatalf("could not bootstrap ip: %s", tc.uc.String()) } }) @@ -355,7 +357,7 @@ func TestUpstreamConfig_Init(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - tc.uc.Init() + tc.uc.Init(context.Background()) tc.uc.uid = "" // we don't care about the uid. assert.Equal(t, tc.expected, tc.uc) }) @@ -396,6 +398,59 @@ func TestUpstreamConfig_VerifyDomain(t *testing.T) { } } +// TestUpstreamConfig_isNextDNS is a regression test for issue-610: NextDNS +// serves its alternative DoH endpoints, the "ultralow" and "anycast" variants, +// under nextdns.io, so recognition follows the parent domain instead of the +// single dns.nextdns.io host. Each case runs through the Domain field and +// through the endpoint-hostname fallback, the two ways an upstream names its +// host. +func TestUpstreamConfig_isNextDNS(t *testing.T) { + tests := []struct { + name string + domain string + want bool + }{ + {"the default endpoint", "dns.nextdns.io", true}, + {"the numbered endpoints", "dns1.nextdns.io", true}, + {"the second numbered endpoint", "dns2.nextdns.io", true}, + {"an ultralow endpoint", "ultralow.dns.nextdns.io", true}, + {"a numbered ultralow endpoint", "ultralow.dns1.nextdns.io", true}, + {"the second numbered ultralow endpoint", "ultralow.dns2.nextdns.io", true}, + {"an anycast endpoint", "anycast.dns.nextdns.io", true}, + {"a numbered anycast endpoint", "anycast.dns1.nextdns.io", true}, + {"the second numbered anycast endpoint", "anycast.dns2.nextdns.io", true}, + {"a mixed case endpoint", "UltraLow.DNS.NextDNS.IO", true}, + {"the parent domain itself", "nextdns.io", true}, + + {"another TLD", "dns.nextdns.com", false}, + {"a domain ending in the same labels", "notnextdns.io", false}, + {"the parent domain as a prefix", "xnextdns.io", false}, + {"the parent domain under another one", "nextdns.io.example.com", false}, + {"an unrelated resolver", "dns.google", false}, + {"a ControlD upstream", "freedns.controld.com", false}, + {"no host at all", "", false}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name+" as a domain", func(t *testing.T) { + t.Parallel() + uc := &UpstreamConfig{Domain: tc.domain} + if got := uc.isNextDNS(); got != tc.want { + t.Errorf("isNextDNS() = %v for domain %q, want %v", got, tc.domain, tc.want) + } + }) + t.Run(tc.name+" as an endpoint", func(t *testing.T) { + t.Parallel() + // Domain is empty, so the lookup falls back to the endpoint host. + uc := &UpstreamConfig{Endpoint: "https://" + tc.domain + "/abc123"} + if got := uc.isNextDNS(); got != tc.want { + t.Errorf("isNextDNS() = %v for endpoint %q, want %v", got, uc.Endpoint, tc.want) + } + }) + } +} + func TestUpstreamConfig_UpstreamSendClientInfo(t *testing.T) { tests := []struct { name string @@ -412,6 +467,16 @@ func TestUpstreamConfig_UpstreamSendClientInfo(t *testing.T) { &UpstreamConfig{Endpoint: "https://freedns.controld.com/p2", Type: ResolverTypeDOH3}, true, }, + { + "default with nextdns upstream DoH", + &UpstreamConfig{Endpoint: "https://dns.nextdns.io/abc123", Type: ResolverTypeDOH}, + true, + }, + { + "default with nextdns ultralow upstream DoH", + &UpstreamConfig{Endpoint: "https://ultralow.dns2.nextdns.io/abc123", Type: ResolverTypeDOH}, + true, + }, { "default with non-ControlD upstream", &UpstreamConfig{Endpoint: "https://dns.google/dns-query", Type: ResolverTypeDOH}, @@ -497,7 +562,7 @@ func TestUpstreamConfig_IsDiscoverable(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - tc.uc.Init() + tc.uc.Init(context.Background()) if got := tc.uc.IsDiscoverable(); got != tc.discoverable { t.Errorf("unexpected result, want: %v, got: %v", tc.discoverable, got) } @@ -505,6 +570,52 @@ func TestUpstreamConfig_IsDiscoverable(t *testing.T) { } } +func TestRebootstrapRace(t *testing.T) { + uc := &UpstreamConfig{ + Name: "test-doh", + Type: ResolverTypeDOH, + Endpoint: "https://example.com/dns-query", + Domain: "example.com", + bootstrapIPs: []string{"1.1.1.1", "1.0.0.1"}, + } + + ctx := LoggerCtx(context.Background(), NopLogger) + + uc.SetupTransport(ctx) + + if uc.transport == nil { + t.Fatal("initial transport should be set") + } + + const goroutines = 100 + + uc.ReBootstrap(ctx) + + started := make(chan struct{}) + go func() { + close(started) + for { + switch uc.rebootstrap.Load() { + case rebootstrapStarted, rebootstrapInProgress: + uc.ReBootstrap(ctx) + default: + return + } + } + }() + + <-started + + var wg sync.WaitGroup + for range goroutines { + wg.Go(func() { + uc.ensureSetupTransport(ctx) + }) + } + + wg.Wait() +} + func ptrBool(b bool) *bool { return &b } diff --git a/config_quic.go b/config_quic.go index 33f56b92..99ac982a 100644 --- a/config_quic.go +++ b/config_quic.go @@ -9,60 +9,57 @@ import ( "runtime" "sync" - "github.com/miekg/dns" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" ) -func (uc *UpstreamConfig) setupDOH3Transport() { - switch uc.IPStack { - case IpStackBoth, "": - uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs) - case IpStackV4: - uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs4) - case IpStackV6: - uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs6) - case IpStackSplit: - uc.http3RoundTripper4 = uc.newDOH3Transport(uc.bootstrapIPs4) - if HasIPv6() { - uc.http3RoundTripper6 = uc.newDOH3Transport(uc.bootstrapIPs6) - } else { - uc.http3RoundTripper6 = uc.http3RoundTripper4 - } - uc.http3RoundTripper = uc.newDOH3Transport(uc.bootstrapIPs) +func (uc *UpstreamConfig) newDOH3Transport(ctx context.Context, addrs []string) http.RoundTripper { + if uc.Type != ResolverTypeDOH3 { + return nil } -} - -func (uc *UpstreamConfig) newDOH3Transport(addrs []string) http.RoundTripper { rt := &http3.Transport{} - rt.TLSClientConfig = &tls.Config{RootCAs: uc.certPool} + rt.TLSClientConfig = &tls.Config{RootCAs: uc.certPool, MinVersion: tls.VersionTLS12} + logger := LoggerFromCtx(ctx) rt.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { _, port, _ := net.SplitHostPort(addr) // if we have a bootstrap ip set, use it to avoid DNS lookup if uc.BootstrapIP != "" { addr = net.JoinHostPort(uc.BootstrapIP, port) - ProxyLogger.Load().Debug().Msgf("sending doh3 request to: %s", addr) + Log(ctx, logger.Debug(), "Sending doh3 request to: %s", addr) udpConn, err := net.ListenUDP("udp", nil) if err != nil { return nil, err } remoteAddr, err := net.ResolveUDPAddr("udp", addr) if err != nil { + udpConn.Close() return nil, err } - return quic.DialEarly(ctx, udpConn, remoteAddr, tlsCfg, cfg) - } - dialAddrs := make([]string, len(addrs)) - for i := range addrs { - dialAddrs[i] = net.JoinHostPort(addrs[i], port) + conn, err := quic.DialEarly(ctx, udpConn, remoteAddr, tlsCfg, cfg) + if err != nil { + udpConn.Close() + return nil, err + } + closeUDPConnWhenDone(conn, udpConn) + return conn, nil } pd := &quicParallelDialer{} - conn, err := pd.Dial(ctx, dialAddrs, tlsCfg, cfg) + conn, err := pd.Dial(ctx, joinHostPorts(addrs, port), tlsCfg, cfg) if err != nil { - return nil, err + direct := uc.directFallbackIPs() + if len(direct) == 0 { + return nil, err + } + Log(ctx, logger.Debug().Err(err), "Bootstrap addresses failed, dialing direct ips %v", direct) + var directErr error + if conn, directErr = pd.Dial(ctx, joinHostPorts(direct, port), tlsCfg, cfg); directErr != nil { + return nil, errors.Join(err, directErr) + } } - ProxyLogger.Load().Debug().Msgf("sending doh3 request to: %s", conn.RemoteAddr()) - return conn, err + Log(ctx, logger.Debug(), "Sending doh3 request to: %s", conn.RemoteAddr()) + // err may still hold the bootstrap failure the direct dial recovered + // from; returning it would make http3 drop the connection. + return conn, nil } runtime.SetFinalizer(rt, func(rt *http3.Transport) { rt.CloseIdleConnections() @@ -70,25 +67,19 @@ func (uc *UpstreamConfig) newDOH3Transport(addrs []string) http.RoundTripper { return rt } -func (uc *UpstreamConfig) doh3Transport(dnsType uint16) http.RoundTripper { - uc.transportOnce.Do(func() { - uc.SetupTransport() - }) - if uc.rebootstrap.CompareAndSwap(true, false) { - uc.SetupTransport() - } - switch uc.IPStack { - case IpStackBoth, IpStackV4, IpStackV6: - return uc.http3RoundTripper - case IpStackSplit: - switch dnsType { - case dns.TypeA: - return uc.http3RoundTripper4 - default: - return uc.http3RoundTripper6 - } - } - return uc.http3RoundTripper +func (uc *UpstreamConfig) doh3Transport(ctx context.Context, dnsType uint16) http.RoundTripper { + uc.ensureSetupTransport(ctx) + return transportByIpStack(uc.IPStack, dnsType, uc.http3RoundTripper, uc.http3RoundTripper4, uc.http3RoundTripper6) +} + +func (uc *UpstreamConfig) doqTransport(ctx context.Context, dnsType uint16) *doqConnPool { + uc.ensureSetupTransport(ctx) + return transportByIpStack(uc.IPStack, dnsType, uc.doqConnPool, uc.doqConnPool4, uc.doqConnPool6) +} + +func (uc *UpstreamConfig) dotTransport(ctx context.Context, dnsType uint16) *dotConnPool { + uc.ensureSetupTransport(ctx) + return transportByIpStack(uc.IPStack, dnsType, uc.dotClientPool, uc.dotClientPool4, uc.dotClientPool6) } // Putting the code for quic parallel dialer here: @@ -96,11 +87,28 @@ func (uc *UpstreamConfig) doh3Transport(dnsType uint16) http.RoundTripper { // - quic dialer is different with net.Dialer // - simplification for quic free version type parallelDialerResult struct { - conn *quic.Conn - err error + conn *quic.Conn + udpConn *net.UDPConn + err error } -type quicParallelDialer struct{} +// closeUDPConnWhenDone closes udpConn once conn terminates. quic.DialEarly does +// not take ownership of the socket it is handed, so the caller must. +func closeUDPConnWhenDone(conn *quic.Conn, udpConn *net.UDPConn) { + go func() { + <-conn.Context().Done() + _ = udpConn.Close() + }() +} + +// quicParallelDialer races DialEarly across a list of remote addresses and +// returns the first successful connection. When transport is non-nil, all dials +// share that transport's UDP socket. When transport is nil, the dialer falls +// back to a fresh UDP socket per attempt; losing sockets are closed as their +// dials settle, and the winning one is closed with the connection it carries. +type quicParallelDialer struct { + transport *quic.Transport +} // Dial performs parallel dialing to the given address list. func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { @@ -110,8 +118,6 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t ctx, cancel := context.WithCancel(ctx) defer cancel() - done := make(chan struct{}) - defer close(done) ch := make(chan *parallelDialerResult, len(addrs)) var wg sync.WaitGroup wg.Add(len(addrs)) @@ -128,22 +134,25 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t ch <- ¶llelDialerResult{conn: nil, err: err} return } - udpConn, err := net.ListenUDP("udp", nil) - if err != nil { - ch <- ¶llelDialerResult{conn: nil, err: err} - return - } - conn, err := quic.DialEarly(ctx, udpConn, remoteAddr, tlsCfg, cfg) - select { - case ch <- ¶llelDialerResult{conn: conn, err: err}: - case <-done: - if conn != nil { - conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeNoError), "") + var ( + conn *quic.Conn + udpConn *net.UDPConn + ) + if d.transport != nil { + conn, err = d.transport.DialEarly(ctx, remoteAddr, tlsCfg, cfg) + } else { + udpConn, err = net.ListenUDP("udp", nil) + if err != nil { + ch <- ¶llelDialerResult{conn: nil, err: err} + return } - if udpConn != nil { + conn, err = quic.DialEarly(ctx, udpConn, remoteAddr, tlsCfg, cfg) + if err != nil { udpConn.Close() + udpConn = nil } } + ch <- ¶llelDialerResult{conn: conn, udpConn: udpConn, err: err} }(addr) } @@ -151,10 +160,40 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t for res := range ch { if res.err == nil { cancel() - return res.conn, res.err + if res.udpConn != nil { + closeUDPConnWhenDone(res.conn, res.udpConn) + } + go closeLosingQuicConns(ch) + return res.conn, nil } errs = append(errs, res.err) } return nil, errors.Join(errs...) } + +// closeLosingQuicConns drains ch and releases everything left in it. +func closeLosingQuicConns(ch <-chan *parallelDialerResult) { + for res := range ch { + if res.conn != nil { + res.conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeNoError), "") + } + if res.udpConn != nil { + _ = res.udpConn.Close() + } + } +} + +func (uc *UpstreamConfig) newDOQConnPool(ctx context.Context, addrs []string) *doqConnPool { + if uc.Type != ResolverTypeDOQ { + return nil + } + return newDOQConnPool(ctx, uc, addrs) +} + +func (uc *UpstreamConfig) newDOTClientPool(ctx context.Context, addrs []string) *dotConnPool { + if uc.Type != ResolverTypeDOT { + return nil + } + return newDOTClientPool(ctx, uc, addrs) +} diff --git a/config_test.go b/config_test.go index cd392d50..42a9599c 100644 --- a/config_test.go +++ b/config_test.go @@ -67,6 +67,27 @@ func TestLoadDefaultConfig(t *testing.T) { assert.Len(t, cfg.Upstream, 2) } +func TestAllowUnprivilegedLogSendConfig(t *testing.T) { + for _, tc := range []struct { + name string + body string + want bool + }{ + {"absent", "[service]\n", false}, + {"disabled", "[service]\nallow_unprivileged_log_send = false\n", false}, + {"enabled", "[service]\nallow_unprivileged_log_send = true\n", true}, + } { + t.Run(tc.name, func(t *testing.T) { + v := viper.New() + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(strings.NewReader(tc.body))) + var cfg ctrld.Config + require.NoError(t, v.Unmarshal(&cfg)) + assert.Equal(t, tc.want, cfg.Service.AllowUnprivilegedLogSend) + }) + } +} + func TestConfigOverride(t *testing.T) { v := viper.NewWithOptions(viper.KeyDelimiter("::")) ctrld.InitConfig(v, "test_load_config") @@ -169,6 +190,72 @@ discover_ptr = false require.False(t, *cfg.Service.DiscoverPtr) } +func TestLogMaxKeys(t *testing.T) { + tests := []struct { + name string + serviceKeys string + wantSize int + wantBackups *int + }{ + {"size and backups set", "log_max_size_mb = 20\nlog_max_backups = 1", 20, intPtr(1)}, + {"size zero, backups absent", "log_max_size_mb = 0", 0, nil}, + {"backups zero", "log_max_backups = 0", 0, intPtr(0)}, + {"the largest values the keys take", "log_max_size_mb = 1024\nlog_max_backups = 64", 1024, intPtr(64)}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := configWithServiceKeys(t, tc.serviceKeys) + validate := validator.New() + require.NoError(t, ctrld.ValidateConfig(validate, cfg)) + require.Equal(t, tc.wantSize, cfg.Service.LogMaxSizeMB) + if tc.wantBackups == nil { + require.Nil(t, cfg.Service.LogMaxBackups) + return + } + require.NotNil(t, cfg.Service.LogMaxBackups) + require.Equal(t, *tc.wantBackups, *cfg.Service.LogMaxBackups) + }) + } +} + +func TestLogMaxKeysOutOfRange(t *testing.T) { + tests := []struct { + name string + serviceKeys string + wantField string + }{ + {"negative size", "log_max_size_mb = -1", "LogMaxSizeMB"}, + {"negative backups", "log_max_backups = -1", "LogMaxBackups"}, + {"size above the limit", "log_max_size_mb = 1025", "LogMaxSizeMB"}, + {"backups above the limit", "log_max_backups = 65", "LogMaxBackups"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := configWithServiceKeys(t, tc.serviceKeys) + validate := validator.New() + err := ctrld.ValidateConfig(validate, cfg) + require.ErrorContains(t, err, tc.wantField) + }) + } +} + +func configWithServiceKeys(t *testing.T, serviceKeys string) *ctrld.Config { + t.Helper() + v := viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "test_log_max_keys") + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(strings.NewReader("[service]\n"+serviceKeys+"\n"))) + var cfg ctrld.Config + require.NoError(t, v.Unmarshal(&cfg)) + return &cfg +} + +func intPtr(n int) *int { + return &n +} + func defaultConfig(t *testing.T) *ctrld.Config { v := viper.New() ctrld.InitConfig(v, "test_load_default_config") diff --git a/desktop_darwin.go b/desktop_darwin.go index 039c0fac..7ba8b6b2 100644 --- a/desktop_darwin.go +++ b/desktop_darwin.go @@ -5,3 +5,6 @@ package ctrld func IsDesktopPlatform() bool { return true } + +// SelfDiscover reports whether ctrld should only do self discover. +func SelfDiscover() bool { return true } diff --git a/desktop_others.go b/desktop_others.go index de486e78..6d6a9a3f 100644 --- a/desktop_others.go +++ b/desktop_others.go @@ -7,3 +7,6 @@ package ctrld func IsDesktopPlatform() bool { return false } + +// SelfDiscover reports whether ctrld should only do self discover. +func SelfDiscover() bool { return false } diff --git a/desktop_windows.go b/desktop_windows.go index 4e9526b9..186a5ffc 100644 --- a/desktop_windows.go +++ b/desktop_windows.go @@ -1,7 +1,22 @@ package ctrld +import "golang.org/x/sys/windows" + // IsDesktopPlatform indicates if ctrld is running on a desktop platform, // currently defined as macOS or Windows workstation. func IsDesktopPlatform() bool { return isWindowsWorkStation() } + +// SelfDiscover reports whether ctrld should only do self discover. +func SelfDiscover() bool { + return isWindowsWorkStation() +} + +// isWindowsWorkStation reports whether ctrld was run on a Windows workstation machine. +func isWindowsWorkStation() bool { + // From https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa + const VER_NT_WORKSTATION = 0x0000001 + osvi := windows.RtlGetVersion() + return osvi.ProductType == VER_NT_WORKSTATION +} diff --git a/discover_user_darwin.go b/discover_user_darwin.go new file mode 100644 index 00000000..0267343d --- /dev/null +++ b/discover_user_darwin.go @@ -0,0 +1,148 @@ +//go:build darwin + +package ctrld + +import ( + "context" + "os/exec" + "strconv" + "strings" +) + +// DiscoverMainUser attempts to find the primary user on macOS systems. +// This is designed to work reliably under RMM deployments where traditional +// environment variables and session detection may not be available. +// +// Priority chain (deterministic, lowest UID wins among candidates): +// 1. Console user from stat -f %Su /dev/console +// 2. Active console session user via scutil +// 3. First user with UID >= 501 from dscl (standard macOS user range) +func DiscoverMainUser(ctx context.Context) string { + logger := LoggerFromCtx(ctx).Debug() + + // Methods 1 and 2: the logged-in console user. + if user := discoverSessionUser(ctx); user != "" { + return user + } + + // Method 3: Find lowest UID >= 501 from directory services + logger.Msg("attempting to discover user via dscl directory scan") + if user := getLowestRegularUser(ctx); user != "" { + logger.Str("method", "dscl").Str("user", user).Msg("found user via dscl scan") + return user + } + + logger.Msg("all user discovery methods failed") + return "unknown" +} + +// discoverSessionUser returns the user of an active login session, or "" when +// nobody is logged in. It never falls back to a guess from the account list. +func discoverSessionUser(ctx context.Context) string { + logger := LoggerFromCtx(ctx).Debug() + + // Method 1: Check console owner via stat + logger.Msg("attempting to discover user via console stat") + if user := getConsoleUser(ctx); user != "" && user != "root" { + logger.Str("method", "stat").Str("user", user).Msg("found user via console stat") + return user + } + + // Method 2: Check active console session via scutil + logger.Msg("attempting to discover user via scutil ConsoleUser") + if user := getScutilConsoleUser(ctx); user != "" && user != "root" { + logger.Str("method", "scutil").Str("user", user).Msg("found user via scutil ConsoleUser") + return user + } + + return "" +} + +// getConsoleUser uses stat to find the owner of /dev/console +func getConsoleUser(ctx context.Context) string { + cmd := exec.CommandContext(ctx, "stat", "-f", "%Su", "/dev/console") + out, err := cmd.Output() + if err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("failed to stat /dev/console") + return "" + } + return strings.TrimSpace(string(out)) +} + +// getScutilConsoleUser uses scutil to get the current console user +func getScutilConsoleUser(ctx context.Context) string { + cmd := exec.CommandContext(ctx, "scutil", "-r", "ConsoleUser") + out, err := cmd.Output() + if err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("failed to get ConsoleUser via scutil") + return "" + } + + lines := strings.Split(string(out), "\n") + for _, line := range lines { + if strings.Contains(line, "Name :") { + parts := strings.Fields(line) + if len(parts) >= 3 { + return strings.TrimSpace(parts[2]) + } + } + } + return "" +} + +// getLowestRegularUser finds the user with the lowest UID >= 501 +func getLowestRegularUser(ctx context.Context) string { + // Get list of all users with UID >= 501 + cmd := exec.CommandContext(ctx, "dscl", ".", "list", "/Users", "UniqueID") + out, err := cmd.Output() + if err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("failed to list users via dscl") + return "" + } + + var candidates []struct { + name string + uid int + } + + lines := strings.Split(string(out), "\n") + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) != 2 { + continue + } + + username := fields[0] + uidStr := fields[1] + + uid, err := strconv.Atoi(uidStr) + if err != nil { + continue + } + + // Only consider regular users (UID >= 501 on macOS) + if uid >= 501 { + candidates = append(candidates, struct { + name string + uid int + }{username, uid}) + } + } + + if len(candidates) == 0 { + return "" + } + + // Find the candidate with the lowest UID (deterministic choice) + lowestUID := candidates[0].uid + result := candidates[0].name + + for _, candidate := range candidates[1:] { + if candidate.uid < lowestUID { + lowestUID = candidate.uid + result = candidate.name + } + } + + return result +} \ No newline at end of file diff --git a/discover_user_linux.go b/discover_user_linux.go new file mode 100644 index 00000000..ea6003fd --- /dev/null +++ b/discover_user_linux.go @@ -0,0 +1,251 @@ +//go:build linux + +package ctrld + +import ( + "bufio" + "context" + "os" + "os/exec" + "strconv" + "strings" +) + +// DiscoverMainUser attempts to find the primary user on Linux systems. +// This is designed to work reliably under RMM deployments where traditional +// environment variables and session detection may not be available. +// +// Priority chain (deterministic, lowest UID wins among candidates): +// 1. Active users from loginctl list-users +// 2. Parse /etc/passwd for users with UID >= 1000, prefer admin group members +// 3. Fallback to lowest UID >= 1000 from /etc/passwd +func DiscoverMainUser(ctx context.Context) string { + logger := LoggerFromCtx(ctx).Debug() + + // Method 1: an active loginctl user. + if user := discoverSessionUser(ctx); user != "" { + return user + } + + // Method 2: Parse /etc/passwd and find admin users first + logger.Msg("attempting to discover user via /etc/passwd with admin preference") + if user := getPasswdUserWithAdminPreference(ctx); user != "" { + logger.Str("method", "passwd+admin").Str("user", user).Msg("found admin user via /etc/passwd") + return user + } + + // Method 3: Fallback to lowest UID >= 1000 from /etc/passwd + logger.Msg("attempting to discover user via /etc/passwd lowest UID") + if user := getLowestPasswdUser(ctx); user != "" { + logger.Str("method", "passwd").Str("user", user).Msg("found user via /etc/passwd") + return user + } + + logger.Msg("all user discovery methods failed") + return "unknown" +} + +// discoverSessionUser returns the user of an active login session, or "" when +// nobody is logged in. It never falls back to a guess from the account list. +func discoverSessionUser(ctx context.Context) string { + logger := LoggerFromCtx(ctx).Debug() + + // Method 1: Check active users via loginctl + logger.Msg("attempting to discover user via loginctl") + if user := getLoginctlUser(ctx); user != "" { + logger.Str("method", "loginctl").Str("user", user).Msg("found user via loginctl") + return user + } + + return "" +} + +// getLoginctlUser uses loginctl to find active users +func getLoginctlUser(ctx context.Context) string { + cmd := exec.CommandContext(ctx, "loginctl", "list-users", "--no-legend") + out, err := cmd.Output() + if err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("failed to run loginctl list-users") + return "" + } + + var candidates []struct { + name string + uid int + } + + lines := strings.Split(string(out), "\n") + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + + uidStr := fields[0] + username := fields[1] + + uid, err := strconv.Atoi(uidStr) + if err != nil { + continue + } + + // Only consider regular users (UID >= 1000 on Linux) + if uid >= 1000 { + candidates = append(candidates, struct { + name string + uid int + }{username, uid}) + } + } + + if len(candidates) == 0 { + return "" + } + + // Return user with lowest UID (deterministic choice) + lowestUID := candidates[0].uid + result := candidates[0].name + + for _, candidate := range candidates[1:] { + if candidate.uid < lowestUID { + lowestUID = candidate.uid + result = candidate.name + } + } + + return result +} + +// getPasswdUserWithAdminPreference parses /etc/passwd and prefers admin group members +func getPasswdUserWithAdminPreference(ctx context.Context) string { + users := parsePasswdFile(ctx) + if len(users) == 0 { + return "" + } + + var adminUsers []struct { + name string + uid int + } + var regularUsers []struct { + name string + uid int + } + + // Separate admin and regular users + for _, user := range users { + if isUserInAdminGroups(ctx, user.name) { + adminUsers = append(adminUsers, user) + } else { + regularUsers = append(regularUsers, user) + } + } + + // Prefer admin users, then regular users + candidates := adminUsers + if len(candidates) == 0 { + candidates = regularUsers + } + + if len(candidates) == 0 { + return "" + } + + // Return user with lowest UID (deterministic choice) + lowestUID := candidates[0].uid + result := candidates[0].name + + for _, candidate := range candidates[1:] { + if candidate.uid < lowestUID { + lowestUID = candidate.uid + result = candidate.name + } + } + + return result +} + +// getLowestPasswdUser returns the user with lowest UID >= 1000 from /etc/passwd +func getLowestPasswdUser(ctx context.Context) string { + users := parsePasswdFile(ctx) + if len(users) == 0 { + return "" + } + + // Return user with lowest UID (deterministic choice) + lowestUID := users[0].uid + result := users[0].name + + for _, user := range users[1:] { + if user.uid < lowestUID { + lowestUID = user.uid + result = user.name + } + } + + return result +} + +// parsePasswdFile parses /etc/passwd and returns users with UID >= 1000 +func parsePasswdFile(ctx context.Context) []struct { + name string + uid int +} { + file, err := os.Open("/etc/passwd") + if err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("failed to open /etc/passwd") + return nil + } + defer file.Close() + + var users []struct { + name string + uid int + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Split(line, ":") + if len(fields) < 3 { + continue + } + + username := fields[0] + uidStr := fields[2] + + uid, err := strconv.Atoi(uidStr) + if err != nil { + continue + } + + // Only consider regular users (UID >= 1000 on Linux) + if uid >= 1000 { + users = append(users, struct { + name string + uid int + }{username, uid}) + } + } + + return users +} + +// isUserInAdminGroups checks if a user is in common admin groups +func isUserInAdminGroups(ctx context.Context, username string) bool { + adminGroups := []string{"sudo", "wheel", "admin"} + + for _, group := range adminGroups { + cmd := exec.CommandContext(ctx, "groups", username) + out, err := cmd.Output() + if err != nil { + continue + } + + if strings.Contains(string(out), group) { + return true + } + } + + return false +} \ No newline at end of file diff --git a/discover_user_others.go b/discover_user_others.go new file mode 100644 index 00000000..afa00f4d --- /dev/null +++ b/discover_user_others.go @@ -0,0 +1,19 @@ +//go:build !windows && !linux && !darwin + +package ctrld + +import "context" + +// DiscoverMainUser returns "unknown" for unsupported platforms. +// This is a stub implementation for platforms where username detection +// is not yet implemented. +func DiscoverMainUser(ctx context.Context) string { + LoggerFromCtx(ctx).Debug().Msg("username discovery not implemented for this platform") + return "unknown" +} + +// discoverSessionUser returns "" because no session source is implemented for +// this platform. +func discoverSessionUser(ctx context.Context) string { + return "" +} diff --git a/discover_user_windows.go b/discover_user_windows.go new file mode 100644 index 00000000..2ed2dbfb --- /dev/null +++ b/discover_user_windows.go @@ -0,0 +1,305 @@ +//go:build windows + +package ctrld + +import ( + "context" + "strconv" + "strings" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/registry" +) + +var ( + wtsapi32 = windows.NewLazySystemDLL("wtsapi32.dll") + procWTSGetActiveConsoleSessionId = wtsapi32.NewProc("WTSGetActiveConsoleSessionId") + procWTSQuerySessionInformation = wtsapi32.NewProc("WTSQuerySessionInformationW") + procWTSFreeMemory = wtsapi32.NewProc("WTSFreeMemory") +) + +const ( + WTSUserName = 5 +) + +// DiscoverMainUser attempts to find the primary user on Windows systems. +// This is designed to work reliably under RMM deployments where traditional +// environment variables and session detection may not be available. +// +// Priority chain (deterministic, lowest RID wins among candidates): +// 1. Active console session user via WTSGetActiveConsoleSessionId +// 2. Registry ProfileList scan for Administrators group members +// 3. Fallback to lowest RID from ProfileList +func DiscoverMainUser(ctx context.Context) string { + logger := LoggerFromCtx(ctx).Debug() + + // Method 1: the active console session user. + if user := discoverSessionUser(ctx); user != "" { + return user + } + + // Method 2: Scan registry for admin users + logger.Msg("attempting to discover user via registry with admin preference") + if user := getRegistryUserWithAdminPreference(ctx); user != "" { + logger.Str("method", "registry+admin").Str("user", user).Msg("found admin user via registry") + return user + } + + // Method 3: Fallback to lowest RID from registry + logger.Msg("attempting to discover user via registry lowest RID") + if user := getLowestRegistryUser(ctx); user != "" { + logger.Str("method", "registry").Str("user", user).Msg("found user via registry") + return user + } + + logger.Msg("all user discovery methods failed") + return "unknown" +} + +// discoverSessionUser returns the user of an active login session, or "" when +// nobody is logged in. It never falls back to a guess from the account list. +func discoverSessionUser(ctx context.Context) string { + logger := LoggerFromCtx(ctx).Debug() + + // Method 1: Check active console session + logger.Msg("attempting to discover user via active console session") + if user := getActiveConsoleUser(ctx); user != "" { + logger.Str("method", "console").Str("user", user).Msg("found user via active console session") + return user + } + + return "" +} + +// getActiveConsoleUser gets the username of the active console session +func getActiveConsoleUser(ctx context.Context) string { + // Guard against missing WTS procedures (e.g., Windows Server Core). + if err := procWTSGetActiveConsoleSessionId.Find(); err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("WTSGetActiveConsoleSessionId not available, skipping console session check") + return "" + } + sessionId, _, _ := procWTSGetActiveConsoleSessionId.Call() + if sessionId == 0xFFFFFFFF { // Invalid session + LoggerFromCtx(ctx).Debug().Msg("no active console session found") + return "" + } + + var buffer uintptr + var bytesReturned uint32 + + if err := procWTSQuerySessionInformation.Find(); err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("WTSQuerySessionInformationW not available") + return "" + } + ret, _, _ := procWTSQuerySessionInformation.Call( + 0, // WTS_CURRENT_SERVER_HANDLE + sessionId, + uintptr(WTSUserName), + uintptr(unsafe.Pointer(&buffer)), + uintptr(unsafe.Pointer(&bytesReturned)), + ) + + if ret == 0 { + LoggerFromCtx(ctx).Debug().Msg("failed to query session information") + return "" + } + defer procWTSFreeMemory.Call(buffer) + + // Convert buffer to string + username := windows.UTF16PtrToString((*uint16)(unsafe.Pointer(buffer))) + if username == "" { + return "" + } + + return username +} + +// getRegistryUserWithAdminPreference scans registry profiles and prefers admin users +func getRegistryUserWithAdminPreference(ctx context.Context) string { + profiles := getRegistryProfiles(ctx) + if len(profiles) == 0 { + return "" + } + + var adminProfiles []registryProfile + var regularProfiles []registryProfile + + // Separate admin and regular users + for _, profile := range profiles { + if isUserInAdministratorsGroup(profile.username) { + adminProfiles = append(adminProfiles, profile) + } else { + regularProfiles = append(regularProfiles, profile) + } + } + + // Prefer admin users, then regular users + candidates := adminProfiles + if len(candidates) == 0 { + candidates = regularProfiles + } + + if len(candidates) == 0 { + return "" + } + + // Return user with lowest RID (deterministic choice) + lowestRID := candidates[0].rid + result := candidates[0].username + + for _, candidate := range candidates[1:] { + if candidate.rid < lowestRID { + lowestRID = candidate.rid + result = candidate.username + } + } + + return result +} + +// getLowestRegistryUser returns the user with lowest RID from registry +func getLowestRegistryUser(ctx context.Context) string { + profiles := getRegistryProfiles(ctx) + if len(profiles) == 0 { + return "" + } + + // Return user with lowest RID (deterministic choice) + lowestRID := profiles[0].rid + result := profiles[0].username + + for _, profile := range profiles[1:] { + if profile.rid < lowestRID { + lowestRID = profile.rid + result = profile.username + } + } + + return result +} + +type registryProfile struct { + username string + rid uint32 + sid string +} + +// getRegistryProfiles scans the registry ProfileList for user profiles +func getRegistryProfiles(ctx context.Context) []registryProfile { + key, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList`, registry.ENUMERATE_SUB_KEYS) + if err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("failed to open ProfileList registry key") + return nil + } + defer key.Close() + + subkeys, err := key.ReadSubKeyNames(-1) + if err != nil { + LoggerFromCtx(ctx).Debug().Err(err).Msg("failed to read ProfileList subkeys") + return nil + } + + var profiles []registryProfile + + for _, subkey := range subkeys { + // Only process SIDs that start with S-1-5-21 (domain/local user accounts) + if !strings.HasPrefix(subkey, "S-1-5-21-") { + continue + } + + profileKey, err := registry.OpenKey(key, subkey, registry.QUERY_VALUE) + if err != nil { + continue + } + + profileImagePath, _, err := profileKey.GetStringValue("ProfileImagePath") + profileKey.Close() + if err != nil { + continue + } + + // Extract username from profile path (e.g., C:\Users\username) + pathParts := strings.Split(profileImagePath, `\`) + if len(pathParts) == 0 { + continue + } + username := pathParts[len(pathParts)-1] + + // Extract RID from SID (last component after final hyphen) + sidParts := strings.Split(subkey, "-") + if len(sidParts) == 0 { + continue + } + ridStr := sidParts[len(sidParts)-1] + rid, err := strconv.ParseUint(ridStr, 10, 32) + if err != nil { + continue + } + + // Only consider regular users (RID >= 1000, excludes built-in accounts). + // rid == 500 is the default Administrator account (DOMAIN_USER_RID_ADMIN). + // See: https://learn.microsoft.com/en-us/windows/win32/secauthz/well-known-sids + if rid == 500 || rid >= 1000 { + profiles = append(profiles, registryProfile{ + username: username, + rid: uint32(rid), + sid: subkey, + }) + } + } + + return profiles +} + +// isUserInAdministratorsGroup checks if a user is in the Administrators group +func isUserInAdministratorsGroup(username string) bool { + // Open the user account + usernamePtr, err := syscall.UTF16PtrFromString(username) + if err != nil { + return false + } + + var userSID *windows.SID + var domain *uint16 + var userSIDSize, domainSize uint32 + var use uint32 + + // First call to get buffer sizes + err = windows.LookupAccountName(nil, usernamePtr, userSID, &userSIDSize, domain, &domainSize, &use) + if err != nil && err != windows.ERROR_INSUFFICIENT_BUFFER { + return false + } + + // Allocate buffers and make actual call + userSID = (*windows.SID)(unsafe.Pointer(&make([]byte, userSIDSize)[0])) + domain = (*uint16)(unsafe.Pointer(&make([]uint16, domainSize)[0])) + + err = windows.LookupAccountName(nil, usernamePtr, userSID, &userSIDSize, domain, &domainSize, &use) + if err != nil { + return false + } + + // Check if user is member of Administrators group (S-1-5-32-544) + adminSID, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + if err != nil { + return false + } + + // Open user token (this is a simplified check) + var token windows.Token + err = windows.OpenProcessToken(windows.CurrentProcess(), windows.TOKEN_QUERY, &token) + if err != nil { + return false + } + defer token.Close() + + // Check group membership + member, err := token.IsMember(adminSID) + if err != nil { + return false + } + + return member +} diff --git a/dns.go b/dns.go index f2b71a5c..f89f824f 100644 --- a/dns.go +++ b/dns.go @@ -14,6 +14,12 @@ func SetCacheReply(answer, msg *dns.Msg, code int) { // See https://datatracker.ietf.org/doc/html/rfc7873#section-4 sCookie.Cookie = cCookie.Cookie[:16] + sCookie.Cookie[16:] } + // NOTE: the answer's EDNS Client Subnet (ECS) is intentionally left as the + // upstream returned it. Correctness across clients is guaranteed by + // partitioning the cache and singleflight keys by ECS (see + // dnscache.CanonicalECS), so a cache hit only ever serves an answer that was + // resolved for the requester's own subnet. Rewriting the ECS option here + // without re-scoping the Answer records would violate RFC 7871 §7.3. } // getEdns0Cookie returns Edns0 cookie from *dns.OPT if present. diff --git a/dns_test.go b/dns_test.go new file mode 100644 index 00000000..9ca76081 --- /dev/null +++ b/dns_test.go @@ -0,0 +1,57 @@ +package ctrld + +import ( + "net" + "testing" + + "github.com/miekg/dns" +) + +// Test_SetCacheReply_DoesNotRewriteECS documents the post-#564 contract: cross-client +// correctness is guaranteed by partitioning the cache/singleflight keys by ECS +// (dnscache.CanonicalECS), NOT by rewriting the cached answer's ECS option. Rewriting the +// ECS metadata while leaving the Answer records scoped to another subnet would violate +// RFC 7871 §7.3 and make forwarders accept a wrong-subnet answer. SetCacheReply must +// therefore leave the answer's ECS untouched. +func Test_SetCacheReply_DoesNotRewriteECS(t *testing.T) { + answer := new(dns.Msg) + answer.SetQuestion(dns.Fqdn("controld.com"), dns.TypeA) + answer.SetEdns0(4096, false) + cachedSubnet := &dns.EDNS0_SUBNET{ + Code: dns.EDNS0SUBNET, + Family: 2, + SourceNetmask: 64, + SourceScope: 64, + Address: net.ParseIP("2001:db8:1::"), + } + answer.IsEdns0().Option = append(answer.IsEdns0().Option, cachedSubnet) + + req := new(dns.Msg) + req.SetQuestion(dns.Fqdn("controld.com"), dns.TypeA) + req.SetEdns0(4096, true) + req.IsEdns0().Option = append(req.IsEdns0().Option, &dns.EDNS0_SUBNET{ + Code: dns.EDNS0SUBNET, + Family: 2, + SourceNetmask: 64, + Address: net.ParseIP("2001:db8:2::"), + }) + + SetCacheReply(answer, req, dns.RcodeSuccess) + + var got *dns.EDNS0_SUBNET + for _, o := range answer.IsEdns0().Option { + if e, ok := o.(*dns.EDNS0_SUBNET); ok { + got = e + break + } + } + if got == nil { + t.Fatal("SetCacheReply dropped the answer's ECS option") + } + if want := net.ParseIP("2001:db8:1::"); !got.Address.Equal(want) { + t.Fatalf("SetCacheReply rewrote the answer ECS to the requester's subnet: got %v, want %v (unchanged)", got.Address, want) + } + if got.SourceScope != 64 { + t.Fatalf("SetCacheReply altered the answer ECS scope: got %d, want 64 (unchanged)", got.SourceScope) + } +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 68d4d6dd..cd32d46b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,18 +1,19 @@ -# Using Debian bullseye for building regular image. +# Using Debian bookworm for building regular image. # Using scratch image for minimal image size. # The final image has: # # - Timezone info file. # - CA certs file. # - /etc/{passwd,group} file. -# - Non-cgo ctrld binary. +# - Non-cgo ctrld-client binary. # -# CI_COMMIT_TAG is used to set the version of ctrld binary. -FROM golang:1.20-bullseye as base +# CI_COMMIT_TAG is used to set the version of the ctrld-client binary. +FROM golang:1.27-bookworm AS base WORKDIR /app -RUN apt-get update && apt-get install -y upx-ucl +RUN echo "deb http://deb.debian.org/debian bookworm-backports main" | tee /etc/apt/sources.list.d/backports.list +RUN apt update && apt install -t bookworm-backports upx-ucl COPY . . @@ -20,6 +21,15 @@ ARG tag=master ENV CI_COMMIT_TAG=$tag RUN CTRLD_NO_QF=yes CGO_ENABLED=0 ./scripts/build.sh +# Resolve the arch-suffixed build artifact to one fixed path, and fail the build +# if it is not there. A COPY glob that matches nothing does not reliably fail, +# which is how a rename can produce an image whose ENTRYPOINT names a binary the +# image does not contain. +RUN set -eu; \ + set -- ctrld-client-linux-*-nocgo; \ + [ "$#" -eq 1 ] && [ -x "$1" ] || { echo >&2 "no single executable build artifact: $*"; exit 1; }; \ + mv -- "$1" /ctrld-client + FROM scratch COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo @@ -27,6 +37,6 @@ COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=base /etc/passwd /etc/passwd COPY --from=base /etc/group /etc/group -COPY --from=base /app/ctrld-linux-*-nocgo ctrld +COPY --from=base /ctrld-client ctrld-client -ENTRYPOINT ["./ctrld", "run"] +ENTRYPOINT ["./ctrld-client", "run"] diff --git a/docker/Dockerfile.debug b/docker/Dockerfile.debug index 2ba36028..0f2376a1 100644 --- a/docker/Dockerfile.debug +++ b/docker/Dockerfile.debug @@ -1,18 +1,19 @@ -# Using Debian bullseye for building regular image. +# Using Debian bookworm for building regular image. # Using scratch image for minimal image size. # The final image has: # # - Timezone info file. # - CA certs file. # - /etc/{passwd,group} file. -# - Non-cgo ctrld binary. +# - Non-cgo ctrld-client binary. # -# CI_COMMIT_TAG is used to set the version of ctrld binary. -FROM golang:bullseye as base +# CI_COMMIT_TAG is used to set the version of the ctrld-client binary. +FROM golang:1.27-bookworm AS base WORKDIR /app -RUN apt-get update && apt-get install -y upx-ucl +RUN echo "deb http://deb.debian.org/debian bookworm-backports main" | tee /etc/apt/sources.list.d/backports.list +RUN apt update && apt install -t bookworm-backports upx-ucl COPY . . @@ -20,6 +21,15 @@ ARG tag=master ENV CI_COMMIT_TAG=$tag RUN CTRLD_NO_QF=yes CGO_ENABLED=0 ./scripts/build.sh +# Resolve the arch-suffixed build artifact to one fixed path, and fail the build +# if it is not there. A COPY glob that matches nothing does not reliably fail, +# which is how a rename can produce an image whose ENTRYPOINT names a binary the +# image does not contain. +RUN set -eu; \ + set -- ctrld-client-linux-*-nocgo; \ + [ "$#" -eq 1 ] && [ -x "$1" ] || { echo >&2 "no single executable build artifact: $*"; exit 1; }; \ + mv -- "$1" /ctrld-client + FROM alpine COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo @@ -27,6 +37,6 @@ COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=base /etc/passwd /etc/passwd COPY --from=base /etc/group /etc/group -COPY --from=base /app/ctrld-linux-*-nocgo ctrld +COPY --from=base /ctrld-client ctrld-client -ENTRYPOINT ["./ctrld", "run"] +ENTRYPOINT ["./ctrld-client", "run"] diff --git a/docs/619-target-ownership-limits.md b/docs/619-target-ownership-limits.md new file mode 100644 index 00000000..7ccad15a --- /dev/null +++ b/docs/619-target-ownership-limits.md @@ -0,0 +1,17 @@ +# Target ownership safety (relates to #619) + +The target cleanup record is not proof of actual service DNS. Reconciliation now reports a tracked/actual mismatch instead of returning an implicit already-installed decision. An external clear or IPv6-only edit does not authorize reinstalling a target; the record is retained to prevent repeated watchdog admission. Usable external IPv4 keeps the existing guarded cleanup path. + +Ownership is written by atomic replacement before target installation. Failure refuses installation, rather than depending on a future restart to recover an untracked target. A failed DNS command may have partly mutated OS state, so its cleanup record is retained. Failed ownership-file removal also retains memory state and does not emit the target-removed success event. + +On external edits, cleanup discards the stale saved-static backup before releasing the record. The full saved-static sweep skips a service with retained target cleanup state; unreadable/malformed ownership state conservatively skips the sweep. No unbounded retry, fatal exit, or machine-stop policy is introduced. + +## Evidence and limits + +- Linux-adapted Darwin tests execute the real target lifecycle and saved-static sweep with only OS boundaries replaced. New regression tests cover persistence failure before mutation, external edit through reconciliation/cleanup/full sweep, ownership clear failure, actual-vs-tracked discrepancy, and failed-cleanup sweep guards. Custom listener values are configured and assertions derive the target from the production selector. +- This is not native macOS PF/CLAT proof. Existing native CLAT admission/removal behavior still needs native validation on the integrated artifact; no PF rules or target selector were changed here. +- A missing target is not automatically repaired: external clearing and accidental loss cannot safely be distinguished from these observations. The diagnostic reports the uncertainty; it does not claim resolution. +- Once explicit cleanup releases a record, a later new admission uses the existing policy for IPv6-only static DNS. This is not a persistent per-service operator opt-out mechanism. +- Retaining cleanup state does not guarantee another cleanup attempt after process exit. The void shutdown API still cannot report aggregate stop failure to its caller; no guaranteed retry or issue closure is claimed. +- Startup/running-interface reset and callback shutdown ordering live in prog.go and are outside this patch. Persistence failure was not a universal restart orphan on the baseline: startup can rescue the current interface, while old/noncurrent-service cases are conditional. +- Network configuration commands are not compare-and-swap transactions. A concurrent external edit after the last read remains a native race; these tests do not prove otherwise. Atomic file replacement does not claim power-loss durability. diff --git a/docs/config.md b/docs/config.md index 99e98c9c..7a98c898 100644 --- a/docs/config.md +++ b/docs/config.md @@ -18,16 +18,12 @@ The config file allows for advanced configuration of the `ctrld` utility to cove - `/etc/controld` on *nix. - User's home directory on Windows. - - Same directory with `ctrld` binary on these routers: - - `ddwrt` - - `merlin` - - `freshtomato` - Current directory. The user can choose to override default value using command line `--config` or `-c`: ```shell -ctrld run --config /path/to/myconfig.toml +ctrld-client run --config /path/to/myconfig.toml ``` If no configuration files found, a default `ctrld.toml` file will be created in the current directory. @@ -109,6 +105,45 @@ The `[service]` section controls general behaviors. log_path = "log.txt" ``` +### allow_unprivileged_log_send + +macOS only. Default: `false`. The configuration that ctrld generates from the +Control D API never enables it. An administrator opts in through the device's +custom ctrld configuration in the Control D dashboard, which ctrld writes to its +root-owned configuration file: + +```toml +[service] + allow_unprivileged_log_send = true +``` + +Restart the service as administrator after enabling, on a new or an existing +install. At startup ctrld rewrites the configuration file whenever the API answer +differs from it, so a custom config added, changed or removed after installation +reaches the file. Standard users can then run +`ctrld log send` without `sudo`. **Every local-user process can trigger a +machine-wide diagnostic upload to Control D**, not just the interactive user. +This grants upload permission, not permission to read logs or administer ctrld. +`--full`, `log view`, `log tail`, and other privileged commands remain restricted. +Other platforms retain their existing behavior. + +This requires Control D mode (`--cd`) with a custom configuration, and file-backed logs. +The config, log files, rotated logs, and every parent directory must be root-owned, +not group/world writable, and free of extended ACLs. Symlinks and hard-linked log +files are refused; only the standard macOS `/var`, `/etc`, and `/tmp` aliases are +canonicalized. A user-owned home directory is not a safe service log location. +Do not loosen log permissions or the administrative control socket to enable this +feature. The service does not repair unsafe directories automatically. + +The setting and source paths are captured at startup. Reload does not activate a +previously disabled socket or change its sources. Removing the setting or setting +it to `false` in the protected config causes the next upload to be refused even +before restart; an upload already in progress may finish. Restart after disabling +to remove the socket. Service stop cancels and joins active delegated requests. + +See [Standard-user uploads on macOS](runtime-internal-logging.md#standard-user-uploads-on-macos) +for bounds, refusal messages, and the native QA release gate. + ### log_level Logging level you wish to enable. @@ -121,10 +156,28 @@ Logging level you wish to enable. ### log_path Relative or absolute path of the log file. +Point `log_path` at a file that no other program rotates. ctrld renames `.1` to `.N` and uploads these files. A lower `log_max_backups` leaves the older numbered `log_path` files in place, but ctrld removes the extra numbered files of its internal logs. + - Type: string - Required: no - Default: "" +### log_max_size_mb +Maximum size of the debug log file in MB. When the file reaches this size, ctrld rotates it. This limit also applies to the `log_path` file. + +- Type: integer +- Required: no +- Valid values: 1 to 1024 +- Default: 10 + +### log_max_backups +Number of rotated debug log files that ctrld keeps. If the value is 0, ctrld keeps no rotated file. This number also applies to the `log_path` file. + +- Type: integer +- Required: no +- Valid values: 0 to 64 +- Default: 4 + ### cache_enable When `cache_enable = true`, all resolved DNS query responses will be cached for duration of the upstream record TTLs. @@ -261,6 +314,21 @@ Specifying the `ip` and `port` of the Prometheus metrics server. The Prometheus - Required: no - Default: "" +### intercept_mode +How ctrld captures the host's DNS traffic. `dns` intercepts it in the OS packet filter (WFP on Windows, pf on macOS) and +forwards VPN internal domains to the VPN's DNS server; `hard` intercepts it with no VPN split routing; `off` uses interface +DNS mode, setting ctrld's listener as the DNS server of the network interfaces. + +The `--intercept-mode` flag takes precedence over this value. When neither is set, the default applies: `dns` on macOS and +Windows desktop editions, interface DNS mode elsewhere, Windows Server included. An installed service that predates the +default keeps interface DNS mode. `ctrld-client start --intercept-mode off` writes `off` here, because an unset value means the +default. See [Intercept Mode Default](intercept-mode-default.md). + +- Type: string +- Required: no +- Valid values: `off`, `dns`, `hard` +- Default: `dns` for new installations on macOS and Windows desktop editions; interface DNS mode elsewhere, Windows Server included, and for services installed before this default + ### dns_watchdog_enabled Watches all physical interfaces for DNS changes and reverts them to ctrld's settings.The DNS watchdog process only runs on Windows and MacOS. @@ -293,7 +361,23 @@ If a remote upstream fails to resolve a query or is unreachable, `ctrld` will fo - Type: boolean - Required: no -- Default: true on Windows, MacOS and non-router Linux. +- Default: true on Windows, MacOS and Linux. + +### nrpt_recovery_max_attempts +Windows DNS intercept mode uses NRPT health probes and recovery when Windows stops routing queries to the local `ctrld` listener. This limits how many consecutive recovery flows can run before `ctrld` enters a cooldown and stops making policy/Dnscache changes. + +Set to `0` to disable this circuit breaker and keep retrying indefinitely. + +- Type: integer +- Required: no +- Default: 0 (unlimited, current behavior) + +### nrpt_recovery_cooldown +Cooldown duration after `nrpt_recovery_max_attempts` consecutive Windows NRPT recovery flows. During cooldown, `ctrld` logs the suppressed recovery and avoids additional `RefreshPolicyEx`, Dnscache `paramchange`, and DNS cache flush calls. + +- Type: time duration string +- Required: no +- Default: 30m ## Upstream The `[upstream]` section specifies the DNS upstream servers that `ctrld` will forward DNS requests to. diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md new file mode 100644 index 00000000..08a072d7 --- /dev/null +++ b/docs/dns-intercept-mode.md @@ -0,0 +1,697 @@ +# DNS Intercept Mode + +## Overview + +DNS intercept mode is an alternative approach to DNS management that uses OS-level packet interception instead of modifying network interface DNS settings. This eliminates race conditions with VPN software, endpoint security tools, and other programs that also manage DNS. + +## The Problem + +By default, ctrld sets DNS to `127.0.0.1` on network interfaces so all queries go through ctrld's local listener. However, VPN software (F5 BIG-IP, Cisco AnyConnect, Palo Alto GlobalProtect, etc.) also overwrites interface DNS settings, creating conflicts: + +1. **DNS Setting War**: ctrld sets DNS to `127.0.0.1`, VPN overwrites to its DNS servers, ctrld's watchdog detects the change and restores `127.0.0.1`, VPN overwrites again — infinitely. + +2. **Bypass Window**: During the watchdog polling interval (up to 20 seconds), DNS queries may go to the VPN's DNS servers, bypassing ctrld's filtering profiles (malware blocking, content filtering, etc.). + +3. **Resolution Failures**: During the brief moments when DNS is being rewritten, queries may fail entirely, causing intermittent connectivity loss. + +## The Solution + +DNS intercept mode works at a lower level than interface settings: + +- **Windows**: Uses NRPT (Name Resolution Policy Table) to route all DNS queries to `127.0.0.1` (ctrld's listener) via the Windows DNS Client service. In `hard` mode, additionally uses WFP (Windows Filtering Platform) to block all outbound DNS (port 53) except to localhost and private ranges, preventing any bypass. VPN software can set interface DNS freely — NRPT's most-specific-match ensures VPN-specific domains still resolve correctly while ctrld handles everything else. + +- **macOS**: Uses pf (packet filter) to redirect all outbound DNS (port 53) traffic to ctrld's listener at `127.0.0.1:53`. Any DNS query, regardless of which DNS server the OS thinks it's using, gets transparently redirected to ctrld. + +## Usage + +On macOS and Windows desktop editions, `dns` is the default mode for new installations: a fresh +start that selects no mode, by flag or by `service.intercept_mode`, intercepts. An existing +Standard installation keeps interface DNS mode across upgrades. `--intercept-mode off` +selects interface DNS mode and is persisted. See +[Intercept Mode Default](intercept-mode-default.md) for the full precedence. + +```bash +# Start ctrld with DNS intercept mode (the default on macOS and Windows desktop editions) +ctrld-client start --cd +ctrld-client start --intercept-mode dns --cd + +# Hard intercept: all DNS through ctrld, no VPN split routing +ctrld-client start --intercept-mode hard --cd + +# Or with a config file +ctrld-client start --intercept-mode dns -c /path/to/ctrld.toml + +# Run in foreground (debug) +ctrld-client run --intercept-mode dns --cd +ctrld-client run --intercept-mode hard --cd + +# Interface DNS mode instead (persisted as intercept_mode = "off") +ctrld-client start --intercept-mode off --cd +``` + +### Intercept Modes + +| Flag | DNS Interception | VPN Split Routing | Captive Portal Recovery | +|------|-----------------|-------------------|------------------------| +| `--intercept-mode dns` | ✅ WFP/pf | ✅ Auto-detect & forward | ✅ Active | +| `--intercept-mode hard` | ✅ WFP/pf | ❌ All through ctrld | ✅ Active | + +**`--intercept-mode dns`** (recommended): Intercepts all DNS via WFP/pf, but automatically discovers search domains from VPN and virtual network adapters (Tailscale, F5, Cisco AnyConnect, etc.) and forwards matching queries to the DNS server on that interface. This allows VPN internal resources (e.g., `*.corp.local`) to resolve correctly while ctrld handles everything else. + +**`--intercept-mode hard`**: Same OS-level interception, but does NOT forward any queries to VPN DNS servers. Every DNS query goes through ctrld's configured upstreams. Use this when you want total DNS control and don't need VPN internal domain resolution. Captive portal recovery still works — network authentication pages are handled automatically. + +## How It Works + +### Windows (NRPT + WFP) + +Windows DNS intercept uses a two-tier architecture with mode-dependent enforcement: + +- **`dns` mode**: NRPT + loopback WFP protect — graceful DNS routing through the Windows DNS Client service, with proactive WFP permit filters that protect the NRPT → localhost path from third-party DNS block filters (e.g., OpenVPN's `block-outside-dns`). +- **`hard` mode**: NRPT + WFP — same NRPT routing, plus WFP kernel-level block filters that prevent any outbound DNS bypass. Equivalent enforcement to macOS pf. + +#### Why This Design? + +WFP can only **block** or **permit** connections — it **cannot redirect** them (redirection requires kernel-mode callout drivers). Without NRPT, WFP blocks outbound DNS but doesn't tell applications where to send queries instead — they see DNS failures. NRPT provides the "positive routing" while WFP provides enforcement. + +Separating them into modes means most users get `dns` mode (safe, can never break DNS) while high-security deployments use `hard` mode (full enforcement, same guarantees as macOS pf). + +#### Startup Sequence (dns mode) + +1. Checks for a non-ctrld GP child whose only namespace is `.` and whose only nameserver is ctrld's actual listener IP. +2. When that candidate exists, preserves adapter DNS, sends a DNS Client probe before any NRPT mutation, and re-reads the same GP child. A matching before/after rule plus a received probe enters **GP-managed mode**; ctrld does not write NRPT, call `RefreshPolicyEx`/`paramchange`, or flush DNS for policy activation. +3. Without a still-matching GP candidate, creates the normal ctrld-owned catch-all, signals DNS Client, and flushes stale cache entries. +4. **Activates loopback WFP protect** — adds 4 permit filters (IPv4/IPv6 × UDP/TCP) for DNS to localhost with `FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT`. These prevent third-party WFP block filters from blocking the NRPT → listener path (see [Loopback WFP Protect](#loopback-wfp-protect) below). Non-fatal if this fails. +5. Starts the 30-second ownership-aware NRPT health monitor. +6. Re-verifies an initially ineffective GP candidate synchronously after WFP setup; ctrld-owned NRPT uses the asynchronous probe-and-heal sequence. + +#### Startup Sequence (hard mode) + +1. Establishes NRPT routing using the same GP-managed adoption or ctrld-owned fallback sequence as `dns` mode. +2. Opens WFP engine with `RPC_C_AUTHN_DEFAULT` (0xFFFFFFFF). +3. Cleans up any stale sublayer from a previous unclean shutdown. +4. Creates sublayer with maximum weight (0xFFFF). +5. Adds **permit** filters (weight 10) for DNS to localhost (`127.0.0.1`/`::1` port 53). +6. Adds **permit** filters (weight 10) for DNS to RFC1918 + CGNAT subnets (10/8, 172.16/12, 192.168/16, 100.64/10). +7. Adds **block** filters (weight 1) for all other outbound DNS (port 53 UDP+TCP). +8. Starts the NRPT/WFP health monitor. + +**Atomic guarantee:** NRPT routing must exist before WFP starts. If WFP setup fails, ctrld rolls back only a rule it owns. A GP-managed child is never deleted, rewritten, or replaced with interface DNS merely because ctrld's WFP setup failed. + +On shutdown, ctrld stops its monitor and WFP session. It removes and signals only ctrld-owned NRPT state; a GP-managed catch-all remains untouched. + +#### NRPT Details + +The **Name Resolution Policy Table** is a Windows feature (originally for DirectAccess) that tells the DNS Client service to route queries matching specific namespace patterns to specific DNS servers. ctrld adds a catch-all rule: + +| Registry Value | Type | Value | Purpose | +|---|---|---|---| +| `Name` | REG_MULTI_SZ | `.` | Namespace pattern (`.` = catch-all, matches everything) | +| `GenericDNSServers` | REG_SZ | `127.0.0.1` | DNS server to use for matching queries | +| `ConfigOptions` | REG_DWORD | `0x8` | Standard DNS resolution (no DirectAccess) | +| `Version` | REG_DWORD | `0x2` | NRPT rule version 2 | + +**Registry path**: the primary key is the local store, written on every start: +`HKLM\SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig\{B2E9A3C1-7F4D-4A8E-9D6B-5C1E0F3A2B8D}`. +The GP-store key +`HKLM\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig\CtrldCatchAll` +is written only when foreign GP rules would otherwise hide the local store, and never when +the foreign rule is an administrator catch-all for this listener. + +**Group Policy refresh**: The DNS Client service only reads NRPT from registry during Group Policy processing cycles (default: every 90 minutes). ctrld calls `RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE)` when activating or repairing rules it owns. While Group Policy remains the owner, ctrld does not run NRPT activation/heal signaling. For the transition that removes a ctrld fallback, ctrld signals once after removing its keys and then probes. If the proof finds no route, ctrld writes back only its local key and signals again. + +#### GP-managed NRPT ownership + +Enterprise deployments may install a computer-scoped GP child before starting ctrld with: + +- exactly one namespace: `.`; +- exactly one `GenericDNSServers` value; and +- that nameserver equal to ctrld's actual loopback listener (`127.0.0.1` or the alternate loopback selected on an AD DNS server). + +At service startup ctrld reads that candidate before the normal adapter reset, probes through Windows DNS Client while its listener is already bound, and re-reads the same child. When the rule remains present and the probe arrives, ctrld records **Group Policy** as the NRPT owner. Adapter DNS stays on the organization's resolvers, and ctrld does not create, delete, refresh, or flush NRPT policy. + +The health monitor keeps using functional probes: + +- matching GP rule + successful probe: observe only; +- matching GP rule + failed probe: retry loopback WFP protection, then report the external policy as ineffective without running NRPT heal signals; +- matching GP rule disappears: create the normal ctrld-owned fallback and verify it, unless another GP catch-all targets a different resolver; +- GP catch-all targets another resolver: report the conflict and do not create a second ambiguous catch-all; +- matching GP rule returns: prove it with a probe, remove only ctrld's deterministic fallback keys, and return ownership to Group Policy. If the proof finds no route while the same rule stays on disk, ctrld puts back only its local-store key, keeps ownership, and does not write its GP-store rule adjacent to the administrator's rule. + +Deploy the GPO **before** starting or restarting ctrld if adapter DNS must remain completely untouched. Remove or unlink the GP rule before intentionally removing the ctrld service. A GP catch-all that remains pointed at loopback while no listener is running causes DNS failure by design; ctrld cannot safely delete an administrator-owned policy during uninstall. + +#### WFP Filter Architecture + +**Filter priority**: Permit filters have weight 10, block filters have weight 1. WFP evaluates higher-weight filters first, so localhost and private-range DNS is always permitted. + +**RFC1918 + CGNAT permits**: Static subnet permit filters allow DNS to private IP ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10). This means VPN DNS servers on private IPs (Tailscale MagicDNS on 100.100.100.100, corporate VPN DNS on 10.x.x.x, etc.) work without needing dynamic per-server exemptions. + +**VPN coexistence**: VPN software can set DNS to whatever it wants on the interface — for public IPs, the WFP block filter prevents those servers from being reached on port 53. For private IPs, the subnet permits allow it. ctrld handles all DNS routing through NRPT and can forward VPN-specific domains to VPN DNS servers through its own upstream mechanism. + +#### Loopback WFP Protect (dns mode) + +Third-party VPN software (e.g., OpenVPN, Securepoint SSL VPN) can install WFP block filters via `block-outside-dns` that block **all** DNS traffic to non-tunnel interfaces — including loopback. This breaks the NRPT → `127.0.0.1:53` path that ctrld depends on, causing DNS resolution to time out. + +ctrld proactively adds 4 WFP "hard permit" filters at startup: + +| Filter | Layer | Protocol | +|---|---|---| +| Permit DNS to localhost (IPv4/UDP) | ALE_AUTH_CONNECT_V4 | UDP | +| Permit DNS to localhost (IPv4/TCP) | ALE_AUTH_CONNECT_V4 | TCP | +| Permit DNS to localhost (IPv6/UDP) | ALE_AUTH_CONNECT_V6 | UDP | +| Permit DNS to localhost (IPv6/TCP) | ALE_AUTH_CONNECT_V6 | TCP | + +**Key properties:** +- **Scope**: Port 53 to `127.0.0.1` (or configured listener IP) and `::1` only +- **Flag**: `FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT` (0x08) — "hard permit" that overrides BLOCK decisions from other sublayers regardless of weight or insertion order +- **Weight**: 15 (above hard mode's permit=10) +- **Sublayer**: ctrld's sublayer at maximum priority (0xFFFF) +- **Lifetime**: Process lifetime — added at startup, removed on shutdown/uninstall + +Because `CLEAR_ACTION_RIGHT` is a cross-sublayer override, the order of filter installation doesn't matter — even if a VPN connects hours later and adds its own WFP block filters, ctrld's hard permit for loopback DNS is never overridden. + +The reactive fallback in `nrptProbeAndHeal()` is preserved as defense-in-depth for edge cases where proactive activation fails at startup. + +See: [Issue #526](https://gitlab.int.windscribe.com/controld/clients/ctrld/-/issues/526) + +#### NRPT Probe and Auto-Heal + +`RefreshPolicyEx` returns immediately — it does NOT wait for the DNS Client service to actually load the NRPT rule. On cold machines (first boot, fresh install), the DNS Client may take several seconds to process the policy refresh. During this window, the NRPT rule exists in the registry but isn't active. + +ctrld verifies NRPT is actually working by sending a probe DNS query (`_nrpt-probe-.nrpt-probe.ctrld.test`) through Go's `net.Resolver` (which calls `GetAddrInfoW` → DNS Client → NRPT path). If ctrld receives the probe on its listener, NRPT is active. + +**Startup probes:** A matching GP candidate is probed synchronously before any NRPT mutation and re-read afterward. ctrld-owned rules keep the asynchronous activation/heal sequence: immediate probe, bounded policy signaling retries, then two-phase delete/re-add recovery. Only one probe sequence runs at a time. + +**Ownership boundary:** When the active owner is Group Policy, a failed probe never enters ctrld's NRPT refresh/delete/re-add sequence. ctrld may repair its narrowly scoped loopback WFP permits, but leaves the external registry child and DNS Client policy signaling to the administrator. + +#### NRPT Health Monitor + +A dedicated background goroutine (`nrptHealthMonitor`) runs every 30 seconds and now performs active probing: + +1. **Ownership check:** Distinguish a matching external GP child from ctrld's deterministic local/GP keys. +2. **Active probe:** Verify Windows DNS Client still routes to the listener. +3. **Transition:** If the external child disappears, activate ctrld's normal fallback. If it returns while the fallback is active, prove it before removing only ctrld's keys. If the proof finds no route while the same rule stays on disk, ctrld puts back only its local-store key, keeps ownership, and does not write its GP-store rule adjacent to the administrator's rule. +4. **Owned recovery:** Restore/heal only when ctrld owns the NRPT rule. +5. **(hard mode)** Verify the WFP sublayer exists and fully restart intercept state on loss. + +This is periodic (not just network-event-driven) because VPN software can clear NRPT at any time. Additionally, `scheduleDelayedRechecks()` (called on network change events) performs immediate NRPT verification at 2s and 4s after changes. + +#### Known Caveats + +- **`nslookup` bypasses NRPT**: `nslookup.exe` uses its own DNS resolver implementation and does NOT go through the Windows DNS Client service, so it ignores NRPT rules entirely. Use `Resolve-DnsName` (PowerShell) or `ping` to verify DNS resolution through NRPT. This is a well-known Windows behavior, not a ctrld bug. +- **`RPC_C_AUTHN_DEFAULT`**: `FwpmEngineOpen0` requires `RPC_C_AUTHN_DEFAULT` (0xFFFFFFFF) for the authentication service parameter. Using `RPC_C_AUTHN_NONE` (0) returns `ERROR_NOT_SUPPORTED` on some configurations (e.g., Parallels VMs). +- **FWP_DATA_TYPE enum**: The `FWP_DATA_TYPE` enum starts at `FWP_EMPTY=0`, making `FWP_UINT8=1`, `FWP_UINT16=2`, etc. Some documentation examples incorrectly start at 0. + +### macOS (pf) + +1. ctrld writes a pf anchor file at `/etc/pf.anchors/com.controld.ctrld` +2. Adds the anchor reference to `/etc/pf.conf` (if not present) +3. Loads the anchor with `pfctl -a com.controld.ctrld -f ` +4. Enables pf with `pfctl -e` (if not already enabled) +5. The anchor redirects all outbound DNS (port 53) on non-loopback interfaces to `127.0.0.1:53` +6. On shutdown, the anchor is flushed, the file removed, and references cleaned from `pf.conf` + +**ctrld's own traffic**: ctrld's upstream queries use DoH (HTTPS on port 443), not plain DNS on port 53, so the pf redirect does not create a loop for DoH upstreams. **Warning:** If an "os" upstream is configured (which uses plain DNS on port 53 to external servers), the pf redirect will capture ctrld's own outbound queries and create a loop. ctrld will log a warning at startup if this is detected. Use DoH upstreams when DNS intercept mode is active. + +## What Changes vs Interface DNS Mode + +| Behavior | Interface DNS Mode | DNS Intercept Mode | +|----------|-------------|-------------------| +| Interface DNS settings | Set to `127.0.0.1` | **Not modified** | +| DNS watchdog | Active (polls every 20s) | **Disabled** | +| VPN DNS conflict | Race condition possible | **Eliminated** | +| Profile bypass window | Up to 20 seconds | **Zero** | +| Requires admin/root | Yes | Yes | +| Additional OS requirements | None | WFP (Windows), pf (macOS) | + +## Logging + +DNS intercept mode produces detailed logs for troubleshooting: + +``` +DNS intercept: initializing Windows Filtering Platform (WFP) +DNS intercept: WFP engine opened (handle: 0x1a2b3c) +DNS intercept: WFP sublayer created (weight: 0xFFFF — maximum priority) +DNS intercept: added permit filter "Permit DNS to localhost (IPv4/UDP)" (ID: 12345) +DNS intercept: added block filter "Block outbound DNS (IPv4/UDP)" (ID: 12349) +DNS intercept: WFP filters active — all outbound DNS (port 53) blocked except to localhost +``` + +On macOS: +``` +DNS intercept: initializing macOS packet filter (pf) redirect +DNS intercept: wrote pf anchor file: /etc/pf.anchors/com.controld.ctrld +DNS intercept: loaded pf anchor "com.controld.ctrld" +DNS intercept: pf anchor "com.controld.ctrld" active with 3 rules +DNS intercept: pf redirect active — all outbound DNS (port 53) redirected to 127.0.0.1:53 +``` + +DNS intercept mode on macOS also runs the wake detector. That detector reports a `Host woke` journal event with the source `detector` and the length of the sleep. +Outside DNS intercept mode the netmon delta is the only wake source, and it reports no length of its own. +The events of the journal are listed in [Network-recovery diagnostics](network-recovery-diagnostics.md#journal-events). + +## Troubleshooting + +### Windows + +```powershell +# Check NRPT rules (should show ctrld's catch-all, . → 127.0.0.1; the primary key is the +# local-store GUID rule, and CtrldCatchAll appears only when ctrld also wrote the GP store) +Get-DnsClientNrptRule + +# Check NRPT registry directly +Get-ChildItem "HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig" + +# Force Group Policy refresh (if NRPT not taking effect) +gpupdate /target:computer /force + +# Check if WFP filters are active +netsh wfp show filters + +# Check ctrld's specific filters (look for "ctrld" in output) +netsh wfp show filters | Select-String "ctrld" + +# Test DNS resolution (use Resolve-DnsName, NOT nslookup!) +# nslookup bypasses DNS Client / NRPT — it will NOT reflect NRPT routing +Resolve-DnsName example.com +ping example.com + +# If you must use nslookup, specify localhost explicitly: +nslookup example.com 127.0.0.1 +``` + +### macOS + +```bash +# Check if pf is enabled +sudo pfctl -si + +# Check ctrld's anchor rules +sudo pfctl -a com.controld.ctrld -sr +sudo pfctl -a com.controld.ctrld -sn + +# Check pf.conf for anchor reference +cat /etc/pf.conf | grep ctrld + +# Test DNS is going through ctrld +dig @127.0.0.1 example.com +``` + +## Limitations + +- **Linux**: Not supported. Linux uses `systemd-resolved` or `/etc/resolv.conf` which don't have the same VPN conflict issues. If needed in the future, `iptables`/`nftables` REDIRECT could be used. + +- **Split DNS for VPN internal domains**: In `--intercept-mode dns` mode, VPN search domains are auto-detected from virtual network adapters and forwarded to the VPN's DNS servers automatically. In `--intercept-mode hard` mode, VPN internal domains (e.g., `*.corp.local`) will NOT resolve unless configured as explicit upstream rules in ctrld's configuration. + +- **Tailscale Split DNS and App Connectors (macOS)**: The sandboxed Tailscale app keeps Split DNS routes with custom resolvers and App Connector domains inside its own resolver (`100.100.100.100`); they do not appear in `scutil --dns`. When a discovered resolver is Tailscale's, ctrld also reads Tailscale's route list from its LocalAPI (`/localapi/v0/dns-config`, Tailscale v1.98.0 or later) and routes those suffixes to `100.100.100.100`. The list is read on each VPN DNS refresh; after changing Split DNS or App Connectors, reconnect Tailscale to apply them. Names routed this way are resolved by the tailnet and not filtered by Control D, as in standard mode. If the LocalAPI is unavailable, only `scutil --dns` domains are used. + - **Loop guard**: a route is taken only when quad-100 can reach its resolvers without crossing pf's port-53 redirect. DoH and App Connector (peerAPI) resolvers always qualify. A plain port-53 resolver qualifies only on a Tailscale IPv4 address and only in non-exit mode, where the VPN DNS interface passthrough exists (section 9). Routes with any other resolver (public, LAN, IPv6, or any port-53 resolver in exit-node mode) keep going to Control D; routing them to quad-100 would loop back to ctrld. + - **Endpoint trust**: root-owned credentials are tried first (Standalone `/Library/Tailscale`, then `/var/run/tailscaled.socket`). The App Store extension is used last, and only when the process holding the `sameuserproof` file is named exactly `IPNExtension`, the file sits in a user's `Library/Group Containers/*.tailscale.ipn.macos/`, and the running process passes a Tailscale code-signature check. + - **Failed reads**: a failed LocalAPI read (for example a 503 while Tailscale starts) keeps the last good route set while quad-100 is still discovered, and retries after 5 s, 15 s and 60 s, then stops until the next network change. + +- **macOS mDNSResponder interaction**: On macOS, ctrld uses a workaround ("mDNSResponder hack") that binds to `0.0.0.0:53` instead of `127.0.0.1:53` and refuses queries from non-localhost sources. In dns-intercept mode, pf's `rdr` rewrites the destination IP to `127.0.0.1:53` but preserves the original source IP (e.g., `192.168.2.73`). The mDNSResponder source-IP check is automatically bypassed in dns-intercept mode because the pf/WFP rules already ensure only legitimate intercepted DNS traffic reaches ctrld's listener. + +- **Other WFP/pf users**: If other software (VPN, firewall, endpoint security) also uses WFP or pf for DNS interception, there may be priority conflicts. ctrld uses maximum sublayer weight on Windows and a named anchor on macOS to minimize this risk. See "VPN App Coexistence" below for macOS-specific defenses. + +## VPN App Coexistence (macOS) + +VPN apps (Windscribe, Cisco AnyConnect, F5 BIG-IP, etc.) often manage pf rules themselves, which can interfere with ctrld's DNS intercept. ctrld uses a multi-layered defense strategy: + +### 1. Anchor Priority Enforcement + +When injecting our anchor reference into the running pf ruleset, ctrld **prepends** both the `rdr-anchor` and `anchor` references before all other anchors. pf evaluates rules top-to-bottom, so our DNS intercept `quick` rules match port 53 traffic before a VPN app's broader rules in their own anchor. + +### 2. Interface-Specific Tunnel Rules + +VPN apps commonly add rules like `pass out quick on ipsec0 inet all` that match ALL traffic on the VPN interface. If their anchor is evaluated before ours (e.g., after a ruleset reload), these broad rules capture DNS. ctrld counters this by adding explicit DNS intercept rules for each active tunnel interface (ipsec*, utun*, ppp*, tap*, tun*). These interface-specific rules match port 53 only, so they take priority over the VPN app's broader "all" match even within the same anchor evaluation pass. + +### 3. Dynamic Tunnel Interface Detection + +The network change monitor (`validInterfacesMap()`) only tracks physical hardware ports (en0, bridge0, etc.) — it doesn't see tunnel interfaces (utun*, ipsec*, etc.) created by VPN software. When a VPN connects and creates a new interface (e.g., utun420 for WireGuard), ctrld detects this through a separate tunnel interface change check and rebuilds the pf anchor to include explicit intercept rules for the new interface. This runs on every network change event that is not noise, even if no physical interface changed. A delta that touches AirDrop or virtual adapters alone is noise, and ctrld skips the handler for it, so no tunnel discovery runs. + +### 4. pf Watchdog + Network Change Hooks + +A background watchdog (30s interval) plus immediate checks on network change events detect when another program replaces the entire pf ruleset (e.g., Windscribe's `pfctl -f /etc/pf.conf`). When detected, ctrld rebuilds its anchor with up-to-date tunnel interface rules and re-injects the anchor reference at the top of the ruleset. A 2-second delayed re-check catches race conditions where the other program clears rules slightly after the network event. + +A noise delta starts no immediate check and no delayed re-check. The watchdog keeps its own 30 s cadence, so it still restores a missing anchor during a delta storm. + +### 4a. Active Interception Probe (pf Translation State Corruption) + +Programs like Parallels Desktop reload `/etc/pf.conf` when creating/destroying virtual network interfaces (bridge100, vmenet0). This can corrupt pf's internal translation engine — rdr rules survive in text form but stop evaluating, causing DNS interception to silently fail while the watchdog reports "intact." + +ctrld detects interface appearance/disappearance and spawns an async probe monitor: + +1. **Probe mechanism:** A subprocess runs with GID=0 (wheel, not `_ctrld`) and sends a DNS query to the OS resolver. If pf interception is working, the query gets redirected to ctrld (127.0.0.1:53) and is detected in the DNS handler. A confirmed send without local receipt within 1s gives `not_intercepted`. +2. **Backoff schedule:** Probes at 0, 0.5, 1, 2, 4 seconds (~8s window) to win the race against async pf reloads by the hypervisor. Only one monitor runs at a time (atomic singleton). +3. **Auto-heal:** Only after `not_intercepted`, `forceReloadPFMainRuleset()` dumps the running ruleset and pipes it back through `pfctl -f -`, resetting pf's translation engine. VPN-safe because it reassembles from the current running state. +4. **Watchdog integration:** The 30s watchdog also runs the probe when rule text checks pass, as a safety net for unknown corruption causes. + +The probe uses IPv4 only. Missing targets and helper failures give `indeterminate`, which does not cause a reload. +See [target selection and diagnostics](network-recovery-diagnostics.md#pf-probes-macos). +A successful probe proves local interception delivery, not remote DNS health. + +### 5. Proactive DoH Connection Pool Reset + +When the watchdog detects a pf ruleset replacement, it force-rebootstraps all upstream transports via `ForceReBootstrap()`. This is necessary because `pfctl -f` flushes the entire pf state table, which kills existing TCP connections (including ctrld's DoH connections to upstream DNS servers like 76.76.2.22:443). + +The force-rebootstrap does two things that the lazy `ReBootstrap()` cannot: +1. **Closes idle connections on the old transport** (`CloseIdleConnections()`), causing in-flight HTTP/2 requests on dead connections to fail immediately instead of waiting for the 5s context deadline +2. **Creates the new transport synchronously**, so it's ready before any DNS queries arrive post-wipe + +Without this, Go's `http.Transport` keeps trying dead connections until each request's context deadline expires (~5s), then the lazy rebootstrap creates a new transport for the *next* request. With force-rebootstrap, the blackout is reduced from ~5s to ~100ms (one fresh TLS handshake). + +### 6. Blanket Process Exemption (group _ctrld) + +ctrld creates a macOS system group (`_ctrld`) and sets its effective GID at startup via `syscall.Setegid()`. The pf anchor includes a blanket rule: + +``` +pass out quick group _ctrld +``` + +This exempts **all** outbound traffic from the ctrld process — not just DNS (port 53), but also DoH (TCP 443), DoT (TCP 853), health checks, and any other connections. This is essential because VPN firewalls like Windscribe load `block drop all` rulesets that would otherwise block ctrld's upstream connections even after the pf anchor is restored. + +Because ctrld's anchor is prepended before all other anchors, and this rule uses `quick`, it evaluates before any VPN firewall rules. The result: ctrld's traffic is never blocked regardless of what other pf rulesets are loaded. + +The per-IP exemptions (OS resolver, VPN DNS) remain as defense-in-depth for the DNS redirect loop prevention — the blanket rule handles everything else. + +### 7. Loopback Outbound Pass Rule + +When `route-to lo0` redirects a DNS packet to loopback, pf re-evaluates the packet **outbound on lo0**. None of the existing route-to rules match on lo0 (they're all `on ! lo0` or `on utunX`), so without an explicit pass rule, the packet falls through to the main ruleset where VPN firewalls' `block drop all` drops it — before it ever reaches the inbound rdr rule. + +``` +pass out quick on lo0 inet proto udp from any to ! 127.0.0.1 port 53 +pass out quick on lo0 inet proto tcp from any to ! 127.0.0.1 port 53 +``` + +This bridges the route-to → rdr gap: route-to sends outbound on lo0 → this rule passes it → loopback reflects it inbound → rdr rewrites destination to 127.0.0.1:53 → ctrld receives the query. Without this rule, DNS intercept fails whenever a `block drop all` firewall (Windscribe, etc.) is active. + +### 8. Response Routing via `reply-to lo0` + +After rdr redirects DNS to 127.0.0.1:53, ctrld responds to the original client source IP (e.g., 100.94.163.168 — a VPN tunnel IP). Without intervention, the kernel routes this response through the VPN tunnel interface (utun420) based on its routing table, and the response is lost. + +``` +pass in quick on lo0 reply-to lo0 inet proto { udp, tcp } from any to 127.0.0.1 port 53 +``` + +`reply-to lo0` tells pf to force response packets for this connection back through lo0, overriding the kernel routing table. The response stays local, rdr reverse NAT rewrites the source from 127.0.0.1 back to the original DNS server IP (e.g., 10.255.255.3), and the client process receives a correctly-addressed response. + +### 9. VPN DNS Split Routing and Exit Mode Detection + +When a VPN like Tailscale MagicDNS is active, two distinct modes require different pf handling: + +#### The Problem: DNS Proxy Loop + +VPN DNS handlers like Tailscale's MagicDNS run as macOS Network Extensions. MagicDNS +listens on 100.100.100.100 and forwards queries to internal upstream nameservers +(e.g., 10.3.112.11, 10.3.112.12) via the VPN tunnel interface (utun13). + +Without special handling, pf's generic `pass out quick on ! lo0 route-to lo0` rule +intercepts MagicDNS's upstream queries on the tunnel interface, routing them back +to ctrld → which matches VPN DNS split routing → forwards to MagicDNS → loop: + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ THE LOOP (without passthrough rules) │ +│ │ +│ 1. dig gitlab.int.windscribe.com │ +│ → pf intercepts → route-to lo0 → rdr → ctrld (127.0.0.1:53) │ +│ │ +│ 2. ctrld: VPN DNS match → forward to 100.100.100.100:53 │ +│ → group _ctrld exempts → reaches MagicDNS │ +│ │ +│ 3. MagicDNS: forward to upstream 10.3.112.11:53 via utun13 │ +│ → pf generic rule matches (utun13 ≠ lo0, 10.3.112.11 ≠ skip) │ +│ → route-to lo0 → rdr → back to ctrld ← LOOP! │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +#### The Fix: Interface Passthrough + Exit Mode Detection + +**Split DNS mode** (VPN handles only specific domains): + +ctrld adds passthrough rules for VPN DNS interfaces that let MagicDNS's upstream +queries flow without interception. A `` table contains the VPN DNS server +IPs (e.g., 100.100.100.100) — traffic TO those IPs is NOT passed through (still +intercepted by pf → ctrld enforces profile): + +``` +table { 100.100.100.100 } + +# MagicDNS upstream queries (to 10.3.112.11 etc.) — pass through +pass out quick on utun13 inet proto udp from any to ! port 53 +pass out quick on utun13 inet proto tcp from any to ! port 53 + +# Queries TO MagicDNS (100.100.100.100) — not matched above, +# falls through to generic rule → intercepted → ctrld → profile enforced +``` + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ SPLIT DNS MODE (with passthrough rules) │ +│ │ +│ Non-VPN domain (popads.net): │ +│ dig popads.net → system routes to 100.100.100.100 on utun13 │ +│ → passthrough rule: dest IS in → NOT matched │ +│ → generic rule: route-to lo0 → rdr → ctrld → profile blocks it ✅ │ +│ │ +│ VPN domain (gitlab.int.windscribe.com): │ +│ dig gitlab.int... → pf intercepts → ctrld │ +│ → VPN DNS match → forward to 100.100.100.100 (group exempt) │ +│ → MagicDNS → upstream 10.3.112.11 on utun13 │ +│ → passthrough rule: dest NOT in → MATCHED → passes ✅ │ +│ → 10.3.112.11 returns correct internal answer (10.3.112.113) │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +**Exit mode** (all traffic through VPN): + +When Tailscale exit node is enabled, MagicDNS becomes the system's **default** +resolver (not just supplemental). If we added passthrough rules, ALL DNS would +bypass ctrld — losing profile enforcement. + +Exit mode is detected using two independent signals (either triggers exit mode): + +**1. Default route detection (primary, most reliable):** +Uses `netmon.DefaultRouteInterface()` to check if the system's default route +(0.0.0.0/0) goes through a VPN DNS interface. If `DefaultRouteInterface` matches +a VPN DNS interface name (e.g., utun13), the VPN owns the default route — it's +exit mode. This is the ground truth: the routing table directly reflects whether +all traffic flows through the VPN, regardless of how the VPN presents itself in +scutil. + +**2. scutil flag detection (secondary, fallback):** +If the VPN DNS server IP appears in a `scutil --dns` resolver entry that has +**no search domains** and **no Supplemental flag**, it's acting as the system's +default resolver (exit mode). This catches edge cases where the default route +hasn't changed yet but scutil already shows the VPN as the default DNS. + +``` +# Non-exit mode — default route on en0, 100.100.100.100 is Supplemental: +$ route -n get 0.0.0.0 | grep interface + interface: en0 ← physical NIC, not VPN +resolver #1 + search domain[0] : int.windscribe.com + nameserver[0] : 100.100.100.100 + flags : Supplemental, Request A records + +# Exit mode — default route on utun13, 100.100.100.100 is default resolver: +$ route -n get 0.0.0.0 | grep interface + interface: utun13 ← VPN interface! +resolver #2 + nameserver[0] : 100.100.100.100 ← MagicDNS is default + flags : Request A records ← no Supplemental! +``` + +In exit mode, NO passthrough rules are generated. pf intercepts all DNS → ctrld +enforces its profile on everything. VPN search domains still resolve correctly +via ctrld's VPN DNS split routing (forwarded to MagicDNS through the group +exemption). + +#### Summary Table + +| Scenario | Passthrough | Profile Enforced | VPN Domains | +|----------|-------------|-----------------|-------------| +| No VPN | None | ✅ All traffic | N/A | +| Split DNS (Tailscale non-exit) | ✅ VPN interface | ✅ Non-VPN domains | ✅ Via MagicDNS | +| Exit mode (Tailscale exit node) | ❌ None | ✅ All traffic | ✅ Via ctrld split routing | +| Zscaler Private Access | None | ✅ All traffic except one health-check name | ✅ Via Client Connector | +| Windscribe | None (different flow) | ✅ All traffic | N/A | +| Hard intercept | None | ✅ All traffic | ❌ Not forwarded | + +### Nuclear Option (Future) + +If anchor ordering + interface rules prove insufficient, an alternative approach is available: inject DNS intercept rules directly into the **main pf ruleset** (not inside an anchor). Main ruleset rules are evaluated before ALL anchors, making them impossible for another app to override without explicitly removing them. This is more invasive and not currently implemented, but documented here as a known escalation path. + +## Known VPN Conflicts + +### F5 BIG-IP APM + +F5 BIG-IP APM VPN is a known source of DNS conflicts with ctrld (Support ticket #1688001). The conflict occurs because F5's VPN client aggressively manages DNS: + +**How the conflict manifests:** + +1. ctrld sets system DNS to `127.0.0.1` / `::1` for local forwarding +2. F5 VPN connects and **overwrites DNS on all interfaces** by prepending its own servers (e.g., `10.50.10.77`, `192.168.208.56`) +3. F5 enforces split DNS patterns (e.g., `*.provisur.local`) and activates its DNS Relay Proxy (`F5FltSrv.exe` / `F5FltSrv.sys`) +4. ctrld's watchdog detects the change and restores `127.0.0.1` — F5 overwrites again +5. This loop causes intermittent resolution failures, slow responses, and VPN disconnects + +**Why `--intercept-mode dns` solves this:** + +- ctrld no longer modifies interface DNS settings — there is nothing for F5 to overwrite +- WFP (Windows) blocks all outbound DNS except to localhost, so F5's prepended DNS servers are unreachable on port 53 +- F5's DNS Relay Proxy (`F5FltSrv`) becomes irrelevant since no queries reach it +- In `--intercept-mode dns` mode, F5's split DNS domains (e.g., `*.provisur.local`) are auto-detected from the VPN adapter and forwarded to F5's DNS servers through ctrld's upstream mechanism + +**F5-side mitigations (if `--intercept-mode dns` is not available):** + +- In APM Network Access DNS settings, enable **"Allow Local DNS Servers"** (`AllowLocalDNSServersAccess = 1`) +- Disable **"Enforce DNS Name Resolution Order"** +- Switch to IP-based split tunneling instead of DNS-pattern-based to avoid activating F5's relay proxy +- Update F5 to version 17.x+ which includes DNS handling fixes (see F5 KB K80231353) + +**Additional considerations:** + +- CrowdStrike Falcon and similar endpoint security with network inspection can compound the conflict (three-way DNS stomping) +- F5's relay proxy (`F5FltSrv`) performs similar functions to ctrld — they are in direct conflict when both active +- The seemingly random failure pattern is caused by timing-dependent race conditions between ctrld's watchdog, F5's DNS enforcement, and (optionally) endpoint security inspection + +### Zscaler Private Access (macOS) + +Zscaler Client Connector will not synthesize addresses for Private Access +applications until it has confirmed that DNS answers reach it on its own DNS +path. It confirms this by resolving one fixed public name, +`dnsechotest.zscaler.com`, and observing the answer. + +**How the conflict manifests:** + +1. `--intercept-mode dns` is active, so pf redirects port 53 to ctrld +2. Client Connector's query for `dnsechotest.zscaler.com` is redirected into ctrld +3. ctrld answers it from the Control D upstream, so Client Connector never sees the + answer arrive on its own DNS path +4. Client Connector treats domain validation as failed and keeps Private Access + disabled — while ordinary Control D DNS stays perfectly healthy, which is why the + symptom looks like "ZPA is broken" rather than "DNS is broken" + +**How ctrld handles it:** + +On macOS in `dns` mode, ctrld routes that one exact name the same way a Control D +"bypass" rule does: with an empty upstream list, which resolves through +`upstream.os` — the system resolver set, which under Intercept Mode still includes +Client Connector's own DNS servers. Because the decision is made on the name alone, +it covers every record type. + +The answer is **not cached**. Client Connector enables Private Access only after it +observes the echo answer arrive on its own DNS path, so an answer ctrld served from +its cache reads as silence to Client Connector — the query has to reach the OS +resolver on every poll. This matters most on reconnect: an answer cached while ZPA +was disconnected would otherwise outlive `InitializeOsResolver()` and keep Private +Access disabled for the rest of that entry's TTL (or `cache_ttl_override`). + +The scope is deliberately narrow — a single fixed public health-check name: + +- Private Access **application** domains are not special-cased. Once Client Connector + has validated its DNS path it answers them itself, so they never need a ctrld + routing exception; anything it does not claim stays on Control D. +- No interface, resolver range, or pf rule is exempted. There is no ZPA interface + detection, no CGNAT probing, and no passthrough rule. +- Everything else, including every other `zscaler.com` name, stays intercepted and + filtered by Control D. +- `--intercept-mode hard` opts out, the same as it opts out of VPN DNS split + routing: all DNS goes through ctrld and ZPA is not accommodated. +- The route does **not** authorize the query. A `Restricted` listener still answers + only sources that match its policy, so the built-in route is not a hole in + source authorization. +- An explicit listener **domain rule** that routes this name to a specific upstream + outranks the built-in route, and is the supported way to opt out without leaving + Intercept Mode. Network- and MAC-policy targets do not: they route every name + from a source and so state nothing about this one. +- A rule that itself selects the OS path — empty targets, which is how a Control D + profile's bypass list reaches ctrld — keeps its own routing and labels but is still + treated as the health check, so it gets the no-cache behavior too. A machine + upgraded while still carrying the bypass-folder workaround therefore gets the full + fix rather than just the routing half. + +Adding `dnsechotest.zscaler.com` to a Control D bypass folder produces the same +routing and remains a valid workaround on releases without this fix. Disabling +Intercept Mode entirely (`ctrld start --intercept-mode off`) also restores ZPA, at +the cost of interface-based DNS handling. + +**Status:** the routing decision is confirmed by a customer running the bypass-folder +equivalent. It has not yet been verified against a real ZPA tenant with a compiled +build; see the implementation tracker for the outstanding smoke test. + +### Cisco AnyConnect + +Cisco AnyConnect exhibits similar DNS override behavior. `--intercept-mode dns` mode prevents the conflict by operating at the packet filter level rather than competing for interface DNS settings. + +### Windscribe Desktop App + +Windscribe's macOS firewall implementation (`FirewallController_mac`) replaces the entire pf ruleset when connecting/disconnecting via `pfctl -f`, which wipes ctrld's anchor references and flushes the pf state table (killing active DoH connections). ctrld handles this with multiple defenses: + +1. **pf watchdog** detects the wipe and restores anchor rules immediately on network change events (or within 30s via periodic check) +2. **DoH transport force-reset** immediately replaces upstream transports when a pf wipe is detected (closing old connections + creating new ones synchronously), reducing the DNS blackout from ~5s to ~100ms +3. **Tunnel interface detection** adds explicit intercept rules for Windscribe's WireGuard interface (e.g., utun420) when it appears +4. **Dual delayed re-checks** (2s + 4s after network event) catch race conditions where VPN apps modify pf rules and DNS settings asynchronously after the initial network change +5. **Deferred pf restore** waits for VPN to finish its pf modifications before restoring ctrld's rules, preventing the reconnect death spiral +6. **Blanket group exemption** (`pass out quick group _ctrld`) ensures all ctrld traffic (including DoH on port 443) passes through VPN firewalls like Windscribe's `block drop all` + +## 7. VPN DNS Lifecycle + +When VPN software connects or disconnects, ctrld must track DNS state changes to ensure correct routing and avoid stale state. + +### Network Change Event Flow (macOS) + +``` +Network change detected (netmon callback) + │ + ├─ Immediate actions: + │ ├─ ensurePFAnchorActive() — verify/restore pf anchor references + │ ├─ checkTunnelInterfaceChanges() — detect new/removed VPN interfaces + │ │ ├─ New tunnel → pfStartStabilization() (wait for VPN to finish pf changes) + │ │ └─ Removed tunnel → rebuild anchor immediately (with VPN DNS exemptions) + │ └─ vpnDNS.Refresh() — re-discover VPN DNS from scutil --dns + │ + ├─ Delayed re-check at 2s: + │ ├─ ensurePFAnchorActive() — catch async pf wipes + │ ├─ checkTunnelInterfaceChanges() + │ ├─ InitializeOsResolver() — clear stale DNS from scutil + │ └─ vpnDNS.Refresh() — clear stale VPN DNS routes + │ + └─ Delayed re-check at 4s: + └─ (same as 2s — catches slower VPN teardowns) +``` + +A noise delta skips this whole chain. ctrld runs no pf read, no tunnel discovery, no OS resolver read, and no VPN DNS refresh for it. The noise class holds the AirDrop interfaces (`awdl*`, `llw*`) and the virtual adapters. The pf watchdog keeps its own 30 s cadence, so it still tests the rules during a delta storm. + +### VPN Connect Sequence + +1. VPN creates tunnel interface (e.g., utun420) +2. Network change fires → `checkTunnelInterfaceChanges()` detects new tunnel +3. **Stabilization mode** activates — suppresses pf restores while VPN modifies rules +4. Stabilization loop polls `pfctl -sr` hash every 1.5s +5. When hash stable for 6s → VPN finished → restore ctrld's pf anchor +6. `vpnDNS.Refresh()` discovers VPN's search domains and DNS servers from `scutil --dns` +7. Anchor rebuild includes VPN DNS exemptions (so ctrld can reach VPN DNS on port 53) + +### VPN Disconnect Sequence + +1. VPN removes tunnel interface +2. Network change fires → `checkTunnelInterfaceChanges()` detects removal +3. Anchor rebuilt immediately (no stabilization needed for removals) +4. VPN app may asynchronously wipe pf rules (`pfctl -f /etc/pf.conf`) +5. VPN app may asynchronously clean up DNS settings from `scutil --dns` +6. **2s delayed re-check**: restores pf anchor if wiped, refreshes OS resolver +7. **4s delayed re-check**: catches slower VPN teardowns +8. `vpnDNS.Refresh()` returns empty → `onServersChanged(nil)` clears stale exemptions +9. `InitializeOsResolver()` re-reads `scutil --dns` → clears stale LAN nameservers + +### Key Design Decisions + +- **`buildPFAnchorRules()` receives VPN DNS servers**: All call sites (tunnel rebuild, watchdog restore, stabilization exit) pass `vpnDNS.CurrentServers()` so exemptions are preserved for still-active VPNs. +- **`onServersChanged` called even when server list is empty**: Ensures stale pf exemptions from a previous VPN session are cleaned up on disconnect. +- **OS resolver refresh in delayed re-checks**: VPN apps often finish DNS cleanup 1-3s after the network change event. The delayed `InitializeOsResolver()` call ensures stale LAN nameservers (e.g., Windscribe's 10.255.255.3) don't cause 2s query timeouts. +- **Ordering: tunnel checks → VPN DNS refresh → delayed re-checks**: Ensures anchor rebuilds from tunnel changes include current VPN DNS exemptions. + +## Related + +- [GitLab Issue #489](https://gitlab.int.windscribe.com/controld/clients/ctrld/-/issues/489) — Original issue and discussion +- F5 BIG-IP APM VPN DNS conflict (Support ticket #1688001) diff --git a/docs/ephemeral_mode.md b/docs/ephemeral_mode.md index b0af5724..154b9f93 100644 --- a/docs/ephemeral_mode.md +++ b/docs/ephemeral_mode.md @@ -3,11 +3,11 @@ ## Launch arguments ```shell -$ ctrld run --help +$ ctrld-client run --help Run the DNS proxy server Usage: - ctrld run [flags] + ctrld-client run [flags] Flags: --base64_config string base64 encoded config @@ -29,7 +29,7 @@ Global Flags: For example: ```shell -ctrld run --listen=127.0.0.1:53 --primary_upstream=https://freedns.controld.com/p2 --secondary_upstream=8.8.8.8:53 --domains=*.company.int,*.net --log /path/to/log.log +ctrld-client run --listen=127.0.0.1:53 --primary_upstream=https://freedns.controld.com/p2 --secondary_upstream=8.8.8.8:53 --domains=*.company.int,*.net --log /path/to/log.log ``` Above command will be translated roughly to this config: @@ -70,5 +70,5 @@ Only `listen` and `primary_upstream` flags are required. `ctrld` can read a complete base64 encoded config via command line flag. This allows you to supply complex configurations. ```shell -ctrld run --base64_config="CltsaXN0ZW5lcl0KCiAgW2xpc3RlbmVyLjBdCiAgICBpcCA9ICIxMjcuMC4wLjEiCiAgICBwb3J0ID0gNTMKICAgIHJlc3RyaWN0ZWQgPSBmYWxzZQoKW25ldHdvcmtdCgogIFtuZXR3b3JrLjBdCiAgICBjaWRycyA9IFsiMC4wLjAuMC8wIl0KICAgIG5hbWUgPSAiTmV0d29yayAwIgoKW3Vwc3RyZWFtXQoKICBbdXBzdHJlYW0uMF0KICAgIGJvb3RzdHJhcF9pcCA9ICI3Ni43Ni4yLjExIgogICAgZW5kcG9pbnQgPSAiaHR0cHM6Ly9mcmVlZG5zLmNvbnRyb2xkLmNvbS9wMSIKICAgIG5hbWUgPSAiQ29udHJvbCBEIC0gQW50aS1NYWx3YXJlIgogICAgdGltZW91dCA9IDUwMDAKICAgIHR5cGUgPSAiZG9oIgoKICBbdXBzdHJlYW0uMV0KICAgIGJvb3RzdHJhcF9pcCA9ICI3Ni43Ni4yLjExIgogICAgZW5kcG9pbnQgPSAicDIuZnJlZWRucy5jb250cm9sZC5jb20iCiAgICBuYW1lID0gIkNvbnRyb2wgRCAtIE5vIEFkcyIKICAgIHRpbWVvdXQgPSAzMDAwCiAgICB0eXBlID0gImRvcSIK" +ctrld-client run --base64_config="CltsaXN0ZW5lcl0KCiAgW2xpc3RlbmVyLjBdCiAgICBpcCA9ICIxMjcuMC4wLjEiCiAgICBwb3J0ID0gNTMKICAgIHJlc3RyaWN0ZWQgPSBmYWxzZQoKW25ldHdvcmtdCgogIFtuZXR3b3JrLjBdCiAgICBjaWRycyA9IFsiMC4wLjAuMC8wIl0KICAgIG5hbWUgPSAiTmV0d29yayAwIgoKW3Vwc3RyZWFtXQoKICBbdXBzdHJlYW0uMF0KICAgIGJvb3RzdHJhcF9pcCA9ICI3Ni43Ni4yLjExIgogICAgZW5kcG9pbnQgPSAiaHR0cHM6Ly9mcmVlZG5zLmNvbnRyb2xkLmNvbS9wMSIKICAgIG5hbWUgPSAiQ29udHJvbCBEIC0gQW50aS1NYWx3YXJlIgogICAgdGltZW91dCA9IDUwMDAKICAgIHR5cGUgPSAiZG9oIgoKICBbdXBzdHJlYW0uMV0KICAgIGJvb3RzdHJhcF9pcCA9ICI3Ni43Ni4yLjExIgogICAgZW5kcG9pbnQgPSAicDIuZnJlZWRucy5jb250cm9sZC5jb20iCiAgICBuYW1lID0gIkNvbnRyb2wgRCAtIE5vIEFkcyIKICAgIHRpbWVvdXQgPSAzMDAwCiAgICB0eXBlID0gImRvcSIK" ``` diff --git a/docs/firewall-mode.md b/docs/firewall-mode.md new file mode 100644 index 00000000..cfccdb53 --- /dev/null +++ b/docs/firewall-mode.md @@ -0,0 +1,420 @@ +# Firewall Mode + +Firewall mode makes DNS policy unbypassable *while ctrld is running* by blocking outbound +connections to any IP that wasn't resolved by ctrld. On Windows, enforcement is tied to the +process lifetime - see [Enforcement lifetime](#enforcement-lifetime-what-happens-when-the-process-dies). This closes the "DNS gap" - where apps use hardcoded +IPs, direct-IP fallbacks, or alternative DNS resolvers to bypass DNS-based filtering. + +## How It Works + +1. **DNS responses feed the allowlist**: Every successful A/AAAA record resolved by ctrld + is added to an in-memory allowlist with TTL-based expiry. + +2. **Outbound connections are checked**: Before any outbound TCP/UDP connection, the + destination IP is checked against the allowlist. If it wasn't resolved by ctrld, the + connection is blocked. + +3. **Permanent entries are always allowed**: Loopback, RFC1918 private ranges, link-local, + CGNAT, multicast, ctrld's own listener, and upstream resolver IPs are always allowed. + +4. **The organization's allowed destinations are always allowed**: managed endpoints receive + an explicit list of destinations from the API that stay reachable without a DNS lookup - + see [Organization Allowed Destination IPs](#organization-allowed-destination-ips). + +## Configuration + +### TOML Config + +```toml +[service] + firewall_mode = "on" # "off" (default) or "on" + intercept_mode = "hard" # Required on desktop for enforcement +``` + +### CLI Flag + +```bash +ctrld-client start --firewall-mode on --intercept-mode hard +``` + +### Remote API + +Firewall mode can be toggled remotely via the ControlD API's `custom_config` field, +which is polled by `apiConfigReload()`. The same response carries the organization's +`destination_ips` list - see +[Organization Allowed Destination IPs](#organization-allowed-destination-ips). + +## Platform-Specific Enforcement + +### macOS (pf) + +When both firewall mode and intercept mode are active, ctrld extends the pf anchor +with a `` table: + +- Default: block all outbound traffic +- Pass: traffic to IPs in the `` table +- Pass: traffic to the organization's allowed destinations in the `` table +- Pass: traffic to loopback and link-local +- Pass: existing DNS intercept rules + +The table is dynamically updated as DNS responses arrive. Updates are batched (200ms +accumulation window) to avoid excessive `pfctl` calls. + +### Windows (WFP) + +When both firewall mode and hard intercept mode are active, ctrld extends the WFP +sublayer with dynamic permit filters: + +- Base: block all outbound traffic (low-weight filter) +- Dynamic: permit filters for each IP in the allowlist +- Static: permits for loopback, RFC1918, ctrld listener +- Organization: permit filters for each entry in the Allowed Destination IP list + +Permit filters are added/removed dynamically as the allowlist changes. + +#### Enforcement lifetime: what happens when the process dies + +ctrld opens its WFP session as a **dynamic** session, so Windows removes every filter it +added - including firewall mode's block-all - as soon as the process exits, however it +exits. That is a deliberate trade, recorded here because it changes what "unbypassable" +means on Windows: + +- **Before**: a hard kill (`taskkill /f`, a crash) left the filters installed with no ctrld + to manage them. The host was unusable rather than unfiltered, and only a reboot or a + manual WFP cleanup recovered it. A replacement ctrld could not even reach the API to + start, so it never got far enough to clean up - the deadlock this session change breaks. +- **Now**: the same kill leaves the host *unfiltered* until the service restarts. + +A clean stop or uninstall behaved this way already, and both need administrator rights, as +does killing a SYSTEM service - so the newly exposed case is specifically the hard kill of +an already-privileged process. What it costs is that an administrator can turn enforcement +off without uninstalling and without a trace beyond the service state. + +Compensating controls: + +1. **Restart policy backs off instead of burning out.** `ConfigureWindowsServiceFailureActions` + uses 5s / 30s / 2m restart delays with a 10-minute reset window, so three failures cannot + spend the whole budget inside 15 seconds and leave the host unfiltered. Repeated kills + still end in a stopped service - a bounded policy has to - but it takes minutes. +2. **Startup cleans up predecessors.** `cleanupStaleDNSInterceptState` removes filters left + by a build that predates session-scoped ownership, so an upgrade from such a build cannot + inherit the old lockout. + +Still open: enforcement stopping while policy should be active is visible only in the local +log. Reporting that state centrally is follow-up work, and is the control that would make +the hard-kill case detectable rather than merely bounded. + +### Linux and Unsupported Platforms + +Kernel enforcement is not implemented yet. On unsupported platforms, `firewall_mode = "on"` currently fails open: ctrld still records allowlist stats, but it does not block outbound traffic. A warning is logged at startup so this is visible. Future work: iptables/nftables rules or eBPF, and possibly a strict mode that fails closed when platform enforcement is unavailable. + +## Permanently Allowed IPs + +These IPs are always allowed regardless of DNS resolution: + +| Range | Reason | +|-------|--------| +| `127.0.0.0/8`, `::1` | Loopback - local services | +| `10.0.0.0/8` | RFC1918 - LAN, printers, NAS | +| `172.16.0.0/12` | RFC1918 - LAN | +| `192.168.0.0/16` | RFC1918 - LAN | +| `169.254.0.0/16`, `fe80::/10` | Link-local - DHCP, mDNS | +| `100.64.0.0/10` | CGNAT - Tailscale, carrier NAT | +| `224.0.0.0/4`, `ff00::/8` | Multicast - mDNS, SSDP | +| ctrld listener IPs | Self - DNS proxy must be reachable | +| Upstream resolver IPs | DoH/DoT/DoQ endpoints | + +## Organization Allowed Destination IPs + +Firewall Mode only permits what ctrld resolved, so a service addressed by literal IP - with +no DNS lookup to observe - is unreachable. An organization can publish a list of destinations +that stay reachable anyway, without having to turn Firewall Mode off. + +The API sends the *effective* list for the endpoint's organization in the `destination_ips` +field of every resolver-config response: the organization's own entries plus any inherited +from a parent organization that applies its settings to sub-organizations. Entries are IPv4 +or IPv6 addresses (bare, e.g. `203.0.113.10`) or CIDRs (e.g. `198.51.100.0/24`, +`2001:db8::/48`). Nothing is configured locally - the list is not a TOML setting. + +How it is applied: + +- **As a set, not as additions.** Every refresh - the scheduled one and a forced + `apiConfigReload` - replaces the previous set. An entry added upstream takes effect on the + next refresh; an entry removed upstream stops bypassing Firewall Mode on the next refresh, + unless it is independently allowed by a DNS resolution or a permanent entry. +- **Without a reload.** Applying the list does not restart listeners or reload the config, + and it is unaffected by the allowlist flushes that follow a profile change or a network + change - unlike DNS-resolved IPs, these entries carry no TTL and are never reaped. +- **Per platform.** macOS puts them in a second pf table, ``, passed by + its own rules; Windows installs a WFP permit filter per entry, at the same weight as the + dynamic permits. Both are kept apart from the DNS-resolved entries so a flush of those + leaves the organization's list installed. On Linux and other unsupported platforms the set + is tracked in memory and reported in stats, but nothing enforces it (see above). +- **Retried until enforcement agrees.** ctrld tracks the set the API asked for separately + from the set pf/WFP has accepted. A failed `pfctl` call or WFP filter operation does not + advance the applied set, so the same change is retried by the next refresh and by a + reconcile every 5 minutes - a rejected addition does not leave an approved destination + blocked, and a rejected removal does not leave a withdrawn one permitted. Until the two + agree the difference is reported as `allowed_destinations_pending` in the stats line, and + the "applied" log line is not written. +- **Enforcement startup replaces, it does not add.** When pf/WFP enforcement comes up, ctrld + knows nothing about what it holds - the macOS table is a `persist` table that outlives the + process, so it can still contain what a previous run put there, including entries the + organization has since withdrawn. The first reconcile therefore replaces the table's whole + contents (an empty list means emptying it), and that replace is retried on the same + schedule until it succeeds; only then does ctrld consider any part of the set applied. +- **Bad entries are dropped individually.** An entry that is not a valid address or CIDR is + logged once and skipped; the rest of the list still applies. +- **Addresses are logged at debug level.** The list is organization network topology, so + Info-level logging - which is persisted and travels in support bundles - carries only + counts. + +Devices with Firewall Mode off are unaffected: there is nothing to make an exception to, so +the list is ignored until the mode is turned on. + +## Live Profile Updates + +When a ControlD profile changes (domain goes from allowed → blocked or vice versa): + +1. ctrld's `apiConfigReload()` detects the change +2. The entire allowlist is flushed +3. Subsequent DNS queries repopulate the allowlist under the new policy +4. Brief connectivity interruption (~seconds) while DNS cache repopulates + +This is the "flush and repopulate" strategy - simple and correct, with a small +tradeoff of a brief connectivity blip on config changes. + +## Network State Changes + +When the device changes networks (WiFi → cellular, between WiFi networks, etc.): + +1. `monitorNetworkChanges()` detects the transition +2. The allowlist is flushed (old IPs may not be routable on new network) +3. DNS cache is also flushed (existing behavior) +4. Both repopulate naturally from new DNS queries + +## Edge Cases + +### CDN IP Rotation +A domain may resolve to different IPs over time. Each resolved IP is added independently +with its own TTL. Multiple IPs can coexist for the same domain. + +### CNAME Chains +For `foo.com` → CNAME → `bar.cdn.com` → A record, the final A/AAAA IPs are allowlisted +and associated with the original query domain (`foo.com`). + +### Short TTLs +Some CDNs use 30-second TTLs. The allowlist enforces a minimum TTL of 30 seconds to +prevent excessive churn. The background reaper runs every 30 seconds. + +### App Startup Race +Apps may attempt connections before their first DNS query reaches ctrld. This is a known +limitation. A "learning mode" grace period at startup is a future enhancement. + +### Cached Responses +When ctrld serves a response from its DNS cache, the allowlist entries are refreshed. +This prevents the case where the DNS cache outlives the allowlist TTL. + +### Long-Lived Connections and Direct-IP Retries +Firewall mode learns allowed destinations from DNS responses. If an app keeps a +long-lived connection open across a firewall/profile refresh, or retries directly +to a previously resolved IP without issuing another DNS query, the reconnect can +remain blocked until the app performs DNS resolution again. This is an accepted +v1 tradeoff and should be called out in release notes and compatibility testing +for common apps. + +## VM / Container Workloads (macOS) + +By default a VM or container resolves DNS through a path the host ctrld does not +observe (the hypervisor's own resolver on the guest bridge, or a resolver the +guest is configured to use). The guest-resolved public IP therefore never enters +``, and the guest's forwarded/NATed egress to that IP is dropped by +the blanket block - DNS "works" inside the guest but TCP/443 fails. (Tracked as +issue #569.) + +Exempting the whole bridge interface would turn the guest into a policy bypass, +so it is intentionally **not** done. Instead ctrld makes those guests first-class +Firewall Mode clients by forcing their DNS through itself. The trusted source +subnets are the **union** of: + +1. **Auto-detected VM networks (default, no config).** At pf-anchor build time an + interface is trusted only when it is up, carries an **RFC1918 IPv4** network, and + its VM ownership can be proven one of two ways: + + - **its own name is vendor-specific** - `vnic` (Parallels), `vboxnet` + (VirtualBox host-only), `vmnet` (legacy kext-based VMware Fusion on Intel); + - **it is a `bridge*` whose member list contains a vendor VM interface** + (typically `vmenet*`). This is the case for every `vmnet.framework` stack - + UTM and other Virtualization.framework guests, Docker Desktop, Multipass, and + Fusion 12.1+ NAT - where the RFC1918 gateway address sits on `bridge10x` and + the vendor-named `vmenet*` interface is an address-less member of it. Matching + on interface name alone never sees those stacks. + + Physical uplinks (`en*`), loopback, VPN tunnels (`utun*`), public ranges, and + IPv6 never qualify. Each auto-trusted subnet is logged at debug level + (`Firewall: auto-detected VM/container network for forwarded DNS`, with the + `reason` field naming the proof), and its pf rules are scoped to the interface it + was detected on (`on `), so an unrelated interface carrying the same + private range is never affected. + + A `bridge*` **name** is still not proof of anything: macOS uses that namespace + for Thunderbolt and aggregated links too (ctrld's own tunnel-change code treats + `bridge0` as physical). Membership is what distinguishes them - a Thunderbolt + bridge has `en*` members and is never trusted, however private its address. + +2. **Configured subnets (opt-in).** Needed for any stack whose ownership + auto-detection cannot prove - a VM network on a plain interface with no vendor + name, a bridge with no vendor member, or a deliberately non-RFC1918 range: + + ```toml + [service] + firewall_mode = "on" + intercept_mode = "hard" + # Only needed when auto-detection cannot prove the network is a VM network. + firewall_forwarded_sources = ["192.168.64.0/24"] + ``` + + Find the subnet with `ifconfig` on the host - for a `vmnet.framework` stack it is + the `bridge1xx` interface serving the VM. Use the network address in CIDR form; + host bits are normalized away. A configured entry matches on the source CIDR + alone - it is an admin opt-in, so it is not tied to one interface. Entries must be + **IPv4**; interception targets ctrld's IPv4 listener, so an IPv6 entry is ignored + with a warning. Config only **adds** to auto-detection; it never disables it. + + A malformed or non-IPv4 entry is dropped with a warning and the rest of the set + still applies - this field is deliberately **not** hard-validated at startup, so a + typo in an MDM-pushed subnet cannot stop ctrld from serving DNS. The warning is + logged when the set of bad entries changes, not on every internal rebuild, so a + standing typo does not fill the log. + +### Guest start/stop and network changes + +VM/container interfaces come and go while ctrld runs, and the pf watchdog does not +rebuild an anchor whose rules are still intact. ctrld therefore tracks the effective +forwarded-source set (auto-detected ∪ configured) and, whenever it changes, +rebuilds the anchor and drops the pf states of the affected subnets (targeted +`pfctl -k `, not a global state flush) so the new policy applies +immediately instead of when old states expire. A guest's in-flight connections are +re-established under the new rules. + +Reconciliation runs on interface appear/disappear, on network changes, on the +delayed post-change re-checks (a new VM network often gets its address slightly +after its interface appears), and on the pf watchdog tick - which bounds how long a +started guest can go untrusted, or a stopped guest stay trusted, to one watchdog +interval even if no network event fires. Each transition is logged with the subnets +that gained and lost trust. + +The set ctrld considers applied only advances once pf has actually accepted the new +anchor. If the write or `pfctl -f` fails, the previous set stays recorded, nothing is +flushed, a warning is logged, and the next reconciliation (at the latest the next +watchdog tick) retries the same transition - so a transient failure cannot leave the +old anchor installed while ctrld believes the change is done. + +### Supported behavior and trust boundary + +For each source subnet (auto-detected or configured), when Firewall Mode + +intercept are active, ctrld: + +- **Forces guest plaintext DNS (port 53) through ctrld** (pf `route-to lo0` onto + the existing loopback redirect). Every guest resolution is policy-enforced and + populates ``, so the guest's egress to allowed destinations is + then permitted by the same allowlist rule as the host. +- **Blocks guest IPv4 DoT (port 853)** so a guest cannot swap in an alternate + encrypted resolver to escape policy. Rules are emitted for the source's own + address family only (all sources are IPv4) - pf refuses to load an entire anchor + containing an `inet6` rule with an IPv4 source, which would take DNS interception + down with it. Guest traffic to an IPv6 DoT resolver is instead covered by the + blanket IPv6 outbound block, since such a resolver never enters ``. + +The boundary is explicit and per-subnet - a guest still **cannot** bypass Control +D policy via a direct public IP (never resolved through ctrld ⇒ never allowlisted) +or an alternate plaintext/DoT resolver. It is not an interface-wide permit. + +### Confirming what is trusted + +At startup (and on every change) ctrld logs the effective set, naming each subnet's +origin, so `firewall_forwarded_sources` can be verified without reading pf rules: + +``` +Firewall: forwarded-workload (VM/container) DNS interception active for these source subnets count=2 sources=["192.168.64.0/24 (auto-detected on bridge100)","192.168.252.0/24 (configured)"] +``` + +The interface named is where the address lives, which for a `vmnet.framework` stack +is the bridge (`bridge100`), not its `vmenet*` member. + +When the set is empty the log says so explicitly, rather than staying silent: + +``` +Firewall: no forwarded-workload (VM/container) sources — guest DNS is not intercepted. Auto-detection needs an up interface with an RFC1918 IPv4 address that is either vendor-named (vnic*, vboxnet*, vmnet*) or a bridge with a VM member (vmenet*); anything else must be listed in service.firewall_forwarded_sources +``` + +Note that `firewall_forwarded_sources` is a **local** config setting. If it is not in +`/etc/controld/ctrld.toml` on the device, ctrld has nothing to act on - check the file +itself, not only the dashboard. + +### Limitations + +- **A resolver running inside the guest is not supported** while Firewall Mode is on. + The design depends on the guest sending plaintext DNS (port 53) that host ctrld can + observe. A guest-side resolver (ctrld, systemd-resolved with DoT, dnscrypt, ...) + sends its upstream queries encrypted instead, so the host learns no addresses and + the guest's egress is blocked - and its DoT is blocked outright by the port-853 rule. + That is the trust boundary working as intended, not a regression: a guest that + resolves privately could otherwise reach any destination it liked. Point the guest + at the host bridge address (its default DHCP resolver) and let host ctrld enforce + policy for it. +- **DoH over 443** inside the guest is indistinguishable from ordinary HTTPS and + is not intercepted. To keep enforcement strict, disable DoH in the guest OS/ + browser, or restrict the guest to the host resolver. +- **IPv6 guest DNS** is not redirected (ctrld's intercept listener is IPv4); the + anchor's existing IPv6 DNS block forces guests to fall back to interceptable + IPv4 DNS. IPv6 forwarded sources are therefore unsupported: an IPv6 + `firewall_forwarded_sources` entry is ignored with a warning rather than emitted + as a rule. +- Auto-detection needs ownership proof: a vendor interface name, or a bridge with a + vendor VM member. A VM network on a plain unrecognized interface, or a bridge whose + hypervisor attaches no vendor-named member, needs an explicit + `firewall_forwarded_sources` entry. Bridge membership is read with `ifconfig`, and + only for a bridge that already carries an RFC1918 IPv4 address. +- macOS only. Windows (WFP) Firewall Mode VM behavior is tracked separately + (#568). + +## Metrics + +Allowlist stats are logged every 5 minutes: + +``` +Firewall allowlist stats allowed_ips=142 permanent_ips=18 allowed_destinations=3 allowed_destinations_pending=0 tracked_domains=89 total_hits=4521 total_misses=23 +``` + +`allowed_destinations` is the number of prefixes in the organization's Allowed Destination +IP list. `allowed_destinations_pending` is how many of its changes platform enforcement has +not accepted yet - normally 0; a non-zero value that persists across reconciles means +`pfctl`/WFP keeps rejecting the change, and the warning that named the error is in the log. + + +## Troubleshooting + +### Everything is blocked +- Check that the upstream resolver IPs are in the permanent allowlist (logged at startup) +- Verify DNS is working: `nslookup example.com 127.0.0.1` +- Check allowlist stats for hit/miss ratio + +### Certain apps don't work +- The app may be using hardcoded IPs (this is the intended behavior - those IPs aren't DNS-resolved). + For an approved service, add its addresses to the organization's Allowed Destination IP list, + then either wait for the next scheduled refresh (`refetch_time`, hourly by default) or force + one by resolving `.verify.controld.com` through ctrld, which is the only trigger that + makes ctrld re-fetch its resolver config on demand. The applied set is logged as `Firewall: + applied organization allowed destination IPs` and counted as `allowed_destinations` in the + stats line. If that line does not appear, check for `could not apply all organization allowed + destinations` (a delta that enforcement rejected) or `could not install organization allowed + destinations, will retry` (the full install done when enforcement starts, or after a resync), + along with the `allowed_destinations_pending` count - enforcement is refusing the change and + the reconcile is retrying it +- Check if the app uses a custom DNS resolver that bypasses ctrld +- RFC1918 traffic is always allowed, so LAN-only apps should work + +### High miss count +- Normal for the first few seconds after startup or network change +- Persistent high misses may indicate apps using hardcoded IPs extensively diff --git a/docs/intercept-mode-default.md b/docs/intercept-mode-default.md new file mode 100644 index 00000000..7c2cf74f --- /dev/null +++ b/docs/intercept-mode-default.md @@ -0,0 +1,149 @@ +# Intercept Mode Default + +On the supported desktop platforms, macOS and Windows desktop (workstation) +editions, a new ctrld installation runs in DNS intercept mode (`dns`) unless +you select a mode explicitly. Everywhere else, Windows Server included, it +keeps interface DNS mode, which sets ctrld's listener as the DNS server of the +network interfaces. An existing installation keeps the mode it had: upgrading +never switches a Standard (interface DNS) install to intercept mode. See +[Upgrades](#upgrades). + +Windows Server is excluded on purpose. A server, and a domain controller +running its own DNS server in particular, is not switched to intercept mode by +default. Intercept mode still works there when selected explicitly. + +This applies to a service that `ctrld-client start` installs and to `ctrld-client run` in +the foreground alike. See [DNS Intercept Mode](dns-intercept-mode.md) for what +intercept mode does. + +## Precedence + +The effective mode is the first of these that is set: + +1. `--intercept-mode` on the command line, or in the service's arguments. +2. `service.intercept_mode` in the configuration the service runs with: + `ctrld.toml`, a `--base64_config`, or a Control D custom configuration. +3. The default: `dns` on macOS and Windows desktop editions; interface DNS + mode elsewhere, Windows Server included. An installed service that predates + the default is the exception, see [Upgrades](#upgrades). + +`off` is a selection like `dns` and `hard`, not an absence of one. At either +explicit level it stops the lookup and selects interface DNS mode, including +over a persisted `intercept_mode = "dns"`. + +| `--intercept-mode` | `service.intercept_mode` | macOS / Windows desktop | Windows Server | Linux and others | +|---|---|---|---|---| +| unset | unset | `dns` (default)² | interface DNS mode | interface DNS mode | +| unset | `dns` / `hard` | that mode | that mode | that mode¹ | +| unset | `off` | interface DNS mode | interface DNS mode | interface DNS mode | +| `dns` / `hard` | any | that mode | that mode | that mode¹ | +| `off` | any | interface DNS mode | interface DNS mode | interface DNS mode | + +² For a fresh install or a foreground run. An installed service that predates +the default keeps interface DNS mode, see [Upgrades](#upgrades). + +¹ Intercept mode is not supported outside Windows and macOS, so an explicit +`dns` or `hard` there fails to start intercept, as before. An unset mode never +tries: those platforms do not log an intercept-start failure just because the +default changed. + +Desktop versus server follows `ctrld.IsDesktopPlatform`, which ctrld already +uses elsewhere: on Windows it is the workstation product type +(`VER_NT_WORKSTATION`), so Windows Server and domain controllers are not +desktop platforms. +iOS is not a desktop platform here either, although it builds ctrld's macOS +code: it has no pf to intercept with. + +The default also keeps interface DNS mode for a configuration that cannot run +in intercept mode. On macOS that is any `os` or untyped upstream: pf would +redirect ctrld's own port-53 queries to it back to ctrld. Such a configuration +worked in interface DNS mode and keeps doing so; the service log says why the +default was not used. Selecting `dns` or `hard` explicitly for it still fails +to start intercept, as before. + +The service decides the default itself, on every start, against the +configuration it runs with. In Control D mode (`--cd`) that is the resolver's +configuration it fetches, so an `os` upstream or an `intercept_mode` in a +custom configuration counts, including one added after installation. + +## Opting out + +Use interface DNS mode explicitly: + +```shell +# Windows (Admin Shell) +ctrld-client.exe start --intercept-mode off --cd RESOLVER_ID_HERE + +# macOS +sudo ctrld-client start --intercept-mode off --cd RESOLVER_ID_HERE +``` + +On an installed service, `ctrld-client start --intercept-mode off` with no other +arguments switches the existing service to interface DNS mode, keeping its +other arguments. + +The selection is stored in two places, so it survives restarts and upgrades: + +- in the service's arguments, as `--intercept-mode off`; +- in `ctrld.toml`, as `intercept_mode = "off"`. + +Earlier builds cleared `intercept_mode` when switching to `off`. That no longer +works as an opt-out: an unset value now means the platform default, so +switching to `off` writes `off`. Older builds read `off` the same way, so a +downgrade keeps interface DNS mode too. + +## Upgrades + +Only a fresh installation gets the `dns` default. An existing Standard +(interface DNS) installation stays in Standard mode across upgrades, whichever +path upgrades it. + +To tell the two apart, `ctrld-client start` marks the services it installs, in the +service's arguments, when no mode is selected: + +- A fresh install records `--intercept-mode-default=true`, an internal flag. + It ranks lowest: the service applies the default only if neither + `--intercept-mode` nor `service.intercept_mode` selects a mode, against the + configuration it fetched. +- A reinstall over an existing service keeps what that service recorded, its + mode or the marker. If it recorded neither, it was installed in Standard mode + before this default, and the reinstall records `--intercept-mode off`, unless + `service.intercept_mode` selects a mode. +- Platforms without the `dns` default record nothing, as before. + +Older ctrld builds ignore flags they do not know in the service's arguments, +so the marker is safe if a device is downgraded. + +An installed service with neither a mode nor the marker therefore predates +the default. It keeps interface DNS mode when the new binary runs it, and the +service log says why. That covers upgrades that rewrite nothing, such as +`ctrld-client upgrade` run by an older binary, which swaps the binary and restarts the +service with its existing arguments. A foreground `ctrld-client run` with no mode +selected uses the default. + +The macOS package pins such a service explicitly when it upgrades it with a +profile that has no `InterceptMode`: it runs `ctrld-client start --intercept-mode +off`, the same as for a `standard` profile. See +[macOS MDM Package](macos-pkg-mdm.md). + +To move an existing Standard installation to intercept mode, select it +explicitly with `ctrld-client start --intercept-mode dns`. + +The Control D installer and dashboard should generate `--intercept-mode off` for +a device where Standard mode is chosen, so a fresh install never gets intercept +mode against that choice. + +## Where the mode is reported + +- `ctrld-client start` output: `DNS intercept mode active ...` once the service runs + in intercept mode, followed, when the default selected it, by how to switch + to interface DNS mode. A reinstall over a Standard install prints `Existing + service selects no intercept mode; keeping interface DNS mode + (--intercept-mode off)`. +- The service log, at startup: `Intercept mode enabled via config` when the + configuration set the mode, `Intercept mode "dns" is the default on this + platform; ...` when the default applied, or `Intercept mode default not + applied; using interface DNS mode` with the reason, for a service that + predates the default or a configuration that cannot run in intercept mode. +- The log header field `intercept_mode`, which reads `off` for interface DNS + mode. diff --git a/docs/internal-domains.md b/docs/internal-domains.md new file mode 100644 index 00000000..449d1bb3 --- /dev/null +++ b/docs/internal-domains.md @@ -0,0 +1,246 @@ +# Organization Internal Domains + +Internal Domains let an organization administrator route a domain suffix, and +every subdomain under it, to a chosen resolver. The selection is made in the +Control D dashboard and reaches the endpoint through the managed resolver +configuration, so no hand-edited `ctrld.toml` is involved and changes apply on +the next configuration refresh. + +## API contract + +The managed resolver configuration carries the list in `resolver.split_dns`: + +```json +{ + "resolver": { + "split_dns": [ + { "domain": "corp.example.com", "mode": "resolvers", "resolvers": ["10.0.0.53", "10.0.0.54"] }, + { "domain": "office.example.com", "mode": "os", "resolvers": [] }, + { "domain": "vault.example.com", "mode": "resolvers_only", "resolvers": ["10.0.0.55"] } + ] + } +} +``` + +`mode` is what the administrator selected, and it decides the routing: + +| `mode` | Behavior | +|---|---| +| `os` | **Network default.** The query goes to `upstream.os`, the endpoint's current system resolver, and follows DHCP and VPN resolver changes. Any addresses still present in `resolvers` are **not** read, so switching a domain back to the OS resolver cannot be undone by leftovers from a previous selection. | +| `resolvers` | **Explicit resolver with network fallback** (the default explicit selection). The query goes to the addresses in `resolvers` first, tried in the order given; if none of them resolves it, it falls back to the network resolvers (see [Network fallback](#network-fallback)). | +| `resolvers_only` | **Explicit resolver only.** The query goes only to the addresses in `resolvers`, and their answer or failure is final. Not offered by the dashboard yet. | +| absent | A deployment that predates the field. Treated as **explicit resolver with network fallback**, the same as `resolvers`. If the entry has no resolvers there is nothing explicit to try, so it uses the network default (`os`). | +| anything else | Fails closed. With resolvers, treated as `resolvers_only`: only the resolvers the administrator chose, so a wire value this build does not know (a renamed strict mode, a future mode) never sends the domain to the Control D upstream or the network. Without resolvers there is nothing to fail closed to, so the entry is dropped. | + +In both explicit modes an empty or entirely unusable list does not become OS +resolution: the entry is dropped instead. A dropped entry leaves its domain on +its normal path and is reported in the `skipped` count. + +Addresses may be IPv4 or IPv6, bare or with a port; a bare address uses port 53. +An address that is not a valid IP is dropped from the list. + +A domain is lowercased and stripped of surrounding whitespace, a leading `*.` +and the root dot, then validated as an ASCII hostname suffix: 1-63 character +labels, letters, digits and hyphens only, no leading or trailing hyphen, 253 +characters overall. Single-label names and punycode are supported. This is the +same rule the API validates against, so anything the dashboard accepted passes. +A value that fails is rejected and counted, never repaired — deleting the +offending characters would produce a different suffix from the configured one, +and routing a private domain to the wrong place is worse than not routing it. + +## Generated configuration + +Each accepted domain contributes two policy rules, `` and `*.`, +which is how ctrld matches a suffix and all of its subdomains. The explicit +modes also generate one upstream per resolver address, named `internal_`. +Each one carries an internal marker recording its mode, which is what tells +`proxy()` that an upstream is a generated Internal Domain resolver and whether +the network fallback may run. The marker is never read from or written to a +configuration file, so the key name is only a name: an upstream a local +`ctrld.toml` or a custom configuration defines as `internal_foo`, or even as an +exact copy of a generated one, is an ordinary upstream. The generated +configuration looks like this: + +```toml +[upstream.internal_0] + type = "legacy" + endpoint = "10.0.0.10:53" + +[listener.0.policy] + rules = [ + { "aws.example.com" = ["upstream.internal_0", "upstream.internal_1"] }, + { "*.aws.example.com" = ["upstream.internal_0", "upstream.internal_1"] }, + { "corp.internal" = [] }, + { "*.corp.internal" = [] }, + ] +``` + +An empty target list is the same representation Magic Folder excludes use, and +`proxy()` resolves it through `upstream.os`. + +## Precedence + +The generated configuration is built in a fixed order, and a rule that already +exists for a source is never overwritten. From highest precedence to lowest: + +1. **Endpoint custom configuration.** A valid `ctrld.custom_config` replaces the + generated configuration outright. Internal Domains are not merged into it, + the same as Magic Folder excludes — if an organization uses a custom config, + its rules are the whole policy. +2. **Magic Folder excludes** (`resolver.exclude`). These are written first, so an + excluded domain keeps its exclusion even if it also appears as an Internal + Domain. Only the exact source collides: an exclude for `aws.example.com` does + not stop the Internal Domain from claiming `*.aws.example.com`. +3. **Internal Domains** (`resolver.split_dns`), most specific suffix first. + The API permits overlapping suffixes and returns the list sorted bytewise by + domain, which puts a parent ahead of its children. Policy rules are matched + in order and the first hit wins, so appending in API order would let + `*.example.com` swallow every query a configured `z.example.com` was meant to + answer. Entries are therefore ordered by label count before generation, and + the ordering is stable, so equally specific entries keep API order and the + first of two identical domains still wins. +4. **Active Directory auto-detection** (Windows). `addExtraSplitDnsRule` runs + after the generated configuration and skips any domain that already has a + rule, so an Internal Domain for the AD domain takes precedence over the + auto-detected one. + + A query under the AD domain whose matched rule resolves to the OS resolver + alone (the auto-detected rule, a Magic Folder exclude, or an OS-resolver + Internal Domain) is a LAN query, so the OS resolver does not race Control D's + public resolver (76.76.2.0) against the domain controller. Before this, a + rule match returned ahead of the LAN classification and every such lookup + (DC hostnames, DC-locator SRV names that carry the machine name, WPAD) also + went to 76.76.2.0 in plaintext; when both failed, the public NXDOMAIN could + be the answer returned. An explicit-resolver rule for the AD domain is + unchanged. The same LAN mark applies wherever an AD name reaches the OS + resolver outside rule handling: the DNS-intercept recovery bypass, and the + OS-resolver retry after an explicit upstream fails with + `leak_on_upstream_failure` enabled. + +**VPN DNS auto-detection** is evaluated inside `proxy()` in DNS-intercept mode, +after policy matching and independently of it, so it needs its own rule: + +- An **explicit-resolver** Internal Domain skips VPN routing ahead of its + configured resolvers. The administrator named the resolvers for that suffix; + VPN suffixes are discovered from the OS, and a discovered route must not + override a named one. In `resolvers` mode the VPN DNS servers are asked + afterwards, as part of the [network fallback](#network-fallback). +- An **OS-resolver** Internal Domain does not skip it, and behaves exactly like + a Magic Folder exclude. It asks for the endpoint's default resolution, which + under intercept mode includes the VPN's own resolver. + +Every domain that is not an Internal Domain keeps its current VPN routing. + +## Network fallback + +In `resolvers` mode the configured resolvers are always asked first, even while +the DNS-intercept recovery bypass is active and ahead of VPN DNS split routing. +A configured resolver that times out, is unreachable, or answers SERVFAIL, +NXDOMAIN, REFUSED or NOTIMP hands the query to the next configured resolver, +and once all of them have, to the network resolvers: + +1. VPN DNS servers whose domains match the query, then VPN DNS servers that + advertise no domains (DNS-intercept mode only, where ctrld tracks them). +2. The OS resolver's **LAN** nameservers: private, loopback, link-local and + CGNAT addresses the active network provides. + +A configured resolver that the upstream monitor has marked down (after 10 +seconds of continuous failures, or 50 failures) is skipped rather than waited +on, so an endpoint away from the organization network reaches the fallback +without paying the resolver's timeout, about 2 seconds per resolver, on every +query. Queries in the first seconds off-site, before the resolver is marked +down, still pay it. A generated resolver comes back up only by answering, so +skipping one also re-checks it in the background, at most once every 30 +seconds, with the query that was skipped. Once it answers it is used again. +`resolvers_only` mode never skips: with no fallback to go to, skipping would +only turn the resolver's answer into SERVFAIL. + +The first NOERROR answer wins. NXDOMAIN triggers the fallback because the +configured resolver may be reachable but not know a name that the local +network's resolver does, for example on another subnet. REFUSED and NOTIMP +trigger it because they mean the resolver will not serve this endpoint, which +is how an organization resolver with an access list answers an endpoint off +the organization network. An empty NOERROR (the name exists but has no record +of the requested type) is final, and so is any other rcode. + +The OS step never sends the name to public DNS. It is a LAN-only query: a +public nameserver in the OS pool is not asked at all, whether DHCP supplied it +(a router handing out `192.168.1.1` plus `8.8.8.8`, a Wi-Fi handing out only +public resolvers) or it is ctrld's own public fallback (`76.76.2.0`). With no +LAN nameserver the OS step sends nothing. The general OS-resolver catch-all, +which does reach public nameservers, is never used either. The VPN step uses +the servers the VPN pushes as they are, including a public address a VPN +pushes as its DNS server. + +While Windows is serving retained VPN DNS state, a VPN transport failure stops +the fallback before the OS resolver, as it does for VPN split routing. + +When no resolver resolves the name, the query ends with the best negative +answer it got (NXDOMAIN before any other), or SERVFAIL if it got none. + +## Unreachable explicit resolvers + +In `resolvers_only` mode, when every configured resolver fails the query returns +SERVFAIL, and an NXDOMAIN or SERVFAIL answer from them is returned as it is. +Either way the query is not retried anywhere else. Administrators who need the +configured resolvers' answer enforced select this mode. + +In neither explicit mode does a failure trigger the endpoint recovery flow. That +flow exists for the loss of general DNS. One unavailable internal server is not +that, and letting it start recovery would reset endpoint DNS settings for every +other domain. + +OS-resolver mode has no such restriction: it is the system resolver, so it +follows DHCP and VPN changes and shares their failure handling. + +## Self-uninstall eligibility + +ctrld treats a plain managed install — one Control D upstream and no endpoint +custom configuration — as eligible for the REFUSED-triggered deletion check, +which is the fast path that notices a device has been deleted server-side. +Generated `internal_*` upstreams do not count toward that, because they come +from the managed configuration itself rather than from an operator choosing a +second upstream. Eligibility is recomputed on every setup and reload, so adding +or removing Internal Domains, or gaining a genuinely custom second upstream, +is reflected instead of latching on the first value seen. + +Discounting is by the internal marker, not by key name or any other +configurable field: an endpoint custom configuration replaces the generated +config outright and could define an upstream `internal_0` that copies a +generated one, which must not buy eligibility. The uninstall itself remains +gated on the API confirming the device is gone. + +## Refresh + +`apiConfigReload` compares the stored `split_dns` against each fetch. Domains +are compared in canonical form and independently of API ordering, so an +unchanged list does not reload; resolver order is significant, because it is the +administrator's failover order. Any add, removal, domain change or resolver +change triggers a reload, which regenerates the configuration from the API +response. Because the generated configuration is rebuilt rather than patched, a +removed domain leaves no rule and no `internal_*` upstream behind. + +## Logging + +Generated `internal_*` upstreams carry the organization's private resolver +addresses, so anything naming one stays at debug, and the level above it carries +only a classification (`timeout`, `refused`, `unreachable`, `source_unavailable`, +`error`). The per-query error sampler applies to the classification line as +well. After 5 lines in one minute for the same upstream and class, the next +classification lines go to debug, and one `Per-query errors sampled` line gives +the count. That covers every consumer, not just query handling: + +- the bootstrap-IP line in `setupUpstream`; +- the transport error from a failed query, which names the endpoint; +- the **loop checker**, which probes every local upstream on a one-minute + ticker — a generated resolver is always a local address, so this path runs + with no user query at all; +- the **recovery checker**, which takes every non-OS upstream. Keeping an + Internal Domain query from *starting* recovery does not keep its resolver out + of recovery started for another reason. + +Info and warning logs report counts only — domains applied, OS-resolver and +explicit-resolver totals (with `explicit_resolver_only` for `resolvers_only`), +generated resolvers, skipped and preempted entries. +Domain names and resolver addresses are organization-private and appear only at +debug level. diff --git a/docs/ipv6-only-dns-target-qa.md b/docs/ipv6-only-dns-target-qa.md new file mode 100644 index 00000000..e37d2f92 --- /dev/null +++ b/docs/ipv6-only-dns-target-qa.md @@ -0,0 +1,29 @@ +# IPv6-only DNS-target validation + +Relates to #617. Native CLAT/PF behavior is not proven by parser tests or cross-compilation. This change must not disable IPv6 DNS enforcement, change PF routing, or reclassify every failed DHCP command as empty DNS. + +## Regression gates + +- Ordinary static IPv4 DNS and successful DHCPv4 discovery retain their existing behavior. +- An explicit non-owned IPv6 loopback resolver (`::1`), alone or mixed with other static DNS, prevents the expanded fallback. Its settings and backup must remain unchanged through reconciliation and cleanup. +- The target-specific DHCP lookup shares a two-second deadline across both commands, with a 100 ms pipe-cleanup grace. Timeout remains an error, not confirmed absence. This does not impose a deadline on legacy global discovery or every operation under the target lock. +- An unknown, truncated, malformed, timed-out or permission-denied native-state read authorizes no new DNS mutation. +- A CLAT-looking address alone is insufficient. Positive native publication must match the current default interface/service, with no contradictory native IPv4 state. +- Confirmed IPv6-only state can reach the existing listener-aware target installation path despite unavailable DHCPv4 data. +- Repeated reconciliation is idempotent. Existing targets, external DNS changes, default-service switches, DHCP return and restart/stop cleanup preserve ownership. +- A failed preservation snapshot cannot authorize the new fallback write. +- Logs distinguish configured IPv6 blocking, observed rule counters, missing evidence, counter resets and successful local receipt. Shared counters cannot identify which packet was blocked. +- Diagnostics are bounded, omit raw PF dumps/query names and do not cause DNS/PF changes or repeated repair actions. + +## Native comparison + +Use an owned disposable Mac with a local console, an identified baseline/candidate and the same profile/listener. Save exact original DNS settings. Do not run host-mutating tests on shared CI hosts; native unit tests must intercept the first mutation boundary. + +1. Verify a healthy baseline on ordinary IPv4/DHCP, then establish actual host-side CLAT using an owned hotspot or isolated NAT64/PREF64 fixture. Record the interface/service mapping, native state, routes, static DNS, raw scutil DNS and both DHCP commands' exit/output. A carrier label or synthetic address assignment is not a fixture. +2. Confirm upstream and direct-listener resolution independently. Keep a pre-existing manual loopback DNS override from masking the target decision, without deleting unrelated configuration. +3. Compare startup already attached, disconnected startup followed by attachment, and a same-process transition to the IPv6-only network. Test sleep/wake separately and exclude intentionally offline/sleep periods from recovery measurements. +4. On the candidate, require the appropriate temporary target and fresh system-resolution success while still on CLAT. Record any independent PF-probe failure rather than treating working system DNS as proof of repaired transparent interception. +5. Return to IPv4/DHCP; verify target removal/restoration. Repeat with explicit static DNS, an existing tracked target, external DNS changes and an injected unknown/read-error state. Stop/uninstall must not leave DNS pointing to a dead listener. +6. Read the retained diagnostics after a failed probe and after restored receipt. Distinguish missing rules/read errors from a present IPv6 block with cumulative traffic counters. A reload or rising shared counter does not prove a particular probe's fate. + +Preserve captures privately and record the exact artifact. A successful owned-lab case supports the scoped fix; a customer retest remains a separate validation gate. diff --git a/docs/known-issues.md b/docs/known-issues.md index 0d13bccf..7c757010 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -14,7 +14,7 @@ This document outlines known issues with ctrld and their current status, workaro **Workarounds**: 1. **Recommended**: Upgrade your macOS system to Darwin 15.6 or later, which has been tested and verified to work correctly with ctrld self-upgrade functionality. -2. **Alternative**: Run `ctrld upgrade prod` directly to manually upgrade ctrld to the latest version on Darwin 15.5. +2. **Alternative**: Run `ctrld-client upgrade prod` directly to manually upgrade ctrld to the latest version on Darwin 15.5. **Affected Versions**: ctrld v1.4.2 and later on macOS Darwin 15.5 @@ -22,6 +22,78 @@ This document outlines known issues with ctrld and their current status, workaro --- +## Windows Issues + +### VPN `block-outside-dns` Breaks DNS When Using ctrld in DNS Mode + +**Issue**: VPN software that uses OpenVPN's `block-outside-dns` directive installs WFP (Windows Filtering Platform) block filters that prevent DNS queries from reaching ctrld's loopback listener. + +**Status**: Fixed in v1.5.1 + +**Description**: When a VPN connects with `block-outside-dns` enabled, OpenVPN adds WFP filters that block all DNS traffic to non-tunnel interfaces — including loopback (`127.0.0.1`). Since ctrld's NRPT catch-all rule routes DNS through the Windows DNS Client to `127.0.0.1:53`, the WFP block filters prevent DNS Client from reaching ctrld, causing all DNS queries to time out. + +This affects any VPN client that implements `block-outside-dns` via WFP, including: +- OpenVPN GUI (community) +- Securepoint SSL VPN +- Any OpenVPN-based client that honors the `block-outside-dns` push directive + +**Fix**: ctrld now proactively adds WFP "hard permit" filters for DNS to localhost at startup. These use `FWPM_FILTER_FLAG_CLEAR_ACTION_RIGHT` to override block decisions from any other WFP sublayer, ensuring the NRPT → loopback path is always available regardless of VPN state. See `docs/dns-intercept-mode.md` for technical details. + +**Affected Versions**: ctrld ≤ v1.5.0 in `dns` intercept mode on Windows + +**Last Updated**: 04/28/2026 + +--- + +### DNS Stalls for Up to ~100 Seconds After Start, and the Install Rolls Back + +**Issue**: On Windows in `dns` intercept mode, a freshly started ctrld can leave every query unanswered for a minute or more. The start self-check fails in the meantime and rolls the install back (`SERVICE_SELFCHECK_FAILED`, exit 53). This happens with `ctrld start --config ` and also with `ctrld start --cd `. + +**Status**: Fixed on the `v1.0` branch in `c40cc8d7`, not yet released. v1.5.7 is affected. + +**Description**: The first DoH request sets up the upstream transport, and that setup calls `ctrld.HasIPv6()`. In the affected versions `HasIPv6()` also started a Tailscale `netmon` monitor inside its `sync.Once`. Building the monitor's first interface state calls `getPACWindows`, which asks WinHTTP for proxy auto-discovery (WPAD). WPAD discovery resolves names through the Windows DNS Client. NRPT already sends those lookups to ctrld, and ctrld cannot answer until the same `sync.Once` returns. So every DNS request waits behind proxy discovery, and proxy discovery waits on DNS, until WinHTTP's own timeouts expire. Then all pending queries complete at once. + +It is a race between WinHTTP's proxy discovery and ctrld installing its NRPT rule: + +- If discovery finishes before the rule is in place, nothing happens. +- With `--cd`, `HasIPv6()` is first called during the API fetch, only about 75 to 95 ms before the NRPT rule becomes active. That window is too short wherever discovery is slow, so `--cd` installs are affected too. +- With `--config`, nothing calls `HasIPv6()` before the rule, so the stall is near-certain. +- A network where the proxy-discovery lookups are answered quickly outside ctrld avoids it. On a domain member, for example, the AD domain's names go to the domain controller through the auto-added AD split rule. +- How long the stall lasts depends on how long WinHTTP's discovery takes on that network. + +A goroutine dump from an affected build (`ecf7955e`) taken during the stall shows the DoH transport setup blocked in the syscall, with 111 DNS request goroutines parked in `sync.Once` behind it: + +``` +UpstreamConfig.dohTransport → ensureSetupTransport (sync.Once) + → SetupTransport → ctrld.HasIPv6 (sync.Once) + → netmon.New → netmon.GetState → getPACWindows [Windows syscall] +``` + +While the stall lasts, ctrld opens no connection to its upstream, although the network and the upstream both answer normally. Windows' certificate service lookups (`ctldl.windowsupdate.com`, Sectigo/USERTrust OCSP and CRL hosts) also queue in ctrld, but they are a consequence of the stall, not its cause. Warming Windows' certificate state before the start does not prevent it. + +**Fix**: `c40cc8d7` ("put the network state of the host in the log journal") removed the monitor from `HasIPv6()`. The process's one network monitor now reports IPv6 state through `SetIPv6Available`, and `HasIPv6()` only runs a bounded IPv6 check. On master (2.0) the same change arrived with `c5ad04ab`. + +**Workarounds** (affected versions): +1. **Recommended**: Add `--skip_self_checks` so the service is not rolled back. DNS then recovers by itself once WinHTTP's proxy discovery times out, after about 100 seconds in the QA lab. +2. **Alternative**: Retry the install. Whether it stalls depends on how quickly proxy discovery finishes, so a second attempt can succeed. +3. Take a build with `c40cc8d7` once one is released. + +**Reproduction** (ctrld-qa lab, workgroup endpoint on the simulated home network): + +- `config-file-upstream-ready` starts ctrld from a local config and requires Control D resolution within 20 seconds. + - It fails on every build that has the monitor in `HasIPv6()`: 10 of 10 runs, on the v1.5.7 release and tag, `ecf7955e`, and 2.0 commits before `c5ad04ab`. + - It passes on every build without the monitor: 9 of 9 runs, on `c40cc8d7`, the `v1.0` tip `cc849871`, and master from `6dcdddc2`. +- `smoke-lifecycle` is a normal `--cd` install. + - The v1.5.7 release failed its start self-check in 4 of 4 runs, each with the same signature: `HasIPv6()` 75 to 95 ms before the NRPT rule, then no upstream reply until the rollback. + - The `v1.0` tip `cc849871` passed. +- On the domain-joined endpoint, `--cd` installs pass. + +**Affected Versions**: ctrld ≤ v1.5.7 on Windows, `dns` intercept mode. Near-certain with `--config`; with `--cd`, it depends on how quickly WinHTTP proxy discovery finishes on the network. + +**Last Updated**: 09/30/2026 + +--- + ## Contributing to Known Issues If you encounter an issue not listed here, please: diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md new file mode 100644 index 00000000..a24a8ebc --- /dev/null +++ b/docs/network-recovery-diagnostics.md @@ -0,0 +1,305 @@ +# Network-recovery diagnostics + +The retained stream is the journal, the file `ctrld-journal.log` in the ctrld home directory. +It keeps every warning and every error, plus every event marked `journal=true`. +The journal is on disk, so it keeps these events across a restart and a self-upgrade. +Its budget is 2 MB with 2 backup files. +Every `ctrld log send` upload holds the whole journal, so a reproduction does not need a continuous `ctrld log tail` capture. +Match timestamps and IDs. The last retained error does not describe the current network in every case. +Each upload starts with a header line that names the build, the intercept mode, and the network at send time. + +## Journal events + +These events describe the host network. Each one carries `journal=true`, so it stays on disk after a rotation and after a restart. +netmon reports every network change as a delta, which holds the state before the change and the state after it. +The table gives the trigger, the level, and the fields of each event. + +| Event | Trigger | Level | Fields | +|---|---|---|---| +| `Network snapshot` | Start, an accepted transition, a recovery begin, and a recovery end. Two limits hold some of them back | info | `trigger` (`start`, `transition`, `recovery_begin`, `recovery_end`), `network` | +| `Network interface changed` | One event per changed interface of a delta that is not noise. A mixed delta gives an event for the interfaces outside the noise class only | info | `transition_id`, `interface`, `action`, `class`, `hardware_port`, `service`, `ips_before`, `ips_after`, `flags`, `mtu`, `is_default_route` | +| `Network transition` | The end of every network callback that is not noise | info, debug for the outcomes `ignored` and `snapshot_superseded` | `transition_id`, `outcome`, `is_major_change`, `changed`, `interface`, `changed_interfaces`, `time_jumped`, `default_route_before`, `default_route_after`, `have_v4_before`, `have_v4_after`, `have_v6_before`, `have_v6_after`, `source_ipv4_before`, `source_ipv4_after`, `source_ipv6_before`, `source_ipv6_after`, `source_snapshot_available` | +| `Network delta noise` | Each noise delta, and one journal line per 10 minutes of a storm | debug, info for the journal line | `interfaces`, `count`, `first_at`, `last_at` | +| `Host woke` | One event per wake: a time jump in a netmon delta, or the macOS wake detector, which runs in DNS intercept mode only | info | `gap_known`, `gap_ms` (only when `gap_known` is true), `source` (`netmon`, `detector`), `default_route`, `interfaces_up` | +| `OS resolver set changed` | A nameserver read that differs from the read before | info | `source` (`scutil`, `dhcp`, `resolv.conf`, `unknown`), `before`, `after`, `default_route`, `reason` (`start`, `recovery`, `delayed_recheck`, `wake_probe`, `vpn_settle`, `unspecified`) | +| `Reinitialized OS resolver with nameservers: …` | ctrld read the nameservers of the system again and built the OS resolver again | info | none, the nameservers are part of the message | +| `DNS configuration changed` | A `scutil --dns` poll that finds a changed resolver, one event per resolver (macOS) | info | `nameservers`, `if_index`, `interface`, `scoped`, `action` (`added`, `changed`, `removed`), `flags`, `search_domain_count`, `order`, `reachable` | +| `Recovery begin` | The start of every recovery | info | `transition_id`, `recovery_generation`, `recovery_reason`, `intercept`, `probe_upstreams` | +| `Recovery skipped: configured upstream is not marked down` | An OS-only failure does not start global recovery because a configured candidate is not marked down. First/changed candidate, then at most once per 5 minutes for an unchanged candidate | info | `recovery_reason`, `healthy_upstream` | +| `Recovery end` | The end of every recovery | info, warn for a canceled recovery and for a recovery with a failure | `transition_id`, `recovery_generation`, `recovery_reason`, `outcome`, `had_failure`, `duration_ms`, `recovered_upstream`, `dhcp_servers`, `dhcp_server_count`, `intercept_target_action`, `bypass_active` | +| `Upstream … recovered …` | An upstream that was down answers again | info | `down_for_ms` | +| `Recovery canceled with no successor …` | A recovery stops and no other recovery follows it | info | none | +| `DNS intercept recovery: enabling DHCP bypass …` | ctrld sends the queries of a recovery to the OS resolver | info | none | +| `DNS intercept recovery complete: disabling DHCP bypass …` | The recovery ends and the normal query path returns | info | none | +| `DNS intercept recovery: found DHCP nameservers: …` | ctrld read the DHCP nameservers of the service | info | `dhcp_servers` | +| `DNS intercept recovery: no DHCP nameservers found` | The service gave no DHCP nameserver | warn | none | +| `intercept DNS target: service … set …` | ctrld wrote the loopback target into a network service | warn | `service`, `target`, `reason` | +| `intercept DNS target: removed …` | ctrld took the loopback target out of the service | info | `service`, `target`, `reason` | +| `intercept DNS target: … DNS changed externally …` | Another program changed the DNS of the service, so the target of ctrld is gone (macOS) | info | `service`, `target`, `reason` (`external_change`) | +| `dns64: discovered NAT64 prefix …` | A NAT64 prefix that differs from the result before | info | none | +| `dns64: no NAT64 prefix present …` | A no-prefix result that differs from the result before | info | none | +| `DNS intercept: interface appeared/disappeared …` | An interface appeared or disappeared, and the probe monitor starts | info | `interface`, `class`, `action` | +| `PF anchor list changed` | A missing anchor, a restored anchor, a stabilization start, and a stabilization end (macOS) | warn for the reason `missing`, info for the rest | `reason`, `anchors`, `anchors_known`, `pf_enabled`, `pf_since` | +| `Tunnel interface changed` | The set of tunnel interfaces changes (macOS). A retry of the same change writes no second event | info | `added`, `removed`, `owner` | +| `DNS intercept: VPN connecting …` | A new tunnel interface starts the stabilization mode (macOS) | info | none | +| `DNS intercept: pf stable for …` | The pf ruleset stops changing and stabilization ends (macOS) | info | none | +| `Query health` | A change of the health class that holds for two evaluations, and a heartbeat every 15 minutes | info | `class`, `window_s`, `queries`, `failed_queries`, `cache_hits`, `failures_by_class`, `upstreams_down`, `bypass_active`, `cache_hit_ratio` | + +No event in this catalog holds a query name, an upstream URL, or a token. +The journal does hold network addresses. It holds the LAN and VPN resolver addresses, the gateways, the addresses of the host, and the DHCP servers of a service. +It holds no search suffix. `DNS configuration changed` counts the suffixes in `search_domain_count` and logs the count alone. +The field `owner` of `Tunnel interface changed` names the VPN product that owns the tunnel. On Windows the field `hardware_port` names the VPN product of an adapter in the same way. +The Wi-Fi network name is never logged. On macOS 14 and later, only a process with the Location Services permission can read it, and a daemon cannot request that permission. + +### The network object + +The `network` object describes the host network at one moment. +The `Network snapshot` event and the header line of every log file render the same object, with these fields: + +- Routes: `default_route_v4`, `default_route_v6`, `gateway_v4`, `gateway_v6` +- Families: `have_v4`, `have_v6` +- Interfaces: `interfaces`, an array of `name`, `class`, `up`, `ips`, `hardware_port`, `service`, and the count `interfaces_omitted` +- Resolvers: `resolvers`, the host and the port of each OS nameserver +- Source addresses: `source_ipv4`, `source_ipv6` +- Intercept state: `intercept_target`, `dns_less_target`, `bypass_active`, `recovery_running`, `pf_stabilizing` +- Link: `link_type`, `tethered`, `clat_present`, `nat64_prefix` + +The field `default_route_interface` is gone. Read `default_route_v4` and `default_route_v6` in its place. +The two routes and the two gateways come from the route table of macOS, one route per address family. A tunnel that owns the default route has no next hop, so its route names the tunnel and its gateway stays empty. +Windows and Linux read no route table in this release. They fill `default_route_v4` from the network monitor and leave `default_route_v6`, `gateway_v4`, and `gateway_v6` empty. +On macOS, `default_route_v4` comes from the network monitor only when the route table gives no route for the two address families. When the route table gives an IPv6 route only, `default_route_v4` stays empty. + +`link_type` is one of `wifi`, `ethernet`, `usb_tether`, `tunnel`, and `unknown`. +`link_type` and `tethered` describe the interface of the IPv4 default route. When no IPv4 route is present, they describe the interface of the IPv6 default route. When the route table gives no route, they describe the default interface of the network monitor. Thus the link fields and the route fields of one record describe the same interface. +macOS reads it from the name of the hardware port. Linux reads `wifi`, `ethernet`, `usb_tether`, and `tunnel` from the kernel. +Windows reads the description of the adapter in place of a port name. The words Ethernet, LAN, and Thunderbolt win over the tunnel class on every platform, so a VPN adapter with a description such as `SSL VPN Virtual Ethernet Adapter` reads as `ethernet`. +Windows gives `tunnel` to the other VPN adapters. The other links are `unknown`. +The field `service` names a macOS network service. It stays empty on Windows and on Linux. +`tethered` is true on every platform when the interface of the default route holds an address of the range 172.20.10.0/28. +It is also true when the default gateway is in that range, and when the hardware port of the default route is `iPhone USB`. + +The list `interfaces` holds the interfaces that are up or hold an address, and it stops at 32 entries. +`interfaces_omitted` counts the interfaces that the list leaves out, the ones over the limit included. +A container host holds hundreds of interfaces, and a list of all of them fills the journal. + +ctrld does not write a snapshot for every trigger. A snapshot equal to the snapshot written last reaches no journal. +Every other snapshot waits 60 s after the snapshot written last. A new value of `default_route_v4`, `default_route_v6`, or `resolvers` passes that wait. +The trigger `start` always writes. + +### Noise class + +A noise delta changes interfaces of the noise class only, and it does not move the default route. +The noise class holds `awdl*` and `llw*` on macOS. +On Linux it holds the names `docker*`, `veth*`, `virbr*`, and `br-*`. A device of the tun or tap type is a tunnel, not a noise interface. +On Windows it holds the virtual adapters without a connector. A Windows adapter whose description names TAP, Wintun, WireGuard, or VPN is a tunnel, not a noise interface. +On Linux and on Windows, an interface that holds a global address on either side of the delta is never noise. +A delta is also not noise when it flips `have_v4` or `have_v6`, or when it carries a time jump. +A noise delta runs no reconcile work. +ctrld starts no `scutil` process, reads no PF rule, discovers no tunnel, refreshes no VPN DNS, and schedules no delayed recheck for it. +A delta that touches a hardware port and a noise interface in the same callback is not noise. +The PF watchdog keeps its own 30 s timer, so it still tests the PF rules during a delta storm. + +Each noise delta logs one debug line. The first noise delta of a storm also writes a journal line with `count` 1. +While the storm lasts, one journal line of the same name arrives every 10 minutes, with the count of that window. +AirDrop in the Finder is the common source of such a storm on macOS. + +### DNS configuration poll (macOS) + +ctrld polls `scutil --dns` on macOS only. The first poll runs at start. +The poll runs every 15 s for 10 minutes after a network change, a wake, or the end of a recovery. +Each of these three events starts the fast cadence at once, even while the poll waits. +At rest the poll runs every 5 minutes. Each changed resolver of a poll gives one `DNS configuration changed` event. + +### Change-only lines + +These lines log at debug level on change only. Each one carries `repeats`, the number of equal results before it: + +- `Got system nameservers` +- `Final available nameservers` +- `pf anchor intact` +- `discovered active tunnel interfaces` +- `VPN DNS refresh completed` + +### PF anchor list (macOS) + +A `PF anchor list changed` event reaches the journal when its reason, its anchor list, or `pf_enabled` differs from the event before it. +The field `pf_since` is no part of that test, because the uptime of pf grows between two reads of one state. +A failed read of the anchor names gives `anchors_known` false and no anchor list. + +### Host woke + +ctrld writes one `Host woke` event per wake. Two sources that report the same wake inside 30 s give one event. +The macOS wake detector runs in DNS intercept mode only. Outside that mode, and on Windows and Linux, the netmon delta is the only source. +The netmon source knows no gap, so its report waits 8 s for the detector. A detector report inside that hold gives the event, with `gap_known` true and the length of the sleep in `gap_ms`. Without a detector report, the netmon report goes out with `gap_known` false and no `gap_ms`. Every later report of the same wake is dropped. + +### Query health + +ctrld grades the query path over a window of 15 minutes and reads the window every minute. +Each failed client query counts one failure, whatever number of upstreams that query tried. +`failures_by_class` keeps one count per failure class. The class `internal_domain` holds the failures of an Internal Domain resolver. +The grade leaves the class `internal_domain` out, and `upstreams_down` counts no Internal Domain resolver. +A laptop away from the company network cannot reach these resolvers. Without this rule the grade stays degraded for the whole day. +A change of the class reports after it holds for two reads of the window. + +## Transitions and recovery + +`Network transition` events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. +The outcome identifies an accepted, ignored, superseded, snapshot_superseded, or unusable change. +A callback whose state read a newer snapshot gives the outcome `snapshot_superseded` at debug level with `transition_id` 0, and it gives no interface event. +Newer accepted changes supersede pending recovery. Ignored changes do not cancel accepted recovery. +Source commits validate addresses against the current network state. +Netmon does not order minor callbacks within one major cache epoch. +For minor callbacks and late major callbacks in that epoch, ctrld reads current interface addresses and flags. +This read does not change host DNS. A failed read defers source writes, but reconciliation continues. + +`Removed stale resolver source` is a warning only when ctrld removes an invalid address. +The warning gives the reason. Every warning enters the journal, so it stays on disk after a debug rotation and after a restart. +`Recovery begin` and `Recovery end` connect the transition to a `recovery_generation`, reason, outcome, and elapsed time. +Only the first upstream failure in each recovery produces a bounded summary. +Every recovery outcome enters the journal. +A canceled recovery and a recovery with a failure log at warn level. +For the outcome `superseded`, `intercept_target_action` is `unchanged` and `bypass_active` is false. The recovery that follows owns both of them. The outcome `failed` means that the preparation of the recovery reported an error, and the event is at warn level. + +These summaries exclude upstream URLs and queried names. +Each PF probe records the recovery generation at its start. +A later recovery cannot receive credit for an earlier probe. + +### Cancellation event migration + +On `master`, `Recovery end` with `outcome=canceled` replaces `Recovery failed; DNS settings remain removed`. +Update support runbooks and log searches to use the structured event. +The cancellation event alone does not prove that interface DNS remains removed. + +Note: On the 1.x (`v1.0`) line, the previous message was `Recovery canceled; DNS settings remain removed`. + +### OS-triggered recovery scope + +An OS-only policy failure does not establish that configured DNS is unavailable. +Before starting OS-triggered recovery, ctrld checks its configured non-OS upstreams, excluding generated Internal Domain resolvers. +If any candidate is not marked down, it leaves shared recovery state and DNS settings unchanged. +The skipped trigger still refreshes in-memory OS-resolver discovery, at most once per 10 seconds while OS failures continue. Concurrent triggers do not queue duplicate discovery work. +This read preserves the existing resolver when discovery yields no nameservers. It does not clear the OS failure state; a successful OS query does that. +It does not remove interface DNS, reload interception rules, or enable bypass. A network change is not required for this refresh. +`Recovery skipped: configured upstream is not marked down` enters the journal with `healthy_upstream`, the candidate that admitted the skip. Generated IDs are retained; operator-defined keys are redacted to `upstream.custom`. +An unchanged candidate is journaled at most once per 5 minutes, rather than once per failed query. A changed candidate is journaled immediately when no refresh is in progress. +If all candidates are down, it uses that same pool for recovery probes instead of waiting only for OS DNS. The recovery still refreshes OS discovery initially, but only configured candidates can end it; the OS-only periodic probe/reinitialization loop is not part of that pool. +With no such configured candidates, it retains OS-only recovery. +`Recovery begin` includes `probe_upstreams` so the exit condition is visible in the journal. + +This is a configured pool, not a reconstruction of every listener's effective default route. +Custom configurations can include policy-only or unused upstreams, and an untested upstream is not marked down. +Ordinary upstream-failure and network-change admission and candidate selection are unchanged. +A success racing with admission can still cause brief bypass; probing the configured pool prevents the old OS-only wait from persisting. +The original OS policy remains OS-routed. + +### DNS-target decision failures (macOS) + +`intercept DNS target: decision unavailable; DNS unchanged` retains the first discovery failure and changes to that condition. +Identical consecutive failures are suppressed, even if recovery IDs change. +A later `decision available again` event closes the episode and reports the cumulative `repeat_count`. +Do not add repeat counts from separate events in the same episode. + +These events include the interface/service, discovery stage, bounded error class, target ownership before/after, and correlation IDs captured before discovery. +`failure_recovery_generation` and `failure_transition_id` identify the episode's first failed decision. +Raw error detail remains in the debug stream, not the journal fields. +A stage can be `system_discovery`, `default_route`, `interface_lookup`, `service_lookup`, `static_dns`, or `dhcp_dns`. +An error class can be `unavailable`, `timeout`, `canceled`, `permission_denied`, `command_failed`, or `read_failed`. + +A DHCP error is not an empty DNS result and never authorizes a DNS write. +Resolution means discovery can make a decision again, not that a DNS write or client lookup succeeded. +`action` describes the change in tracked target ownership; it is not independent proof of an OS write. +The target's existing set/remove events still report those operations. +The journal retains these decisions separately from the bounded debug upload. + +## PF probes (macOS) + +`DNS intercept probe result` includes these fields: + +- `probe_id` and `recovery_generation` +- Numeric `resolver_target` and `resolver_family` +- `stage`, bounded `error_code`, and `outcome` +- `repair_eligible`, `repeated_results`, and elapsed milliseconds + +A changed unsuccessful condition produces a warning. The condition consists of the stage, error code, outcome, and target. +Equal results stay at debug level. The next changed condition includes the preceding repeat count. +The first restored receipt also produces a warning, which enters the journal. This event prevents an older failure from appearing current. +Normal receipts and shutdown cancellation stay at debug level. +The journal budget is finite and the journal rotates by size. It does not keep these events without a limit. + +The outcomes have these meanings: + +- `intercepted`: the registered query reached the local handler. This proves interception delivery, not remote DNS health. +- `not_intercepted`: the helper confirmed a successful UDP write, but the handler did not receive the query before the deadline. + This result permits bounded repair. It does not identify an external PF writer or explain the delivery failure. +- `indeterminate`: the probe has no suitable target, cannot send, receives an invalid helper acknowledgement, or stops before a conclusive result. + This result alone does not cause a PF reload or transport reset. The existing monitor or watchdog retries later. + Independent evidence of missing rules still permits ordinary repair. + +The literal stages are `target`, `start`, `decode`, `dial`, `deadline`, `write`, `sent`, `helper`, `delivery`, and `received`. +`sent` is the helper acknowledgement before the final delivery result. +The error codes are `unavailable`, `invalid_argument`, `unreachable`, `source_unavailable`, `permission`, `timeout`, `io`, `invalid_status`, and `canceled`. +Successful results have an empty error code. No raw subprocess output enters these fields. + +The sender runs outside the `_ctrld` group exemption. +ctrld blocks outgoing IPv6 DNS with its own PF rule and does not redirect it. +The selector preserves a usable first IPv4 target, including an original public resolver. +Otherwise, it selects the first usable IPv4 address in the leading LAN group. +It stops at the public group, which can contain a synthetic fallback. It does not blacklist a particular public IP. +Loopback, unspecified, and multicast addresses are not usable targets. + +`PF repair` events carry the causal `probe_id` and `recovery_generation` for all four callers. +They record `started`, `not_run`, or `reload_completed`. Completed means that the reload ran, not that interception recovered. +A confirming probe adds its own ID and its interception outcome to the original repair correlation. +The watchdog leaves confirmation to its next scheduled probe. +`budget_exhausted` records a wake repair that the existing budget prevents. +A probe-triggered reload does not count as a missing-anchor repair. + +See [PF DNS interception](pf-dns-intercept.md) and [DNS intercept mode](dns-intercept-mode.md) for the rule lifecycle. + +## IPv6-only target fallback and PF diagnostics + +On macOS, a failed DHCPv4 read remains unknown unless configd positively publishes +CLAT for the current primary service/interface and the live interface addresses +agree. A CLAT-looking address alone is insufficient. Unknown or unsupported native +state leaves DNS unchanged. The fallback reuses the existing listener-aware target +and cleanup paths; it does not change PF enforcement or treat every command error +as empty DNS. Native output compatibility and the real hotspot scenario require +[owned-Mac validation](ipv6-only-dns-target-qa.md). + +`DNS intercept IPv6 block diagnostic` is a change-only debug event after unsuccessful +probes and their first restored receipt. It reads only the ctrld anchor using +`pfctl -a com.controld.ctrld -vvsr`, at most once per minute. The command has a +500 ms deadline and a 64 KiB output cap. A restored receipt inside the interval can +refer to the cached capture; `capture_cached`, `capture_age_ms` and +`capture_generation` make this explicit. Collection is synchronous before the +caller can repair: the 500 ms bound covers the command, not its pipe-cleanup grace +or the subsequent kernel route/interface reads. + +`pf_read_status` distinguishes unavailable/failed/unrecognized data from recognized +rules and counters. `ipv6_dns_udp_rule_observed` and its TCP equivalent mean the +exact generated rule was observed, not that it received the probe. A missing rule +match does not prove that no other IPv6 blocking exists. Packet fields are emitted +only when their counters are known. Delta status is `unknown`, `reset` or +`shared_traffic`; decreases are never reported as negative packet counts. + +Counters include all matching traffic, not just the probe, and unobserved reloads +can reset them between captures. A counter increase does not establish that CLAT +translated or PF blocked a particular packet. The event names the probe transport, +resolver target/family, recovery generation and default-route interfaces. +`clat_address_present` is an address hint, not the positive native evidence used to +authorize DNS changes. Raw rules, customer query names and interface-address lists +are not included. + +These debug events do not guarantee journal retention. !1020 separately handles +retained DNS-target decision failures. Reload completion, a configured block rule, +and successful local probe receipt remain distinct observations. + +## Reproduction boundaries + +Compare the affected release with the exact candidate artifact on an owned host. +Keep the configuration unchanged. Capture ordinary DNS and a direct query to the actual listener and port. +Measure recovery after a usable route and resolver return. Exclude unavoidable offline time. +Record source-address changes, PF observations, and upstream results. Success at one layer does not prove success at another. + +Use the recovery mode, ownership, outcome, and effective network-service DNS configuration to assess cancellation. +Never flush global PF state to collect diagnostics. diff --git a/docs/pf-dns-intercept.md b/docs/pf-dns-intercept.md new file mode 100644 index 00000000..3e53e75d --- /dev/null +++ b/docs/pf-dns-intercept.md @@ -0,0 +1,326 @@ +# macOS pf DNS Interception — Technical Reference + +## Overview + +ctrld uses macOS's built-in packet filter (pf) to intercept all DNS traffic at the kernel level, redirecting it to ctrld's local listener at `127.0.0.1:53`. This operates below interface DNS settings, making it immune to VPN software (F5, Cisco, GlobalProtect, etc.) that overwrites DNS on network interfaces. + +## How pf Works (Relevant Basics) + +pf is a stateful packet filter built into macOS (and BSD). It processes packets through a pipeline with **strict rule ordering**: + +``` +options (set) → normalization (scrub) → queueing → translation (nat/rdr) → filtering (pass/block) +``` + +**Anchors** are named rule containers that allow programs to manage their own rules without modifying the global ruleset. Each anchor type must appear in the correct section: + +| Anchor Type | Section | Purpose | +|-------------|---------|---------| +| `scrub-anchor` | Normalization | Packet normalization | +| `nat-anchor` | Translation | NAT rules (not used by ctrld) | +| `rdr-anchor` | Translation | Redirect rules | +| `anchor` | Filtering | Pass/block rules | + +**Critical constraint:** If you place a `rdr-anchor` line after an `anchor` line, pf rejects the entire config with "Rules must be in order." + +## Why We Can't Just Use `rdr on ! lo0` + +The obvious approach: +``` +rdr pass on ! lo0 proto udp from any to any port 53 -> 127.0.0.1 port 53 +``` + +**This doesn't work.** macOS pf `rdr` rules only apply to *forwarded/routed* traffic — packets passing through the machine to another destination. DNS queries originating from the machine itself (locally-originated) are never matched by `rdr` on non-loopback interfaces. + +This is a well-known pf limitation on macOS/BSD. It means the VPN client's DNS queries would be redirected (if routed through the machine), but the user's own applications querying DNS directly would not. + +## Our Approach: route-to + rdr (Two-Step) + +We use a two-step technique to intercept locally-originated DNS: + +``` +Step 1: Force outbound DNS through loopback + pass out quick on ! lo0 route-to lo0 inet proto udp from any to ! 127.0.0.1 port 53 + +Step 2: Pass the packet outbound on lo0 (needed when VPN firewalls have "block drop all") + pass out quick on lo0 inet proto udp from any to ! 127.0.0.1 port 53 no state + +Step 3: Redirect it on loopback to ctrld's listener + rdr on lo0 inet proto udp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 53 + +Step 4: Accept and create state for response routing + pass in quick on lo0 reply-to lo0 inet proto { udp, tcp } from any to 127.0.0.1 port 53 +``` + +> **State handling is critical for VPN firewall coexistence:** +> - **route-to**: `keep state` (default). State is interface-bound on macOS — doesn't match on lo0. +> - **pass out lo0**: `no state`. If this created state, it would match inbound on lo0 and bypass rdr. +> - **rdr**: no `pass` keyword. Packet must go through filter so `pass in` can create response state. +> - **pass in lo0**: `keep state` (default). Creates the ONLY state on lo0 — handles response routing. + +### Packet Flow + +``` +Application queries 10.255.255.3:53 (e.g., VPN DNS server) + ↓ +Kernel: outbound on en0 (or utun420 for VPN) + ↓ +pf filter: "pass out route-to lo0 ... port 53" → redirects to lo0, creates state on en0 + ↓ +pf filter (outbound lo0): "pass out on lo0 ... no state" → passes, NO state created + ↓ +Loopback reflects packet inbound on lo0 + ↓ +pf rdr (inbound lo0): "rdr on lo0 ... port 53 -> 127.0.0.1:53" → rewrites destination + ↓ +pf filter (inbound lo0): "pass in reply-to lo0 ... to 127.0.0.1:53" → creates state + reply route + ↓ +ctrld receives query on 127.0.0.1:53 + ↓ +ctrld resolves via DoH (port 443, exempted by group _ctrld) + ↓ +Response from ctrld: 127.0.0.1:53 → 100.94.163.168:54851 + ↓ +reply-to lo0: forces response through lo0 (without this, kernel routes via utun420 → lost in VPN tunnel) + ↓ +pf applies rdr reverse NAT: src 127.0.0.1 → 10.255.255.3 + ↓ +Application receives response from 10.255.255.3:53 ✓ +``` + +### Why This Works + +1. `route-to lo0` forces the packet onto loopback at the filter stage +2. `pass out on lo0 no state` gets past VPN "block drop all" without creating state +3. No state on lo0 means rdr gets fresh evaluation on the inbound pass +4. `reply-to lo0` on `pass in` forces the response through lo0 — without it, the kernel routes the response to VPN tunnel IPs via the VPN interface and it's lost +4. `rdr` (without `pass`) redirects then hands off to filter rules +5. `pass in keep state` creates the response state — the only state on the lo0 path +6. Traffic already destined for `127.0.0.1` is excluded (`to ! 127.0.0.1`) to prevent loops +7. ctrld's own upstream queries use DoH (port 443), bypassing port 53 rules entirely + +### Why Each State Decision Matters + +| Rule | State | Why | +|------|-------|-----| +| route-to on en0/utun | keep state | Needed for return routing. Interface-bound, won't match on lo0. | +| pass out on lo0 | **no state** | If stateful, it would match inbound lo0 → bypass rdr → DNS broken | +| rdr on lo0 | N/A (no pass) | Must go through filter so pass-in creates response state | +| pass in on lo0 | keep state + reply-to lo0 | Creates lo0 state. `reply-to` forces response through lo0 (not VPN tunnel). | + +## Rule Ordering Within the Anchor + +pf requires translation rules before filter rules, even within an anchor: + +```pf +# === Translation rules (MUST come first) === +rdr on lo0 inet proto udp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 53 +rdr on lo0 inet proto tcp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 53 + +# === Exemptions (filter phase, scoped to _ctrld group) === +pass out quick on ! lo0 inet proto { udp, tcp } from any to port 53 group _ctrld +pass out quick on ! lo0 inet proto { udp, tcp } from any to port 53 group _ctrld + +# === Main intercept (filter phase) === +pass out quick on ! lo0 route-to lo0 inet proto udp from any to ! 127.0.0.1 port 53 +pass out quick on ! lo0 route-to lo0 inet proto tcp from any to ! 127.0.0.1 port 53 + +# === Allow redirected traffic on loopback === +pass in quick on lo0 reply-to lo0 inet proto { udp, tcp } from any to 127.0.0.1 port 53 +``` + +### Exemption Mechanism (Group-Scoped) + +Some IPs must bypass the redirect: + +- **OS resolver nameservers** (e.g., DHCP-assigned DNS): ctrld's recovery/bootstrap path may query these on port 53. Without exemption, these queries loop back to ctrld. +- **VPN DNS servers**: When ctrld forwards VPN-specific domains (split DNS) to the VPN's internal DNS, those queries must reach the VPN DNS server directly. + +Exemptions use `pass out quick` with `group _ctrld` **before** the `route-to` rule. The `group _ctrld` constraint ensures that **only ctrld's own process** can bypass the redirect — other applications cannot circumvent DNS interception by querying the exempted IPs directly. Because pf evaluates filter rules in order and `quick` terminates evaluation, the exempted packet goes directly out the real interface and never hits the `route-to` or `rdr`. + +### The `_ctrld` Group + +To scope pf exemptions to ctrld's process only, we use a dedicated macOS system group: + +1. **Creation**: On startup, `ensureCtrldGroup()` creates a `_ctrld` system group via `dscl` (macOS Directory Services) if it doesn't already exist. The GID is chosen from the 350-450 range to avoid conflicts with Apple's reserved ranges. The function is idempotent. + +2. **Process GID**: Before loading pf rules, ctrld sets its effective GID to `_ctrld` via `syscall.Setegid()`. All sockets created by ctrld after this point are tagged with this GID. + +3. **pf matching**: Exemption rules include `group _ctrld`, so pf only allows bypass for packets from processes with this effective GID. Other processes querying the same exempt IPs are still redirected to ctrld. + +4. **Lifecycle**: The group is **never removed** on shutdown or uninstall. It's a harmless system group, and leaving it avoids race conditions during rapid restart cycles. It is recreated (no-op if exists) on every start. + +## Anchor Injection into pf.conf + +The trickiest part. macOS only processes anchors declared in the active pf ruleset. We must inject our anchor references into the running config. + +### What We Do + +1. Read `/etc/pf.conf` +2. If our anchor reference already exists, reload as-is +3. Otherwise, inject `rdr-anchor "com.controld.ctrld"` in the translation section and `anchor "com.controld.ctrld"` in the filter section +4. Write to a **temp file** and load with `pfctl -f ` +5. **We never modify `/etc/pf.conf` on disk** — changes are runtime-only and don't survive reboot (ctrld re-injects on every start) + +### Injection Logic + +Finding the right insertion point requires understanding the existing pf.conf structure. The algorithm: + +1. **Scan** for existing `rdr-anchor`/`nat-anchor`/`binat-anchor` lines (translation section) and `anchor` lines (filter section) +2. **Insert `rdr-anchor`**: + - Before the first existing `rdr-anchor` line (if any exist) + - Else before the first `anchor` line (translation must come before filtering) + - Else before the first `pass`/`block` line + - Last resort: append (but this should never happen with a valid pf.conf) +3. **Insert `anchor`**: + - Before the first existing `anchor` line (if any) + - Else before the first `pass`/`block` line + - Last resort: append + +### Real-World pf.conf Scenarios + +We test against these configurations: + +#### Default macOS (Sequoia/Sonoma) +``` +scrub-anchor "com.apple/*" +nat-anchor "com.apple/*" +rdr-anchor "com.apple/*" +anchor "com.apple/*" +load anchor "com.apple" from "/etc/pf.anchors/com.apple" +``` +Our `rdr-anchor` goes before `rdr-anchor "com.apple/*"`, our `anchor` goes before `anchor "com.apple/*"`. + +#### Little Snitch +Adds `rdr-anchor "com.obdev.littlesnitch"` and `anchor "com.obdev.littlesnitch"` in the appropriate sections. Our anchors coexist — pf processes multiple anchors in order. + +#### Lulu Firewall (Objective-See) +Adds `anchor "com.objective-see.lulu"`. We insert `rdr-anchor` before it (translation before filtering) and `anchor` before it. + +#### Cisco AnyConnect +Adds `nat-anchor "com.cisco.anyconnect"`, `rdr-anchor "com.cisco.anyconnect"`, `anchor "com.cisco.anyconnect"`. Our anchors insert alongside Cisco's in their respective sections. + +#### Minimal pf.conf (no anchors) +Just `set skip on lo0` and `pass all`. We insert `rdr-anchor` and `anchor` before the `pass` line. + +#### Empty pf.conf +Both anchors appended. This is a degenerate case that shouldn't occur in practice. + +## Failure Modes and Safety + +### What happens if our injection fails? +- `ensurePFAnchorReference` returns an error, logged as a warning +- ctrld continues running but DNS interception may not work +- The anchor file and rules are cleaned up on shutdown +- **No damage to existing pf config** — we never modify files on disk + +### What happens if ctrld crashes (SIGKILL)? +- pf anchor rules persist in kernel memory +- DNS is redirected to 127.0.0.1:53 but nothing is listening → DNS breaks +- On next `ctrld-client start`, we detect the stale anchor file, flush the anchor, and start fresh +- Without ctrld restart: `sudo pfctl -a com.controld.ctrld -F all` manually clears it + +### What if another program flushes all pf rules? +- Our anchor references are removed from the running config +- DNS interception stops (traffic goes direct again — fails open, not closed) +- The periodic watchdog (30s) detects missing rules and restores them +- ctrld continues working for queries sent to 127.0.0.1 directly + +### What if another program reloads pf.conf (corrupting translation state)? +Programs like Parallels Desktop reload `/etc/pf.conf` when creating or destroying +virtual network interfaces (bridge100, vmenet0). This can corrupt pf's internal +translation engine — **rdr rules survive in text form but stop evaluating**. +The watchdog's rule-text checks say "intact" while DNS is silently broken. + +**Detection:** ctrld detects interface appearance/disappearance in the network +change handler and spawns an asynchronous interception probe monitor: + +1. A subprocess sends a DNS query WITHOUT the `_ctrld` group GID, so pf + intercept rules apply to it +2. If ctrld receives the query → pf interception is working +3. A successful UDP write without local receipt within 1s gives `not_intercepted`. +4. Only this meaningful failure permits probe-triggered repair: `forceReloadPFMainRuleset()` does `pfctl -f -` with the current + running ruleset, resetting pf's translation engine + +A missing target or helper failure gives `indeterminate` and does not cause a reload. +The probe uses IPv4 only. See [target selection and diagnostics](network-recovery-diagnostics.md#pf-probes-macos). + +The monitor probes with exponential backoff (0, 0.5, 1, 2, 4s) to win the race +against async pf reloads. Only one monitor runs at a time (singleton). The +watchdog also runs the probe every 30s as a safety net. + +A network delta that touches AirDrop or virtual adapters alone is noise. The +handler returns at once for it, so no probe monitor starts and no pf read runs. +A delta that moves the default route, that flips the v4 or the v6 family, or +that carries a time jump is not noise, whatever its interfaces are. +The watchdog keeps its own 30 s cadence, so it still tests the rules during a +delta storm. + +The full pf reload is VPN-safe: it reassembles from `pfctl -sr` + `pfctl -sn` +(the current running state), preserving all existing anchors and rules. + +### What if another program adds conflicting rdr rules? +- pf processes anchors in declaration order +- If another program redirects port 53 before our anchor, their redirect wins +- If after, ours wins (first match with `quick` or `rdr pass`) +- Our maximum-weight sublayer approach on Windows (WFP) doesn't apply to pf — pf uses rule ordering, not weights + +### What about `set skip on lo0`? +Some pf.conf files include `set skip on lo0` which tells pf to skip ALL processing on loopback. **This would break our approach** since both the `rdr on lo0` and `pass in on lo0` rules would be skipped. + +**Mitigation:** the interception probe. `probePFIntercept()` sends a real query from +outside the `_ctrld` group and confirms the listener received the redirect, which cannot +succeed while pf bypasses loopback. A confirmed send without receipt permits a full reload. +An unsent or unavailable probe is indeterminate and does not trigger repair. + +**Not implemented, contrary to earlier versions of this document:** ctrld does *not* +strip `lo0` from `set skip on` directives, and the watchdog does *not* inspect skip +state. Apple's `pfctl` offers no way to read it — `pfctl(8)` accepts `-s` nat, queue, +rules, Anchors, states, Sources, info, References, labels, timeouts, memory, Tables, +osfp, Interfaces, all, with no options or skip modifier — so text-based detection is not +available on macOS. + +Adding an explicit check is tracked as follow-up: `pfctl(8)` documents +`-s Interfaces -v` as additionally listing which interfaces have skip rules activated, +which is the query to build on once its output shape is confirmed on a host that has a +skip configured. + +## Cleanup + +On shutdown (`stopDNSIntercept`): +1. `pfctl -a com.controld.ctrld -F all` — flush all rules from our anchor +2. Remove `/etc/pf.anchors/com.controld.ctrld` anchor file +3. `pfctl -f /etc/pf.conf` — reload original pf.conf, removing our injected anchor references from the running config + +This is clean: no files modified on disk, no residual rules. + +## Comparison with Other Approaches + +| Approach | Intercepts local DNS? | Survives VPN DNS override? | Risk of loops? | Complexity | +|----------|----------------------|---------------------------|----------------|------------| +| `rdr on ! lo0` | ❌ No | Yes | Low | Low | +| `route-to lo0` + `rdr on lo0` | ✅ Yes | Yes | Medium (need exemptions) | Medium | +| `/etc/resolver/` | Partial (per-domain only) | No (VPN can overwrite) | Low | Low | +| `NEDNSProxyProvider` | ✅ Yes | Yes | Low | High (needs app bundle) | +| NRPT (Windows only) | N/A | Partial | Low | Medium | + +We chose `route-to + rdr` as the best balance of effectiveness and deployability (no app bundle needed, no kernel extension, works with existing ctrld binary). + +## Key pf Nuances Learned + +1. **`rdr` doesn't match locally-originated traffic** — this is the biggest gotcha +2. **Rule ordering is enforced** — translation before filtering, always +3. **Anchors must be declared in the main ruleset** — just loading an anchor file isn't enough +4. **`rdr` without `pass`** — redirected packets must go through filter rules so `pass in keep state` can create response state. `rdr pass` alone is insufficient for response delivery. +5. **State handling is nuanced** — route-to uses `keep state` (state is floating). `pass out on lo0` must use `no state` (prevents rdr bypass). `pass in on lo0` uses `keep state` + `reply-to lo0` (creates response state AND forces response through loopback instead of VPN tunnel). Getting any of these wrong breaks either the forward or return path. +6. **`quick` terminates evaluation** — exemption rules must use `quick` and appear before the route-to rule +7. **Piping to `pfctl -f -` can fail** — special characters in pf.conf content cause issues; use temp files +8. **`set skip on lo0` would break us** — but it's not in default macOS pf.conf +9. **`pass out quick` exemptions work with route-to** — they fire in the same phase (filter), so `quick` + rule ordering means exempted packets never hit the route-to rule +10. **pf cannot cross-AF redirect** — `rdr on lo0 inet6 ... -> 127.0.0.1` is invalid. IPv6 DNS must be handled by an `[::1]` listener. +11. **`block return` doesn't work for IPv6 DNS** — BSD doesn't deliver ICMPv6 unreachable to unconnected UDP sockets (`sendto`). Apps timeout waiting for a response that never comes. +12. **sendmsg from `::1` to global unicast fails on macOS** — unlike IPv4 where `127.0.0.1` can send to any local address, `::1` cannot send to the machine's own global IPv6 address (`EINVAL`). This is the fundamental asymmetry that makes IPv6 DNS interception infeasible. +13. **`nat on lo0` doesn't fire for `route-to`'d packets** — pf runs translation on the original outbound interface (en0), then skips it on lo0's outbound pass. `rdr` works because lo0 inbound is a genuinely new direction. Any lo0 address (including ULAs) can't route to global unicast — the kernel segregates lo0's routing table. +14. **Raw IPv6 sockets bypass routing validation but pf doesn't match them** — `SOCK_RAW` can send from `::1` to global unicast, but pf treats raw socket packets as new connections (not matching rdr state), so reverse-translation doesn't happen. The client sees `::1` as the source, not the original DNS server. +15. **`DIOCNATLOOK` can find the original dest but you can't use it** — The ioctl returns the pre-rdr destination, but `bind()` fails with `EADDRNOTAVAIL` because it's not a local address. macOS IPv6 raw sockets don't support `IPV6_HDRINCL` for source spoofing. +16. **Blocking IPv6 DNS is the pragmatic solution** — macOS automatically retries over IPv4. The ~1s penalty on the first blocked query is negligible compared to the complexity of working around the kernel's IPv6 loopback restrictions. diff --git a/docs/provisioning-failure-codes.md b/docs/provisioning-failure-codes.md new file mode 100644 index 00000000..bff84b46 --- /dev/null +++ b/docs/provisioning-failure-codes.md @@ -0,0 +1,182 @@ +# Provisioning failure codes + +When ctrld hits a terminal failure during provisioning, it reports the same +stable code on three surfaces: + +- **Result file** — `provision_result.json` in the ctrld home directory + (next to the persisted internal `ctrld.log`). JSON with `stage`, `code`, + `exit_code`, `message`, and for listener failures a bounded + `detail.attempts` list of `{addr, proto, os_error}`. Written atomically, + removed on the next successful provisioning. Never contains provision + tokens, resolver/device IDs, or configuration contents. +- **Output line** — one fixed-format line on the CLI output: + `provisioning failed: stage= code= (exit )`. + Installer or MDM wrappers can extract exactly this line (fixed charset: + `stage=[a-z]* code=[A-Z_]* (exit [0-9]*)`) into their own logs without + risking token leakage from other output. +- **Exit code** — stage-scoped: input 20–29, bootstrap 30–39, + listener 40–49, service 50–59. This does not change other existing + exit-code contracts: `ctrld-client status` still exits 0–3, and an invalid + deactivation pin still exits 126. + +A customer or administrator only needs to report the code (or the whole +output line). The table below is the maintained support mapping; it must +stay in sync with `cmd/cli/provision_result.go` and changes in the same MR. + +## Codes + +For every code below, first run `sudo ctrld-client diag` (`sudo ctrld-client diag --json` +gives a paste-safe, machine-readable copy). Get that output before you do +anything else. The command needs no running service. Without root, the +last-provisioning-result and service-state sections report permission +denied, so run it with `sudo`. It repeats the same stage, code, and message +the customer already reported. +The **Next action / evidence** column lists what to look at next. + +| Code | Stage | Exit | Failure scenario | Next action / evidence | +|---|---|---|---|---| +| `PROVISION_TOKEN_MALFORMED` | input | 21 | The `--cd-org` value is not a provisioning code: it is empty, too short, too long, or it contains whitespace or control characters. ctrld checks this before any network call. A missing `org-v1-` prefix is not this code. That case only logs a warning, because legacy codes can lack the prefix. | Make sure that the provisioning code was copied in full, with no extra whitespace. The result file never echoes the value. | +| `CUSTOM_HOSTNAME_INVALID` | input | 22 | The `--custom-hostname` value fails ctrld's own hostname rule (`validHostname`): it is not 3 to 64 characters, or it is not in RFC1123 hostname format. ctrld checks this before any network call. The API itself does not reject a bad hostname during provisioning. When ControlD registers the device name, it folds or strips characters like space, `+`, and `.`. Because of this, a value ctrld accepts can still register under an adjusted name. That case only logs a notice, not a failure. | Fix `--custom-hostname` following the message: it names the offending characters and the allowed format. | +| `INTERCEPT_MODE_INVALID` | input | 23 | The `--intercept-mode` value is not one of `off`, `dns`, or `hard`. ctrld checks this before it installs the service. | Re-run with a valid `--intercept-mode` value. | +| `INVALID_FLAG_COMBINATION` | input | 24 | `--cd` or `--cd-org` is used together with `--nextdns`, or `--proto` is set to anything other than `doh`/`doh3` while `--cd` is in play. ctrld checks this before any network call or service install. | Re-run without the conflicting flag, or fix the invalid value. The message names the exact flags involved. | +| `API_UNREACHABLE` | bootstrap | 30 | The Control D API could not be reached or answered with a retryable error (network failure, proxy interference, 5xx, timeout) and retries ran out. The service manager may retry the service later. | Make sure that the device has network access to `api.controld.com` (DNS, proxy, firewall, captive portal). `ctrld-client diag`'s `api_reachability` block already ran this same probe. Ask whether other TLS traffic works. | +| `API_REJECTED` | bootstrap | 31 | The API answered and permanently rejected the configuration (4xx other than 408/429): bad or revoked token, malformed request. ctrld exits without burning service-manager restarts because retrying cannot change the answer. This is also the fallback for the four `TOKEN_*` codes below. Those codes need the API response to carry `error.metadata.reason`. An API deployment that does not yet populate that field reports every provisioning-code rejection as `API_REJECTED` instead. | Verify the provision token / org configuration in the Control D dashboard. Re-push after fixing credentials. Evidence: HTTP status in the result file `message`. | +| `API_DEVICE_INVALID` | bootstrap | 32 | The API reports the device/resolver no longer exists (error code 40402). When the daemon's own bootstrap preflight discovers this, it self-uninstalls, because the identity is gone server-side. The direct `--cd ` install path (`ctrld-client start --cd`) can also hit this code before the service exists. On that path there is nothing to uninstall yet. | Confirm the device was deleted or re-provisioned in the dashboard; re-provision with a current token. No local evidence needed beyond the code. | +| `API_MAINTENANCE` | bootstrap | 37 | The API answered that it is in hard maintenance (HTTP 503, `error.code` 50302, or on older deployments the message `Maintenance is in progress.`), and this host had no previously working configuration on disk to fall back on. A host that does have one keeps serving DNS from it and never reaches this code. Maintenance is temporary and says nothing about the configuration, so ctrld does not self-uninstall and does not treat it as a permanent rejection. | Wait for the maintenance window to end and re-run. Nothing to fix on the device: the same request succeeds once the API returns. A device that was already provisioned keeps resolving throughout. | +| `TOKEN_INVALID` | bootstrap | 33 | The API rejected the `--cd-org` provisioning code with reason `token_invalid`: the code is not recognized. This code needs the deployed API to send `error.metadata.reason: "token_invalid"`. See `API_REJECTED` above for the fallback when it does not. | Make sure that the code is correct. Re-enter it exactly as given. | +| `TOKEN_EXPIRED` | bootstrap | 34 | The API rejected the `--cd-org` provisioning code with reason `token_expired`. This has the same API-reason dependency as `TOKEN_INVALID`. | Get a new provisioning code from your administrator. | +| `TOKEN_LIMIT_REACHED` | bootstrap | 35 | The API rejected the `--cd-org` provisioning code with reason `token_limit_reached`: it reached its device limit. This has the same API-reason dependency as `TOKEN_INVALID`. | Free up a device slot or use a different provisioning code. | +| `TOKEN_DISABLED` | bootstrap | 36 | The API rejected the `--cd-org` provisioning code with reason `token_disabled`: the code was invalidated. This has the same API-reason dependency as `TOKEN_INVALID`. | Download a profile from an active provisioning code. | +| `LISTENER_BIND_FAILED` | listener | 41 | No listen address could be bound after all fallbacks (configured address, 0.0.0.0:53, localhost:53, random) were exhausted. `detail.attempts` records each tried address with the UDP/TCP OS error, e.g. `address already in use` (another DNS service owns the port) or `can't assign requested address` (address not on any interface). | Read `detail.attempts`: `address already in use` → find the process owning the port (`sudo lsof -i :53 -nP`); `can't assign requested address` → the configured IP is not present on the device. Then fix the conflict or the listener config. | +| `LISTENER_CONFIGURED_ADDR_UNAVAILABLE` | listener | 42 | An explicitly configured listener address could not be bound and configuration checks forbid falling back to another address, or (macOS intercept mode) the required explicit address is unavailable. | The configured `ip:port` in the listener config is wrong for this device or occupied. Verify the address exists on an interface and nothing else binds it; correct the config rather than expecting fallback. | +| `SERVICE_INSTALL_FAILED` | service | 51 | The OS service manager refused to install the service (launchd/systemd/SCM registration failed). | Check OS-level constraints: permissions/elevation, MDM policy blocking daemon installation, corrupted previous install. Evidence: result file `message` (service manager error), plus `launchctl print system/ctrld-client` / `systemctl status ctrld-client` / SCM state. | +| `SERVICE_START_FAILED` | service | 52 | The service installed but the service manager could not start it. | Check the service manager's own log for the start error, then the ctrld home dir `ctrld.log`. Often permissions or a binary quarantined by security tooling. | +| `SERVICE_SELFCHECK_FAILED` | service | 53 | The service started but never became healthy: no fresher failure was reported by the daemon, and the post-install DNS self-check failed. On a fresh install or an upgrade, the just-installed service is rolled back (uninstalled). A restart of an already installed service keeps that service installed. If the daemon itself recorded a more specific failure (e.g. a listener code), that code is reported instead of this one. | Ask for the drained service log printed by `ctrld-client start` and the result file. If the service was running but unreachable, check host firewall rules intercepting DNS to the listener. | +| `UNCLASSIFIED` | service | 59 | This code covers terminal failures on the provisioning boundary that predate a dedicated code. Examples include a config unmarshal, and a file-system or environment failure such as writing the config file, reading `socketDir`, or respawning as a daemon. Other examples are a service-argument update failing mid-upgrade, and a DNS-intercept failure the interface-DNS fallback cannot safely take over from. Every one of these used to be a bare crash with nothing to read. Now they all persist a result file. | Read the result file `message`: it names the specific operation that failed and the underlying OS error. When you report it, treat this the same as any other stage-scoped failure. | + +The macOS installer script (`scripts/pkg/postinstall`) emits the package rows +below, not ctrld itself, before ctrld ever runs. They only ever exit 1 (the +script's own exit code), and they never appear in `provision_result.json`. +The same script also reuses `PROVISION_TOKEN_MALFORMED` and +`CUSTOM_HOSTNAME_INVALID` from the table above, for its own pre-flight +checks on the managed-prefs `ProvisionToken`/`CustomHostname` values. It +uses the same rule, and also logs it as `stage=package (exit 1)`. So those +two codes can appear with either stage, depending on which side rejected +the value. + +| `PROFILE_PREFS_MISSING` | package | 1 | The MDM configuration profile (managed prefs domain `com.controld.ctrld`) has no `ProvisionToken`, or the value is empty. This shows up after the postinstall script's wait loop ends: 12 attempts, 10 seconds apart, about 2 minutes. | Scope the `com.controld.ctrld` configuration profile to the device. If the profile is already on the device, set its `ProvisionToken` value. Then reinstall the package, or run `sudo ctrld-client start --cd-org=` by hand. | +| `INTERCEPT_MODE_INVALID` | package | 1 | On a fresh install, the profile's `InterceptMode` is set, but it is not one of the managed-prefs values (empty, `standard`, `intercept-dns`). A fresh install fails visibly, instead of silently falling back, because a managed deployment must not provision into an unintended mode. On an upgrade, the same bad value only logs a warning and keeps the existing service mode. So a bad profile edit cannot break an already-working fleet. | Fix `InterceptMode` in the profile to one of the allowed values. Then reinstall the package. | +| `SERVICE_RELOAD_FAILED` | package | 1 | An upgrade reused the existing service mode (no managed `InterceptMode` override), and `launchctl load` on the installed plist failed. | Run `sudo launchctl load /Library/LaunchDaemons/ctrld-client.plist` by hand, and read the error it gives. | +| `TEMP_FILE_UNAVAILABLE` | package | 1 | The script failed to create the private temp file it uses to capture ctrld's output (`/tmp` or `TMPDIR` is full or unwritable). Because ctrld never runs, the script has no diagnostics to add. | Free space in `/tmp` or `TMPDIR`, or fix its permissions. Then reinstall the package. | + +## Customer-visible message shape + +Every failure shows the same two-part shape: the fixed identifier line, then +one free-text line specific to the code. Neither part ever contains the +provision token, the resolver/device ID, or config contents. + +- **Input stage** — names the offending flag or value and the rule it + broke. For example: `--cd-org provisioning code is malformed: must be + 6-64 characters with no whitespace or control characters`. For + `CUSTOM_HOSTNAME_INVALID`, the message names the specific offending + characters and the allowed format. +- **Bootstrap, `TOKEN_*`** — one fixed sentence per code (see the table + above). For example: `the provisioning code has expired; get a new + provisioning code from your administrator`. +- **Bootstrap, `API_UNREACHABLE`/`API_REJECTED`/`API_DEVICE_INVALID`** — + When the failure surfaces during a running install's periodic resolver + refresh, the message is `ControlD API rejected this configuration (HTTP + status )`. When it surfaces during the initial provisioning call, the + message is `provision token exchange failed: ` or `failed to fetch + resolver config: `. Neither ever includes the request or response + body. +- **Listener** — names the address or addresses tried. `detail.attempts` in + the result file carries the OS error for each address and protocol. +- **Service** — For `SERVICE_INSTALL_FAILED` or `SERVICE_START_FAILED`, the + message is ` failed: ` (`` is `Install` or `Start`). + When the daemon itself reported no fresher failure, `SERVICE_SELFCHECK_FAILED` + shows one of three messages, depending on what the self-check saw. When + there is no error and the service is not running, the message is `ctrld + service did not pass its post-start self-check`. When the test query + itself errored, the message is `An error occurred while performing test + query: `. When the service was running but the query silently + failed, the message is `ctrld service was running, but a DNS query could + not be sent to its listener; check firewall rules + blocking/intercepting/redirecting DNS queries`. +- **Package** — the plain-language second line the installer script logs + right after the identifier line. Each package row's *Next action* cell + above quotes this line verbatim, because the script has no separate + result file to carry it. + +## Where each code surfaces + +- **Every ctrld-native code** (every row above except `package`): the CLI + output / persisted `ctrld.log`, `provision_result.json`, and `ctrld-client diag` + (which reads that same file). +- **Package-stage codes** (`PROFILE_PREFS_MISSING`, `SERVICE_RELOAD_FAILED`, + `TEMP_FILE_UNAVAILABLE`, the package's own `INTERCEPT_MODE_INVALID` check + on the managed-prefs value, and the package pre-flight's reused + `PROVISION_TOKEN_MALFORMED` and `CUSTOM_HOSTNAME_INVALID`): only + `/var/log/install.log`. ctrld never ran, so no result file exists yet. + `sudo ctrld-client diag` run right after this reports "none recorded" for the last + provisioning result. That is still useful: it shows that the package + script's own gate rejected the profile before ctrld had a chance to run. +- **Any ctrld-native code can also reach `/var/log/install.log`.** The + postinstall script invokes ctrld in two cases: to provision on a fresh + install, or to reapply a managed intercept mode on an upgrade. Either + time, it greps ctrld's own identifier line out of the captured output. + It re-logs that line, and appends `(ctrld exit )`. So a bootstrap or + listener code, not only the three package-only codes, can appear in the + MDM install log. Right after any installer-script failure, it also + appends a fresh `sudo ctrld-client diag` run, each line prefixed `diag:`. + +## Reading the result file + +macOS and Linux (default service home is `/etc/controld`): + +```sh +sudo cat /etc/controld/provision_result.json +``` + +On Windows the file sits next to `ctrld-client.exe` in the install directory. A +custom `homedir` config moves it accordingly; routers and mobile use their +platform home directory. + +The file sits in the same directory as the persisted internal log +(`ctrld.log`) for the user the service runs as. On a healthy install the +file is absent. + +## Support: `ctrld-client diag` + +Run `sudo ctrld-client diag` (or `sudo ctrld-client diag --json`) to collect the facts +support needs for a provisioning ticket, in one copy-paste-safe command. It +collects the client version and the MDM-managed preferences (macOS only, +the token reported only as present or absent, an empty value counts as +absent). It also collects the last provisioning result, the service state, +and whether the Control D API is reachable. It needs no config. It also +runs without root, but then the last-provisioning-result and the +service-state sections report permission denied, and the text report names +the step that gets the rights it needs. Even when it finds a failure, it +still exits 0: that failure is part of the report, not a command failure. +It never asks for or prints the provisioning token itself. + +The `provision_result.status` field in the JSON report is one of `none` +(no result file), `recorded`, `untrusted` (a file that fails the contract +check), `unreadable` (permission denied), or `corrupt` (a file that could +not be parsed). + +## Rules for maintainers + +- Codes are append-only once released. Never rename, renumber, or reuse a + code or exit number; add a new one and note the deprecation here. +- API rejection reasons are append-only too: `tokenFailureCodeForReason` + maps a fixed set of `error.metadata.reason` strings to their `TOKEN_*` + code. An unrecognized or absent reason falls back to `API_REJECTED` (see + `apiFailureCode`). So a new reason string does nothing until both a + matching code and a row here exist. +- Every code added in `cmd/cli/provision_result.go` needs a row here in the + same MR. Tests enforce the code/stage/exit maps and that this table has + exactly one `stage != package` row per ctrld code. The `package` rows + belong to `scripts/pkg/postinstall` and are not part of that count. +- Detail must stay bounded and free of secrets: the constructor strips the + provision token and cd UID and caps sizes; do not bypass it. diff --git a/docs/runtime-internal-logging.md b/docs/runtime-internal-logging.md new file mode 100644 index 00000000..fd713063 --- /dev/null +++ b/docs/runtime-internal-logging.md @@ -0,0 +1,170 @@ +# Runtime internal logging + +## Purpose + +ctrld writes its own log when the configuration sets no log file. The log gives Control D support and ctrld developers the evidence they need after a problem. Two of the three streams are files on disk. They keep their content across a restart and a self-upgrade. + +## When it is on + +Internal logging is on when these three conditions are true at the same time: + +- ctrld runs in Control D mode, with the `--cd` flag +- The configuration sets no `log_path` +- Silent mode is off + +When `log_path` is set, ctrld writes that file and starts no internal stream. The journal needs internal logging, so `log_path` mode has no journal. + +## Streams and files + +ctrld writes three streams. The debug file and the journal file live in the ctrld home directory, next to each other. + +| Stream | Level filter | File | Budget | Survives a restart | +|---|---|---|---|---| +| Console | Notice and above. The `-v` flag adds the lower levels | none | none | No | +| Debug | Every level | `ctrld.log` | 10 MB, 4 backup files | Yes | +| Journal | Warning and above, notice included, plus every event with `journal=true` | `ctrld-journal.log` | 2 MB, 2 backup files | Yes | + +The `-v` flag changes the console level and the level of the `log_path` file. Without the flag, the console shows notice and above. With `-v` it shows info and above. With `-vv` it shows every line. The debug file and the journal keep their filters in each case. + +A backup file carries a number suffix. The debug backup files are `ctrld.log.1` to `ctrld.log.4`, and `ctrld.log.1` holds the newest rotated content. The journal backup files are `ctrld-journal.log.1` and `ctrld-journal.log.2`. + +Rotation is by size. A write that crosses the size limit rotates the file first. ctrld deletes the file beyond the backup count. + +Each stream also keeps a memory buffer: 5 MB for the debug stream and 1 MB for the journal. These buffers serve `ctrld log tail` and the fallback path when a file cannot open. The files hold the history. + +A restart deletes the memory buffers and keeps the files. The new process appends a header line, which marks the restart point inside each file. When the header would make a file larger than its size limit, ctrld rotates the file first, and the header starts the new file. If the file cannot move, the header goes to the end of the current file. After an open error, ctrld opens the file again and appends the header with no rotation, so that file can pass its limit by one header. + +## Header and rotation events + +Every debug file and every journal file starts with one JSON line, the header. ctrld writes the header when it opens a file at start and after every rotation. On an existing file at start, ctrld appends the header. The journal holds JSON lines. The other lines of the debug file have the text format of the console. + +The message of the line is `Log header`. The line carries these fields: + +- Build: `version` and `commit` +- Host: `os`, `arch`, `pid`, and `start_time` +- Mode: `intercept_mode`, `listeners`, `upstream_count`, and `upstream_types` +- Identity: `resolver_uid` +- Files: `log_files` +- Network: `network`, the network snapshot of the host + +The header holds no provisioning token, no upstream endpoint URL, and no query name. `upstream_types` holds the type words, for example `doh` or `os`. `listeners` holds the listen addresses. + +The `network` object holds the routes, the gateways, the interfaces, the resolvers, the link type, and the intercept state. It has no `default_route_interface` field. The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every field of the object. + +After each rotation ctrld logs one `Log rotated` event at info level, with the fields `file`, `bytes_written`, `first_event_at`, `last_event_at`, and `backups`. This event is a journal event, so it lands in the debug stream and in the journal. + +## Network state events + +The journal holds the events that describe the host network. `Network snapshot` renders the `network` object at start, at each accepted transition, and at each begin and end of a recovery. +A snapshot equal to the snapshot written last reaches no journal, and one snapshot per 60 s is the limit. A new default route and a new resolver set pass that limit. `Network interface changed`, `Network transition`, `Host woke`, `OS resolver set changed`, and `DNS configuration changed` name each change of the network. `Recovery begin`, `Recovery end`, `PF anchor list changed`, and `Tunnel interface changed` name the repair work. + +AirDrop and virtual adapters produce a storm of network callbacks. ctrld puts these interfaces in a noise class. A delta that touches noise interfaces alone skips the handler. The first delta of a storm writes one `Network delta noise` journal line, and one more line covers each 10 minutes of the storm. + +`Query health` grades the query path over a window of 15 minutes. It logs at each change between the classes `healthy`, `degraded`, and `failing`, and once every 15 minutes as a heartbeat. A class change reports after it holds for two reads of the window. + +The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every event with its trigger, its level, and its fields. + +## Sampling of per-query errors + +A broken network produces one error line per query. ctrld bounds this flood for each error class and each upstream. The first 5 lines in a 60 s window stay at error level. Later lines in the same window drop to debug level, so the debug stream stays complete. The console and the `log_path` file show these lines only at debug level. The debug file keeps them. + +When a window closes, ctrld logs one `Per-query errors sampled` summary at error level. The summary carries `class`, `upstream`, `count`, `suppressed`, and `window_s`. ctrld logs the summary only when the window suppressed at least one line. + +An upstream logs one `Upstream state changed` event per transition. The down event is at warn level and carries `state` `down`, `failure_count`, and `reason`. The up event is at info level and carries `state` `up`, `down_for_ms`, and `failure_count`. Both are journal events. + +## Configuration keys + +The `log_level` key does not change the internal streams. The debug file keeps every level, and the journal keeps the warnings, the notices, the errors, and the journal events. A notice shares the level value of a warning. The `log_level` key changes the `log_path` file only. The `-v` flag changes the console level and the level of the `log_path` file, as the section "Streams and files" explains. + +Two keys in the `[service]` section size the debug stream: + +- The key `log_max_size_mb` sets the size limit of `ctrld.log` in MB. The default is 10. +- The key `log_max_backups` sets the number of backup files. The default is 4. The value 0 keeps no backup file. + +Both keys also apply to the `log_path` file, which rotates the same way. The journal budget is fixed. + +A change to the keys takes effect with no restart. This includes the first start, when the keys come from the Control D API, and a managed configuration refresh. When the backup count decreases, ctrld deletes the extra backups of `ctrld.log`. ctrld does not delete backups of the `log_path` file, because that directory can hold files of other programs. + +## Uploads + +`ctrld log send` uploads the log to Control D. The body holds these parts, in this order: + +- One header line rendered at send time, with the field `trigger` set to `send` +- The newest 10 MB of the debug files, oldest file first, cut at a line boundary +- The marker `=== LOG_END ===` +- Every journal file, oldest first + +The `network` object of the send-time header comes from a fresh read of the interfaces and of the route table. The resolvers come from the list that ctrld stored at its last resolver initialization, not from a fresh read. `ctrld log send --full` sends every debug file in place of the newest 10 MB. The reported size equals the number of bytes uploaded. + +`ctrld log view` prints the same composition on the terminal and takes the same `--full` flag. ctrld accepts one upload per minute. The body is plain text without gzip. + +In `log_path` mode the body holds the send-time header, then the `log_path` file and its backup files, oldest first. This body has no marker and no journal. + +`ctrld log tail` follows the live debug stream and prints each new line, journal events included. + +## Standard-user uploads on macOS + +Administrator opt-in `service.allow_unprivileged_log_send` defaults to `false`. +See the [configuration and activation requirements](config.md#allow_unprivileged_log_send). +Enabling it allows **all local-user processes** to request machine-wide diagnostic +uploads. Diagnostics can include other users' activity. It does not expose log +contents to the requester and is not a general authorization framework. + +The CLI connects directly to `/private/var/ctrld-log-send/send.sock`, without +checking the user's launchd service status or searching their home directory. +Its root-owned protected parent and connected peer credentials are checked. +The daemon creates the leaf directory without changing existing ancestor +permissions. It does not use macOS's group-writable `/private/var/run`. +The separate socket permits only a bodyless `POST /log/send`, with no query +parameters. Its responses contain status codes only. It cannot select files, +identity, upload destination, commands, or configuration. The administrative +socket remains restricted to mode 0600; `--full` still requires elevation. + +Unlike privileged uploads described above, delegated uploads use only checked, +file-backed source snapshots: up to 10 MiB of newest debug bytes and, in internal +logging mode, up to 6 MiB of journal bytes separated by `=== LOG_END ===`. +There is no fresh send-time header, network-command collection, memory fallback, +or line-boundary scan. Existing file headers remain in the retained bytes. +Source/config trust is checked again for every attempt. Unsafe sources fail closed +instead of being skipped. The daemon fixes the device identity and destination. +Trusted empty logs do not prevent socket startup, but an upload requires actual +log bytes; the separator alone is not an upload. + +Both privileged and delegated sends share one single-flight gate. The gate is +reserved before collection, and every admitted attempt, successful or failed, +starts a one-minute cooldown on completion. Reloading logs does not reset it. +Delegated requests have a five-minute operation deadline, short header/read +timeouts, bounded headers, at most eight accepted connections, and no keep-alive. +Admitted attempts record caller UID and outcome, without log contents or API error +text. Rejected/malformed/cooldown requests do not produce a per-request audit log, +so local callers cannot force diagnostic rotation by flooding rejections. + +Refusals and recovery: + +- `log send --full requires administrator privileges`: use the administrator path. +- `log send unavailable: ask an administrator ...`: the opt-in socket may be + disabled, unavailable, unsafe, or absent on an older daemon. Have an administrator + check configuration and restart; the CLI never falls back to a user-owned socket. +- `log send busy or cooling down; retry after one minute`: wait for the current + operation and its cooldown; parallel callers do not bypass the shared limit. +- `administrator must check configuration and log permissions`: the opt-in was + removed or config/log trust checks failed. Correct the protected paths as admin. +- Other failures return a generic message directing the administrator to the + daemon log, without returning upload payloads or upstream credentials. + +### Native macOS validation + +Before release, verify default-off, enable/restart, disable-before-restart, +restart socket removal, shutdown during upload, standard-user success, `--full` +refusal, privileged command compatibility, older/unavailable daemon behavior, +forbidden direct IPC, and socket impersonation. Verify root-owned default service +paths and actual Darwin peer-credential/ACL syscalls, including extended-ACL +refusals. Linux tests and Darwin cross-compilation do not satisfy this gate. +Automated boundary tests use only temporary synthetic files/sockets and upload +stubs; they do not run service, DNS, or PF operations or upload real diagnostics. + +## Notes + +- Internal logging is not a replacement for a configured `log_path` file in production. +- The journal is small on purpose. It keeps the events that describe a problem, not the events of each query. +- Support reads the upload without access to the host. The header line names the build, the mode, and the network, so support does not have to ask for them. diff --git a/docs/username-detection.md b/docs/username-detection.md new file mode 100644 index 00000000..8da71d7d --- /dev/null +++ b/docs/username-detection.md @@ -0,0 +1,154 @@ +# Username Detection in ctrld + +## Overview + +The ctrld client needs to detect the primary user of a system for telemetry and configuration purposes. This is particularly challenging in RMM (Remote Monitoring and Management) deployments where traditional session-based detection methods fail. + +## The Problem + +In traditional desktop environments, username detection is straightforward using environment variables like `$USER`, `$LOGNAME`, or `$SUDO_USER`. However, RMM deployments present unique challenges: + +- **No active login session**: RMM agents often run as system services without an associated user session +- **Missing environment variables**: Common user environment variables are not available in service contexts +- **Root/SYSTEM execution**: The ctrld process may run with elevated privileges, masking the actual user + +## Solution Approach + +ctrld implements a multi-tier, deterministic username detection system through the `DiscoverMainUser()` function with platform-specific implementations: + +### Key Principles + +1. **Deterministic selection**: No randomness - always returns the same result for the same system state +2. **Priority chain**: Multiple detection methods with clear fallback order +3. **Lowest UID/RID wins**: Among multiple candidates, select the user with the lowest identifier (typically the first user created) +4. **Fast execution**: All operations complete in <100ms using local system resources +5. **Debug logging**: Each decision point logs its rationale for troubleshooting + +## Platform-Specific Implementation + +### macOS (`discover_user_darwin.go`) + +**Detection chain:** +1. **Console owner** (`stat -f %Su /dev/console`) - Most reliable for active GUI sessions +2. **scutil ConsoleUser** - Alternative session detection via System Configuration framework +3. **Directory Services scan** (`dscl . list /Users UniqueID`) - Scan all users with UID ≥ 501, select lowest + +**Rationale**: macOS systems typically have a primary user who owns the console. Service contexts can still access device ownership information. + +### Linux (`discover_user_linux.go`) + +**Detection chain:** +1. **loginctl active users** (`loginctl list-users`) - systemd's session management +2. **Admin user preference** - Parse `/etc/passwd` for UID ≥ 1000, prefer sudo/wheel/admin group members +3. **Lowest UID fallback** - From `/etc/passwd`, select user with UID ≥ 1000 and lowest UID + +**Rationale**: Linux systems may have multiple regular users. Prioritize users in administrative groups as they're more likely to be primary system users. + +### Windows (`discover_user_windows.go`) + +**Detection chain:** +1. **Active console session** (`WTSGetActiveConsoleSessionId` + `WTSQuerySessionInformation`) - Direct Windows API for active user +2. **Registry admin preference** - Scan `HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ProfileList`, prefer Administrators group members +3. **Lowest RID fallback** - From ProfileList, select user with RID ≥ 1000 and lowest RID + +**Rationale**: Windows has well-defined APIs for session management. Registry ProfileList provides a complete view of all user accounts when no active session exists. + +### Other Platforms (`discover_user_others.go`) + +Returns `"unknown"` - placeholder for unsupported platforms. + +## Implementation Details + +### Error Handling + +- Individual detection methods log failures at Debug level and continue to next method +- Only final failure (all methods failed) is noteworthy +- Graceful degradation ensures the system continues operating with `"unknown"` user + +### Performance Considerations + +- Registry/file parsing uses native Go where possible +- External command execution limited to necessary cases +- No network calls or blocking operations +- Timeout context honored for all operations + +### Security + +- No privilege escalation required +- Read-only operations on system resources +- No user data collected beyond username +- Respects system access controls + +## Testing Scenarios + +This implementation addresses these common RMM scenarios: + +1. **Windows Service context**: No interactive user session, service running as SYSTEM +2. **Linux systemd service**: No login session, running as root daemon +3. **macOS LaunchDaemon**: No GUI user context, running as root +4. **Multi-user systems**: Multiple valid candidates, deterministic selection +5. **Minimalist systems**: Limited user accounts, fallback to available options + +## Metadata Submission Strategy + +System metadata (OS, chassis, username, domain) is sent to the Control D API via POST `/utility`. To avoid duplicate submissions and minimize EDR-triggering user discovery, ctrld uses a tiered approach: + +### When metadata is sent + +| Scenario | Metadata sent? | Username included? | +|---|---|---| +| `ctrld-client start` with `--cd-org` (provisioning via `cdUIDFromProvToken`) | ✅ Full | ✅ Yes | +| `ctrld-client run` managed startup (`processCDFlags`) | ✅ Full | ⚠️ Only from an active login session | +| Mobile library start (`RunMobile` → `processCDFlags`) | ✅ Full | ❌ No (see below) | +| `ctrld-client start --cd` install validation (`doValidateCdRemoteConfig`) | ✅ Lightweight | ❌ No | +| `ctrld-client restart` configuration validation (`doValidateCdRemoteConfig`) | ✅ Lightweight | ❌ No | +| Runtime config reload (`doReloadApiConfig`) | ✅ Lightweight | ❌ No | +| Runtime self-uninstall check | ✅ Lightweight | ❌ No | +| Runtime deactivation pin refresh | ✅ Lightweight | ❌ No | + +Provisioning runs the full `DiscoverMainUser` chain. Each managed daemon start runs only the active-session part of that chain (`SystemMetadataStartup`): the console user on macOS, the active console session on Windows, or an active `loginctl` user on Linux. The daemon usually starts at boot, before anyone logs in. When no session user is found, the start omits `username`, and the API keeps the stored value instead of replacing it with an account-list guess or `"unknown"`. Full daemon-start metadata also includes the supported hostname hints. Periodic, forced, self-uninstall, deactivation-PIN, and install/restart validation requests use `SystemMetadataRuntime()` and omit username discovery and hostname hints. + +A `--cd-org` install therefore sends two full snapshots: the token exchange (`cdUIDFromProvToken`), then the first check-in of the daemon that `start` launches with `--cd=`. Full discovery runs for the first and session discovery for the second. The API merges metadata objects, so the second snapshot does not erase fields from the first. + +The mobile library starts through `processCDFlags` too. iOS builds use `GOOS=darwin`, but the `stat` and `scutil` commands that macOS session discovery runs are not available, and Android has no `loginctl` session. Session discovery on mobile finds no user, so the mobile start omits `username`. + +### Host metadata wire contract (version 1) + +Every managed `/utility` request body carries `uid`, an optional `client_id`, and `metadata`. `metadata` is a JSON object of string values, or `null` when the caller supplies none. `UpdateCustomLastFailed` sends `null`, which the API treats as metadata omitted. All resolver-config callers supply metadata. + +Columns: **Provisioning** is the `--cd-org` token exchange; **Start** is the first check-in of each managed daemon start; **Runtime** is every other request in the table above. + +| Key | Source | Platforms | Provisioning | Start | Runtime | Value when the source is unavailable | +|---|---|---|---|---|---|---| +| `os` | `osinfo` | All | ✅ | ✅ | ✅ | Always present | +| `chassis_type` | `system.GetChassisInfo` (macOS model identifier) | All | ✅ | ✅ | ✅ | `""` | +| `chassis_vendor` | `system.GetChassisInfo` (`Apple Inc.` on macOS) | All | ✅ | ✅ | ✅ | `""` | +| `username` | Provisioning: `DiscoverMainUser`. Start: active login session only | All | ✅ | ⚠️ Session user only | ❌ | Provisioning: `"unknown"`. Start: key omitted | +| `domain_or_workgroup` | Windows domain-join status | Windows; `"false"` elsewhere | ✅ | ✅ | ✅ | `"false"` | +| `domain` | Active Directory domain | Windows; `""` elsewhere | ✅ | ✅ | ✅ | `""` | +| `hostname_ComputerName` | `scutil --get ComputerName` | macOS only | ✅ | ✅ | ❌ | Key omitted | +| `hostname_LocalHostName` | `scutil --get LocalHostName` | macOS only | ✅ | ✅ | ❌ | Key omitted | +| `hostname_HostName` | `scutil --get HostName` | macOS only | ✅ | ✅ | ❌ | Key omitted | +| `hostname_os.Hostname` | `os.Hostname()` | All | ✅ | ✅ | ❌ | Key omitted | + +The API accepts only these ten keys. It ignores unknown keys, non-string values, and any value longer than 255 bytes, field by field, without failing the request. An omitted key keeps the stored value and a present empty string clears it. The server side of this contract is [`docs/ctrld-utility-host-metadata.md` in controld-api](https://gitlab.int.windscribe.com/controld/backend/api/-/blob/master/docs/ctrld-utility-host-metadata.md). Bump the version above when a key, its platforms, or its unavailable value changes. + +### Runtime metadata (`SystemMetadataRuntime`) + +Runtime API calls (config reload, self-uninstall check, deactivation pin refresh, install/restart validation) use `SystemMetadataRuntime()` which includes OS and chassis info but **skips username discovery**. This avoids: + +- **EDR false positives**: Repeated user enumeration (registry scans, WTS queries, loginctl calls) can trigger endpoint detection and response alerts +- **Unnecessary work**: Username is unlikely to change while the service is running + +## Migration Notes + +The previous `currentLoginUser()` function has been replaced by `DiscoverMainUser()` with these changes: + +- **Removed dependencies**: No longer uses `logname(1)`, environment variables as primary detection +- **Added platform specificity**: Separate files for each OS with optimized detection logic +- **Improved RMM compatibility**: Designed specifically for service/daemon contexts +- **Maintained compatibility**: Returns same format (string username or "unknown") + +## Future Extensions + +This architecture allows easy addition of new platforms by creating additional `discover_user_.go` files following the same interface pattern. \ No newline at end of file diff --git a/docs/v2.0.0-breaking-changes.md b/docs/v2.0.0-breaking-changes.md new file mode 100644 index 00000000..5de5cedb --- /dev/null +++ b/docs/v2.0.0-breaking-changes.md @@ -0,0 +1,135 @@ +# ctrld v2.0.0 Breaking Changes + +This document outlines the breaking changes introduced in ctrld v2.0.0 and provides migration guidance for affected users. + +## Overview + +ctrld v2.0.0 removes automatic configuration support for router and server platforms. This means ctrld will no longer perform "magic" configuration to automatically set itself up as an upstream for existing DNS software on these platforms. + +## What's Changing + +### Removed Platform Support + +**Router Platforms:** +- ctrld will no longer automatically configure itself as an upstream for dnsmasq or other DNS software +- No automatic detection and configuration of router-specific DNS settings + +**Server Platforms:** +- ctrld will no longer automatically configure Windows Server DNS forwarder settings +- No automatic integration with server DNS services + +### What Remains Supported + +**Desktop Platforms:** +- Windows Desktop +- macOS Desktop +- Linux Desktop + +These platforms continue to receive full automatic configuration support. + +## Stay on v1.x.x + +ctrld v1.x.x will continue to be supported for router and server platforms: +- Important bug fixes (regression or security) will be cherry-picked to v1.x.x branch +- New features may still be added (but may take longer to implement) +- Long-term support for these platforms + +## Migration Path for Router and Server Users + +If you're currently using ctrld v1.x.x on router or server platforms, you need to follow these steps to migrate to v2.0.0: + +### Step 1: Downloading the ctrld-client binary + +To download the new client, follow these steps: + +Stop the current ctrld service: + +```sh +ctrld stop +``` + +Or uninstall the current version: + +```sh +ctrld uninstall +``` + +Download the appropriate binary for your platform: https://dl.controld.com/linux-amd64/ctrld-client + +> **Note**: Replace `amd64` with your platform architecture as needed. + +Verify that the binary was updated correctly: + +```sh +ctrld-client --version +``` + +Expected output: +``` +ctrld-client version v1.0.0 +``` + +### Step 2: Start ctrld-client without self-checking + +You have two ways to start ctrld-client: + +**Option A: Use Remote Configuration (Recommended)** +1. **Export your current configuration:** + - Copy the contents of your current `ctrld.toml` file + +2. **Import to Control D Dashboard:** + - Log into your Control D dashboard + - Use the remote configuration feature to upload your configuration + +3. **Start ctrld-client with remote config:** + ```bash + sudo ctrld-client service start --cd= --skip_self_checks + ``` + +> **Note**: You must use `ctrld-client service start` to prevent DNS being set automatically by ctrld-client. + +**Option B: Use Local Configuration** +```bash +sudo ctrld-client service start --skip_self_checks +``` + +### Step 3: Configure DNS Software to Use ctrld-client as Upstream + +**For dnsmasq users:** +1. Configure dnsmasq to use ctrld-client as upstream: + ```bash + # Add to dnsmasq.conf + no-resolv + server=127.0.0.1#5354 + add-mac + add-subnet=32,128 + # Disable cache or set max-cache-ttl=0 + # to prevent queries from caching + cache-size=0 + # max-cache-ttl=0 + ``` +2. Restart dnsmasq: + ```bash + sudo service dnsmasq restart + ``` + +**For Windows Server users:** +1. Configure DNS forwarder in Windows Server: + - Open DNS Manager + - Right-click on your server name + - Select "Properties" → "Forwarders" tab + - Add `` as a forwarder + +## Getting Help + +If you encounter any issues during migration or have questions about the v2.0.0 changes: + +1. **File an issue:** [GitHub Issues](https://github.com/Control-D-Inc/ctrld/issues) +2. **Contact support:** Email help@controld.com. +3. **Check documentation:** Review the [configuration documentation](config.md) for detailed setup instructions + +## Summary + +While ctrld v2.0.0 removes automatic configuration for router and server platforms, it provides a more focused experience for desktop users while still allowing router/server users to continue using ctrld with manual configuration or by staying on the v1.x.x branch. + +The migration path is designed to be straightforward, with multiple options to suit different use cases and technical comfort levels. diff --git a/docs/wfp-dns-intercept.md b/docs/wfp-dns-intercept.md new file mode 100644 index 00000000..28d2c5e6 --- /dev/null +++ b/docs/wfp-dns-intercept.md @@ -0,0 +1,480 @@ +# Windows DNS Intercept — Technical Reference + +## Overview + +On Windows, DNS intercept mode uses a two-layer architecture: + +- **`dns` mode (default)**: NRPT only — graceful DNS routing via the Windows DNS Client service +- **`hard` mode**: NRPT + WFP — full enforcement with kernel-level block filters + +`dns` mode avoids ctrld's outbound block filters and therefore degrades more gracefully when owned NRPT is removed. Resolution still depends on the active NRPT target being reachable; an administrator-owned GP catch-all intentionally remains fail-closed if its loopback listener is stopped. + +## Architecture: dns vs hard Mode + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ dns mode (NRPT + loopback WFP protect) │ +│ │ +│ App DNS query → DNS Client service → NRPT lookup │ +│ → "." catch-all matches → forward to 127.0.0.1 (ctrld) │ +│ │ +│ Loopback WFP protect: 4 hard-permit filters (port 53 to │ +│ localhost, CLEAR_ACTION_RIGHT) prevent third-party VPN WFP │ +│ blocks (e.g., OpenVPN block-outside-dns) from breaking NRPT. │ +│ │ +│ Owned rule missing → restore; GP missing → owned fallback │ +│ GP rule + dead listener remains intentionally fail-closed │ +└─────────────────────────────────────────────────────────────────┘ + +┌─────────────────────────────────────────────────────────────────┐ +│ hard mode (NRPT + WFP) │ +│ │ +│ App DNS query → DNS Client service → NRPT → 127.0.0.1 (ctrld)│ +│ │ +│ Bypass attempt (raw 8.8.8.8:53) → WFP BLOCK filter │ +│ VPN DNS on private IP → WFP subnet PERMIT filter → allowed │ +│ │ +│ NRPT route is established before WFP starts │ +│ WFP failure rolls back only ctrld-owned NRPT; GP is untouched │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## NRPT (Name Resolution Policy Table) + +### What It Does + +NRPT is a Windows feature (originally for DirectAccess) that tells the DNS Client +service to route queries matching specific namespace patterns to specific DNS servers. +ctrld adds a catch-all rule that routes ALL DNS to `127.0.0.1`: + +| Registry Value | Type | Value | Purpose | +|---|---|---|---| +| `Name` | REG_MULTI_SZ | `.` | Namespace (`.` = catch-all) | +| `GenericDNSServers` | REG_SZ | `127.0.0.1` | Target DNS server | +| `ConfigOptions` | REG_DWORD | `0x8` | Standard DNS resolution | +| `Version` | REG_DWORD | `0x2` | NRPT rule version 2 | +| `Comment` | REG_SZ | `` | Empty (matches PowerShell behavior) | +| `DisplayName` | REG_SZ | `` | Empty (matches PowerShell behavior) | +| `IPSECCARestriction` | REG_SZ | `` | Empty (matches PowerShell behavior) | + +### Registry Paths — GP vs Local (Critical) + +Windows NRPT has two registry paths with **all-or-nothing** precedence: + +| Path | Name | Mode | +|---|---|---| +| `HKLM\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig` | **GP path** | Group Policy mode | +| `HKLM\SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig` | **Local path** | Local/service store mode | + +**Precedence rule**: If ANY rules exist in the GP path (from IT policy, VPN, MDM, +or our own earlier builds), DNS Client enters "GP mode" and **ignores ALL local-path +rules entirely**. This is not per-rule — it's a binary switch. + +**Consequence**: On non-domain-joined (WORKGROUP) machines, `RefreshPolicyEx` is +unreliable. If we write to the GP path, DNS Client enters GP mode but the rules +never activate — resulting in `Get-DnsClientNrptPolicy` returning empty even though +`Get-DnsClientNrptRule` shows the rule in registry. + +ctrld uses an adaptive strategy (matching [Tailscale's approach](https://github.com/tailscale/tailscale/blob/main/net/dns/nrpt_windows.go)) when it owns NRPT: + +1. **Always write to the local path** using a deterministic GUID key name + (`{B2E9A3C1-7F4D-4A8E-9D6B-5C1E0F3A2B8D}`). This is the baseline that works + on all non-domain machines. +2. **Check if other software has GP NRPT rules** (`otherGPRulesExist()`). If + foreign GP rules are present (IT policy, VPN), DNS Client is already in GP mode + and our local rule would be invisible — so we also write to the GP path. ctrld does + not write to the GP path when the foreign rule is an administrator catch-all for this + listener: a second catch-all in that store is never written, whatever the cost. +3. **If no foreign GP rules exist**, clean any stale ctrld GP rules and delete + the empty GP parent key. This ensures DNS Client stays in "local mode" where + the local-path rule activates immediately via `paramchange`. + +### Adopting an Organization-Owned GP Catch-All + +Before applying that ctrld-owned strategy, service startup looks for a non-ctrld GP child with exactly `Name=["."]` and one `GenericDNSServers` value equal to the actual listener IP. The registry match is only an ownership candidate: ctrld sends its unique DNS Client probe before any NRPT write and re-reads the same child afterward. + +Intercept state stays unpublished while that happens. Startup publishes nothing until it has fully succeeded, which is why the probe and heal flows take the state as an argument instead of reading the published field — publishing early would expose a half-built `wfpState`, with no engine handle and filter IDs still being assigned, to callers such as the VPN DNS exemption path. The one deliberate exception is hard mode when WFP setup fails while GP-managed NRPT is verified routing: that state is published so the health monitor can keep retrying WFP, and the service start is still reported as failed. + +When the rule remains unchanged and the probe arrives, Group Policy owns NRPT: + +- the startup adapter reset is skipped; +- no ctrld NRPT key is created; +- `RefreshPolicyEx`, Dnscache `paramchange`, and cache flush are not used for that policy; +- shutdown/uninstall leave the GP child untouched. + +A matching GP child that remains present but fails its probe is reported as ineffective and is not rewritten. If the proof finds no route while the same rule stays on disk, ctrld puts back only its local-store key, keeps ownership, and does not write its GP-store rule adjacent to the administrator's rule. If the child disappears, ctrld activates its normal owned fallback. A GP catch-all that instead changes to another resolver is reported as a conflict; ctrld does not create a second ambiguous catch-all. If the matching rule later returns, ctrld probes first, removes only its deterministic fallback keys, signals the ownership transition once, and resumes observing Group Policy. + +**Deployment ordering:** apply the GPO before starting ctrld to guarantee adapter DNS is never reset. Remove/unlink it before intentionally stopping or uninstalling ctrld. A GP catch-all still targeting loopback with no listener running is a deliberate fail-closed state and will break DNS. + +### Reproducing the Empty GP Parent Case + +This is a production code reference, so the temporary repro script is not kept in +the repository. For MR !942 review, the test script and exact before/after steps +are posted in the MR discussion. The scenario to compare is: + +1. Run the same approved PowerShell repro script against a pre-fix build and this + branch with the same ctrld config. +2. Create an empty GP NRPT parent key while ctrld is running in DNS intercept mode. +3. Confirm pre-fix logs can spend policy refresh/paramchange retries while the GP + parent remains empty. +4. Confirm post-fix logs clean the empty GP parent, send one NRPT-change signal, + and re-probe before normal retries. + +### VPN Coexistence + +NRPT uses most-specific-match. VPN NRPT rules for specific domains (e.g., +`*.corp.local` → `10.20.30.1`) take priority over ctrld's `.` catch-all. +This means VPN split DNS works naturally — VPN-specific domains go to VPN DNS, +everything else goes to ctrld. No exemptions or special handling needed. + +### DNS Client Notification + +After writing NRPT rules, DNS Client must be notified to reload: + +1. **`paramchange`**: `sc control dnscache paramchange` — signals DNS Client to + re-read configuration. Works for local-path rules on most machines. +2. **`RefreshPolicyEx`**: `RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE)` from + `userenv.dll` — triggers GP refresh for GP-path rules. Unreliable on non-domain + machines (WORKGROUP). Fallback: `gpupdate /target:computer /force`. +3. **DNS cache flush**: `DnsFlushResolverCache` from `dnsapi.dll` or `ipconfig /flushdns` + — clears stale cached results from before NRPT was active. + +### DNS Cache Flush + +After NRPT changes, stale DNS cache entries could bypass the new routing. ctrld flushes: + +1. **Primary**: `DnsFlushResolverCache` from `dnsapi.dll` +2. **Fallback**: `ipconfig /flushdns` (subprocess) + +### Known Limitation: nslookup + +`nslookup.exe` implements its own DNS resolver and does NOT use the Windows DNS Client +service. It ignores NRPT entirely. Use `Resolve-DnsName` (PowerShell) or `ping` to +verify DNS resolution through NRPT. This is a well-known Windows behavior. + +## WFP (Windows Filtering Platform) — hard Mode Only + +### Filter Stack + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Sublayer: "ctrld DNS Intercept" (weight 0xFFFF — max priority) │ +│ │ +│ ┌─ Permit Filters (weight 10) ─────────────────────────────┐ │ +│ │ • IPv4/UDP to 127.0.0.1:53 → PERMIT │ │ +│ │ • IPv4/TCP to 127.0.0.1:53 → PERMIT │ │ +│ │ • IPv6/UDP to ::1:53 → PERMIT │ │ +│ │ • IPv6/TCP to ::1:53 → PERMIT │ │ +│ │ • RFC1918 + CGNAT subnets:53 → PERMIT (VPN DNS) │ │ +│ │ • VPN DNS exemptions (dynamic) → PERMIT │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─ Block Filters (weight 1) ───────────────────────────────┐ │ +│ │ • All IPv4/UDP to *:53 → BLOCK │ │ +│ │ • All IPv4/TCP to *:53 → BLOCK │ │ +│ │ • All IPv6/UDP to *:53 → BLOCK │ │ +│ │ • All IPv6/TCP to *:53 → BLOCK │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ │ +│ Filter evaluation: higher weight wins → permits checked first │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Why WFP Can't Work Alone + +WFP operates at the connection authorization layer (`FWPM_LAYER_ALE_AUTH_CONNECT`). +It can only **block** or **permit** connections — it **cannot redirect** them. +Redirection requires kernel-mode callout drivers (`FwpsCalloutRegister` in +`fwpkclnt.lib`) using `FWPM_LAYER_ALE_CONNECT_REDIRECT_V4/V6`, which are not +accessible from userspace. + +Without NRPT, WFP blocks outbound DNS but doesn't tell applications where to send +queries instead — they just see DNS failures. This is why `hard` mode requires NRPT +to be active first, and why WFP is rolled back if NRPT setup fails. + +### Sublayer Priority + +Weight `0xFFFF` (maximum) ensures ctrld's filters take priority over any other WFP +sublayers from VPN software, endpoint security, or Windows Defender Firewall. + +### RFC1918 + CGNAT Subnet Permits + +Static permit filters for private IP ranges (10.0.0.0/8, 172.16.0.0/12, +192.168.0.0/16, 100.64.0.0/10) allow VPN DNS servers on private IPs to work +without dynamic per-server exemptions. This covers Tailscale MagicDNS +(100.100.100.100), corporate VPN DNS (10.x.x.x), and similar. + +### VPN DNS Exemption Updates + +When `vpnDNSManager.Refresh()` discovers VPN DNS servers on public IPs: + +1. Delete all existing VPN permit filters (by stored IDs) +2. For each VPN DNS server IP: + - IPv4: `addWFPPermitIPFilter()` on `ALE_AUTH_CONNECT_V4` + - IPv6: `addWFPPermitIPv6Filter()` on `ALE_AUTH_CONNECT_V6` + - Both UDP and TCP for each IP +3. Store new filter IDs for next cleanup cycle + +**In `dns` mode, VPN DNS exemptions are skipped** — there are no ctrld WFP block +filters to exempt from. The loopback WFP protect filters only permit localhost +DNS; VPN DNS traffic goes through the tunnel interface and is already permitted +by the VPN's own WFP rules. + +### Session Lifecycle + +**Startup (hard mode):** +``` +1. Adopt a proven matching GP catch-all, or install ctrld-owned NRPT +2. FwpmEngineOpen0() with RPC_C_AUTHN_DEFAULT (0xFFFFFFFF) +3. Delete stale sublayer (crash recovery) +4. FwpmSubLayerAdd0() — weight 0xFFFF +5. Add 4 localhost permit filters +6. Add 4 block filters +7. Add RFC1918 + CGNAT subnet permits +8. Start ownership-aware NRPT/WFP health monitor +``` + +**Startup (dns mode):** +``` +1. Adopt a proven matching GP catch-all, or install ctrld-owned NRPT +2. Activate loopback WFP protect (4 hard-permit filters for localhost DNS) +3. Start ownership-aware NRPT health monitor +``` + +**Shutdown:** +``` +1. Stop NRPT health monitor +2. Remove + signal only ctrld-owned NRPT; leave GP-managed policy untouched +3. Clean up ctrld WFP filters, sublayer, and engine session +``` + +**Crash Recovery:** +On startup, `cleanupStaleDNSInterceptState()` calls `FwpmSubLayerDeleteByKey0` to +remove any stale sublayer from a previous unclean shutdown (the deterministic GUID +ensures we only ever target our own). It opens a **non-dynamic** session, because a +dynamic one cannot delete objects that were not added in a dynamic session — which is +what a build predating session-scoped ownership leaves behind. + +Whether deleting the sublayer also removes the filters inside it is **not documented** +either way, and is unresolved (see the note on `cleanupStaleDNSInterceptState`). If it +does not, the call returns `FWP_E_IN_USE` and the orphaned filters need deleting first; +that case is logged as a warning rather than reported as a successful cleanup, so it is +visible in a support log instead of silently leaving enforcement active. + +Since every current build uses a dynamic session, the OS removes ctrld's filters when +the process dies, so this recovery path only matters for state left by an older build. + +## NRPT Probe and Auto-Heal + +### The Problem: Async GP Refresh Race + +`RefreshPolicyEx` triggers a Group Policy refresh but returns immediately — it does +NOT wait for the DNS Client service to actually reload NRPT from the registry. On +cold machines (first boot, fresh install, long sleep), the DNS Client may take +several seconds to process the policy refresh. During this window, NRPT rules exist +in the registry but the DNS Client hasn't loaded them — queries bypass ctrld. + +### The Solution: Active Probing + +After writing NRPT to the registry, ctrld sends a probe DNS query through the +Windows DNS Client path to verify NRPT is actually working: + +1. Generate a unique probe domain: `_nrpt-probe-.nrpt-probe.ctrld.test` +2. Send it via Go's `net.Resolver` (calls `GetAddrInfoW` → DNS Client → NRPT) +3. If NRPT is active, DNS Client routes it to 127.0.0.1 → ctrld receives it +4. ctrld's DNS handler recognizes the probe prefix and signals success +5. If the probe times out (2s), NRPT isn't loaded yet → retry with remediation + +### Startup Probes + +For a matching GP candidate, startup blocks for one 2-second probe before any NRPT mutation and then re-reads the same child. A received query plus the unchanged rule proves both routing and external ownership. If that first probe fails, ctrld installs its normal WFP protection and performs one more ownership-safe probe before advertising startup readiness. + +ctrld-owned NRPT retains the asynchronous sequence: + +``` +Immediate probe + ├─ Success → "NRPT verified working", done + └─ Timeout + ├─ Empty GP parent → clean once, signal once, re-probe + └─ Otherwise → bounded 1s/2s/4s signal + probe retries + └─ Still failing → two-phase remove/signal/re-add/final probe +``` + +### GP-Managed Probe Failure + +A matching GP child still owns Windows' effective NRPT store even when its probe fails. Writing a local rule cannot override that precedence, and rewriting the GP child would violate administrator ownership. ctrld therefore retries only its loopback WFP permit protection, reports the ineffective external policy, and leaves NRPT registry values and policy signals untouched. + +### Health Monitor Integration + +The 30s periodic health monitor now does actual probing, not just registry checks: + +``` +Every 30s: + ├─ GP-managed owner + │ ├─ Matching child + probe success → observe only + │ ├─ Matching child + probe failure → WFP-only retry; no NRPT mutation + │ └─ Matching child gone → activate ctrld-owned fallback + verify + │ + ├─ ctrld-owned owner + │ ├─ Working matching GP child returns → remove only ctrld keys; adopt GP + │ │ └─ Probe finds no route, rule still on disk → write back only the + │ │ local-store key, keep ownership, write no GP sibling + │ ├─ ctrld key missing → restore + signal + verify + │ └─ ctrld key present → probe; run owned heal sequence on failure + │ + └─ (hard mode) Check WFP sublayer; full intercept restart if missing +``` + +**Singleton guard:** Only one asynchronous probe-and-heal sequence runs at a time (atomic bool). Startup's GP-candidate probe completes before the health monitor starts; direct periodic probes finish before they schedule a heal sequence. + +**Why periodic, not just network-event?** VPN software or Group Policy updates can +clear NRPT at any time, not just during network changes. A 30s periodic check ensures +recovery within a bounded window. + +**Hard mode safety:** The health monitor verifies NRPT before checking WFP. If NRPT +is gone, it's restored first. WFP is never running without NRPT — this prevents +DNS blackholes where WFP blocks everything but NRPT isn't routing to ctrld. + +## DNS Flow Diagrams + +### Normal Resolution (both modes) + +``` +App → DNS Client → NRPT lookup → "." matches → 127.0.0.1 → ctrld + → Control D DoH (port 443, not affected by WFP port-53 rules) + → response flows back +``` + +### VPN Split DNS (both modes) + +``` +App → DNS Client → NRPT lookup: + VPN domain (*.corp.local) → VPN's NRPT rule wins → VPN DNS server + Everything else → ctrld's "." catch-all → 127.0.0.1 → ctrld + → VPN domain match → forward to VPN DNS (port 53) + → (hard mode: WFP subnet permit allows private IP DNS) +``` + +### Bypass Attempt (hard mode only) + +``` +App → raw socket to 8.8.8.8:53 → WFP ALE_AUTH_CONNECT → BLOCK +``` + +In `dns` mode, this query would succeed (no WFP) — the tradeoff for never +breaking DNS. + +## Key Differences from macOS (pf) + +| Aspect | macOS (pf) | Windows dns mode | Windows hard mode | +|--------|-----------|------------------|-------------------| +| **Routing** | `rdr` redirect | NRPT policy | NRPT policy | +| **Enforcement** | `route-to` + block rules | Loopback WFP protect | WFP block filters | +| **Can break DNS?** | Yes (pf corruption) | No | Yes (if NRPT lost) | +| **VPN coexistence** | Watchdog + stabilization | NRPT + loopback hard-permit | Same + WFP permits | +| **Bypass protection** | pf catches all packets | None | WFP catches all connections | +| **Recovery** | Probe + auto-heal | Health monitor re-adds | Full restart on sublayer loss | + +## WFP API Notes + +### Struct Layouts + +WFP C API structures are manually defined in Go (`golang.org/x/sys/windows` doesn't +include WFP types). Field alignment must match the C ABI exactly — any mismatch +causes access violations or silent corruption. + +### FWP_DATA_TYPE Enum + +``` +FWP_EMPTY = 0 +FWP_UINT8 = 1 +FWP_UINT16 = 2 +FWP_UINT32 = 3 +FWP_UINT64 = 4 +... +``` + +**⚠️** Some documentation examples incorrectly start at 1. The enum starts at 0 +(`FWP_EMPTY`), making all subsequent values offset by 1 from what you might expect. + +### GC Safety + +When passing Go heap objects to WFP syscalls via `unsafe.Pointer`, use +`runtime.KeepAlive()` to prevent garbage collection during the call: + +```go +conditions := make([]fwpmFilterCondition0, 3) +filter.filterCondition = &conditions[0] +r1, _, _ := procFwpmFilterAdd0.Call(...) +runtime.KeepAlive(conditions) +``` + +### Authentication + +`FwpmEngineOpen0` requires `RPC_C_AUTHN_DEFAULT` (0xFFFFFFFF) for the authentication +service parameter. `RPC_C_AUTHN_NONE` (0) returns `ERROR_NOT_SUPPORTED` on some +configurations (e.g., Parallels VMs). + +### Elevation + +WFP requires admin/SYSTEM privileges. `FwpmEngineOpen0` fails with HRESULT 0x32 +when run non-elevated. Services running as SYSTEM have this automatically. + +## Debugging + +### Check NRPT Rules + +```powershell +# PowerShell — show active NRPT rules +Get-DnsClientNrptRule + +# Check registry directly +Get-ChildItem "HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig" +``` + +### Check WFP Filters (hard mode) + +```powershell +# Show all WFP filters (requires admin) — output is XML +netsh wfp show filters + +# Search for ctrld's filters +Select-String "ctrld" filters.xml +``` + +### Verify DNS Resolution + +```powershell +# Use Resolve-DnsName, NOT nslookup (nslookup bypasses NRPT) +Resolve-DnsName example.com +ping example.com + +# If you must use nslookup, specify localhost: +nslookup example.com 127.0.0.1 + +# Force GP refresh (if NRPT not loading) +gpupdate /target:computer /force + +# Verify service registration +sc qc ctrld-client +``` + +### Service Verification + +After install, verify the Windows service is correctly registered: + +```powershell +# Check binary path and start type +sc qc ctrld-client + +# Should show: +# BINARY_PATH_NAME: "C:\...\ctrld-client.exe" run --cd xxxxx --intercept-mode dns +# START_TYPE: AUTO_START +``` + +## Related + +- [DNS Intercept Mode Overview](dns-intercept-mode.md) — cross-platform documentation +- [pf DNS Intercept](pf-dns-intercept.md) — macOS technical reference +- [Microsoft WFP Documentation](https://docs.microsoft.com/en-us/windows/win32/fwp/windows-filtering-platform-start-page) +- [Microsoft NRPT Documentation](https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-r2-and-2012/dn593632(v=ws.11)) diff --git a/doh.go b/doh.go index 3459cb8a..70c4f9df 100644 --- a/doh.go +++ b/doh.go @@ -25,6 +25,16 @@ const ( dohOsHeader = "x-cd-os" dohClientIDPrefHeader = "x-cd-cpref" headerApplicationDNS = "application/dns-message" + + // dohMaxResponseSize caps the response body read from a DoH/DoH3 + // upstream. A DNS message is bounded by the protocol's 16-bit length + // field; anything larger cannot be a valid response. The cap stops a + // malicious or compromised upstream from driving ctrld into unbounded + // memory growth via io.ReadAll on attacker-controlled bytes. + dohMaxResponseSize = dns.MaxMsgSize + // dohMaxErrorBodySize bounds how much of a non-200 response body is + // read for inclusion in the returned error. + dohMaxErrorBodySize = 1024 ) // EncodeOsNameMap provides mapping from OS name to a shorter string, used for encoding x-cd-os value. @@ -53,6 +63,9 @@ var EncodeArchNameMap = map[string]string{ var DecodeArchNameMap = map[string]string{} func init() { + // Create reverse mappings for OS and architecture names + // This is needed because the API expects encoded values, but we need to decode + // them back to their original form for processing for k, v := range EncodeOsNameMap { DecodeOsNameMap[v] = k } @@ -85,8 +98,15 @@ type dohResolver struct { // Resolve performs DNS query with given DNS message using DOH protocol. func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + if err := validateMsg(msg); err != nil { + return nil, err + } + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "DoH resolver query started") + data, err := msg.Pack() if err != nil { + Log(ctx, logger.Error().Err(err), "Failed to pack DNS message") return nil, err } @@ -98,6 +118,7 @@ func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro endpoint.RawQuery = query.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) if err != nil { + Log(ctx, logger.Error().Err(err), "Could not create HTTP request") return nil, fmt.Errorf("could not create request: %w", err) } addHeader(ctx, req, r.uc) @@ -105,45 +126,53 @@ func (r *dohResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro if len(msg.Question) > 0 { dnsTyp = msg.Question[0].Qtype } - c := http.Client{Transport: r.uc.dohTransport(dnsTyp)} + c := http.Client{Transport: r.uc.dohTransport(ctx, dnsTyp)} if r.isDoH3 { - transport := r.uc.doh3Transport(dnsTyp) + transport := r.uc.doh3Transport(ctx, dnsTyp) if transport == nil { + Log(ctx, logger.Error(), "DoH3 is not supported") return nil, errors.New("DoH3 is not supported") } c.Transport = transport } + + Log(ctx, logger.Debug(), "Sending DoH request to: %s", endpoint.String()) resp, err := c.Do(req) - if err != nil && r.uc.FallbackToDirectIP() { + if err != nil && r.uc.FallbackToDirectIP(ctx) { retryCtx, cancel := r.uc.Context(context.WithoutCancel(ctx)) defer cancel() - Log(ctx, ProxyLogger.Load().Warn().Err(err), "retrying request after fallback to direct ip") + logger := LoggerFromCtx(ctx) + logger.Warn().Err(err).Msg("Retrying request after fallback to direct ip") resp, err = c.Do(req.Clone(retryCtx)) } if err != nil { err = wrapUrlError(err) - if r.isDoH3 { - if closer, ok := c.Transport.(io.Closer); ok { - closer.Close() - } - } + Log(ctx, logger.Error().Err(err), "DoH request failed") return nil, fmt.Errorf("could not perform request: %w", err) } defer resp.Body.Close() - buf, err := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, dohMaxErrorBodySize)) + return nil, fmt.Errorf("wrong response from DOH server, got: %s, status: %d", string(body), resp.StatusCode) + } + + buf, err := io.ReadAll(io.LimitReader(resp.Body, dohMaxResponseSize+1)) if err != nil { + Log(ctx, logger.Error().Err(err), "Could not read response body") return nil, fmt.Errorf("could not read message from response: %w", err) } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("wrong response from DOH server, got: %s, status: %d", string(buf), resp.StatusCode) + if len(buf) > dohMaxResponseSize { + return nil, fmt.Errorf("DoH response exceeds %d-byte maximum DNS message size", dohMaxResponseSize) } answer := new(dns.Msg) if err := answer.Unpack(buf); err != nil { + Log(ctx, logger.Error().Err(err), "Failed to unpack DNS answer") return nil, fmt.Errorf("answer.Unpack: %w", err) } + + Log(ctx, logger.Debug(), "DoH resolver query successful") return answer, nil } @@ -163,7 +192,8 @@ func addHeader(ctx context.Context, req *http.Request, uc *UpstreamConfig) { } } if printed { - Log(ctx, ProxyLogger.Load().Debug(), "sending request header: %v", dohHeader) + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "Sending request header: %v", dohHeader) } dohHeader.Set("Content-Type", headerApplicationDNS) dohHeader.Set("Accept", headerApplicationDNS) diff --git a/doh_http2_healthcheck_test.go b/doh_http2_healthcheck_test.go new file mode 100644 index 00000000..94d7bd7c --- /dev/null +++ b/doh_http2_healthcheck_test.go @@ -0,0 +1,164 @@ +package ctrld + +import ( + "context" + "crypto/tls" + "io" + "net" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" +) + +// freezableProxy forwards TCP traffic between the client and target until +// freeze is called. After that it stops relaying server-to-client bytes while +// still draining the client side, so the client sees a silent peer and only +// its own HTTP/2 health check can notice. clientClosed is closed when the +// client side of the connection ends. +type freezableProxy struct { + ln net.Listener + target string + mu sync.Mutex + frozen bool + clientClosed chan struct{} + once sync.Once +} + +func newFreezableProxy(t *testing.T, target string) *freezableProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + p := &freezableProxy{ln: ln, target: target, clientClosed: make(chan struct{})} + t.Cleanup(func() { ln.Close() }) + go p.serve() + return p +} + +func (p *freezableProxy) isFrozen() bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.frozen +} + +func (p *freezableProxy) freeze() { + p.mu.Lock() + p.frozen = true + p.mu.Unlock() +} + +func (p *freezableProxy) serve() { + for { + c, err := p.ln.Accept() + if err != nil { + return + } + go p.handle(c) + } +} + +func (p *freezableProxy) handle(client net.Conn) { + defer client.Close() + server, err := net.Dial("tcp", p.target) + if err != nil { + return + } + defer server.Close() + go func() { + buf := make([]byte, 32*1024) + for { + n, err := server.Read(buf) + if err != nil { + return + } + if p.isFrozen() { + continue + } + if _, err := client.Write(buf[:n]); err != nil { + return + } + } + }() + buf := make([]byte, 32*1024) + for { + n, err := client.Read(buf) + if err != nil { + if p.isFrozen() { + p.once.Do(func() { close(p.clientClosed) }) + } + return + } + if p.isFrozen() { + continue + } + if _, err := server.Write(buf[:n]); err != nil { + return + } + } +} + +// TestDoHTransport_HTTP2HealthCheckClosesSilentConnection verifies that the +// DoH transport negotiates HTTP/2 and that its HTTP/2 health check closes a +// connection whose peer stops answering, so a wedged connection is not kept +// in the pool (https://github.com/golang/go/issues/36026). +func TestDoHTransport_HTTP2HealthCheckClosesSilentConnection(t *testing.T) { + oldSend, oldPing := dohSendPingTimeout, dohPingTimeout + dohSendPingTimeout, dohPingTimeout = 200*time.Millisecond, 200*time.Millisecond + t.Cleanup(func() { dohSendPingTimeout, dohPingTimeout = oldSend, oldPing }) + + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + testCert := generateTestCertificate(t) + srv.TLS = &tls.Config{ + Certificates: []tls.Certificate{testCert.tlsCert}, + NextProtos: []string{"h2", "http/1.1"}, + MinVersion: tls.VersionTLS12, + } + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + + proxy := newFreezableProxy(t, srv.Listener.Addr().String()) + _, port, _ := net.SplitHostPort(proxy.ln.Addr().String()) + + uc := dohUpstreamForTLSServer(t, srv) + uc.Endpoint = "https://127.0.0.1:" + port + "/dns-query" + uc.BootstrapIP = "127.0.0.1" + uc.SetupTransport(context.Background()) + t.Cleanup(uc.transport.CloseIdleConnections) + + if uc.transport.HTTP2 == nil { + t.Fatal("DoH transport has no HTTP/2 config") + } + + resp, err := uc.transport.RoundTrip(mustNewRequest(t, uc.Endpoint)) + if err != nil { + t.Fatalf("RoundTrip: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.ProtoMajor != 2 { + t.Fatalf("DoH transport negotiated HTTP/%d, want HTTP/2", resp.ProtoMajor) + } + + proxy.freeze() + + select { + case <-proxy.clientClosed: + case <-time.After(5 * time.Second): + t.Fatal("DoH transport kept a silent HTTP/2 connection open: the health check PING did not close it") + } +} + +func mustNewRequest(t *testing.T, url string) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + return req +} diff --git a/doh_test.go b/doh_test.go index 92fa79f8..3ed0be0c 100644 --- a/doh_test.go +++ b/doh_test.go @@ -157,20 +157,21 @@ func Test_ClientCertificateVerificationError(t *testing.T) { }, } + ctx := context.Background() for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - tc.uc.Init() - tc.uc.SetupBootstrapIP() - r, err := NewResolver(tc.uc) + tc.uc.Init(ctx) + tc.uc.SetupBootstrapIP(ctx) + r, err := NewResolver(ctx, tc.uc) if err != nil { t.Fatal(err) } msg := new(dns.Msg) msg.SetQuestion("verify.controld.com.", dns.TypeA) msg.RecursionDesired = true - _, err = r.Resolve(context.Background(), msg) + _, err = r.Resolve(ctx, msg) // Verify the error contains the expected certificate information if err == nil { t.Fatal("expected certificate verification error, got nil") @@ -196,6 +197,7 @@ func testTLSServer(t *testing.T, handler http.Handler) (*httptest.Server, *x509. server := httptest.NewUnstartedServer(handler) server.TLS = &tls.Config{ Certificates: []tls.Certificate{testCert.tlsCert}, + MinVersion: tls.VersionTLS12, } server.StartTLS() @@ -232,6 +234,7 @@ func newTestHTTP3Server(t *testing.T, handler http.Handler) *testHTTP3Server { tlsConfig := &tls.Config{ Certificates: []tls.Certificate{testCert.tlsCert}, NextProtos: []string{"h3"}, // HTTP/3 protocol identifier + MinVersion: tls.VersionTLS12, } // Create HTTP/3 server @@ -264,3 +267,215 @@ func newTestHTTP3Server(t *testing.T, handler http.Handler) *testHTTP3Server { return h3Server } + +// blockingBodyHandler writes exactly nbytes of body with the given status, +// flushes them, then blocks until release is closed WITHOUT ever returning. +// Because the handler does not return, the response stream is never terminated +// (no EOF/FIN). A client that stops after a bounded prefix therefore completes, +// while a client that reads to EOF blocks. Tests set nbytes to the exact read +// cap so the client consumes the whole written body (no half-written frame is +// left blocking on flow control) yet still never sees EOF. +func blockingBodyHandler(status, nbytes int, release <-chan struct{}) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", headerApplicationDNS) + w.WriteHeader(status) + if _, err := w.Write(make([]byte, nbytes)); err != nil { + return + } + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + <-release + } +} + +// requireBoundedResolve asserts that r.Resolve returns the expected size/status +// error while the server is still withholding EOF (the handler is blocked in +// blockingBodyHandler). Returning under those conditions proves ctrld read only +// a bounded prefix of the body: a resolver that instead read to EOF would block +// on the withheld stream and trip the deadline. This is the deterministic +// regression guard for the issue-312 OOM protections, replacing the earlier +// flaky server-side byte counter (issue-561). +func requireBoundedResolve(t *testing.T, r Resolver, msg *dns.Msg, wantErrSubstr string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + type result struct { + answer *dns.Msg + err error + } + done := make(chan result, 1) + go func() { + answer, err := r.Resolve(ctx, msg) + done <- result{answer, err} + }() + + select { + case res := <-done: + if res.err == nil { + t.Fatalf("Resolve unexpectedly succeeded; answer=%v", res.answer) + } + if !strings.Contains(res.err.Error(), wantErrSubstr) { + t.Fatalf("error %q does not contain %q", res.err, wantErrSubstr) + } + if res.answer != nil { + t.Fatalf("Resolve returned non-nil answer alongside error: %v", res.answer) + } + case <-time.After(5 * time.Second): + t.Fatal("Resolve did not return while the server withheld EOF: the body is being read to EOF instead of a bounded prefix (issue-312 OOM protection missing)") + } +} + +// dohUpstreamForTLSServer wires an UpstreamConfig at a local httptest TLS +// server, trusting its self-signed certificate. BootstrapIP is set so no +// real DNS lookup runs. +func dohUpstreamForTLSServer(t *testing.T, srv *httptest.Server) *UpstreamConfig { + t.Helper() + pool := x509.NewCertPool() + pool.AddCert(srv.Certificate()) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + uc := &UpstreamConfig{ + Name: "doh-oversize", + Type: ResolverTypeDOH, + Endpoint: srv.URL + "/dns-query", + BootstrapIP: u.Hostname(), + Timeout: 2000, + } + uc.SetCertPool(pool) + uc.Init(context.Background()) + return uc +} + +// doh3UpstreamForAddr wires an UpstreamConfig at a local HTTP/3 server, +// trusting its self-signed certificate. +func doh3UpstreamForAddr(t *testing.T, addr string, cert *x509.Certificate) *UpstreamConfig { + t.Helper() + pool := x509.NewCertPool() + pool.AddCert(cert) + host, _, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("split host/port %q: %v", addr, err) + } + uc := &UpstreamConfig{ + Name: "doh3-oversize", + Type: ResolverTypeDOH3, + Endpoint: "h3://" + addr + "/dns-query", + BootstrapIP: host, + Timeout: 5000, + } + uc.SetCertPool(pool) + uc.Init(context.Background()) + return uc +} + +// TestDoHResolve_OversizedBody_Rejected locks in the fix for +// github.com/Control-D-Inc/ctrld/issues/312: a malicious DoH upstream +// returning a body larger than the DNS protocol allows must be rejected +// with an explicit size error rather than buffered into ctrld memory. +func TestDoHResolve_OversizedBody_Rejected(t *testing.T) { + // Write exactly the LimitReader cap, then withhold EOF. ctrld's bounded + // read (io.LimitReader of dohMaxResponseSize+1) returns after this prefix; + // an unbounded read would block on the missing EOF and trip the deadline. + release := make(chan struct{}) + defer close(release) + srv := httptest.NewUnstartedServer(blockingBodyHandler(http.StatusOK, dohMaxResponseSize+1, release)) + testCert := generateTestCertificate(t) + srv.TLS = &tls.Config{ + Certificates: []tls.Certificate{testCert.tlsCert}, + NextProtos: []string{"h2", "http/1.1"}, + MinVersion: tls.VersionTLS12, + } + srv.StartTLS() + t.Cleanup(srv.Close) + + uc := dohUpstreamForTLSServer(t, srv) + r, err := NewResolver(context.Background(), uc) + if err != nil { + t.Fatalf("NewResolver: %v", err) + } + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + requireBoundedResolve(t, r, msg, "maximum DNS message size") +} + +// TestDoHResolve_NonOKStatus_BoundedErrorBody locks in that a non-200 +// response with a huge body does not pull the body fully into ctrld +// memory just to format an error string. +func TestDoHResolve_NonOKStatus_BoundedErrorBody(t *testing.T) { + // Same synchronization as the oversized-body test, but at the error-body + // cap: the non-200 path reads through an io.LimitReader of + // dohMaxErrorBodySize, so it must return after this prefix without EOF. + release := make(chan struct{}) + defer close(release) + srv := httptest.NewUnstartedServer(blockingBodyHandler(http.StatusBadGateway, dohMaxErrorBodySize, release)) + testCert := generateTestCertificate(t) + srv.TLS = &tls.Config{ + Certificates: []tls.Certificate{testCert.tlsCert}, + NextProtos: []string{"h2", "http/1.1"}, + MinVersion: tls.VersionTLS12, + } + srv.StartTLS() + t.Cleanup(srv.Close) + + uc := dohUpstreamForTLSServer(t, srv) + r, err := NewResolver(context.Background(), uc) + if err != nil { + t.Fatalf("NewResolver: %v", err) + } + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + requireBoundedResolve(t, r, msg, "status: 502") +} + +// TestDoHResolve_OversizedBody_DoH3 mirrors the DoH oversized-body check +// on the HTTP/3 transport, since github-312 specifically reproduced the +// OOM via DoH3. +func TestDoHResolve_OversizedBody_DoH3(t *testing.T) { + release := make(chan struct{}) + defer close(release) + testCert := generateTestCertificate(t) + udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0}) + if err != nil { + t.Fatalf("udp listen: %v", err) + } + h3 := &http3.Server{ + Handler: blockingBodyHandler(http.StatusOK, dohMaxResponseSize+1, release), + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{testCert.tlsCert}, + NextProtos: []string{"h3"}, + MinVersion: tls.VersionTLS12, + }, + } + go func() { + if err := h3.Serve(udpConn); err != nil && !errors.Is(err, http.ErrServerClosed) { + t.Logf("h3 server: %v", err) + } + }() + t.Cleanup(func() { + _ = h3.Close() + _ = udpConn.Close() + }) + time.Sleep(100 * time.Millisecond) + + uc := doh3UpstreamForAddr(t, udpConn.LocalAddr().String(), testCert.cert) + r, err := NewResolver(context.Background(), uc) + if err != nil { + t.Fatalf("NewResolver: %v", err) + } + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + requireBoundedResolve(t, r, msg, "maximum DNS message size") +} diff --git a/doq.go b/doq.go index 0903411c..b3b1edb3 100644 --- a/doq.go +++ b/doq.go @@ -5,103 +5,398 @@ package ctrld import ( "context" "crypto/tls" + "errors" + "fmt" "io" "net" + "runtime" + "sync" "time" "github.com/miekg/dns" "github.com/quic-go/quic-go" ) +// doqMaxResponseSize caps the bytes read from a DoQ stream: a 2-byte +// length prefix plus a DNS message bounded by dns.MaxMsgSize. Anything +// larger cannot be a valid response and is rejected before buffering more +// data from the upstream. +const doqMaxResponseSize = 2 + dns.MaxMsgSize + type doqResolver struct { uc *UpstreamConfig } func (r *doqResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { - endpoint := r.uc.Endpoint - tlsConfig := &tls.Config{NextProtos: []string{"doq"}} - ip := r.uc.BootstrapIP - if ip == "" { - dnsTyp := uint16(0) - if msg != nil && len(msg.Question) > 0 { - dnsTyp = msg.Question[0].Qtype - } - ip = r.uc.bootstrapIPForDNSType(dnsTyp) + if err := validateMsg(msg); err != nil { + return nil, err + } + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "DoQ resolver query started") + + // Get the appropriate connection pool based on DNS type and IP stack + dnsTyp := uint16(0) + if msg != nil && len(msg.Question) > 0 { + dnsTyp = msg.Question[0].Qtype + } + + pool := r.uc.doqTransport(ctx, dnsTyp) + if pool == nil { + Log(ctx, logger.Error(), "DoQ connection pool is not available") + return nil, errors.New("DoQ connection pool is not available") + } + + answer, err := pool.Resolve(ctx, msg) + if err != nil { + Log(ctx, logger.Error().Err(err), "DoQ request failed") + } else { + Log(ctx, logger.Debug(), "DoQ resolver query successful") } - tlsConfig.ServerName = r.uc.Domain - _, port, _ := net.SplitHostPort(endpoint) - endpoint = net.JoinHostPort(ip, port) - return resolve(ctx, msg, endpoint, tlsConfig) + return answer, err } -func resolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls.Config) (*dns.Msg, error) { - // DoQ quic-go server returns io.EOF error after running for a long time, - // even for a good stream. So retrying the query for 5 times before giving up. - for i := 0; i < 5; i++ { - answer, err := doResolve(ctx, msg, endpoint, tlsConfig) +const doqPoolSize = 16 + +// doqConnPool manages a pool of QUIC connections for DoQ queries using a buffered channel. +// A single quic.Transport (and its UDP socket) is shared by every connection in the pool, +// so the OS socket lifecycle is tied to the pool rather than to each dial. Without this +// ownership model, a strict DoQ upstream that triggers reconnect churn would leak one +// caller-owned UDP socket per dial — see github.com/Control-D-Inc/ctrld/issues/309. +type doqConnPool struct { + uc *UpstreamConfig + addrs []string + port string + tlsConfig *tls.Config + quicConfig *quic.Config + conns chan *doqConn + + transportMu sync.Mutex + transport *quic.Transport + transportConn *net.UDPConn + transportErr error + transportInit bool + closed bool +} + +type doqConn struct { + conn *quic.Conn +} + +func newDOQConnPool(_ context.Context, uc *UpstreamConfig, addrs []string) *doqConnPool { + _, port, _ := net.SplitHostPort(uc.Endpoint) + if port == "" { + port = "853" + } + + tlsConfig := &tls.Config{ + NextProtos: []string{"doq"}, + RootCAs: uc.certPool, + ServerName: uc.Domain, + MinVersion: tls.VersionTLS12, + } + + quicConfig := &quic.Config{ + KeepAlivePeriod: 15 * time.Second, + } + + pool := &doqConnPool{ + uc: uc, + addrs: addrs, + port: port, + tlsConfig: tlsConfig, + quicConfig: quicConfig, + conns: make(chan *doqConn, doqPoolSize), + } + + // Use SetFinalizer here because we need to call a method on the pool itself. + // AddCleanup would require passing the pool as arg (which panics) or capturing + // it in a closure (which prevents GC). SetFinalizer is appropriate for this case. + runtime.SetFinalizer(pool, func(p *doqConnPool) { + p.CloseIdleConnections() + }) + + return pool +} + +// Resolve performs a DNS query using a pooled QUIC connection. +func (p *doqConnPool) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + // Retry logic for transient errors: io.EOF (connection reset), + // IdleTimeoutError (stale pooled connection timed out), and + // StreamLimitReachedError (stream credit exhausted before server MAX_STREAMS arrived). + for range 5 { + answer, err := p.doResolve(ctx, msg) if err == io.EOF { continue } + var idleErr *quic.IdleTimeoutError + if errors.As(err, &idleErr) { + continue + } + var streamLimitErr quic.StreamLimitReachedError + if errors.As(err, &streamLimitErr) { + continue + } if err != nil { return nil, wrapCertificateVerificationError(err) } return answer, nil } - return nil, &quic.ApplicationError{ErrorCode: quic.ApplicationErrorCode(quic.InternalError), ErrorMessage: quic.InternalError.Message()} + return nil, &quic.ApplicationError{ + ErrorCode: quic.ApplicationErrorCode(quic.InternalError), + ErrorMessage: quic.InternalError.Message(), + } } -func doResolve(ctx context.Context, msg *dns.Msg, endpoint string, tlsConfig *tls.Config) (*dns.Msg, error) { - session, err := quic.DialAddr(ctx, endpoint, tlsConfig, nil) +func (p *doqConnPool) doResolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + conn, err := p.getConn(ctx) if err != nil { return nil, err } - defer session.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") + // Pack the DNS message msgBytes, err := msg.Pack() if err != nil { + p.putConn(conn, false) return nil, err } - stream, err := session.OpenStream() - if err != nil { - return nil, err - } - + // Ensure the context has a deadline before calling OpenStreamSync, which + // blocks until the server sends a MAX_STREAMS update. Without a deadline the + // call could block indefinitely when the server never sends the update. deadline, ok := ctx.Deadline() if !ok { - deadline = time.Now().Add(5 * time.Second) + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + deadline, _ = ctx.Deadline() + } + + // OpenStreamSync blocks until the server's MAX_STREAMS credit arrives, + // avoiding the StreamLimitReachedError race that OpenStream (non-blocking) + // triggers when the credit replenishment frame is still in flight. + stream, err := conn.OpenStreamSync(ctx) + if err != nil { + p.putConn(conn, false) + return nil, err } _ = stream.SetDeadline(deadline) + // Write message length (2 bytes) followed by message var msgLen = uint16(len(msgBytes)) var msgLenBytes = []byte{byte(msgLen >> 8), byte(msgLen & 0xFF)} if _, err := stream.Write(msgLenBytes); err != nil { + stream.Close() + p.putConn(conn, false) return nil, err } if _, err := stream.Write(msgBytes); err != nil { + stream.Close() + p.putConn(conn, false) return nil, err } - buf, err := io.ReadAll(stream) - if err != nil { + // RFC 9250 section 4.2 requires the client to indicate end-of-request by + // closing the send side of the stream (STREAM FIN). Servers may defer + // processing until FIN arrives, so the close must happen before reading. + // Stream.Close closes only the send direction; the receive direction + // remains open for the response. + if err := stream.Close(); err != nil { + p.putConn(conn, false) return nil, err } - _ = stream.Close() + // A DoQ response is a 2-byte length prefix followed by a DNS message. + // The DNS message is bounded by the protocol at dns.MaxMsgSize, so a + // well-formed response is at most doqMaxResponseSize bytes. Read one + // byte past that cap to distinguish "at limit" from "over limit" and + // reject oversized responses before they can drive memory growth from + // a malicious or compromised upstream. + buf, err := io.ReadAll(io.LimitReader(stream, doqMaxResponseSize+1)) + if err != nil { + p.putConn(conn, false) + return nil, err + } - // io.ReadAll hide the io.EOF error returned by quic-go server. - // Once we figure out why quic-go server sends io.EOF after running - // for a long time, we can have a better way to handle this. For now, - // make sure io.EOF error returned, so the caller can handle it cleanly. + // io.ReadAll hides io.EOF error, so check for empty buffer. if len(buf) == 0 { + p.putConn(conn, false) return nil, io.EOF } + if len(buf) > doqMaxResponseSize { + p.putConn(conn, false) + return nil, fmt.Errorf("DoQ response exceeds %d-byte maximum", doqMaxResponseSize) + } + + // RFC 9250: each DoQ DNS message is encoded as a 2-octet length field + // followed by the DNS message. Reject responses that are shorter than + // the prefix or whose prefix declares more bytes than were received, + // and retire the misbehaving connection. Without this guard, buf[2:] + // would panic when len(buf) < 2. + if len(buf) < 2 { + p.putConn(conn, false) + return nil, fmt.Errorf("malformed DoQ response: %d byte(s), need >= 2 for length prefix", len(buf)) + } + respLen := int(buf[0])<<8 | int(buf[1]) + if 2+respLen > len(buf) { + p.putConn(conn, false) + return nil, fmt.Errorf("malformed DoQ response: length prefix %d exceeds payload %d", respLen, len(buf)-2) + } + + p.putConn(conn, true) + + // Unpack DNS response (skip 2-byte length prefix). answer := new(dns.Msg) - if err := answer.Unpack(buf[2:]); err != nil { + if err := answer.Unpack(buf[2 : 2+respLen]); err != nil { return nil, err } - answer.SetReply(msg) + // RFC 9250 section 4.2.1 requires the DNS Message ID to be 0 on the wire, + // so restore the downstream transaction ID for the client. Do NOT use + // SetReply here: it rewrites the RCODE to NOERROR and overwrites the + // Question with the request's, which would mask upstream failures from the + // failover logic (a SERVFAIL would look like success) and let a + // wrong-question answer pass validation and poison the cache. Preserve the + // upstream RCODE, Question, and answer sections untouched so the proxy can + // evaluate them. See github.com/Control-D-Inc/ctrld/issues/322. + answer.Id = msg.Id return answer, nil } + +// getConn gets a QUIC connection from the pool or creates a new one. +// A connection is taken from the channel while in use; putConn returns it. +func (p *doqConnPool) getConn(ctx context.Context) (*quic.Conn, error) { + for { + select { + case dc := <-p.conns: + if dc.conn != nil && dc.conn.Context().Err() == nil { + return dc.conn, nil + } + if dc.conn != nil { + dc.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") + } + default: + _, conn, err := p.dialConn(ctx) + if err != nil { + return nil, err + } + return conn, nil + } + } +} + +// putConn returns a connection to the pool for reuse by other goroutines. +func (p *doqConnPool) putConn(conn *quic.Conn, isGood bool) { + if !isGood || conn == nil || conn.Context().Err() != nil { + if conn != nil { + conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") + } + return + } + dc := &doqConn{conn: conn} + select { + case p.conns <- dc: + default: + // Channel full, close the connection + dc.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") + } +} + +// dialConn creates a new QUIC connection using parallel dialing like DoH3. +// All connections from the pool multiplex on a single pool-owned UDP socket, +// so reconnect churn cannot grow the host's FD count. +func (p *doqConnPool) dialConn(ctx context.Context) (string, *quic.Conn, error) { + logger := LoggerFromCtx(ctx) + + tr, err := p.getOrInitTransport() + if err != nil { + return "", nil, err + } + + // If we have a bootstrap IP, use it directly + if p.uc.BootstrapIP != "" { + addr := net.JoinHostPort(p.uc.BootstrapIP, p.port) + Log(ctx, logger.Debug(), "Sending DoQ request to: %s", addr) + remoteAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return "", nil, err + } + conn, err := tr.DialEarly(ctx, remoteAddr, p.tlsConfig, p.quicConfig) + if err != nil { + return "", nil, err + } + return addr, conn, nil + } + + // Use parallel dialing like DoH3 + dialAddrs := make([]string, len(p.addrs)) + for i := range p.addrs { + dialAddrs[i] = net.JoinHostPort(p.addrs[i], p.port) + } + + pd := &quicParallelDialer{transport: tr} + conn, err := pd.Dial(ctx, dialAddrs, p.tlsConfig, p.quicConfig) + if err != nil { + return "", nil, err + } + + addr := conn.RemoteAddr().String() + Log(ctx, logger.Debug(), "Sending DoQ request to: %s", addr) + return addr, conn, nil +} + +// getOrInitTransport returns the pool's shared quic.Transport, initialising it +// on first call. Once the pool has been closed it permanently returns an error +// so that callers cannot resurrect a dead pool. +func (p *doqConnPool) getOrInitTransport() (*quic.Transport, error) { + p.transportMu.Lock() + defer p.transportMu.Unlock() + if p.closed { + return nil, errors.New("doq pool closed") + } + if p.transportInit { + return p.transport, p.transportErr + } + p.transportInit = true + udpConn, err := net.ListenUDP("udp", nil) + if err != nil { + p.transportErr = err + return nil, err + } + p.transportConn = udpConn + p.transport = &quic.Transport{Conn: udpConn} + return p.transport, nil +} + +// CloseIdleConnections closes all idle connections, the shared quic.Transport, +// and the pool's UDP socket. Connections currently checked out (in use) get +// terminated by the transport close as well — without that, the OS socket +// would remain bound to a goroutine that the caller cannot reach to clean up. +func (p *doqConnPool) CloseIdleConnections() { +drain: + for { + select { + case dc := <-p.conns: + if dc.conn != nil { + dc.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") + } + default: + break drain + } + } + p.transportMu.Lock() + if p.closed { + p.transportMu.Unlock() + return + } + p.closed = true + tr := p.transport + udpConn := p.transportConn + p.transportMu.Unlock() + if tr != nil { + _ = tr.Close() + } + if udpConn != nil { + _ = udpConn.Close() + } +} diff --git a/doq_test.go b/doq_test.go index 14055dd0..407d0f4a 100644 --- a/doq_test.go +++ b/doq_test.go @@ -1,4 +1,3 @@ -// test_helpers.go package ctrld import ( @@ -8,8 +7,11 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "io" "math/big" "net" + "os" + "runtime" "strings" "testing" "time" @@ -99,6 +101,7 @@ func newTestQUICServer(t *testing.T) *testQUICServer { tlsConfig := &tls.Config{ Certificates: []tls.Certificate{testCert.tlsCert}, NextProtos: []string{"doq"}, + MinVersion: tls.VersionTLS12, } // Create QUIC listener @@ -221,3 +224,595 @@ func (s *testQUICServer) handleStream(t *testing.T, stream *quic.Stream) { return } } + +// malformedDoQServer is a test QUIC server that drains the client's DoQ +// request and writes caller-supplied raw bytes back. The bytes are not +// required to be a well-framed DoQ response, which is what lets the +// regression tests exercise malformed-response handling. +type malformedDoQServer struct { + listener *quic.Listener + cert *x509.Certificate + addr string + response []byte +} + +func newMalformedDoQServer(t *testing.T, response []byte) *malformedDoQServer { + t.Helper() + + testCert := generateTestCertificate(t) + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{testCert.tlsCert}, + NextProtos: []string{"doq"}, + } + + listener, err := quic.ListenAddr("127.0.0.1:0", tlsConfig, nil) + if err != nil { + t.Fatalf("failed to create QUIC listener: %v", err) + } + + s := &malformedDoQServer{ + listener: listener, + cert: testCert.cert, + addr: listener.Addr().String(), + response: response, + } + + go s.serve() + t.Cleanup(func() { _ = listener.Close() }) + return s +} + +func (s *malformedDoQServer) serve() { + for { + conn, err := s.listener.Accept(context.Background()) + if err != nil { + return + } + go s.handleConn(conn) + } +} + +func (s *malformedDoQServer) handleConn(conn *quic.Conn) { + for { + stream, err := conn.AcceptStream(context.Background()) + if err != nil { + return + } + go s.handleStream(stream) + } +} + +func (s *malformedDoQServer) handleStream(stream *quic.Stream) { + defer stream.Close() + + // Drain the client's DoQ-framed request so the client's writes complete + // cleanly before we reply with our attacker-controlled bytes. Using + // io.ReadFull because a single Read on a QUIC stream may return short. + lenBuf := make([]byte, 2) + if _, err := io.ReadFull(stream, lenBuf); err != nil { + return + } + msgLen := uint16(lenBuf[0])<<8 | uint16(lenBuf[1]) + if msgLen > 0 { + discard := make([]byte, msgLen) + if _, err := io.ReadFull(stream, discard); err != nil { + return + } + } + + if len(s.response) > 0 { + _, _ = stream.Write(s.response) + } +} + +// newMalformedDoQUpstream builds an UpstreamConfig wired to a local +// malformed test server with the test certificate trusted via a custom +// cert pool. We bypass SetupBootstrapIP by setting BootstrapIP directly, +// so the pool dials 127.0.0.1 without any DNS lookup. +func newMalformedDoQUpstream(t *testing.T, cert *x509.Certificate, addr string) *UpstreamConfig { + t.Helper() + + pool := x509.NewCertPool() + pool.AddCert(cert) + + host, _, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("split host/port %q: %v", addr, err) + } + + uc := &UpstreamConfig{ + Name: "doq-malformed", + Type: ResolverTypeDOQ, + Endpoint: addr, + Domain: host, + BootstrapIP: host, + Timeout: 2000, + } + uc.SetCertPool(pool) + return uc +} + +// TestDoQResolve_MalformedResponse verifies that DoQ upstream +// responses violating RFC 9250 framing — fewer than 2 bytes, or a +// length prefix declaring more payload than was received — return a +// handled error instead of panicking on the length-prefix slice. +func TestDoQResolve_MalformedResponse(t *testing.T) { + tests := []struct { + name string + response []byte + }{ + // Empty stream is already handled via io.EOF; locked in so a + // future change that drops that branch is caught. + {"empty response", nil}, + + // One byte: too short to hold the 2-octet length prefix. + {"single byte response", []byte{0x00}}, + + // Length prefix declares 16 bytes; payload is absent. + {"length prefix only", []byte{0x00, 0x10}}, + + // Length prefix declares 65535 bytes; only 1 byte of payload + // arrived. + {"length prefix larger than payload", []byte{0xFF, 0xFF, 0x00}}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + server := newMalformedDoQServer(t, tt.response) + uc := newMalformedDoQUpstream(t, server.cert, server.addr) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pool := newDOQConnPool(ctx, uc, []string{"127.0.0.1"}) + t.Cleanup(pool.CloseIdleConnections) + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + answer, err := pool.Resolve(ctx, msg) + if err == nil { + t.Fatalf("Resolve unexpectedly succeeded for malformed response %v; answer=%v", tt.response, answer) + } + if answer != nil { + t.Fatalf("Resolve returned non-nil answer alongside error: answer=%v err=%v", answer, err) + } + }) + } +} + +// strictDoQServer accepts DoQ queries but defers the response until the +// client signals end-of-request with STREAM FIN, as required by RFC 9250 +// section 4.2. It exists to lock in the fix for +// github.com/Control-D-Inc/ctrld/issues/309 where a client +// that never closes its send side caused the server to wait forever and the +// client to churn through reconnects. +type strictDoQServer struct { + listener *quic.Listener + cert *x509.Certificate + addr string +} + +func newStrictDoQServer(t *testing.T) *strictDoQServer { + t.Helper() + + testCert := generateTestCertificate(t) + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{testCert.tlsCert}, + NextProtos: []string{"doq"}, + MinVersion: tls.VersionTLS12, + } + + listener, err := quic.ListenAddr("127.0.0.1:0", tlsConfig, nil) + if err != nil { + t.Fatalf("failed to create QUIC listener: %v", err) + } + + s := &strictDoQServer{ + listener: listener, + cert: testCert.cert, + addr: listener.Addr().String(), + } + go s.serve() + t.Cleanup(func() { _ = listener.Close() }) + return s +} + +func (s *strictDoQServer) serve() { + for { + conn, err := s.listener.Accept(context.Background()) + if err != nil { + return + } + go s.handleConn(conn) + } +} + +func (s *strictDoQServer) handleConn(conn *quic.Conn) { + for { + stream, err := conn.AcceptStream(context.Background()) + if err != nil { + return + } + go s.handleStream(stream) + } +} + +func (s *strictDoQServer) handleStream(stream *quic.Stream) { + defer stream.Close() + + // Drain until the client closes the send side. This is the behaviour + // that triggered the bug: if the client never sends STREAM FIN, this + // read blocks until the stream's deadline fires. + body, err := io.ReadAll(stream) + if err != nil { + return + } + if len(body) < 2 { + return + } + msgLen := uint16(body[0])<<8 | uint16(body[1]) + if int(msgLen) != len(body)-2 { + return + } + + msg := new(dns.Msg) + if err := msg.Unpack(body[2:]); err != nil { + return + } + + response := new(dns.Msg) + response.SetReply(msg) + response.Authoritative = true + if len(msg.Question) > 0 && msg.Question[0].Qtype == dns.TypeA { + response.Answer = append(response.Answer, &dns.A{ + Hdr: dns.RR_Header{ + Name: msg.Question[0].Name, + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: 300, + }, + A: net.ParseIP("192.0.2.1"), + }) + } + + respBytes, err := response.Pack() + if err != nil { + return + } + respLen := uint16(len(respBytes)) + if _, err := stream.Write([]byte{byte(respLen >> 8), byte(respLen & 0xFF)}); err != nil { + return + } + if _, err := stream.Write(respBytes); err != nil { + return + } +} + +func newStrictDoQUpstream(t *testing.T, cert *x509.Certificate, addr string, useBootstrap bool) *UpstreamConfig { + t.Helper() + + pool := x509.NewCertPool() + pool.AddCert(cert) + + host, _, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("split host/port %q: %v", addr, err) + } + + uc := &UpstreamConfig{ + Name: "doq-strict", + Type: ResolverTypeDOQ, + Endpoint: addr, + Domain: host, + Timeout: 3000, + } + if useBootstrap { + uc.BootstrapIP = host + } + uc.SetCertPool(pool) + return uc +} + +// TestDoQResolve_StrictServerWaitsForFIN exercises the RFC 9250 client-FIN +// requirement. With the bug present, the server's io.ReadAll blocks until +// the stream deadline expires and the client sees a timeout, so a successful +// resolve here proves that the client now sends STREAM FIN before reading. +func TestDoQResolve_StrictServerWaitsForFIN(t *testing.T) { + t.Parallel() + + server := newStrictDoQServer(t) + uc := newStrictDoQUpstream(t, server.cert, server.addr, true) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + host, _, _ := net.SplitHostPort(server.addr) + pool := newDOQConnPool(ctx, uc, []string{host}) + t.Cleanup(pool.CloseIdleConnections) + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + answer, err := pool.Resolve(ctx, msg) + if err != nil { + t.Fatalf("Resolve failed against strict DoQ server: %v", err) + } + if answer == nil || len(answer.Answer) == 0 { + t.Fatalf("Resolve returned no answer records: %+v", answer) + } + a, ok := answer.Answer[0].(*dns.A) + if !ok || !a.A.Equal(net.ParseIP("192.0.2.1")) { + t.Fatalf("unexpected answer: %+v", answer.Answer[0]) + } +} + +// TestDoQResolve_ParallelDialPathStrictFIN exercises the parallel-dial path +// (no BootstrapIP) against the same FIN-strict server, so that both the +// single-dial branch and the parallel-dial branch are covered. +func TestDoQResolve_ParallelDialPathStrictFIN(t *testing.T) { + t.Parallel() + + server := newStrictDoQServer(t) + uc := newStrictDoQUpstream(t, server.cert, server.addr, false) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + host, _, _ := net.SplitHostPort(server.addr) + pool := newDOQConnPool(ctx, uc, []string{host}) + t.Cleanup(pool.CloseIdleConnections) + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + answer, err := pool.Resolve(ctx, msg) + if err != nil { + t.Fatalf("Resolve (parallel-dial path) failed against strict DoQ server: %v", err) + } + if answer == nil || len(answer.Answer) == 0 { + t.Fatalf("Resolve (parallel-dial path) returned no answer records: %+v", answer) + } +} + +// TestDoQPool_ChurnDoesNotGrowFDs exercises the reconnect-churn scenario +// described in github.com/Control-D-Inc/ctrld/issues/309: repeated dials +// against a server that closes existing connections must not grow the process +// FD count, because the pool now shares one UDP socket via quic.Transport instead +// of allocating one per dial. Linux-only because /proc/self/fd is the cheapest +// portable proxy for "what's still open." +func TestDoQPool_ChurnDoesNotGrowFDs(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("FD accounting via /proc/self/fd is linux-only") + } + t.Parallel() + + server := newStrictDoQServer(t) + uc := newStrictDoQUpstream(t, server.cert, server.addr, true) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + host, _, _ := net.SplitHostPort(server.addr) + pool := newDOQConnPool(ctx, uc, []string{host}) + t.Cleanup(pool.CloseIdleConnections) + + makeQuery := func(i int) *dns.Msg { + msg := new(dns.Msg) + // Vary the question so any caching layer cannot short-circuit. + msg.SetQuestion(dns.Fqdn(strings.Repeat("a", 1+i%8)+".example.com"), dns.TypeA) + msg.RecursionDesired = true + return msg + } + + // Warm the pool so the steady-state transport and at least one + // connection are open. Without this, the first resolve in the measured + // loop would inflate the baseline. + if _, err := pool.Resolve(ctx, makeQuery(0)); err != nil { + t.Fatalf("warm-up Resolve failed: %v", err) + } + + baseline := countOpenFDs(t) + + // Force reconnect churn by closing the connection between each query. + // Without the fix this would leak one UDP socket per round; with the + // fix the pool's shared transport keeps a single socket open. + const rounds = 20 + for i := 1; i <= rounds; i++ { + // Drain any pooled connection so the next Resolve has to redial. + drainPooledConns(pool) + + if _, err := pool.Resolve(ctx, makeQuery(i)); err != nil { + t.Fatalf("Resolve in churn loop iteration %d failed: %v", i, err) + } + } + + // Give quic-go a moment to drop any background goroutines that hold + // references to closed sockets. + time.Sleep(200 * time.Millisecond) + + after := countOpenFDs(t) + + // Allow a small slack for transient FDs (goroutine wake-ups, qlog, + // etc.) but reject anything that scales with the number of rounds. + const slack = 5 + if after > baseline+slack { + t.Fatalf("FD count grew under DoQ churn: baseline=%d after=%d rounds=%d (slack=%d)", baseline, after, rounds, slack) + } +} + +// drainPooledConns removes any idle pooled connections so the next Resolve +// is forced to dial a fresh one. It does not close the pool's transport. +func drainPooledConns(p *doqConnPool) { + for { + select { + case dc := <-p.conns: + if dc.conn != nil { + dc.conn.CloseWithError(quic.ApplicationErrorCode(quic.NoError), "") + } + default: + return + } + } +} + +func countOpenFDs(t *testing.T) int { + t.Helper() + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + t.Fatalf("read /proc/self/fd: %v", err) + } + return len(entries) +} + +// TestDoQResolve_OversizedResponse_Rejected locks in the fix for +// github.com/Control-D-Inc/ctrld/issues/312 on the DoQ transport: a +// malicious upstream that writes a response larger than the DNS protocol +// allows must be rejected with an explicit size error, not buffered +// without bound into ctrld memory. +func TestDoQResolve_OversizedResponse_Rejected(t *testing.T) { + t.Parallel() + + // doqMaxResponseSize is 2 + dns.MaxMsgSize. Send something well past + // that. 256 KiB is enough to exceed the cap while keeping the test + // fast on loopback. + response := make([]byte, 256*1024) + // A well-formed length prefix isn't required: the size cap should + // fire before any framing check runs. Use a non-zero prefix so the + // test also documents that the order of validation is "size first, + // framing later." + response[0] = 0xFF + response[1] = 0xFF + + server := newMalformedDoQServer(t, response) + uc := newMalformedDoQUpstream(t, server.cert, server.addr) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pool := newDOQConnPool(context.Background(), uc, []string{"127.0.0.1"}) + t.Cleanup(pool.CloseIdleConnections) + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + answer, err := pool.Resolve(ctx, msg) + if err == nil { + t.Fatalf("Resolve unexpectedly succeeded for oversized response; answer=%v", answer) + } + if !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("error %q does not surface the size cap", err) + } + if answer != nil { + t.Fatalf("Resolve returned non-nil answer alongside error: %v", answer) + } +} + +// frameDoQResponse packs msg and prepends the RFC 9250 2-octet length prefix, +// producing the exact bytes a DoQ server writes on the wire. +func frameDoQResponse(t *testing.T, msg *dns.Msg) []byte { + t.Helper() + b, err := msg.Pack() + if err != nil { + t.Fatalf("pack response: %v", err) + } + n := uint16(len(b)) + return append([]byte{byte(n >> 8), byte(n & 0xFF)}, b...) +} + +// TestDoQResolve_PreservesRcode locks in the fix for +// github.com/Control-D-Inc/ctrld/issues/322: the DoQ resolver must not rewrite +// an upstream response with SetReply, which would clobber a SERVFAIL into +// NOERROR and hide the failure from the proxy's failover logic. The upstream +// RCODE must survive; only the transaction ID is restored for the client. +func TestDoQResolve_PreservesRcode(t *testing.T) { + t.Parallel() + + // RFC 9250 puts the DNS Message ID at 0 on the wire. + resp := new(dns.Msg) + resp.SetQuestion("example.com.", dns.TypeA) + resp.Response = true + resp.Id = 0 + resp.Rcode = dns.RcodeServerFailure + + server := newMalformedDoQServer(t, frameDoQResponse(t, resp)) + uc := newMalformedDoQUpstream(t, server.cert, server.addr) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pool := newDOQConnPool(ctx, uc, []string{"127.0.0.1"}) + t.Cleanup(pool.CloseIdleConnections) + + msg := new(dns.Msg) + msg.SetQuestion("example.com.", dns.TypeA) + msg.RecursionDesired = true + + answer, err := pool.Resolve(ctx, msg) + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if answer.Rcode != dns.RcodeServerFailure { + t.Fatalf("upstream SERVFAIL was rewritten to %s; failover would be bypassed", + dns.RcodeToString[answer.Rcode]) + } + if answer.Id != msg.Id { + t.Fatalf("transaction ID not restored: got %d, want %d", answer.Id, msg.Id) + } +} + +// TestDoQResolve_PreservesWrongQuestion locks in the fix for +// github.com/Control-D-Inc/ctrld/issues/322: when an upstream answers a +// different name than asked, the resolver must preserve the upstream's +// question rather than rewriting it to the request's question (as SetReply +// did). Rewriting would hide the mismatch and let wrong-domain records poison +// the shared cache. +func TestDoQResolve_PreservesWrongQuestion(t *testing.T) { + t.Parallel() + + resp := new(dns.Msg) + resp.SetQuestion("attacker.example.", dns.TypeA) + resp.Response = true + resp.Id = 0 + resp.Answer = append(resp.Answer, &dns.A{ + Hdr: dns.RR_Header{ + Name: "attacker.example.", + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: 300, + }, + A: net.ParseIP("192.0.2.1"), + }) + + server := newMalformedDoQServer(t, frameDoQResponse(t, resp)) + uc := newMalformedDoQUpstream(t, server.cert, server.addr) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + pool := newDOQConnPool(ctx, uc, []string{"127.0.0.1"}) + t.Cleanup(pool.CloseIdleConnections) + + msg := new(dns.Msg) + msg.SetQuestion("victim.example.", dns.TypeA) + msg.RecursionDesired = true + + answer, err := pool.Resolve(ctx, msg) + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if len(answer.Question) == 0 || !strings.EqualFold(answer.Question[0].Name, "attacker.example.") { + t.Fatalf("upstream question was rewritten; got %v, want the upstream's attacker.example.", + answer.Question) + } + if answer.Id != msg.Id { + t.Fatalf("transaction ID not restored: got %d, want %d", answer.Id, msg.Id) + } +} diff --git a/dot.go b/dot.go index 295134c9..6dc9644a 100644 --- a/dot.go +++ b/dot.go @@ -3,7 +3,12 @@ package ctrld import ( "context" "crypto/tls" + "errors" + "io" "net" + "runtime" + "sync" + "time" "github.com/miekg/dns" ) @@ -13,30 +18,320 @@ type dotResolver struct { } func (r *dotResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + if err := validateMsg(msg); err != nil { + return nil, err + } + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "DoT resolver query started") + + dnsTyp := uint16(0) + if msg != nil && len(msg.Question) > 0 { + dnsTyp = msg.Question[0].Qtype + } + + pool := r.uc.dotTransport(ctx, dnsTyp) + if pool == nil { + Log(ctx, logger.Error(), "DoT client pool is not available") + return nil, errors.New("DoT client pool is not available") + } + + answer, err := pool.Resolve(ctx, msg) + if err != nil { + Log(ctx, logger.Error().Err(err), "DoT request failed") + } else { + Log(ctx, logger.Debug(), "DoT resolver query successful") + } + return answer, err +} + +const dotPoolSize = 16 + +// dotConnPool manages a pool of TCP/TLS connections for DoT queries. +// mu serializes retirement, ownership and idle queue operations. +type dotConnPool struct { + uc *UpstreamConfig + addrs []string + port string + tlsConfig *tls.Config + dialer *net.Dialer + conns chan *dotConn + + mu sync.Mutex + closed bool + owned map[*tls.Conn]struct{} // handshaking, checked-out and idle connections + ctx context.Context + cancel context.CancelFunc +} + +type dotConn struct { + conn *tls.Conn +} + +func newDOTClientPool(_ context.Context, uc *UpstreamConfig, addrs []string) *dotConnPool { + _, port, _ := net.SplitHostPort(uc.Endpoint) + if port == "" { + port = "853" + } + // The dialer is used to prevent bootstrapping cycle. - // If r.endpoint is set to dns.controld.dev, we need to resolve + // If endpoint is set to dns.controld.dev, we need to resolve // dns.controld.dev first. By using a dialer with custom resolver, // we ensure that we can always resolve the bootstrap domain // regardless of the machine DNS status. dialer := newDialer(net.JoinHostPort(controldPublicDns, "53")) - dnsTyp := uint16(0) - if msg != nil && len(msg.Question) > 0 { - dnsTyp = msg.Question[0].Qtype + + // Every dial path verifies the server as uc.Domain: a bootstrap_ip, the + // bootstrap list, and also the empty-list path, which dials the endpoint. + // tls.Client needs ServerName there too; without it every handshake fails. + tlsConfig := &tls.Config{ + RootCAs: uc.certPool, + MinVersion: tls.VersionTLS12, + ServerName: uc.Domain, + } + + ctx, cancel := context.WithCancel(context.Background()) + pool := &dotConnPool{ + uc: uc, + addrs: addrs, + port: port, + tlsConfig: tlsConfig, + dialer: dialer, + conns: make(chan *dotConn, dotPoolSize), + owned: make(map[*tls.Conn]struct{}), + ctx: ctx, + cancel: cancel, + } + + // Use SetFinalizer here because we need to call a method on the pool itself. + // AddCleanup would require passing the pool as arg (which panics) or capturing + // it in a closure (which prevents GC). SetFinalizer is appropriate for this case. + runtime.SetFinalizer(pool, func(p *dotConnPool) { + p.Close() + }) + + return pool +} + +// Resolve performs a DNS query using a pooled TCP/TLS connection. +func (p *dotConnPool) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + if msg == nil { + return nil, errors.New("nil DNS message") + } + + conn, err := p.getConn(ctx) + if err != nil { + return nil, wrapCertificateVerificationError(err) + } + + client := dns.Client{Net: "tcp-tls"} + answer, _, err := client.ExchangeWithConnContext(ctx, msg, &dns.Conn{Conn: conn}) + isGood := err == nil + p.putConn(conn, isGood) + + if err != nil { + return nil, wrapCertificateVerificationError(err) + } + + return answer, nil +} + +// getConn gets a TCP/TLS connection from the pool or creates a new one. +// Checked-out connections remain owned until putConn discards them or Close runs. +func (p *dotConnPool) getConn(ctx context.Context) (net.Conn, error) { + for { + if err := ctx.Err(); err != nil { + return nil, err + } + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return nil, net.ErrClosed + } + select { + case dc := <-p.conns: + p.mu.Unlock() + if dc.conn != nil && isAlive(dc.conn) { + return dc.conn, nil + } + if dc.conn != nil { + p.putConn(dc.conn, false) + } + default: + p.mu.Unlock() + _, conn, err := p.dialConn(ctx) + if err != nil { + return nil, err + } + return conn, nil + } + } +} + +// putConn cannot repopulate a retired pool, even for a successful late response. +func (p *dotConnPool) putConn(conn net.Conn, isGood bool) { + if conn == nil { + return + } + dc := &dotConn{conn: conn.(*tls.Conn)} + p.mu.Lock() + if isGood && !p.closed { + select { + case p.conns <- dc: + p.mu.Unlock() + return + default: + } + } + delete(p.owned, dc.conn) + p.mu.Unlock() + _ = dc.conn.Close() +} + +// dialConn creates a new TCP/TLS connection. Retirement cancels pending TCP +// dials and handshakes; finishing a parallel race cancels and closes its losers. +func (p *dotConnPool) dialConn(ctx context.Context) (string, *tls.Conn, error) { + ctx, cancel := context.WithCancel(ctx) + stop := context.AfterFunc(p.ctx, cancel) + defer stop() + defer cancel() + if p.ctx.Err() != nil { + return "", nil, net.ErrClosed + } + logger := LoggerFromCtx(ctx) + if p.uc.BootstrapIP != "" { + endpoint := net.JoinHostPort(p.uc.BootstrapIP, p.port) + Log(ctx, logger.Debug(), "Sending DoT request to: %s", endpoint) + conn, err := p.dialTLSConn(ctx, endpoint, p.tlsConfig) + return endpoint, conn, err + } + if len(p.addrs) > 0 { + type result struct { + conn *tls.Conn + addr string + err error + } + // An unbuffered handoff leaves every losing connection with its dialer. + ch := make(chan result) + for _, addr := range p.addrs { + go func(addr string) { + endpoint := net.JoinHostPort(addr, p.port) + tlsConfig := p.tlsConfig.Clone() + tlsConfig.ServerName = p.uc.Domain + conn, err := p.dialTLSConn(ctx, endpoint, tlsConfig) + select { + case ch <- result{conn: conn, addr: endpoint, err: err}: + case <-ctx.Done(): + if conn != nil { + p.putConn(conn, false) + } + } + }(addr) + } + errs := make([]error, 0, len(p.addrs)) + for range len(p.addrs) { + select { + case res := <-ch: + if res.err == nil && res.conn != nil { + Log(ctx, logger.Debug(), "Sending DoT request to: %s", res.addr) + return res.addr, res.conn, nil + } + if res.err != nil { + errs = append(errs, res.err) + } + case <-ctx.Done(): + return "", nil, ctx.Err() + } + } + return "", nil, errors.Join(errs...) + } + endpoint := p.uc.Endpoint + Log(ctx, logger.Debug(), "Sending DoT request to: %s", endpoint) + conn, err := p.dialTLSConn(ctx, endpoint, p.tlsConfig) + return endpoint, conn, err +} + +// Register ownership before the TLS handshake so Close interrupts stalled peers. +// A TCP dial racing retirement cannot publish a socket into the closed pool. +func (p *dotConnPool) dialTLSConn(ctx context.Context, endpoint string, cfg *tls.Config) (*tls.Conn, error) { + conn, err := p.dialer.DialContext(ctx, "tcp", endpoint) + if err != nil { + return nil, err + } + tlsConn := tls.Client(conn, cfg) + p.mu.Lock() + if p.closed { + p.mu.Unlock() + _ = conn.Close() + return nil, net.ErrClosed + } + p.owned[tlsConn] = struct{}{} + p.mu.Unlock() + if err := tlsConn.HandshakeContext(ctx); err != nil { + p.putConn(tlsConn, false) + return nil, err + } + return tlsConn, nil +} + +// CloseIdleConnections closes only queued connections. The pool remains usable +// and outstanding queries may return their connections. Replacement uses Close. +func (p *dotConnPool) CloseIdleConnections() { + p.mu.Lock() + defer p.mu.Unlock() + for { + select { + case dc := <-p.conns: + if dc.conn != nil { + delete(p.owned, dc.conn) + dc.conn.Close() + } + default: + return + } + } +} + +// Close permanently retires this pool. Closing the underlying TCP sockets also +// interrupts TLS reads/writes without waiting to send a close_notify. +// Keep the channel open: outstanding borrowers may still call putConn. +func (p *dotConnPool) Close() { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return } - tcpNet, _ := r.uc.netForDNSType(dnsTyp) - dnsClient := &dns.Client{ - Net: tcpNet, - Dialer: dialer, - TLSConfig: &tls.Config{RootCAs: r.uc.certPool}, + p.closed = true + p.cancel() + for conn := range p.owned { + _ = conn.NetConn().Close() + delete(p.owned, conn) } - endpoint := r.uc.Endpoint - if r.uc.BootstrapIP != "" { - dnsClient.TLSConfig.ServerName = r.uc.Domain - dnsClient.Net = "tcp-tls" - _, port, _ := net.SplitHostPort(endpoint) - endpoint = net.JoinHostPort(r.uc.BootstrapIP, port) + for len(p.conns) > 0 { + <-p.conns + } +} + +func isAlive(c *tls.Conn) bool { + // Set a very short deadline for the read + c.SetReadDeadline(time.Now().Add(1 * time.Millisecond)) + + // Try to read 1 byte without consuming it (using a small buffer) + one := make([]byte, 1) + _, err := c.Read(one) + + // Reset the deadline for future operations + c.SetReadDeadline(time.Time{}) + + if err == io.EOF { + return false // Connection is definitely closed + } + + // If we get a timeout, it means no data is waiting, + // but the connection is likely still "up." + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true } - answer, _, err := dnsClient.ExchangeContext(ctx, msg, endpoint) - return answer, wrapCertificateVerificationError(err) + return err == nil } diff --git a/go.mod b/go.mod index 2280eb65..67dcc405 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,11 @@ module github.com/Control-D-Inc/ctrld -go 1.23.0 - -toolchain go1.23.7 +go 1.26.0 require ( github.com/Masterminds/semver/v3 v3.2.1 github.com/ameshkov/dnsstamps v1.0.3 + github.com/brunogui0812/sysprofiler v0.5.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/cuonglm/osinfo v0.0.0-20230921071424-e0e1b1e0bbbf github.com/docker/go-units v0.5.0 @@ -16,7 +15,8 @@ require ( github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/illarion/gonotify/v2 v2.0.3 - github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 + github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82 + github.com/jaypipes/ghw v0.21.0 github.com/jaytaylor/go-hostsfile v0.0.0-20220426042432-61485ac1fa6c github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 github.com/kardianos/service v1.2.1 @@ -29,16 +29,15 @@ require ( github.com/prometheus/client_golang v1.19.1 github.com/prometheus/client_model v0.5.0 github.com/prometheus/prom2json v1.3.3 - github.com/quic-go/quic-go v0.54.0 - github.com/rs/zerolog v1.28.0 - github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 + github.com/quic-go/quic-go v0.59.1 + github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.16.0 - github.com/stretchr/testify v1.9.0 - github.com/vishvananda/netlink v1.2.1-beta.2 - golang.org/x/net v0.38.0 - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.31.0 + github.com/stretchr/testify v1.11.1 + github.com/vishvananda/netlink v1.3.1 + go.uber.org/zap v1.27.0 + golang.org/x/net v0.60.0 + golang.org/x/sync v0.23.0 + golang.org/x/sys v0.48.0 golang.zx2c4.com/wireguard/windows v0.5.3 tailscale.com v1.74.0 ) @@ -57,15 +56,15 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/groob/plist v0.0.0-20200425180238-0f631f258c01 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jaypipes/pcidb v1.1.1 // indirect github.com/jsimonetti/rtnetlink v1.4.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/netlink v1.7.2 // indirect @@ -77,26 +76,30 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/qpack v0.6.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/spakin/awk v1.0.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e // indirect - github.com/vishvananda/netns v0.0.4 // indirect - go.uber.org/mock v0.5.0 // indirect + github.com/vishvananda/netns v0.0.5 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/multierr v1.11.0 // indirect go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect - golang.org/x/crypto v0.36.0 // indirect + golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/mod v0.19.0 // indirect - golang.org/x/text v0.23.0 // indirect - golang.org/x/tools v0.23.0 // indirect + golang.org/x/mod v0.41.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/tools v0.49.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + howett.net/plist v1.0.2-0.20250314012144-ee69052608d9 // indirect ) replace github.com/mr-karan/doggo => github.com/Windscribe/doggo v0.0.0-20220919152748-2c118fc391f8 diff --git a/go.sum b/go.sum index 56a71e19..c2d3b843 100644 --- a/go.sum +++ b/go.sum @@ -42,14 +42,14 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Windscribe/zerolog v0.0.0-20241206130353-cc6e8ef5397c h1:UqFsxmwiCh/DBvwJB0m7KQ2QFDd6DdUkosznfMppdhE= -github.com/Windscribe/zerolog v0.0.0-20241206130353-cc6e8ef5397c/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/ameshkov/dnsstamps v1.0.3 h1:Srzik+J9mivH1alRACTbys2xOxs0lRH9qnTA7Y1OYVo= github.com/ameshkov/dnsstamps v1.0.3/go.mod h1:Ii3eUu73dx4Vw5O4wjzmT5+lkCwovjzaEZZ4gKyIH5A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/brunogui0812/sysprofiler v0.5.0 h1:AUekplOKG/VKH6sPSBRxsKOA9Uv5OsI8qolXM73dXPU= +github.com/brunogui0812/sysprofiler v0.5.0/go.mod h1:lLd7gvylgd4nsTSC8exq1YY6qhLWXkgnalxjVzdlbEM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -64,7 +64,7 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cuonglm/osinfo v0.0.0-20230921071424-e0e1b1e0bbbf h1:40DHYsri+d1bnroFDU2FQAeq68f3kAlOzlQ93kCf26Q= github.com/cuonglm/osinfo v0.0.0-20230921071424-e0e1b1e0bbbf/go.mod h1:G45410zMgmnSjLVKCq4f6GpbYAzoP2plX9rPwgx6C24= @@ -91,6 +91,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 h1:ymLjT4f35nQbASLnvxEde4XOBL+Sn7rFuV+FOJqkljg= github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= @@ -165,6 +166,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/groob/plist v0.0.0-20200425180238-0f631f258c01 h1:0T3XGXebqLj7zSVLng9wX9axQzTEnvj/h6eT7iLfUas= +github.com/groob/plist v0.0.0-20200425180238-0f631f258c01/go.mod h1:itkABA+w2cw7x5nYUS/pLRef6ludkZKOigbROmCTaFw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= @@ -179,10 +182,15 @@ github.com/illarion/gonotify/v2 v2.0.3 h1:B6+SKPo/0Sw8cRJh1aLzNEeNVFfzE3c6N+o+vy github.com/illarion/gonotify/v2 v2.0.3/go.mod h1:38oIJTgFqupkEydkkClkbL6i5lXV/bxdH9do5TALPEE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2 h1:9K06NfxkBh25x56yVhWWlKFE8YpicaSfHwoV8SFbueA= -github.com/insomniacslk/dhcp v0.0.0-20231206064809-8c70d406f6d2/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI= +github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82 h1:y5aU8Uvl7eyM5WNgdQvRxbMJb+zo7pD+S72/Yo4pvnQ= +github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82/go.mod h1:qfvBmyDNp+/liLEYWRvqny/PEz9hGe2Dz833eXILSmo= +github.com/jaypipes/ghw v0.21.0 h1:ClG2xWtYY0c1ud9jZYwVGdSgfCI7AbmZmZyw3S5HHz8= +github.com/jaypipes/ghw v0.21.0/go.mod h1:GPrvwbtPoxYUenr74+nAnWbardIZq600vJDD5HnPsPE= +github.com/jaypipes/pcidb v1.1.1 h1:QmPhpsbmmnCwZmHeYAATxEaoRuiMAJusKYkUncMC0ro= +github.com/jaypipes/pcidb v1.1.1/go.mod h1:x27LT2krrUgjf875KxQXKB0Ha/YXLdZRVmw6hH0G7g8= github.com/jaytaylor/go-hostsfile v0.0.0-20220426042432-61485ac1fa6c h1:kbTQ8oGf+BVFvt/fM+ECI+NbZDCqoi0vtZTfB2p2hrI= github.com/jaytaylor/go-hostsfile v0.0.0-20220426042432-61485ac1fa6c/go.mod h1:k6+89xKz7BSMJ+DzIerBdtpEUeTlBMugO/hcVSzahog= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86 h1:elKwZS1OcdQ0WwEDBeqxKwb7WB62QX8bvZ/FJnVXIfk= github.com/josharian/native v1.1.1-0.20230202152459-5c7d0dd6ab86/go.mod h1:aFAMtuldEgx/4q7iSGazk22+IcgvtiC+HIimFO9XlS8= @@ -207,12 +215,6 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -259,10 +261,10 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/prom2json v1.3.3 h1:IYfSMiZ7sSOfliBoo89PcufjWO4eAR0gznGcETyaUgo= github.com/prometheus/prom2json v1.3.3/go.mod h1:Pv4yIPktEkK7btWsrUTWDDDrnpUrAELaOCj+oFwlgmc= -github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= -github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= -github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= +github.com/quic-go/quic-go v0.59.1/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -272,18 +274,19 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spakin/awk v1.0.0 h1:5ulBVgJhdN3XoFGNVv/MOHOIUfPVPvMCIlLH6O6ZqU4= +github.com/spakin/awk v1.0.0/go.mod h1:e7FnxcIEcRqdKwStPYWonox4n9DpharWk+3nnn1IqJs= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -299,29 +302,36 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e h1:BA9O3BmlTmpjbvajAwzWx4Wo2TRVdpPXZEeemGQcajw= github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= -github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= -github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= -github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= +github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org/mem v0.0.0-20220726221520-4f986261bf13 h1:CbZeCBZ0aZj8EfVgnqQcYZgf0lpZ3H9rmp5nkDTAst8= go4.org/mem v0.0.0-20220726221520-4f986261bf13/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= @@ -336,8 +346,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -373,8 +383,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -407,8 +417,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -428,8 +438,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -439,6 +449,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -447,7 +458,6 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -456,7 +466,6 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -472,14 +481,13 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -490,8 +498,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -542,8 +550,8 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -659,6 +667,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +howett.net/plist v1.0.2-0.20250314012144-ee69052608d9 h1:eeH1AIcPvSc0Z25ThsYF+Xoqbn0CI/YnXVYoTLFdGQw= +howett.net/plist v1.0.2-0.20250314012144-ee69052608d9/go.mod h1:fyFX5Hj5tP1Mpk8obqA9MZgXT416Q5711SDT7dQLTLk= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/certs/root_ca_test.go b/internal/certs/root_ca_test.go index fcfd16e5..295be56e 100644 --- a/internal/certs/root_ca_test.go +++ b/internal/certs/root_ca_test.go @@ -11,7 +11,8 @@ func TestCACertPool(t *testing.T) { c := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ - RootCAs: CACertPool(), + RootCAs: CACertPool(), + MinVersion: tls.VersionTLS12, }, }, Timeout: 2 * time.Second, diff --git a/internal/clientinfo/arp_unix.go b/internal/clientinfo/arp_unix.go index f5d8f884..51c934ae 100644 --- a/internal/clientinfo/arp_unix.go +++ b/internal/clientinfo/arp_unix.go @@ -20,6 +20,8 @@ func (a *arpDiscover) scan() { } // trim brackets + // Unix "arp -an" output formats IP addresses with parentheses like "(192.168.1.1)" + // We need to remove these brackets for proper IP parsing ip := strings.ReplaceAll(fields[1], "(", "") ip = strings.ReplaceAll(ip, ")", "") diff --git a/internal/clientinfo/arp_windows.go b/internal/clientinfo/arp_windows.go index 016b752f..c037b29f 100644 --- a/internal/clientinfo/arp_windows.go +++ b/internal/clientinfo/arp_windows.go @@ -17,10 +17,14 @@ func (a *arpDiscover) scan() { continue // empty lines } if line[0] != ' ' { + // Mark that we've found an interface header line + // Windows "arp -a" output has interface headers followed by ARP entries header = true // "Interface:" lines, next is header line. continue } if header { + // Skip the header line that follows interface names + // These lines contain column headers like "Internet Address" and "Physical Address" header = false // header lines continue } diff --git a/internal/clientinfo/client_info.go b/internal/clientinfo/client_info.go index f69b670f..e0d2b575 100644 --- a/internal/clientinfo/client_info.go +++ b/internal/clientinfo/client_info.go @@ -79,10 +79,9 @@ type Table struct { initOnce sync.Once stopOnce sync.Once refreshInterval int + logger *ctrld.Logger dhcp *dhcp - merlin *merlinDiscover - ubios *ubiosDiscover arp *arpDiscover ndp *ndpDiscover ptr *ptrDiscover @@ -98,11 +97,18 @@ type Table struct { ptrNameservers []string } -func NewTable(cfg *ctrld.Config, selfIP, cdUID string, ns []string) *Table { +func NewTable(cfg *ctrld.Config, selfIP, cdUID string, ns []string, logger *ctrld.Logger) *Table { refreshInterval := cfg.Service.DiscoverRefreshInterval + // Set default refresh interval if not configured + // This ensures client discovery continues to work even without explicit configuration if refreshInterval <= 0 { refreshInterval = 2 * 60 // 2 minutes } + // Use no-op logger if none provided + // This prevents nil pointer dereferences when logging is not configured + if logger == nil { + logger = ctrld.NopLogger + } return &Table{ svcCfg: cfg.Service, quitCh: make(chan struct{}), @@ -111,6 +117,7 @@ func NewTable(cfg *ctrld.Config, selfIP, cdUID string, ns []string) *Table { cdUID: cdUID, ptrNameservers: ns, refreshInterval: refreshInterval, + logger: logger, } } @@ -173,13 +180,15 @@ func (t *Table) SetSelfIP(ip string) { t.selfIPLock.Lock() defer t.selfIPLock.Unlock() t.selfIP = ip - t.dhcp.selfIP = t.selfIP - t.dhcp.addSelf() + if t.dhcp != nil { + t.dhcp.selfIP = t.selfIP + t.dhcp.addSelf() + } } // initSelfDiscover initializes necessary client metadata for self query. func (t *Table) initSelfDiscover() { - t.dhcp = &dhcp{selfIP: t.selfIP} + t.dhcp = &dhcp{selfIP: t.selfIP, logger: t.logger} t.dhcp.addSelf() t.ipResolvers = append(t.ipResolvers, t.dhcp) t.macResolvers = append(t.macResolvers, t.dhcp) @@ -189,48 +198,24 @@ func (t *Table) initSelfDiscover() { func (t *Table) init() { // Custom client ID presents, use it as the only source. if _, clientID := controld.ParseRawUID(t.cdUID); clientID != "" { - ctrld.ProxyLogger.Load().Debug().Msg("start self discovery with custom client id") + t.logger.Debug().Msg("Start self discovery with custom client id") t.initSelfDiscover() return } // If we are running on platforms that should only do self discover, use it as the only source, too. if ctrld.SelfDiscover() { - ctrld.ProxyLogger.Load().Debug().Msg("start self discovery on desktop platforms") + t.logger.Debug().Msg("Start self discovery on desktop platforms") t.initSelfDiscover() return } - // Otherwise, process all possible sources in order, that means - // the first result of IP/MAC/Hostname lookup will be used. - // - // Routers custom clients: - // - Merlin - // - Ubios - if t.discoverDHCP() || t.discoverARP() { - t.merlin = &merlinDiscover{} - t.ubios = &ubiosDiscover{} - discovers := map[string]interface { - refresher - HostnameResolver - }{ - "Merlin": t.merlin, - "Ubios": t.ubios, - } - for platform, discover := range discovers { - if err := discover.refresh(); err != nil { - ctrld.ProxyLogger.Load().Warn().Err(err).Msgf("failed to init %s discover", platform) - } - t.hostnameResolvers = append(t.hostnameResolvers, discover) - t.refreshers = append(t.refreshers, discover) - } - } // Hosts file mapping. if t.discoverHosts() { - t.hf = &hostsFile{} - ctrld.ProxyLogger.Load().Debug().Msg("start hosts file discovery") + t.hf = &hostsFile{logger: t.logger} + t.logger.Debug().Msg("Start hosts file discovery") if err := t.hf.init(); err != nil { - ctrld.ProxyLogger.Load().Error().Err(err).Msg("could not init hosts file discover") + t.logger.Error().Err(err).Msg("Could not init hosts file discover") } else { t.hostnameResolvers = append(t.hostnameResolvers, t.hf) t.refreshers = append(t.refreshers, t.hf) @@ -239,10 +224,10 @@ func (t *Table) init() { } // DHCP lease files. if t.discoverDHCP() { - t.dhcp = &dhcp{selfIP: t.selfIP} - ctrld.ProxyLogger.Load().Debug().Msg("start dhcp discovery") + t.dhcp = &dhcp{selfIP: t.selfIP, logger: t.logger} + t.logger.Debug().Msg("Start dhcp discovery") if err := t.dhcp.init(); err != nil { - ctrld.ProxyLogger.Load().Error().Err(err).Msg("could not init DHCP discover") + t.logger.Error().Err(err).Msg("Could not init dhcp discover") } else { t.ipResolvers = append(t.ipResolvers, t.dhcp) t.macResolvers = append(t.macResolvers, t.dhcp) @@ -253,8 +238,8 @@ func (t *Table) init() { // ARP/NDP table. if t.discoverARP() { t.arp = &arpDiscover{} - t.ndp = &ndpDiscover{} - ctrld.ProxyLogger.Load().Debug().Msg("start arp discovery") + t.ndp = &ndpDiscover{logger: t.logger} + t.logger.Debug().Msg("Start arp discovery") discovers := map[string]interface { refresher IpResolver @@ -266,7 +251,7 @@ func (t *Table) init() { for protocol, discover := range discovers { if err := discover.refresh(); err != nil { - ctrld.ProxyLogger.Load().Error().Err(err).Msgf("could not init %s discover", protocol) + t.logger.Error().Err(err).Msgf("Could not init %s discover", protocol) } else { t.ipResolvers = append(t.ipResolvers, discover) t.macResolvers = append(t.macResolvers, discover) @@ -283,7 +268,10 @@ func (t *Table) init() { } // PTR lookup. if t.discoverPTR() { - t.ptr = &ptrDiscover{resolver: ctrld.NewPrivateResolver()} + t.ptr = &ptrDiscover{ + resolver: ctrld.NewPrivateResolver(context.Background()), + logger: t.logger, + } if len(t.ptrNameservers) > 0 { nss := make([]string, 0, len(t.ptrNameservers)) for _, ns := range t.ptrNameservers { @@ -292,21 +280,22 @@ func (t *Table) init() { host, port = h, p } // Only use valid ip:port pair. + // Invalid nameservers can cause PTR discovery to fail silently if _, portErr := strconv.Atoi(port); portErr == nil && port != "0" && net.ParseIP(host) != nil { nss = append(nss, net.JoinHostPort(host, port)) } else { - ctrld.ProxyLogger.Load().Warn().Msgf("ignoring invalid nameserver for ptr discover: %q", ns) + t.logger.Warn().Msgf("Ignoring invalid nameserver for ptr discover: %q", ns) } } if len(nss) > 0 { t.ptr.resolver = ctrld.NewResolverWithNameserver(nss) - ctrld.ProxyLogger.Load().Debug().Msgf("using nameservers %v for ptr discovery", nss) + t.logger.Debug().Msgf("Using nameservers %v for ptr discovery", nss) } } - ctrld.ProxyLogger.Load().Debug().Msg("start ptr discovery") + t.logger.Debug().Msg("Start ptr discovery") if err := t.ptr.refresh(); err != nil { - ctrld.ProxyLogger.Load().Error().Err(err).Msg("could not init PTR discover") + t.logger.Error().Err(err).Msg("Could not init ptr discover") } else { t.hostnameResolvers = append(t.hostnameResolvers, t.ptr) t.refreshers = append(t.refreshers, t.ptr) @@ -314,10 +303,10 @@ func (t *Table) init() { } // mdns. if t.discoverMDNS() { - t.mdns = &mdns{} - ctrld.ProxyLogger.Load().Debug().Msg("start mdns discovery") + t.mdns = &mdns{logger: t.logger} + t.logger.Debug().Msg("Start mdns discovery") if err := t.mdns.init(t.quitCh); err != nil { - ctrld.ProxyLogger.Load().Error().Err(err).Msg("could not init mDNS discover") + t.logger.Error().Err(err).Msg("Could not init mdns discover") } else { t.hostnameResolvers = append(t.hostnameResolvers, t.mdns) } @@ -483,6 +472,7 @@ func (t *Table) ListClients() []*Client { for _, c := range ipMap { // If we found a client with empty hostname, use hostname from // an existed client which has the same MAC address. + // This helps fill in missing hostnames when multiple IPs share the same MAC if cFromMac := clientsByMAC[c.Mac]; cFromMac != nil && c.Hostname == "" { c.Hostname = cFromMac.Hostname } diff --git a/internal/clientinfo/client_info_test.go b/internal/clientinfo/client_info_test.go index b5bdfa57..1df50a90 100644 --- a/internal/clientinfo/client_info_test.go +++ b/internal/clientinfo/client_info_test.go @@ -1,9 +1,66 @@ package clientinfo import ( + "sync" "testing" + + "github.com/Control-D-Inc/ctrld" ) +// TestTable_SetSelfIP_NilDHCP ensures SetSelfIP does not panic when t.dhcp is +// nil, which happens when DHCP discovery is disabled and the network-change +// callback fires before or without initialisation. +func TestTable_SetSelfIP_NilDHCP(t *testing.T) { + table := &Table{} // dhcp is nil + // Must not panic. + table.SetSelfIP("192.168.1.1") + if got := table.SelfIP(); got != "192.168.1.1" { + t.Fatalf("SelfIP() = %q, want %q", got, "192.168.1.1") + } +} + +// TestTable_SetSelfIP_UpdatesDHCP ensures SetSelfIP propagates the new IP to +// the dhcp discover and calls addSelf when dhcp is initialised. +func TestTable_SetSelfIP_UpdatesDHCP(t *testing.T) { + table := &Table{ + dhcp: &dhcp{selfIP: "10.0.0.1"}, + } + table.SetSelfIP("10.0.0.2") + if got := table.SelfIP(); got != "10.0.0.2" { + t.Fatalf("SelfIP() = %q, want %q", got, "10.0.0.2") + } + if table.dhcp.selfIP != "10.0.0.2" { + t.Fatalf("dhcp.selfIP = %q, want %q", table.dhcp.selfIP, "10.0.0.2") + } +} + +// TestTable_SetSelfIP_Concurrent ensures concurrent calls to SetSelfIP do not +// race, regardless of whether dhcp is nil or not. +func TestTable_SetSelfIP_Concurrent(t *testing.T) { + for _, tc := range []struct { + name string + table *Table + }{ + {"nil dhcp", &Table{}}, + {"with dhcp", &Table{dhcp: &dhcp{}}}, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var wg sync.WaitGroup + for range 10 { + wg.Add(1) + go func() { + defer wg.Done() + tc.table.SetSelfIP("192.168.1.1") + _ = tc.table.SelfIP() + }() + } + wg.Wait() + }) + } +} + func Test_normalizeIP(t *testing.T) { tests := []struct { name string @@ -28,8 +85,9 @@ func Test_normalizeIP(t *testing.T) { func TestTable_LookupRFC1918IPv4(t *testing.T) { table := &Table{ - dhcp: &dhcp{}, - arp: &arpDiscover{}, + dhcp: &dhcp{}, + arp: &arpDiscover{}, + logger: ctrld.NopLogger, } table.ipResolvers = append(table.ipResolvers, table.dhcp) diff --git a/internal/clientinfo/dhcp.go b/internal/clientinfo/dhcp.go index 5d11d5eb..efe44ed9 100644 --- a/internal/clientinfo/dhcp.go +++ b/internal/clientinfo/dhcp.go @@ -13,13 +13,11 @@ import ( "strings" "sync" - "tailscale.com/net/netmon" - "github.com/fsnotify/fsnotify" + "tailscale.com/net/netmon" "tailscale.com/util/lineread" "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router" ) type dhcp struct { @@ -30,6 +28,7 @@ type dhcp struct { watcher *fsnotify.Watcher selfIP string + logger *ctrld.Logger } func (d *dhcp) init() error { @@ -39,10 +38,6 @@ func (d *dhcp) init() error { } d.addSelf() d.watcher = watcher - for file, format := range clientInfoFiles { - // Ignore errors for default lease files. - _ = d.addLeaseFile(file, format) - } return nil } @@ -50,11 +45,7 @@ func (d *dhcp) watchChanges() { if d.watcher == nil { return } - if dir := router.LeaseFilesDir(); dir != "" { - if err := d.watcher.Add(dir); err != nil { - ctrld.ProxyLogger.Load().Err(err).Str("dir", dir).Msg("could not watch lease dir") - } - } + for { select { case event, ok := <-d.watcher.Events: @@ -64,7 +55,7 @@ func (d *dhcp) watchChanges() { if event.Has(fsnotify.Create) { if format, ok := clientInfoFiles[event.Name]; ok { if err := d.addLeaseFile(event.Name, format); err != nil { - ctrld.ProxyLogger.Load().Err(err).Str("file", event.Name).Msg("could not add lease file") + d.logger.Err(err).Str("file", event.Name).Msg("Could not add lease file") } } continue @@ -72,14 +63,14 @@ func (d *dhcp) watchChanges() { if event.Has(fsnotify.Write) || event.Has(fsnotify.Rename) || event.Has(fsnotify.Chmod) || event.Has(fsnotify.Remove) { format := clientInfoFiles[event.Name] if err := d.readLeaseFile(event.Name, format); err != nil && !os.IsNotExist(err) { - ctrld.ProxyLogger.Load().Err(err).Str("file", event.Name).Msg("leases file changed but failed to update client info") + d.logger.Err(err).Str("file", event.Name).Msg("Leases file changed but failed to update client info") } } case err, ok := <-d.watcher.Errors: if !ok { return } - ctrld.ProxyLogger.Load().Err(err).Msg("could not watch client info file") + d.logger.Err(err).Msg("Could not watch client info file") } } @@ -150,6 +141,9 @@ func (d *dhcp) lookupIPByHostname(name string, v6 bool) string { return true } if addr, err := netip.ParseAddr(key.(string)); err == nil && addr.Is6() == v6 { + // Categorize addresses into RFC1918 (private) and public + // RFC1918 addresses are prioritized because they're more likely to be + // the actual client IP in most network configurations if addr.IsPrivate() { rfc1918Addrs = append(rfc1918Addrs, addr) } else { @@ -222,7 +216,7 @@ func (d *dhcp) dnsmasqReadClientInfoReader(reader io.Reader) error { } ip := normalizeIP(string(fields[2])) if net.ParseIP(ip) == nil { - ctrld.ProxyLogger.Load().Warn().Msgf("invalid ip address entry: %q", ip) + d.logger.Warn().Msgf("Invalid ip address entry: %q", ip) ip = "" } @@ -273,13 +267,17 @@ func (d *dhcp) iscDHCPReadClientInfoReader(reader io.Reader) error { } switch fields[0] { case "lease": + // Normalize IP address to lowercase for consistent comparison + // DHCP lease files may contain mixed-case IP addresses ip = normalizeIP(strings.ToLower(fields[1])) if net.ParseIP(ip) == nil { - ctrld.ProxyLogger.Load().Warn().Msgf("invalid ip address entry: %q", ip) + d.logger.Warn().Msgf("Invalid ip address entry: %q", ip) ip = "" } case "hardware": if len(fields) >= 3 { + // Convert MAC to lowercase and remove trailing semicolon + // DHCP lease files use semicolon-terminated MAC addresses mac = strings.ToLower(strings.TrimRight(fields[2], ";")) if _, err := net.ParseMAC(mac); err != nil { // Invalid dhcp, skip. @@ -287,6 +285,8 @@ func (d *dhcp) iscDHCPReadClientInfoReader(reader io.Reader) error { } } case "client-hostname": + // Remove quotes and semicolons from hostname + // DHCP lease files may quote hostnames and add semicolons hostname = strings.Trim(fields[1], `";`) } } @@ -328,7 +328,7 @@ func (d *dhcp) keaDhcp4ReadClientInfoReader(r io.Reader) error { } ip := normalizeIP(record[0]) if net.ParseIP(ip) == nil { - ctrld.ProxyLogger.Load().Warn().Msgf("invalid ip address entry: %q", ip) + d.logger.Warn().Msgf("Invalid ip address entry: %q", ip) ip = "" } @@ -350,7 +350,7 @@ func (d *dhcp) keaDhcp4ReadClientInfoReader(r io.Reader) error { func (d *dhcp) addSelf() { hostname, err := os.Hostname() if err != nil { - ctrld.ProxyLogger.Load().Err(err).Msg("could not get hostname") + d.logger.Err(err).Msg("Could not get hostname") return } hostname = normalizeHostname(hostname) @@ -390,22 +390,4 @@ func (d *dhcp) addSelf() { } } }) - for _, netIface := range router.SelfInterfaces() { - mac := netIface.HardwareAddr.String() - if mac == "" { - return - } - d.mac2name.Store(mac, hostname) - addrs, _ := netIface.Addrs() - for _, addr := range addrs { - ipNet, ok := addr.(*net.IPNet) - if !ok { - continue - } - ip := ipNet.IP - d.mac.LoadOrStore(ip.String(), mac) - d.ip.LoadOrStore(mac, ip.String()) - d.ip2name.Store(ip.String(), hostname) - } - } } diff --git a/internal/clientinfo/dhcp_lease_files.go b/internal/clientinfo/dhcp_lease_files.go index 34aabf3a..3f1c5ac8 100644 --- a/internal/clientinfo/dhcp_lease_files.go +++ b/internal/clientinfo/dhcp_lease_files.go @@ -3,18 +3,5 @@ package clientinfo import "github.com/Control-D-Inc/ctrld" // clientInfoFiles specifies client info files and how to read them on supported platforms. -var clientInfoFiles = map[string]ctrld.LeaseFileFormat{ - "/tmp/dnsmasq.leases": ctrld.Dnsmasq, // ddwrt - "/tmp/dhcp.leases": ctrld.Dnsmasq, // openwrt - "/var/lib/misc/dnsmasq.leases": ctrld.Dnsmasq, // merlin - "/mnt/data/udapi-config/dnsmasq.lease": ctrld.Dnsmasq, // UDM Pro - "/data/udapi-config/dnsmasq.lease": ctrld.Dnsmasq, // UDR - "/etc/dhcpd/dhcpd-leases.log": ctrld.Dnsmasq, // Synology - "/tmp/var/lib/misc/dnsmasq.leases": ctrld.Dnsmasq, // Tomato - "/run/dnsmasq-dhcp.leases": ctrld.Dnsmasq, // EdgeOS - "/run/dhcpd.leases": ctrld.IscDhcpd, // EdgeOS - "/var/dhcpd/var/db/dhcpd.leases": ctrld.IscDhcpd, // Pfsense - "/home/pi/.router/run/dhcp/dnsmasq.leases": ctrld.Dnsmasq, // Firewalla - "/var/lib/kea/dhcp4.leases": ctrld.KeaDHCP4, // Pfsense - "/var/db/dnsmasq.leases": ctrld.Dnsmasq, // OPNsense -} +// TODO: cleanup this after server support removal. +var clientInfoFiles = map[string]ctrld.LeaseFileFormat{} diff --git a/internal/clientinfo/hostsfile.go b/internal/clientinfo/hostsfile.go index d96229df..003e1b81 100644 --- a/internal/clientinfo/hostsfile.go +++ b/internal/clientinfo/hostsfile.go @@ -27,6 +27,7 @@ type hostsFile struct { watcher *fsnotify.Watcher mu sync.Mutex m map[string][]string + logger *ctrld.Logger } // init performs initialization works, which is necessary before hostsFile can be fully operated. @@ -55,7 +56,7 @@ func (hf *hostsFile) refresh() error { // override hosts file with host_entries.conf content if present. hem, err := parseHostEntriesConf(hostEntriesConfPath) if err != nil && !os.IsNotExist(err) { - ctrld.ProxyLogger.Load().Debug().Err(err).Msg("could not read host_entries.conf file") + hf.logger.Debug().Err(err).Msg("Could not read host_entries.conf file") } for k, v := range hem { hf.m[k] = v @@ -77,14 +78,14 @@ func (hf *hostsFile) watchChanges() { } if event.Has(fsnotify.Write) || event.Has(fsnotify.Rename) || event.Has(fsnotify.Chmod) || event.Has(fsnotify.Remove) { if err := hf.refresh(); err != nil && !os.IsNotExist(err) { - ctrld.ProxyLogger.Load().Err(err).Msg("hosts file changed but failed to update client info") + hf.logger.Err(err).Msg("Hosts file changed but Failed to update client info") } } case err, ok := <-hf.watcher.Errors: if !ok { return } - ctrld.ProxyLogger.Load().Err(err).Msg("could not watch client info file") + hf.logger.Err(err).Msg("Could not watch client info file") } } @@ -164,6 +165,8 @@ func parseHostEntriesConfFromReader(r io.Reader) map[string][]string { for scanner.Scan() { line := scanner.Text() if after, found := strings.CutPrefix(line, "local-zone:"); found { + // Extract local zone name for domain suffix removal + // This is needed because unbound appends the local zone to hostnames after = strings.TrimSpace(after) fields := strings.Fields(after) if len(fields) > 1 { @@ -176,6 +179,8 @@ func parseHostEntriesConfFromReader(r io.Reader) map[string][]string { if !found { continue } + // Clean up the parsed data by removing whitespace and quotes + // This ensures consistent formatting for hostname processing after = strings.TrimSpace(after) after = strings.Trim(after, `"`) fields := strings.Fields(after) @@ -183,6 +188,8 @@ func parseHostEntriesConfFromReader(r io.Reader) map[string][]string { continue } ip := fields[0] + // Remove local zone suffix from hostname for cleaner lookups + // Unbound adds the local zone to hostnames, but we want just the base name name := strings.TrimSuffix(fields[1], "."+localZone) hostsMap[ip] = append(hostsMap[ip], name) } diff --git a/internal/clientinfo/mdns.go b/internal/clientinfo/mdns.go index a09d7296..981de124 100644 --- a/internal/clientinfo/mdns.go +++ b/internal/clientinfo/mdns.go @@ -34,7 +34,8 @@ var ( ) type mdns struct { - name sync.Map // ip => hostname + name sync.Map // ip => hostname + logger *ctrld.Logger } func (m *mdns) LookupHostnameByIP(ip string) string { @@ -74,10 +75,7 @@ func (m *mdns) lookupIPByHostname(name string, v6 bool) string { if value == name { if addr, err := netip.ParseAddr(key.(string)); err == nil && addr.Is6() == v6 { ip = addr.String() - if addr.IsLoopback() { // Continue searching if this is loopback address. - return true - } - return false + return addr.IsLoopback() // Continue searching if this is loopback address. } } return true @@ -92,9 +90,9 @@ func (m *mdns) init(quitCh chan struct{}) error { } // Check if IPv6 is available once and use the result for the rest of the function. - ctrld.ProxyLogger.Load().Debug().Msgf("checking for IPv6 availability in mdns init") + m.logger.Debug().Msgf("Checking for ipv6 availability in mdns init") ipv6 := ctrldnet.IPv6Available(context.Background()) - ctrld.ProxyLogger.Load().Debug().Msgf("IPv6 is %v in mdns init", ipv6) + m.logger.Debug().Msgf("ipv6 is %v in mdns init", ipv6) v4ConnList := make([]*net.UDPConn, 0, len(ifaces)) v6ConnList := make([]*net.UDPConn, 0, len(ifaces)) @@ -128,11 +126,11 @@ func (m *mdns) probeLoop(conns []*net.UDPConn, remoteAddr net.Addr, quitCh chan for { err := m.probe(conns, remoteAddr) if shouldStopProbing(err) { - ctrld.ProxyLogger.Load().Warn().Msgf("stop probing %q: %v", remoteAddr, err) + m.logger.Warn().Msgf("Stop probing %q: %v", remoteAddr, err) break } if err != nil { - ctrld.ProxyLogger.Load().Warn().Err(err).Msg("error while probing mdns") + m.logger.Warn().Err(err).Msg("Error while probing mdns") bo.BackOff(context.Background(), errors.New("mdns probe backoff")) continue } @@ -160,7 +158,7 @@ func (m *mdns) readLoop(conn *net.UDPConn) { if errors.Is(err, net.ErrClosed) { return } - ctrld.ProxyLogger.Load().Debug().Err(err).Msg("mdns readLoop error") + m.logger.Debug().Err(err).Msg("Mdns readLoop error") return } @@ -183,11 +181,11 @@ func (m *mdns) readLoop(conn *net.UDPConn) { if ip != "" && name != "" { name = normalizeHostname(name) if val, loaded := m.name.LoadOrStore(ip, name); !loaded { - ctrld.ProxyLogger.Load().Debug().Msgf("found hostname: %q, ip: %q via mdns", name, ip) + m.logger.Debug().Msgf("Found hostname: %q, ip: %q via mdns", name, ip) } else { old := val.(string) if old != name { - ctrld.ProxyLogger.Load().Debug().Msgf("update hostname: %q, ip: %q, old: %q via mdns", name, ip, old) + m.logger.Debug().Msgf("Update hostname: %q, ip: %q, old: %q via mdns", name, ip, old) m.name.Store(ip, name) } } @@ -217,6 +215,8 @@ func (m *mdns) probe(conns []*net.UDPConn, remoteAddr net.Addr) error { for _, conn := range conns { _ = conn.SetWriteDeadline(time.Now().Add(time.Second * 30)) if _, werr := conn.WriteTo(buf, remoteAddr); werr != nil { + // Capture the last write error for reporting + // Multiple connections may fail, but we only report the last error err = werr } } @@ -226,7 +226,7 @@ func (m *mdns) probe(conns []*net.UDPConn, remoteAddr net.Addr) error { // getDataFromAvahiDaemonCache reads entries from avahi-daemon cache to update mdns data. func (m *mdns) getDataFromAvahiDaemonCache() { if _, err := exec.LookPath("avahi-browse"); err != nil { - ctrld.ProxyLogger.Load().Debug().Err(err).Msg("could not find avahi-browse binary, skipping.") + m.logger.Debug().Err(err).Msg("Could not find avahi-browse binary, skipping.") return } // Run avahi-browse to discover services from cache: @@ -236,7 +236,7 @@ func (m *mdns) getDataFromAvahiDaemonCache() { // - "-c" -> read from cache. out, err := exec.Command("avahi-browse", "-a", "-r", "-p", "-c").Output() if err != nil { - ctrld.ProxyLogger.Load().Debug().Err(err).Msg("could not browse services from avahi cache") + m.logger.Debug().Err(err).Msg("Could not browse services from avahi cache") return } m.storeDataFromAvahiBrowseOutput(bytes.NewReader(out)) @@ -256,7 +256,7 @@ func (m *mdns) storeDataFromAvahiBrowseOutput(r io.Reader) { name := normalizeHostname(fields[6]) // Only using cache value if we don't have existed one. if _, loaded := m.name.LoadOrStore(ip, name); !loaded { - ctrld.ProxyLogger.Load().Debug().Msgf("found hostname: %q, ip: %q via avahi cache", name, ip) + m.logger.Debug().Msgf("Found hostname: %q, ip: %q via avahi cache", name, ip) } } } diff --git a/internal/clientinfo/mdns_test.go b/internal/clientinfo/mdns_test.go index e6f86989..28c23d9f 100644 --- a/internal/clientinfo/mdns_test.go +++ b/internal/clientinfo/mdns_test.go @@ -3,6 +3,8 @@ package clientinfo import ( "strings" "testing" + + "github.com/Control-D-Inc/ctrld" ) func Test_mdns_storeDataFromAvahiBrowseOutput(t *testing.T) { @@ -11,7 +13,7 @@ func Test_mdns_storeDataFromAvahiBrowseOutput(t *testing.T) { =;wlp0s20f3;IPv6;Foo\032\0402\041;_companion-link._tcp;local;Foo-2.local;192.168.1.123;64842;"rpBA=00:00:00:00:00:01" "rpHI=e6ae2cbbca0e" "rpAD=36566f4d850f" "rpVr=510.71.1" "rpHA=0ddc20fdddc8" "rpFl=0x30000" "rpHN=1d4a03afdefa" "rpMac=0" =;wlp0s20f3;IPv4;Foo\032\0402\041;_companion-link._tcp;local;Foo-2.local;192.168.1.123;64842;"rpBA=00:00:00:00:00:01" "rpHI=e6ae2cbbca0e" "rpAD=36566f4d850f" "rpVr=510.71.1" "rpHA=0ddc20fdddc8" "rpFl=0x30000" "rpHN=1d4a03afdefa" "rpMac=0" ` - m := &mdns{} + m := &mdns{logger: ctrld.NopLogger} m.storeDataFromAvahiBrowseOutput(strings.NewReader(content)) ip := "192.168.1.123" val, loaded := m.name.LoadOrStore(ip, "") diff --git a/internal/clientinfo/merlin.go b/internal/clientinfo/merlin.go deleted file mode 100644 index 8a39398f..00000000 --- a/internal/clientinfo/merlin.go +++ /dev/null @@ -1,71 +0,0 @@ -package clientinfo - -import ( - "strings" - "sync" - - "github.com/Control-D-Inc/ctrld/internal/router" - "github.com/Control-D-Inc/ctrld/internal/router/merlin" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -const merlinNvramCustomClientListKey = "custom_clientlist" - -type merlinDiscover struct { - hostname sync.Map // mac => hostname -} - -func (m *merlinDiscover) refresh() error { - if router.Name() != merlin.Name { - return nil - } - out, err := nvram.Run("get", merlinNvramCustomClientListKey) - if err != nil { - return err - } - ctrld.ProxyLogger.Load().Debug().Msg("reading Merlin custom client list") - m.parseMerlinCustomClientList(out) - return nil -} - -func (m *merlinDiscover) LookupHostnameByIP(ip string) string { - return "" -} - -func (m *merlinDiscover) LookupHostnameByMac(mac string) string { - val, ok := m.hostname.Load(mac) - if !ok { - return "" - } - return val.(string) -} - -// "nvram get custom_clientlist" output: -// -// 00:00:00:00:00:01>0>4>>00:00:00:00:00:02>0>24>>... -// -// So to parse it, do the following steps: -// -// - Split by "<" => entries -// - For each entry, split by ">" => parts -// - Empty parts => skip -// - Empty parts[0] => skip empty hostname -// - Empty parts[1] => skip empty MAC -func (m *merlinDiscover) parseMerlinCustomClientList(data string) { - entries := strings.Split(data, "<") - for _, entry := range entries { - parts := strings.SplitN(string(entry), ">", 3) - if len(parts) < 2 || len(parts[0]) == 0 || len(parts[1]) == 0 { - continue - } - hostname := normalizeHostname(parts[0]) - mac := strings.ToLower(parts[1]) - m.hostname.Store(mac, hostname) - } -} - -func (m *merlinDiscover) String() string { - return "merlin" -} diff --git a/internal/clientinfo/merlin_test.go b/internal/clientinfo/merlin_test.go deleted file mode 100644 index 0437035a..00000000 --- a/internal/clientinfo/merlin_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package clientinfo - -import ( - "testing" -) - -func TestParseMerlinCustomClientList(t *testing.T) { - tests := []struct { - name string - clientList string - macList []string - hostnameList []string - macNotPresentList []string - }{ - { - "normal", - "00:00:00:00:00:01>0>4>>", - []string{"00:00:00:00:00:01"}, - []string{"client1"}, - nil, - }, - { - "multiple clients", - "00:00:00:00:00:01>0>4>>00:00:00:00:00:02>0>24>>", - []string{"00:00:00:00:00:01", "00:00:00:00:00:02"}, - []string{"client1", "client2"}, - nil, - }, - { - "empty hostname", - "00:00:00:00:00:01>0>4>><>00:00:00:00:00:02>0>24>>", - []string{"00:00:00:00:00:01"}, - []string{"client1"}, - []string{"00:00:00:00:00:02"}, - }, - { - "empty dhcp", - "00:00:00:00:00:01>0>4>>>>", - []string{"00:00:00:00:00:01"}, - []string{"client1"}, - []string{""}, - }, - { - "invalid", - "qwerty", - nil, - nil, - nil, - }, - { - "empty", - "", - - nil, - nil, - nil, - }, - } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - m := &merlinDiscover{} - m.parseMerlinCustomClientList(tc.clientList) - for i, mac := range tc.macList { - val, ok := m.hostname.Load(mac) - if !ok { - t.Errorf("missing hostname: %s", mac) - } - hostname := val.(string) - if hostname != tc.hostnameList[i] { - t.Errorf("hostname mismatch, want: %q, got: %q", tc.hostnameList[i], hostname) - } - } - for _, mac := range tc.macNotPresentList { - if _, ok := m.hostname.Load(mac); ok { - t.Errorf("mac2name address %q should not be present", mac) - } - } - }) - } -} diff --git a/internal/clientinfo/ndp.go b/internal/clientinfo/ndp.go index 9d9155d7..f53e7fe1 100644 --- a/internal/clientinfo/ndp.go +++ b/internal/clientinfo/ndp.go @@ -20,8 +20,9 @@ import ( // ndpDiscover provides client discovery functionality using NDP protocol. type ndpDiscover struct { - mac sync.Map // ip => mac - ip sync.Map // mac => ip + mac sync.Map // ip => mac + ip sync.Map // mac => ip + logger *ctrld.Logger } // refresh re-scans the NDP table. @@ -97,7 +98,7 @@ func (nd *ndpDiscover) saveInfo(ip, mac string) { func (nd *ndpDiscover) listen(ctx context.Context) { ifis, err := allInterfacesWithV6LinkLocal() if err != nil { - ctrld.ProxyLogger.Load().Debug().Err(err).Msg("failed to find valid ipv6 interfaces") + nd.logger.Debug().Err(err).Msg("Failed to find valid ipv6 interfaces") return } for _, ifi := range ifis { @@ -110,11 +111,11 @@ func (nd *ndpDiscover) listen(ctx context.Context) { func (nd *ndpDiscover) listenOnInterface(ctx context.Context, ifi *net.Interface) { c, ip, err := ndp.Listen(ifi, ndp.Unspecified) if err != nil { - ctrld.ProxyLogger.Load().Debug().Err(err).Msg("ndp listen failed") + nd.logger.Debug().Err(err).Msg("Ndp listen failed") return } defer c.Close() - ctrld.ProxyLogger.Load().Debug().Msgf("listening ndp on: %s", ip.String()) + nd.logger.Debug().Msgf("Listening ndp on: %s", ip.String()) for { select { case <-ctx.Done(): @@ -128,7 +129,7 @@ func (nd *ndpDiscover) listenOnInterface(ctx context.Context, ifi *net.Interface if errors.As(readErr, &opErr) && (opErr.Timeout() || opErr.Temporary()) { continue } - ctrld.ProxyLogger.Load().Debug().Err(readErr).Msg("ndp read loop error") + nd.logger.Debug().Err(readErr).Msg("Ndp read loop error") return } @@ -173,6 +174,9 @@ func (nd *ndpDiscover) scanUnix(r io.Reader) { } if mac := parseMAC(fields[1]); mac != "" { ip := fields[0] + // Remove interface suffix from IPv6 addresses + // Unix systems append interface names to IPv6 addresses (e.g., "fe80::1%eth0") + // This suffix needs to be removed for proper IP parsing if idx := strings.IndexByte(ip, '%'); idx != -1 { ip = ip[:idx] } @@ -191,11 +195,15 @@ func normalizeMac(mac string) string { return mac } // Windows use "-" instead of ":" as separator. + // This normalization is needed because different operating systems use different + // separators for MAC addresses, but net.ParseMAC expects ":" format mac = strings.ReplaceAll(mac, "-", ":") parts := strings.Split(mac, ":") if len(parts) != 6 { return "" } + // Pad single-digit hex values with leading zero + // This ensures consistent formatting for MAC address parsing for i, c := range parts { if len(c) == 1 { parts[i] = "0" + c diff --git a/internal/clientinfo/ndp_linux.go b/internal/clientinfo/ndp_linux.go index ebd416f0..fb3aacd2 100644 --- a/internal/clientinfo/ndp_linux.go +++ b/internal/clientinfo/ndp_linux.go @@ -5,15 +5,13 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" - - "github.com/Control-D-Inc/ctrld" ) // scan populates NDP table using information from system mappings. func (nd *ndpDiscover) scan() { neighs, err := netlink.NeighList(0, netlink.FAMILY_V6) if err != nil { - ctrld.ProxyLogger.Load().Warn().Err(err).Msg("could not get neigh list") + nd.logger.Warn().Err(err).Msg("Could not get neighbor list") return } @@ -34,7 +32,7 @@ func (nd *ndpDiscover) subscribe(ctx context.Context) { done := make(chan struct{}) defer close(done) if err := netlink.NeighSubscribe(ch, done); err != nil { - ctrld.ProxyLogger.Load().Err(err).Msg("could not perform neighbor subscribing") + nd.logger.Err(err).Msg("Could not perform neighbor subscribing") return } for { @@ -47,7 +45,7 @@ func (nd *ndpDiscover) subscribe(ctx context.Context) { } ip := normalizeIP(nu.IP.String()) if nu.Type == unix.RTM_DELNEIGH { - ctrld.ProxyLogger.Load().Debug().Msgf("removing NDP neighbor: %s", ip) + nd.logger.Debug().Msgf("Removing ndp neighbor: %s", ip) nd.mac.Delete(ip) continue } @@ -56,7 +54,7 @@ func (nd *ndpDiscover) subscribe(ctx context.Context) { case netlink.NUD_REACHABLE: nd.saveInfo(ip, mac) case netlink.NUD_FAILED: - ctrld.ProxyLogger.Load().Debug().Msgf("removing NDP neighbor with failed state: %s", ip) + nd.logger.Debug().Msgf("Removing ndp neighbor with failed state: %s", ip) nd.mac.Delete(ip) } } diff --git a/internal/clientinfo/ndp_others.go b/internal/clientinfo/ndp_others.go index 007407b8..70d0c90b 100644 --- a/internal/clientinfo/ndp_others.go +++ b/internal/clientinfo/ndp_others.go @@ -7,8 +7,6 @@ import ( "context" "os/exec" "runtime" - - "github.com/Control-D-Inc/ctrld" ) // scan populates NDP table using information from system mappings. @@ -17,14 +15,14 @@ func (nd *ndpDiscover) scan() { case "windows": data, err := exec.Command("netsh", "interface", "ipv6", "show", "neighbors").Output() if err != nil { - ctrld.ProxyLogger.Load().Warn().Err(err).Msg("could not query ndp table") + nd.logger.Warn().Err(err).Msg("Could not query ndp table") return } nd.scanWindows(bytes.NewReader(data)) default: data, err := exec.Command("ndp", "-an").Output() if err != nil { - ctrld.ProxyLogger.Load().Warn().Err(err).Msg("could not query ndp table") + nd.logger.Warn().Err(err).Msg("Could not query ndp table") return } nd.scanUnix(bytes.NewReader(data)) diff --git a/internal/clientinfo/ptr_lookup.go b/internal/clientinfo/ptr_lookup.go index 9a1d10c4..aa6d5ec4 100644 --- a/internal/clientinfo/ptr_lookup.go +++ b/internal/clientinfo/ptr_lookup.go @@ -17,6 +17,7 @@ type ptrDiscover struct { hostname sync.Map // ip => hostname resolver ctrld.Resolver serverDown atomic.Bool + logger *ctrld.Logger } func (p *ptrDiscover) refresh() error { @@ -73,14 +74,14 @@ func (p *ptrDiscover) lookupHostname(ip string) string { msg := new(dns.Msg) addr, err := dns.ReverseAddr(ip) if err != nil { - ctrld.ProxyLogger.Load().Info().Str("discovery", "ptr").Err(err).Msg("invalid ip address") + p.logger.Info().Str("discovery", "ptr").Err(err).Msg("Invalid ip address") return "" } msg.SetQuestion(addr, dns.TypePTR) ans, err := p.resolver.Resolve(ctx, msg) if err != nil { if p.serverDown.CompareAndSwap(false, true) { - ctrld.ProxyLogger.Load().Info().Str("discovery", "ptr").Err(err).Msg("could not perform PTR lookup") + p.logger.Info().Str("discovery", "ptr").Err(err).Msg("Could not perform ptr lookup") go p.checkServer() } return "" @@ -104,10 +105,9 @@ func (p *ptrDiscover) lookupIPByHostname(name string, v6 bool) string { if value == name { if addr, err := netip.ParseAddr(key.(string)); err == nil && addr.Is6() == v6 { ip = addr.String() - if addr.IsLoopback() { // Continue searching if this is loopback address. - return true - } - return false + // Continue searching if this is a loopback address + // We prefer non-loopback addresses as they're more likely to be the actual client IP + return addr.IsLoopback() // Continue searching if this is loopback address. } } return true diff --git a/internal/clientinfo/ubios.go b/internal/clientinfo/ubios.go deleted file mode 100644 index 0ffd6e59..00000000 --- a/internal/clientinfo/ubios.go +++ /dev/null @@ -1,79 +0,0 @@ -package clientinfo - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "os/exec" - "strings" - "sync" - - "github.com/Control-D-Inc/ctrld/internal/router" - "github.com/Control-D-Inc/ctrld/internal/router/ubios" -) - -// ubiosDiscover provides client discovery functionality on Ubios routers. -type ubiosDiscover struct { - hostname sync.Map // mac => hostname -} - -// refresh reloads unifi devices from database. -func (u *ubiosDiscover) refresh() error { - if router.Name() != ubios.Name { - return nil - } - return u.refreshDevices() -} - -// LookupHostnameByIP returns hostname for given IP. -func (u *ubiosDiscover) LookupHostnameByIP(ip string) string { - return "" -} - -// LookupHostnameByMac returns unifi device custom hostname for the given MAC address. -func (u *ubiosDiscover) LookupHostnameByMac(mac string) string { - val, ok := u.hostname.Load(mac) - if !ok { - return "" - } - return val.(string) -} - -// refreshDevices updates unifi devices name from local mongodb. -func (u *ubiosDiscover) refreshDevices() error { - cmd := exec.Command("/usr/bin/mongo", "localhost:27117/ace", "--quiet", "--eval", ` - DBQuery.shellBatchSize = 256; - db.user.find({name: {$exists: true, $ne: ""}}, {_id:0, mac:1, name:1});`) - b, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("out: %s, err: %w", string(b), err) - } - return u.storeDevices(bytes.NewReader(b)) -} - -// storeDevices saves unifi devices name for caching. -func (u *ubiosDiscover) storeDevices(r io.Reader) error { - decoder := json.NewDecoder(r) - device := struct { - MAC string - Name string - }{} - for { - err := decoder.Decode(&device) - if err == io.EOF { - break - } - if err != nil { - return err - } - mac := strings.ToLower(device.MAC) - u.hostname.Store(mac, normalizeHostname(device.Name)) - } - return nil -} - -// String returns human-readable format of ubiosDiscover. -func (u *ubiosDiscover) String() string { - return "ubios" -} diff --git a/internal/clientinfo/ubios_test.go b/internal/clientinfo/ubios_test.go deleted file mode 100644 index 657cf180..00000000 --- a/internal/clientinfo/ubios_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package clientinfo - -import ( - "strings" - "testing" -) - -func Test_ubiosDiscover_storeDevices(t *testing.T) { - ud := &ubiosDiscover{} - r := strings.NewReader(`{ "mac": "00:00:00:00:00:01", "name": "device 1" } -{ "mac": "00:00:00:00:00:02", "name": "device 2" } -`) - if err := ud.storeDevices(r); err != nil { - t.Fatal(err) - } - - tests := []struct { - name string - mac string - hostname string - }{ - {"device 1", "00:00:00:00:00:01", "device 1"}, - {"device 2", "00:00:00:00:00:02", "device 2"}, - {"non-existed", "00:00:00:00:00:03", ""}, - } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - if got := ud.LookupHostnameByMac(tc.mac); got != tc.hostname { - t.Errorf("hostname mismatched, want: %q, got: %q", tc.hostname, got) - } - }) - } - - // Test for invalid input. - r = strings.NewReader(`{ "mac": "00:00:00:00:00:01", "name": "device 1"`) - if err := ud.storeDevices(r); err == nil { - t.Fatal("expected error, got nil") - } else { - t.Log(err) - } -} diff --git a/internal/controld/api_dial_test.go b/internal/controld/api_dial_test.go new file mode 100644 index 00000000..be24bab2 --- /dev/null +++ b/internal/controld/api_dial_test.go @@ -0,0 +1,178 @@ +package controld + +import ( + "errors" + "net" + "slices" + "strings" + "syscall" + "testing" +) + +// TestJoinAttemptErrorsKeepsEveryAttempt pins the diagnosis the incident lost. +// +// The transport dials several address families in turn. The IPv4 attempt is the +// one that says "the host is blocking ctrld"; the last attempt is usually an IPv6 +// address that is simply unroutable and reports "no route to host". Returning only +// the last error is what turned a self-inflicted block into a phantom routing +// problem in the logs, and sent the investigation after a network fault that did +// not exist. +func TestJoinAttemptErrorsKeepsEveryAttempt(t *testing.T) { + blocked := &net.OpError{Op: "dial", Net: "tcp4", Err: wsaEACCES} + unroutable := &net.OpError{Op: "dial", Net: "tcp6", Err: syscall.EHOSTUNREACH} + + err := joinAttemptErrors([]error{ + wrapAttempt("resolved ipv4", blocked), + wrapAttempt("direct ipv6", unroutable), + }) + if err == nil { + t.Fatal("joinAttemptErrors() = nil for two failed attempts") + } + + msg := err.Error() + for _, want := range []string{"resolved ipv4", "direct ipv6"} { + if !strings.Contains(msg, want) { + t.Errorf("error text does not name the %q attempt: %s", want, msg) + } + } + if !errors.Is(err, wsaEACCES) { + t.Errorf("the IPv4 socket denial did not survive; a caller can no longer tell a local block from a routing failure: %s", msg) + } + if !errors.Is(err, syscall.EHOSTUNREACH) { + t.Errorf("the last attempt's error did not survive: %s", msg) + } + if strings.Contains(msg, "\n") { + t.Errorf("the joined error spans lines, which breaks one-record-per-failure logging: %q", msg) + } +} + +// TestJoinAttemptErrorsSingleAndEmpty covers the degenerate inputs: one attempt is +// returned untouched, and no attempt at all still has to be an error rather than a +// nil the dialer would hand back as a successful connection. +func TestJoinAttemptErrorsSingleAndEmpty(t *testing.T) { + only := errors.New("only attempt") + if got := joinAttemptErrors([]error{only}); !errors.Is(got, only) { + t.Errorf("joinAttemptErrors() = %v, want the single attempt unwrapped", got) + } + if got := joinAttemptErrors(nil); got == nil { + t.Error("joinAttemptErrors(nil) = nil; the dialer would report success with no connection") + } +} + +// TestAPIDialStagesAlwaysDialTheDirectIPs is the guarantee the direct addresses +// exist for: when DNS is unusable, ctrld must still reach the API. +// +// Whatever resolution returns - nothing, stale addresses, one family only - every +// direct address is dialed. The only thing the duplicate trim removes is a second +// dial of an address an earlier stage already covers. +func TestAPIDialStagesAlwaysDialTheDirectIPs(t *testing.T) { + const ( + directV4 = apiDomainComIPv4 + directV6 = apiDomainComIPv6 + ) + v4, v6 := []string{directV4}, []string{directV6} + + tests := []struct { + name string + resolved []string + }{ + {"resolution returned nothing", nil}, + {"resolution returned the direct ips", []string{directV4, directV6}}, + {"resolution returned stale ips", []string{"203.0.113.10", "2001:db8::1"}}, + {"resolution returned ipv4 only", []string{"203.0.113.10"}}, + {"resolution returned ipv6 only", []string{"2001:db8::1"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stages := apiDialStages(tt.resolved, v4, v6) + + var dialed []string + for _, stage := range stages { + if len(stage.ips) == 0 { + t.Errorf("stage %q has no address; it would dial nothing", stage.what) + } + dialed = append(dialed, stage.ips...) + } + for _, direct := range []string{directV4, directV6} { + if !slices.Contains(dialed, direct) { + t.Errorf("the direct address %s is never dialed; the API is unreachable without DNS", direct) + } + if n := count(dialed, direct); n != 1 { + t.Errorf("the direct address %s is dialed %d times, want exactly 1", direct, n) + } + } + for _, ip := range tt.resolved { + if !slices.Contains(dialed, ip) { + t.Errorf("the resolved address %s is never dialed", ip) + } + } + }) + } +} + +// TestAPIDialStagesTryIPv4First pins the order: the IPv4 stages come before the +// IPv6 ones. IPv6 at these hosts is commonly unroutable, and its "no route to +// host" is what used to be the only error a failure reported. +func TestAPIDialStagesTryIPv4First(t *testing.T) { + stages := apiDialStages([]string{"203.0.113.10", "2001:db8::1"}, + []string{apiDomainComIPv4}, []string{apiDomainComIPv6}) + + var order []string + for _, stage := range stages { + order = append(order, stage.network) + } + want := []string{"tcp4", "tcp4", "tcp6", "tcp6"} + if len(order) != len(want) { + t.Fatalf("stage networks = %v, want %v", order, want) + } + for i := range want { + if order[i] != want[i] { + t.Fatalf("stage networks = %v, want %v", order, want) + } + } +} + +func count(haystack []string, needle string) int { + var n int + for _, s := range haystack { + if s == needle { + n++ + } + } + return n +} + +// TestNotInSkipsAlreadyDialedAddresses covers the duplicate-dial trim: LookupIP +// normally answers with the direct addresses, so dialing both lists doubles every +// failure for no added chance of success. +func TestNotInSkipsAlreadyDialedAddresses(t *testing.T) { + if got := notIn([]string{apiDomainComIPv4}, []string{apiDomainComIPv4}); len(got) != 0 { + t.Errorf("notIn() = %v, want empty: the address was already dialed", got) + } + if got := notIn([]string{apiDomainComIPv4}, []string{"203.0.113.10"}); len(got) != 1 { + t.Errorf("notIn() = %v, want the direct address kept when it was not dialed", got) + } + if got := notIn([]string{apiDomainComIPv4}, nil); len(got) != 1 { + t.Errorf("notIn() = %v, want the direct address kept when nothing resolved", got) + } +} + +// TestAPIEndpointIPsCoverEveryDialedAddress ties the Firewall Mode allowlist to the +// transport. Firewall Mode permits APIEndpointIPs; the transport dials +// apiDirectIPs. If one grows an address the other does not, ctrld starts blocking +// its own control plane again, which is precisely the 38-hour outage. +func TestAPIEndpointIPsCoverEveryDialedAddress(t *testing.T) { + for _, dev := range []bool{false, true} { + permitted := APIEndpointIPs(dev) + v4, v6 := apiDirectIPs(dev) + for _, ip := range append(append([]string{}, v4...), v6...) { + if !slices.Contains(permitted, ip) { + t.Errorf("cdDev=%v: the transport dials %s but APIEndpointIPs does not report it, so Firewall Mode will not permit it", dev, ip) + } + } + if len(permitted) != len(v4)+len(v6) { + t.Errorf("cdDev=%v: APIEndpointIPs = %v, but the split halves are %v/%v", dev, permitted, v4, v6) + } + } +} diff --git a/internal/controld/config.go b/internal/controld/config.go index 595e758e..4e073dd4 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -12,14 +12,13 @@ import ( "net/http" "os" "runtime" + "slices" "strings" "time" "github.com/Control-D-Inc/ctrld" "github.com/Control-D-Inc/ctrld/internal/certs" ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" - "github.com/Control-D-Inc/ctrld/internal/router" - "github.com/Control-D-Inc/ctrld/internal/router/ddwrt" ) const ( @@ -39,6 +38,55 @@ const ( sendLogTimeout = 300 * time.Second ) +// Provisioning-token rejection reasons the API sends in error.metadata.reason +// (HTTP 400, code 40003). This list can grow; a value outside it is not an +// error, just one cmd/cli does not classify yet. +const ( + ReasonTokenInvalid = "token_invalid" + ReasonTokenExpired = "token_expired" + ReasonTokenLimitReached = "token_limit_reached" + ReasonTokenDisabled = "token_disabled" +) + +// MaintenanceCode is the error.code the API answers with in hard maintenance +// mode (Application::ERROR_HARD_MAINTENANCE), on HTTP 503, for every public +// API request including /utility and /logs. +// +// The code decides. The message match in IsMaintenance stays as the +// compatibility path for deployments that answer without it. +// +// Two neighbouring codes are deliberately not maintenance. 50301 is the +// generic "service unavailable" (a provisioning rate limit, Internal Domains or +// log storage down), a failure rather than a maintenance window. 50003 is +// read-only mode, which only gates authenticated controllers and so never +// answers /utility or /logs; its message ("Maintenance is in progress, cannot +// make any modifications.") does not match MaintenanceMessage either. +const MaintenanceCode = 50302 + +// MaintenanceMessage is what the API says while it is in hard maintenance. It +// classifies an answer that carries no MaintenanceCode. Matched as a substring because the answer carries guidance after it +// ("Please try again later.") that is not part of the condition itself. +const MaintenanceMessage = "Maintenance is in progress." + +// IsMaintenance reports whether err is the Control D API saying it is in +// maintenance, rather than refusing this particular request. +// +// Maintenance is temporary and says nothing about the configuration that was +// asked for, so a caller must not treat it as a permanent rejection: the same +// request succeeds once the window ends. The HTTP status cannot classify it on +// its own, because a maintenance answer can arrive with a client-error status +// that otherwise means "this request will be refused again". +func IsMaintenance(err error) bool { + var uer *ErrorResponse + if !errors.As(err, &uer) { + return false + } + if uer.ErrorField.Code == MaintenanceCode { + return true + } + return strings.Contains(uer.ErrorField.Message, MaintenanceMessage) +} + // ResolverConfig represents Control D resolver data. type ResolverConfig struct { DOH string `json:"doh"` @@ -47,11 +95,50 @@ type ResolverConfig struct { CustomLastUpdate int64 `json:"custom_last_update"` VersionTarget string `json:"version_target"` } `json:"ctrld"` - Exclude []string `json:"exclude"` + Exclude []string `json:"exclude"` + SplitDNS []SplitDNS `json:"split_dns"` + // DestinationIPs is the organization's effective Allowed Destination IP list: + // the entries configured for this endpoint's organization plus any inherited + // from a parent organization. Each entry is an IPv4/IPv6 address or a CIDR + // (the API reports single-host entries as bare addresses, not /32 or /128). + // Under Firewall Mode these destinations stay reachable without a prior DNS + // lookup; see cmd/cli/firewall.go. + DestinationIPs []string `json:"destination_ips"` UID string `json:"uid"` DeactivationPin *int64 `json:"deactivation_pin,omitempty"` } +// Internal Domain resolution modes. Mode is what the administrator selected; +// Resolvers is only meaningful under SplitDNSModeResolvers and +// SplitDNSModeResolversOnly. +const ( + // SplitDNSModeOS follows the endpoint's own OS/default resolver. + SplitDNSModeOS = "os" + // SplitDNSModeResolvers resolves the domain through the addresses in + // Resolvers first. When none of them answers, or they answer SERVFAIL or + // NXDOMAIN, the query falls back to the resolvers of the endpoint's active + // network and VPN. This is the default explicit selection. + SplitDNSModeResolvers = "resolvers" + // SplitDNSModeResolversOnly resolves the domain through the addresses in + // Resolvers and nothing else: their answer, or their failure, is final. + SplitDNSModeResolversOnly = "resolvers_only" +) + +// SplitDNS is one organization Internal Domain: a domain suffix, the resolution +// mode the organization administrator selected for it, and the resolver +// addresses that mode may need. +// +// Mode decides the routing. Resolvers is read only under the explicit modes, +// so addresses left behind by an earlier selection cannot resurrect themselves +// once the administrator switches back to the OS resolver. An empty Mode is a +// deployment that predates the field; the caller infers the mode from +// Resolvers in that case. +type SplitDNS struct { + Domain string `json:"domain"` + Mode string `json:"mode"` + Resolvers []string `json:"resolvers"` +} + type utilityResponse struct { Success bool `json:"success"` Body struct { @@ -59,26 +146,116 @@ type utilityResponse struct { } `json:"body"` } +// errorMetadata carries additive, optional detail on top of Code/Message. +// Older API deployments omit it, so it must decode to its zero value rather +// than fail the whole response. Its custom UnmarshalJSON gives the same +// tolerance to a malformed value: a metadata that is not an object, or a +// Reason that is not a string (a number, an object, or null), degrades to +// the zero value rather than failing the response that contains it. +type errorMetadata struct { + // Reason is a machine-readable rejection reason sent on provisioning-token + // errors (HTTP 400, code 40003): token_invalid, token_expired, + // token_limit_reached, or token_disabled. Empty when absent or malformed; + // callers must treat any other value as unknown rather than reject the + // response. + Reason string `json:"reason"` +} + +func (m *errorMetadata) UnmarshalJSON(data []byte) error { + var raw struct { + Reason json.RawMessage `json:"reason"` + } + // Best-effort: a metadata that is not an object, or a reason that is not + // a string (number, object, null), leaves the zero value instead of + // failing this decode. Code and Message still classify the failure. + if err := json.Unmarshal(data, &raw); err != nil { + *m = errorMetadata{} + return nil + } + _ = json.Unmarshal(raw.Reason, &m.Reason) + return nil +} + type ErrorResponse struct { ErrorField struct { - Message string `json:"message"` - Code int `json:"code"` + Message string `json:"message"` + Code int `json:"code"` + Metadata errorMetadata `json:"metadata"` } `json:"error"` + // StatusCode is the HTTP status the API answered with. It is not part of the JSON + // body: this type is built for *any* non-200 whose body decodes, so the body alone + // cannot tell a permanent rejection of the request from a transient server-side + // failure, and callers that act differently on the two need the status to tell them + // apart. Zero means the status was not recorded. + StatusCode int `json:"-"` } func (u ErrorResponse) Error() string { return u.ErrorField.Message } +// apiErrorFromResponse builds the error for a non-200 API answer, recording the HTTP +// status alongside the decoded body. +// +// The status is what tells a caller whether the answer will change on a retry: this type +// is built for every non-200 whose body decodes, so a 502 from a load balancer and a 404 +// for a deleted device are otherwise indistinguishable. Both response paths go through +// here so neither can decode a body and forget to record it. +func apiErrorFromResponse(statusCode int, d *json.Decoder) (*ErrorResponse, error) { + errResp := &ErrorResponse{StatusCode: statusCode} + if err := d.Decode(errResp); err != nil { + return nil, err + } + // Decode fills exported fields from the body; StatusCode is json:"-", so it survives. + errResp.StatusCode = statusCode + return errResp, nil +} + type utilityRequest struct { - UID string `json:"uid"` - ClientID string `json:"client_id,omitempty"` + UID string `json:"uid"` + ClientID string `json:"client_id,omitempty"` + Metadata map[string]string `json:"metadata"` } // UtilityOrgRequest contains request data for calling Org API. type UtilityOrgRequest struct { - ProvToken string `json:"prov_token"` - Hostname string `json:"hostname"` + ProvToken string `json:"prov_token"` + Hostname string `json:"hostname"` + Metadata map[string]string `json:"metadata"` +} + +// ResolverConfigRequest contains request data for fetching resolver config. +type ResolverConfigRequest struct { + RawUID string + Version string + Metadata map[string]string + IncludeHostnameHints bool +} + +var hostnameHintsFn = hostnameHints + +func resolverConfigUtilityRequest(req *ResolverConfigRequest, uid, clientID string) utilityRequest { + var metadata map[string]string + if req.Metadata != nil { + metadata = make(map[string]string, len(req.Metadata)) + for key, value := range req.Metadata { + metadata[key] = value + } + } + if req.IncludeHostnameHints { + if metadata == nil { + metadata = make(map[string]string) + } + for key, value := range hostnameHintsFn() { + metadata["hostname_"+key] = value + } + } + + return utilityRequest{ + UID: uid, + ClientID: clientID, + Metadata: metadata, + } } // LogsRequest contains request data for sending runtime logs to API. @@ -87,118 +264,197 @@ type LogsRequest struct { Data io.ReadCloser `json:"-"` } -// FetchResolverConfig fetch Control D config for given uid. -func FetchResolverConfig(rawUID, version string, cdDev bool) (*ResolverConfig, error) { - uid, clientID := ParseRawUID(rawUID) - req := utilityRequest{UID: uid} +// FetchResolverConfig fetch Control D config for a given request. +func FetchResolverConfig(ctx context.Context, req *ResolverConfigRequest, cdDev bool) (*ResolverConfig, error) { + logger := ctrld.LoggerFromCtx(ctx) + ctrld.Log(ctx, logger.Debug(), "Fetching ControlD resolver configuration") + + uid, clientID := ParseRawUID(req.RawUID) + ctrld.Log(ctx, logger.Debug(), "Parsed UID: %s, ClientID: %s", uid, clientID) + + uReq := resolverConfigUtilityRequest(req, uid, clientID) if clientID != "" { - req.ClientID = clientID + ctrld.Log(ctx, logger.Debug(), "Including client ID in request") } - body, _ := json.Marshal(req) - return postUtilityAPI(version, cdDev, false, bytes.NewReader(body)) + body, _ := json.Marshal(uReq) + ctrld.Log(ctx, logger.Debug(), "Sending resolver config request to ControlD API") + return postUtilityAPI(ctx, req.Version, cdDev, false, bytes.NewReader(body)) } -// FetchResolverUID fetch resolver uid from provision token. -func FetchResolverUID(req *UtilityOrgRequest, version string, cdDev bool) (*ResolverConfig, error) { +// FetchResolverUID fetch resolver uid from a given request. +func FetchResolverUID(ctx context.Context, req *UtilityOrgRequest, version string, cdDev bool) (*ResolverConfig, error) { + logger := ctrld.LoggerFromCtx(ctx) + ctrld.Log(ctx, logger.Debug(), "Fetching resolver UID from provision token") + if req == nil { + ctrld.Log(ctx, logger.Error(), "Invalid request: request is nil") return nil, errors.New("invalid request") } + hostname := req.Hostname - if hostname == "" { - hostname, _ = os.Hostname() + if req.Hostname == "" { + hostname, _ = preferredHostname() + ctrld.Log(ctx, logger.Debug(), "Using system hostname: %s", hostname) + req.Hostname = hostname + } else { + ctrld.Log(ctx, logger.Debug(), "Using provided hostname: %s", hostname) + } + + // Include all hostname sources in metadata so the API can pick the + // best one if the primary looks generic (e.g., "Mac", "Mac.lan"). + if req.Metadata == nil { + req.Metadata = make(map[string]string) + } + for k, v := range hostnameHints() { + req.Metadata["hostname_"+k] = v } - body, _ := json.Marshal(UtilityOrgRequest{ProvToken: req.ProvToken, Hostname: hostname}) - return postUtilityAPI(version, cdDev, false, bytes.NewReader(body)) + + ctrld.Log(ctx, logger.Debug(), "Sending UID request to ControlD API") + body, _ := json.Marshal(req) + return postUtilityAPI(ctx, version, cdDev, false, bytes.NewReader(body)) } // UpdateCustomLastFailed calls API to mark custom config is bad. -func UpdateCustomLastFailed(rawUID, version string, cdDev, lastUpdatedFailed bool) (*ResolverConfig, error) { +func UpdateCustomLastFailed(ctx context.Context, rawUID, version string, cdDev, lastUpdatedFailed bool) (*ResolverConfig, error) { uid, clientID := ParseRawUID(rawUID) req := utilityRequest{UID: uid} if clientID != "" { req.ClientID = clientID } body, _ := json.Marshal(req) - return postUtilityAPI(version, cdDev, true, bytes.NewReader(body)) + return postUtilityAPI(ctx, version, cdDev, lastUpdatedFailed, bytes.NewReader(body)) } -func postUtilityAPI(version string, cdDev, lastUpdatedFailed bool, body io.Reader) (*ResolverConfig, error) { +func postUtilityAPI(ctx context.Context, version string, cdDev, lastUpdatedFailed bool, body io.Reader) (*ResolverConfig, error) { + logger := ctrld.LoggerFromCtx(ctx) + ctrld.Log(ctx, logger.Debug(), "Posting utility API request") + apiUrl := resolverDataURLCom if cdDev { apiUrl = resolverDataURLDev + ctrld.Log(ctx, logger.Debug(), "Using development API URL: %s", apiUrl) + } else { + ctrld.Log(ctx, logger.Debug(), "Using production API URL: %s", apiUrl) } - req, err := http.NewRequest("POST", apiUrl, body) + + ctrld.Log(ctx, logger.Debug(), "Creating HTTP request") + // Context-bound so an in-flight request is abandoned when the caller is + // cancelled - a service stop during API preflight must not wait out the + // request timeout, let alone keep retrying. + req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, body) if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to create HTTP request: %v", err) return nil, fmt.Errorf("http.NewRequest: %w", err) } + + ctrld.Log(ctx, logger.Debug(), "Setting request parameters") q := req.URL.Query() q.Set("platform", "ctrld") q.Set("version", version) if lastUpdatedFailed { q.Set("custom_last_failed", "1") + ctrld.Log(ctx, logger.Debug(), "Marking custom config as failed") } req.URL.RawQuery = q.Encode() req.Header.Add("Content-Type", "application/json") - transport := apiTransport(cdDev) + + ctrld.Log(ctx, logger.Debug(), "Setting up API transport") + transport := apiTransport(ctx, cdDev) client := &http.Client{ Timeout: defaultTimeout, Transport: transport, } - resp, err := doWithFallback(client, req, apiServerIP(cdDev)) + + ctrld.Log(ctx, logger.Debug(), "Sending request to ControlD API") + resp, err := doWithFallback(ctx, client, req, apiServerIP(cdDev)) if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to send request to ControlD API: %v", err) return nil, fmt.Errorf("postUtilityAPI client.Do: %w", err) } defer resp.Body.Close() + + ctrld.Log(ctx, logger.Debug(), "Processing API response") d := json.NewDecoder(resp.Body) if resp.StatusCode != http.StatusOK { - errResp := &ErrorResponse{} - if err := d.Decode(errResp); err != nil { + errResp, err := apiErrorFromResponse(resp.StatusCode, d) + if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to decode error response: %v", err) return nil, err } + ctrld.Log(ctx, logger.Error(), "ControlD API returned error: %s", errResp.Error()) return nil, errResp } ur := &utilityResponse{} if err := d.Decode(ur); err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to decode utility response: %v", err) return nil, err } + + ctrld.Log(ctx, logger.Debug(), "Successfully received resolver configuration") return &ur.Body.Resolver, nil } // SendLogs sends runtime log to ControlD API. -func SendLogs(lr *LogsRequest, cdDev bool) error { +func SendLogs(ctx context.Context, lr *LogsRequest, cdDev bool) error { + logger := ctrld.LoggerFromCtx(ctx) + ctrld.Log(ctx, logger.Debug(), "Sending runtime logs to ControlD API") + defer lr.Data.Close() apiUrl := logURLCom if cdDev { apiUrl = logURLDev } - req, err := http.NewRequest("POST", apiUrl, lr.Data) + + body, size, getBody, cleanup, err := spoolLogBody(lr.Data) + if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to spool the log upload: %v", err) + return err + } + defer cleanup() + + ctrld.Log(ctx, logger.Debug(), "Creating HTTP request for log upload") + req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, body) if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to create HTTP request: %v", err) return fmt.Errorf("http.NewRequest: %w", err) } + req.ContentLength = size + req.GetBody = getBody q := req.URL.Query() q.Set("uid", lr.UID) req.URL.RawQuery = q.Encode() req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - transport := apiTransport(cdDev) + + ctrld.Log(ctx, logger.Debug(), "Setting up API transport") + transport := apiTransport(ctx, cdDev) client := &http.Client{ Timeout: sendLogTimeout, Transport: transport, } - resp, err := doWithFallback(client, req, apiServerIP(cdDev)) + + ctrld.Log(ctx, logger.Debug(), "Sending log data to ControlD API") + resp, err := doWithFallback(ctx, client, req, apiServerIP(cdDev)) if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to send logs to ControlD API: %v", err) return fmt.Errorf("SendLogs client.Do: %w", err) } defer resp.Body.Close() + + ctrld.Log(ctx, logger.Debug(), "Processing API response") d := json.NewDecoder(resp.Body) if resp.StatusCode != http.StatusOK { - errResp := &ErrorResponse{} - if err := d.Decode(errResp); err != nil { + errResp, err := apiErrorFromResponse(resp.StatusCode, d) + if err != nil { + ctrld.Log(ctx, logger.Error(), "Failed to decode error response: %v", err) return err } + ctrld.Log(ctx, logger.Error(), "ControlD API returned error: %s", errResp.Error()) return errResp } _, _ = io.Copy(io.Discard, resp.Body) + + ctrld.Log(ctx, logger.Debug(), "Runtime logs sent successfully to ControlD API") return nil } @@ -212,69 +468,185 @@ func ParseRawUID(rawUID string) (string, string) { return uid, clientID } +// APIDomain returns the ControlD API hostname for the environment. +func APIDomain(cdDev bool) string { + if cdDev { + return apiDomainDev + } + return apiDomainCom +} + +// APIEndpointIPs returns the addresses the API transport dials directly when the +// hostname cannot be resolved. +// +// Exported because Firewall Mode has to permit them: it blocks every destination +// ctrld did not resolve through its own listener, and the API is resolved through +// the OS resolver by LookupIP instead, so nothing ever teaches the allowlist about +// it. Left unpermitted, ctrld's own block-all filters deny its API socket - which +// is what stranded the 2026-08-16 Windows run with 920 WSAEACCES denials and not +// one successful configuration refresh in 38 hours. +func APIEndpointIPs(cdDev bool) []string { + if cdDev { + return []string{apiDomainDevIPv4} + } + return []string{apiDomainComIPv4, apiDomainComIPv6} +} + +// apiDirectIPs splits APIEndpointIPs into its IPv4 and IPv6 halves. +func apiDirectIPs(cdDev bool) (v4, v6 []string) { + for _, ip := range APIEndpointIPs(cdDev) { + if strings.Contains(ip, ":") { + v6 = append(v6, ip) + } else { + v4 = append(v4, ip) + } + } + return v4, v6 +} + // apiTransport returns an HTTP transport for connecting to ControlD API endpoint. -func apiTransport(cdDev bool) *http.Transport { +func apiTransport(loggerCtx context.Context, cdDev bool) *http.Transport { transport := http.DefaultTransport.(*http.Transport).Clone() transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - apiDomain := apiDomainCom - apiIpsV4 := []string{apiDomainComIPv4} - apiIpsV6 := []string{apiDomainComIPv6} - apiIPs := []string{apiDomainComIPv4, apiDomainComIPv6} - if cdDev { - apiDomain = apiDomainDev - apiIpsV4 = []string{apiDomainDevIPv4} - apiIpsV6 = []string{} - apiIPs = []string{apiDomainDevIPv4} - } + apiDomain := APIDomain(cdDev) + apiIpsV4, apiIpsV6 := apiDirectIPs(cdDev) + apiIPs := APIEndpointIPs(cdDev) - ips := ctrld.LookupIP(apiDomain) + ips := ctrld.LookupIP(loggerCtx, apiDomain) if len(ips) == 0 { - ctrld.ProxyLogger.Load().Warn().Msgf("No IPs found for %s, use direct IPs: %v", apiDomain, apiIPs) + logger := ctrld.LoggerFromCtx(loggerCtx) + logger.Warn().Msgf("No ips found for %s, use direct ips: %v", apiDomain, apiIPs) ips = apiIPs } - // Separate IPv4 and IPv6 addresses - var ipv4s, ipv6s []string - for _, ip := range ips { - if strings.Contains(ip, ":") { - ipv6s = append(ipv6s, ip) - } else { - ipv4s = append(ipv4s, ip) - } - } - dial := func(ctx context.Context, network string, addrs []string) (net.Conn, error) { d := &ctrldnet.ParallelDialer{} - return d.DialContext(ctx, network, addrs, ctrld.ProxyLogger.Load()) + logger := ctrld.LoggerFromCtx(loggerCtx) + return d.DialContext(ctx, network, addrs, logger.Logger) } _, port, _ := net.SplitHostPort(addr) - // Try IPv4 first - if len(ipv4s) > 0 { - if conn, err := dial(ctx, "tcp4", addrsFromPort(ipv4s, port)); err == nil { + var attempts []error + for _, stage := range apiDialStages(ips, apiIpsV4, apiIpsV6) { + conn, err := dial(ctx, stage.network, addrsFromPort(stage.ips, port)) + if err == nil { return conn, nil } + attempts = append(attempts, wrapAttempt(stage.what, err)) } - // Fallback to direct IPv4 - if conn, err := dial(ctx, "tcp4", addrsFromPort(apiIpsV4, port)); err == nil { - return conn, nil + // Every attempt is reported, not just the last one. The stage that + // diagnoses a local block is the IPv4 one - on Windows a firewall denying + // ctrld's own socket surfaces there as WSAEACCES - while the last stage is + // an IPv6 address that is commonly unroutable and fails with a bare "no + // route to host". Returning only that turned a self-inflicted block into a + // phantom routing problem and sent an incident investigation the wrong way. + return nil, joinAttemptErrors(attempts) + } + if runtime.GOOS == "android" { + transport.TLSClientConfig = &tls.Config{RootCAs: certs.CACertPool(), MinVersion: tls.VersionTLS12} + } + return transport +} + +// apiDialStage is one attempt in the API transport's fallback order. +type apiDialStage struct { + what string + network string + ips []string +} + +// apiDialStages plans the dial order for one API connection: resolved IPv4, the +// direct IPv4, then the same for IPv6. The families are attempted separately +// because a host can have working connectivity to one and not the other. +// +// Every direct address is always dialed. It is the address that has to work when +// DNS does not, so it is dropped from its own stage only when it is already in +// the resolved list and the earlier stage therefore dials it anyway - dialing it +// twice doubles the failures without adding a chance of success. If resolution +// returns nothing, or returns addresses that are stale or wrong, the direct +// stages still carry the full direct list. +func apiDialStages(resolved, directV4, directV6 []string) []apiDialStage { + // Different network stacks may have different connectivity to IPv4 vs IPv6. + var ipv4s, ipv6s []string + for _, ip := range resolved { + if strings.Contains(ip, ":") { + ipv6s = append(ipv6s, ip) + } else { + ipv4s = append(ipv4s, ip) } + } + stages := []apiDialStage{ + {"resolved ipv4", "tcp4", ipv4s}, + {"direct ipv4", "tcp4", notIn(directV4, ipv4s)}, + {"resolved ipv6", "tcp6", ipv6s}, + {"direct ipv6", "tcp6", notIn(directV6, ipv6s)}, + } + out := make([]apiDialStage, 0, len(stages)) + for _, stage := range stages { + if len(stage.ips) > 0 { + out = append(out, stage) + } + } + return out +} - // Fallback to IPv6 if available - if len(ipv6s) > 0 { - if conn, err := dial(ctx, "tcp6", addrsFromPort(ipv6s, port)); err == nil { - return conn, nil - } +// notIn returns the members of ips that are absent from seen. +// +// The direct-IP stages exist for when the hostname does not resolve, and LookupIP +// usually answers with those very addresses, so dialing both lists doubles the +// failures for no added chance of success. +func notIn(ips, seen []string) []string { + if len(seen) == 0 { + return ips + } + var out []string + for _, ip := range ips { + if !slices.Contains(seen, ip) { + out = append(out, ip) } - // Fallback to direct IPv6 - return dial(ctx, "tcp6", addrsFromPort(apiIpsV6, port)) } - if router.Name() == ddwrt.Name || runtime.GOOS == "android" { - transport.TLSClientConfig = &tls.Config{RootCAs: certs.CACertPool()} + return out +} + +// wrapAttempt labels one dial attempt's failure with the stage that produced it, +// so a joined error says which family and which address list failed how. +func wrapAttempt(what string, err error) error { + return fmt.Errorf("%s: %w", what, err) +} + +// joinAttemptErrors combines the dial attempts into one error. +func joinAttemptErrors(attempts []error) error { + switch len(attempts) { + case 0: + return errors.New("no api address to dial") + case 1: + return attempts[0] } - return transport + return &dialAttemptsError{attempts: attempts} +} + +// dialAttemptsError carries every attempt the API dialer made. +// +// errors.Join would do the same for errors.Is, but renders one attempt per line, +// and these end up in a single log record; this keeps them on one line. Unwrap +// returns all of them, so a caller testing for a specific errno - a local socket +// denial rather than an unroutable address - finds it wherever in the sequence it +// happened, not only if it happened last. +type dialAttemptsError struct { + attempts []error +} + +func (e *dialAttemptsError) Error() string { + msgs := make([]string, 0, len(e.attempts)) + for _, err := range e.attempts { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") } +// Unwrap exposes every attempt to errors.Is and errors.As. +func (e *dialAttemptsError) Unwrap() []error { return e.attempts } + func addrsFromPort(ips []string, port string) []string { addrs := make([]string, len(ips)) for i, ip := range ips { @@ -283,16 +655,68 @@ func addrsFromPort(ips []string, port string) []string { return addrs } -func doWithFallback(client *http.Client, req *http.Request, apiIp string) (*http.Response, error) { +// doWithFallback sends req, retrying against apiIp directly if the first attempt +// fails (typically because DNS is not usable yet). +// +// Both failures are reported. The first attempt carries the diagnosis - on Windows +// a local firewall denying the socket surfaces there as WSAEACCES ("An attempt was +// made to access a socket in a way forbidden by its access permissions"), which +// says the host is blocking ctrld rather than that the network is down. Returning +// only the fallback error hid that behind a bare "no route to host" from the IPv6 +// attempt and sent the Firewall Mode incident investigation after a routing +// problem that did not exist. +func doWithFallback(ctx context.Context, client *http.Client, req *http.Request, apiIp string) (*http.Response, error) { resp, err := client.Do(req) + if err == nil { + return resp, nil + } + logger := ctrld.LoggerFromCtx(ctx) + logger.Warn().Err(err).Msgf("Failed to send request, fallback to direct ip: %s", apiIp) + ipReq := req.Clone(req.Context()) + ipReq.Host = apiIp + ipReq.URL.Host = apiIp + // The first attempt consumed the body. A request that can give its body + // again sends the whole body a second time. + if req.GetBody != nil { + body, bodyErr := req.GetBody() + if bodyErr != nil { + return nil, fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", err, apiIp, bodyErr) + } + ipReq.Body = body + } + resp, fallbackErr := client.Do(ipReq) + if fallbackErr != nil { + return nil, fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", err, apiIp, fallbackErr) + } + return resp, nil +} + +// spoolLogBody copies an upload to a temporary file, so the request can go +// out a second time to the direct IP of the API. A log reader yields its +// bytes once, and the first attempt consumes them. The caller runs cleanup +// after the response arrived. +func spoolLogBody(data io.Reader) (body *os.File, size int64, getBody func() (io.ReadCloser, error), cleanup func(), err error) { + f, err := os.CreateTemp("", "ctrld-log-upload-*") + if err != nil { + return nil, 0, nil, nil, fmt.Errorf("creating the upload spool: %w", err) + } + cleanup = func() { + _ = f.Close() + _ = os.Remove(f.Name()) + } + size, err = io.Copy(f, data) if err != nil { - ctrld.ProxyLogger.Load().Warn().Err(err).Msgf("failed to send request, fallback to direct IP: %s", apiIp) - ipReq := req.Clone(req.Context()) - ipReq.Host = apiIp - ipReq.URL.Host = apiIp - resp, err = client.Do(ipReq) + cleanup() + return nil, 0, nil, nil, fmt.Errorf("writing the upload spool: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + cleanup() + return nil, 0, nil, nil, fmt.Errorf("rewinding the upload spool: %w", err) } - return resp, err + // The transport closes the body of a failed attempt, so the second body + // is a new handle on the same file. + getBody = func() (io.ReadCloser, error) { return os.Open(f.Name()) } + return f, size, getBody, cleanup, nil } // apiServerIP returns the direct IP to connect to API server. @@ -302,3 +726,10 @@ func apiServerIP(cdDev bool) string { } return apiDomainComIPv4 } + +// DoWithFallbackForTest exposes doWithFallback so tests outside this package can drive +// the real two-attempt composition through the real retry predicate, rather than +// asserting a copy of this error shape against another copy of it. +func DoWithFallbackForTest(ctx context.Context, client *http.Client, req *http.Request, apiIp string) (*http.Response, error) { + return doWithFallback(ctx, client, req, apiIp) +} diff --git a/internal/controld/config_metadata_test.go b/internal/controld/config_metadata_test.go new file mode 100644 index 00000000..80e0d465 --- /dev/null +++ b/internal/controld/config_metadata_test.go @@ -0,0 +1,41 @@ +package controld + +import "testing" + +func TestResolverConfigUtilityRequestMetadataScope(t *testing.T) { + oldHostnameHints := hostnameHintsFn + t.Cleanup(func() { hostnameHintsFn = oldHostnameHints }) + hostnameHintsFn = func() map[string]string { + return map[string]string{"ComputerName": "workstation"} + } + + baseMetadata := map[string]string{"os": "test-os", "username": "test-user"} + full := resolverConfigUtilityRequest(&ResolverConfigRequest{ + Metadata: baseMetadata, + IncludeHostnameHints: true, + }, "uid", "client") + if got := full.Metadata["hostname_ComputerName"]; got != "workstation" { + t.Errorf("full metadata hostname hint = %q, want workstation", got) + } + if got := full.Metadata["username"]; got != "test-user" { + t.Errorf("full metadata username = %q, want test-user", got) + } + if _, ok := baseMetadata["hostname_ComputerName"]; ok { + t.Error("building the utility request mutated the caller metadata") + } + + runtime := resolverConfigUtilityRequest(&ResolverConfigRequest{ + Metadata: map[string]string{"os": "test-os"}, + }, "uid", "") + if _, ok := runtime.Metadata["hostname_ComputerName"]; ok { + t.Error("runtime metadata included hostname hints") + } + if _, ok := runtime.Metadata["username"]; ok { + t.Error("runtime metadata included username") + } + + empty := resolverConfigUtilityRequest(&ResolverConfigRequest{}, "uid", "") + if empty.Metadata != nil { + t.Error("request without metadata changed from null to an observed empty object") + } +} diff --git a/internal/controld/config_test.go b/internal/controld/config_test.go index b266142d..04cb1323 100644 --- a/internal/controld/config_test.go +++ b/internal/controld/config_test.go @@ -1,9 +1,13 @@ package controld import ( + "encoding/json" + "net/http" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_parseUID(t *testing.T) { @@ -29,3 +33,214 @@ func Test_parseUID(t *testing.T) { }) } } + +// TestAPIErrorRecordsHTTPStatus pins the plumbing the caller's exit decision rests on. +// +// cmd/cli treats a 4xx as "this configuration is refused, restarting cannot help" and +// exits cleanly, while a 5xx keeps the abnormal exit so the service manager retries. Both +// readings need the status, and it is not in the JSON body - so a decode path that +// forgets to record it would quietly send every API error down the retry branch, +// including a deleted device that should self-uninstall and stop. +func TestAPIErrorRecordsHTTPStatus(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantCode int + wantMsg string + }{ + { + name: "deleted device", + statusCode: http.StatusNotFound, + body: `{"error":{"message":"device does not exist","code":40402}}`, + wantCode: InvalidConfigCode, + wantMsg: "device does not exist", + }, + { + // A gateway error body carries no error object at all, which decodes + // cleanly into the zero value - so the status is the only thing that + // distinguishes it from a real rejection. + name: "gateway error with an empty body", + statusCode: http.StatusBadGateway, + body: `{}`, + }, + { + name: "service unavailable", + statusCode: http.StatusServiceUnavailable, + body: `{"error":{"message":"try again later","code":0}}`, + wantMsg: "try again later", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(tc.statusCode, d) + if err != nil { + t.Fatalf("unexpected decode error: %v", err) + } + if errResp.StatusCode != tc.statusCode { + t.Errorf("StatusCode = %d, want %d: the caller cannot tell a permanent rejection from a transient failure without it", errResp.StatusCode, tc.statusCode) + } + if errResp.ErrorField.Code != tc.wantCode { + t.Errorf("code = %d, want %d", errResp.ErrorField.Code, tc.wantCode) + } + if errResp.Error() != tc.wantMsg { + t.Errorf("message = %q, want %q", errResp.Error(), tc.wantMsg) + } + }) + } + + t.Run("an undecodable body is reported as a decode failure", func(t *testing.T) { + d := json.NewDecoder(strings.NewReader("502 Bad Gateway")) + if _, err := apiErrorFromResponse(http.StatusBadGateway, d); err == nil { + t.Error("expected a decode error for a non-JSON body") + } + }) +} + +// TestAPIErrorDecodesRejectionReason pins the additive metadata.reason field the +// API sends on provisioning-token rejections. cmd/cli maps known reasons to their +// own failure codes, and must fall back cleanly when the field is absent or holds +// a value this build does not recognize yet. +func TestAPIErrorDecodesRejectionReason(t *testing.T) { + tests := []struct { + name string + body string + wantReason string + }{ + { + name: "known reason", + body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":"token_disabled"}}}`, + wantReason: "token_disabled", + }, + { + name: "reason absent", + body: `{"error":{"message":"invalid token","code":40003}}`, + wantReason: "", + }, + { + name: "unknown reason value", + body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":"something_new"}}}`, + wantReason: "something_new", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(http.StatusBadRequest, d) + if err != nil { + t.Fatalf("unexpected decode error: %v", err) + } + if errResp.ErrorField.Metadata.Reason != tc.wantReason { + t.Errorf("reason = %q, want %q", errResp.ErrorField.Metadata.Reason, tc.wantReason) + } + }) + } +} + +// TestAPIErrorToleratesMalformedRejectionReason pins the fix for a decode error +// confined to metadata.reason: a reason sent as the wrong JSON type must not +// discard the rest of the response. Before this fix, apiErrorFromResponse +// returned the raw decode error and nothing else, which cmd/cli's +// apiFailureCode cannot recognize as an *ErrorResponse - it falls back to +// API_UNREACHABLE (a retryable bootstrap failure) instead of the permanent +// rejection the HTTP status and code actually describe. +func TestAPIErrorToleratesMalformedRejectionReason(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "reason as a number", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":12345}}}`}, + {name: "reason as an object", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":{"inner":"value"}}}}`}, + {name: "reason as null", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":null}}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(http.StatusBadRequest, d) + if err != nil { + t.Fatalf("a malformed reason must not fail the whole decode: %v", err) + } + if errResp.ErrorField.Message != "invalid token" { + t.Errorf("message = %q, want it to survive the malformed reason", errResp.ErrorField.Message) + } + if errResp.ErrorField.Code != 40003 { + t.Errorf("code = %d, want it to survive the malformed reason", errResp.ErrorField.Code) + } + if errResp.ErrorField.Metadata.Reason != "" { + t.Errorf("reason = %q, want empty for a malformed value", errResp.ErrorField.Metadata.Reason) + } + }) + } +} + +// TestAPIErrorToleratesMalformedMetadata pins the same tolerance one level +// up: a metadata field that is not a JSON object must not discard the rest +// of the response. Code and Message still classify the failure, and Reason +// stays empty. +func TestAPIErrorToleratesMalformedMetadata(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "metadata as a string", body: `{"error":{"message":"invalid token","code":40003,"metadata":"foo"}}`}, + {name: "metadata as a number", body: `{"error":{"message":"invalid token","code":40003,"metadata":7}}`}, + {name: "metadata as an array", body: `{"error":{"message":"invalid token","code":40003,"metadata":["reason"]}}`}, + {name: "metadata as a boolean", body: `{"error":{"message":"invalid token","code":40003,"metadata":true}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(http.StatusBadRequest, d) + if err != nil { + t.Fatalf("unexpected decode error: %v", err) + } + if errResp.ErrorField.Code != 40003 { + t.Errorf("code = %d, want 40003", errResp.ErrorField.Code) + } + if errResp.ErrorField.Message != "invalid token" { + t.Errorf("message = %q, want %q", errResp.ErrorField.Message, "invalid token") + } + if errResp.ErrorField.Metadata.Reason != "" { + t.Errorf("reason = %q, want empty for a malformed metadata", errResp.ErrorField.Metadata.Reason) + } + }) + } +} + +// TestUtilityResponseDecodesDestinationIPs pins the API field that carries the +// organization's effective Allowed Destination IP list. The list is enforced as a +// set of Firewall Mode exceptions, so a silent decode change - a renamed field, a +// nesting change - would leave endpoints blocking destinations the organization +// approved, with nothing in the logs to say why. +func TestUtilityResponseDecodesDestinationIPs(t *testing.T) { + tests := []struct { + name string + body string + want []string + }{ + { + name: "addresses and CIDRs of both families", + body: `{"body":{"resolver":{"doh":"https://dns.controld.dev/abc","destination_ips":["203.0.113.10","198.51.100.0/24","2606:1a40::1","2001:db8::/48"]}},"success":true}`, + want: []string{"203.0.113.10", "198.51.100.0/24", "2606:1a40::1", "2001:db8::/48"}, + }, + { + name: "empty list - the API always sends the field", + body: `{"body":{"resolver":{"doh":"https://dns.controld.dev/abc","destination_ips":[]}},"success":true}`, + want: []string{}, + }, + { + name: "field absent", + body: `{"body":{"resolver":{"doh":"https://dns.controld.dev/abc"}},"success":true}`, + want: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ur := &utilityResponse{} + require.NoError(t, json.Unmarshal([]byte(tc.body), ur)) + assert.Equal(t, tc.want, ur.Body.Resolver.DestinationIPs) + }) + } +} diff --git a/internal/controld/controld_test.go b/internal/controld/controld_test.go index 2c00247d..80762e36 100644 --- a/internal/controld/controld_test.go +++ b/internal/controld/controld_test.go @@ -3,10 +3,13 @@ package controld import ( + "context" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" ) func TestFetchResolverConfig(t *testing.T) { @@ -20,11 +23,18 @@ func TestFetchResolverConfig(t *testing.T) { {"valid dev", "p2", true, false}, {"invalid uid", "abcd1234", false, true}, } + + ctx := context.Background() for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - got, err := FetchResolverConfig(tc.uid, "dev-test", tc.dev) + req := &ResolverConfigRequest{ + RawUID: tc.uid, + Version: "dev-test", + Metadata: ctrld.SystemMetadata(ctx), + } + got, err := FetchResolverConfig(ctx, req, tc.dev) require.False(t, (err != nil) != tc.wantErr, err) if !tc.wantErr { assert.NotEmpty(t, got.DOH) diff --git a/internal/controld/fallback_test.go b/internal/controld/fallback_test.go new file mode 100644 index 00000000..7ab0b33a --- /dev/null +++ b/internal/controld/fallback_test.go @@ -0,0 +1,156 @@ +package controld + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "strings" + "syscall" + "testing" +) + +// errRoundTripper fails the hostname attempt and the direct-ip attempt with +// different errors, mimicking the Firewall Mode incident: the hostname attempt is +// denied by a local firewall (WSAEACCES on Windows) while the direct-ip fallback +// reports an unreachable IPv6 route. +type errRoundTripper struct { + hostname string + firstErr error + fbErr error + fbCalled bool +} + +func (rt *errRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Host == rt.hostname { + return nil, &net.OpError{Op: "dial", Net: "tcp4", Err: rt.firstErr} + } + rt.fbCalled = true + if rt.fbErr == nil { + return &http.Response{ + StatusCode: http.StatusOK, + Body: http.NoBody, + Request: req, + }, nil + } + return nil, &net.OpError{Op: "dial", Net: "tcp6", Err: rt.fbErr} +} + +// wsaEACCES is WSAEACCES (10013): "An attempt was made to access a socket in a way +// forbidden by its access permissions." The value is what Windows reports when a +// WFP filter denies the connect; it is used here as a plain errno so the test runs +// on every platform. +const wsaEACCES = syscall.Errno(10013) + +func TestDoWithFallbackPreservesFirstError(t *testing.T) { + const ( + hostname = "api.controld.com" + apiIP = "147.185.34.1" + ) + rt := &errRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := doWithFallback(context.Background(), &http.Client{Transport: rt}, req, apiIP) + if err == nil { + t.Fatalf("expected an error, got response %v", resp) + } + if !rt.fbCalled { + t.Error("direct-ip fallback was not attempted") + } + + // The actionable failure must survive: an operator reading this error has to be + // able to tell "the host is blocking us" from "the network is down". + if !errors.Is(err, wsaEACCES) { + t.Errorf("first-attempt error (WSAEACCES) was lost, got: %v", err) + } + if !errors.Is(err, syscall.EHOSTUNREACH) { + t.Errorf("fallback error was lost, got: %v", err) + } + if got := err.Error(); !strings.Contains(got, apiIP) { + t.Errorf("error does not mention the fallback ip %q: %v", apiIP, got) + } +} + +func TestDoWithFallbackSucceedsOnFallback(t *testing.T) { + const hostname = "api.controld.com" + rt := &errRoundTripper{hostname: hostname, firstErr: wsaEACCES} + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := doWithFallback(context.Background(), &http.Client{Transport: rt}, req, "147.185.34.1") + if err != nil { + t.Fatalf("expected the fallback to succeed, got: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("StatusCode = %d, want %d", resp.StatusCode, http.StatusOK) + } +} + +func TestDoWithFallbackNoFallbackOnSuccess(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPost, srv.URL, nil) + if err != nil { + t.Fatal(err) + } + resp, err := doWithFallback(context.Background(), srv.Client(), req, "127.0.0.2") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Errorf("StatusCode = %d, want %d", resp.StatusCode, http.StatusOK) + } +} + +// TestDoWithFallbackComposesHostnameAttemptFirst pins the order of the composed error. +// +// The order is not cosmetic. cmd/cli's preflight retry predicate classifies this error +// with errors.As, which returns the first match in the tree, so whichever attempt is +// wrapped first decides whether processCDFlags keeps backing off or fails fast. That +// predicate lives in another package and cannot be called from here, so this test +// guards the property it depends on: the hostname attempt - the one that carries the +// diagnosis - must come first. +func TestDoWithFallbackComposesHostnameAttemptFirst(t *testing.T) { + const hostname = "api.controld.com" + rt := &errRoundTripper{ + hostname: hostname, + firstErr: wsaEACCES, + fbErr: syscall.EHOSTUNREACH, + } + req, err := http.NewRequest(http.MethodPost, "https://"+hostname+"/utility", nil) + if err != nil { + t.Fatal(err) + } + + _, gotErr := doWithFallback(context.Background(), &http.Client{Transport: rt}, req, "147.185.34.1") + if gotErr == nil { + t.Fatal("expected both attempts to fail") + } + + var opErr *net.OpError + if !errors.As(gotErr, &opErr) { + t.Fatalf("no net.OpError in the chain: %v", gotErr) + } + if !errors.Is(opErr.Err, wsaEACCES) { + t.Errorf("first OpError in the chain is %v, want the hostname attempt (%v)", opErr.Err, wsaEACCES) + } + // The tcp4/tcp6 split distinguishes the two attempts in the fake transport. + if opErr.Net != "tcp4" { + t.Errorf("first OpError is from the %s attempt, want tcp4 (hostname)", opErr.Net) + } +} diff --git a/internal/controld/hostname_darwin.go b/internal/controld/hostname_darwin.go new file mode 100644 index 00000000..0b8eb52c --- /dev/null +++ b/internal/controld/hostname_darwin.go @@ -0,0 +1,44 @@ +package controld + +import ( + "os" + "os/exec" + "strings" +) + +// preferredHostname returns the best available hostname on macOS. +// It prefers scutil --get ComputerName which is the user-configured name +// from System Settings → General → About → Name. This is immune to +// DHCP/network state that can cause os.Hostname() and even LocalHostName +// to return generic names like "Mac.lan" on Sequoia with Private Wi-Fi +// Address enabled. +// +// Fallback chain: ComputerName → LocalHostName → os.Hostname() +func preferredHostname() (string, error) { + for _, key := range []string{"ComputerName", "LocalHostName"} { + if out, err := exec.Command("scutil", "--get", key).Output(); err == nil { + if name := strings.TrimSpace(string(out)); name != "" { + return name, nil + } + } + } + return os.Hostname() +} + +// hostnameHints returns all available hostname sources on macOS for +// diagnostic/fallback purposes. The API can use these to pick the +// best hostname if the primary one looks generic (e.g., "Mac"). +func hostnameHints() map[string]string { + hints := make(map[string]string) + for _, key := range []string{"ComputerName", "LocalHostName", "HostName"} { + if out, err := exec.Command("scutil", "--get", key).Output(); err == nil { + if name := strings.TrimSpace(string(out)); name != "" { + hints[key] = name + } + } + } + if h, err := os.Hostname(); err == nil { + hints["os.Hostname"] = h + } + return hints +} diff --git a/internal/controld/hostname_others.go b/internal/controld/hostname_others.go new file mode 100644 index 00000000..8aa03bc3 --- /dev/null +++ b/internal/controld/hostname_others.go @@ -0,0 +1,19 @@ +//go:build !darwin + +package controld + +import "os" + +// preferredHostname returns the system hostname on non-Darwin platforms. +func preferredHostname() (string, error) { + return os.Hostname() +} + +// hostnameHints returns available hostname sources for diagnostic purposes. +func hostnameHints() map[string]string { + hints := make(map[string]string) + if h, err := os.Hostname(); err == nil { + hints["os.Hostname"] = h + } + return hints +} diff --git a/internal/controld/log_upload_test.go b/internal/controld/log_upload_test.go new file mode 100644 index 00000000..c15ce6f1 --- /dev/null +++ b/internal/controld/log_upload_test.go @@ -0,0 +1,49 @@ +package controld + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestDoWithFallbackResendsTheSpooledBody drives the two-attempt upload: the +// first attempt fails on a closed port and the fallback to the direct IP must +// carry the whole body, which the first attempt consumed. +func TestDoWithFallbackResendsTheSpooledBody(t *testing.T) { + payload := strings.Repeat("log line\n", 4096) + var got string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + got = string(b) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + body, size, getBody, cleanup, err := spoolLogBody(strings.NewReader(payload)) + if err != nil { + t.Fatalf("spoolLogBody: %v", err) + } + defer cleanup() + if size != int64(len(payload)) { + t.Fatalf("size = %d, want %d", size, len(payload)) + } + // Port 1 refuses the connection, so the first attempt fails at once. + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://127.0.0.1:1/logs", body) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + req.ContentLength = size + req.GetBody = getBody + + resp, err := doWithFallback(context.Background(), srv.Client(), req, strings.TrimPrefix(srv.URL, "http://")) + if err != nil { + t.Fatalf("doWithFallback: %v", err) + } + resp.Body.Close() + if got != payload { + t.Fatalf("the fallback carried %d bytes, want %d", len(got), len(payload)) + } +} diff --git a/internal/controld/maintenance_test.go b/internal/controld/maintenance_test.go new file mode 100644 index 00000000..182fe1d8 --- /dev/null +++ b/internal/controld/maintenance_test.go @@ -0,0 +1,85 @@ +package controld + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "testing" +) + +func maintenanceErr(status, code int, message string) *ErrorResponse { + e := &ErrorResponse{StatusCode: status} + e.ErrorField.Code = code + e.ErrorField.Message = message + return e +} + +// The message the API answered with during the reported incident, verbatim. +const reportedMaintenanceMessage = "Maintenance is in progress. Please try again later." + +func TestIsMaintenanceByMessage(t *testing.T) { + for _, tc := range []struct { + name string + err error + want bool + }{ + {"the reported answer", maintenanceErr(http.StatusServiceUnavailable, 0, reportedMaintenanceMessage), true}, + {"carried on a client-error status", maintenanceErr(http.StatusBadRequest, 40003, reportedMaintenanceMessage), true}, + {"wrapped", fmt.Errorf("fetching config: %w", maintenanceErr(503, 0, reportedMaintenanceMessage)), true}, + {"a device rejection is not maintenance", maintenanceErr(http.StatusNotFound, InvalidConfigCode, "device does not exist"), false}, + {"an empty message is not maintenance", maintenanceErr(http.StatusBadGateway, 0, ""), false}, + {"a plain error is not maintenance", errors.New("dial tcp: connection refused"), false}, + {"no error is not maintenance", nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := IsMaintenance(tc.err); got != tc.want { + t.Fatalf("IsMaintenance() = %v, want %v", got, tc.want) + } + }) + } +} + +// TestIsMaintenanceByCode covers the dedicated code. It must classify an +// answer that carries no recognizable message, must not stop recognizing the +// message form, which older deployments keep sending, and must not swallow +// the neighbouring codes that mean something else. +func TestIsMaintenanceByCode(t *testing.T) { + for _, tc := range []struct { + name string + err error + want bool + }{ + {"hard maintenance code", maintenanceErr(http.StatusServiceUnavailable, 50302, "scheduled downtime"), true}, + {"message form without the code", maintenanceErr(http.StatusServiceUnavailable, 0, reportedMaintenanceMessage), true}, + {"service unavailable is a failure", maintenanceErr(http.StatusServiceUnavailable, 50301, "Service unavailable"), false}, + {"read-only mode", maintenanceErr(http.StatusInternalServerError, 50003, + "Maintenance is in progress, cannot make any modifications. Please try again soon."), false}, + {"deleted device", maintenanceErr(http.StatusNotFound, InvalidConfigCode, "device does not exist"), false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := IsMaintenance(tc.err); got != tc.want { + t.Fatalf("IsMaintenance() = %v, want %v", got, tc.want) + } + }) + } +} + +// The API's hard maintenance answer, byte for byte as api_public/index.php +// writes it. "body" is an empty JSON array rather than an object, so the error +// decode must not depend on it. +func TestIsMaintenanceFromHardMaintenanceResponse(t *testing.T) { + const body = `{"body":[],"success":false,"error":{"date":"Mon, 05 Oct 2026 10:00:00 +0000",` + + `"message":"Maintenance is in progress. Please try again later.","code":50302}}` + errResp, err := apiErrorFromResponse(http.StatusServiceUnavailable, json.NewDecoder(strings.NewReader(body))) + if err != nil { + t.Fatalf("decoding the maintenance answer: %v", err) + } + if errResp.StatusCode != http.StatusServiceUnavailable || errResp.ErrorField.Code != MaintenanceCode { + t.Fatalf("errResp = %+v", errResp) + } + if !IsMaintenance(fmt.Errorf("fetching config: %w", errResp)) { + t.Fatal("the hard maintenance answer was not classified as maintenance") + } +} diff --git a/internal/controld/reachability.go b/internal/controld/reachability.go new file mode 100644 index 00000000..27cd9e7e --- /dev/null +++ b/internal/controld/reachability.go @@ -0,0 +1,33 @@ +package controld + +import ( + "context" + "io" + "net/http" +) + +// ProbeReachability makes one lightweight request to the ControlD API host, +// reusing the same transport and IP-fallback logic real provisioning traffic +// takes. Any HTTP response, even an error status, counts as reachable: this +// checks the network path, not whether the endpoint accepts the request. +// +// The caller controls how long to wait via ctx; there is no timeout here +// beyond what ctx enforces. +func ProbeReachability(ctx context.Context, cdDev bool) error { + apiURL := apiURLCom + if cdDev { + apiURL = apiURLDev + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return err + } + client := &http.Client{Transport: apiTransport(ctx, cdDev)} + resp, err := doWithFallback(ctx, client, req, apiServerIP(cdDev)) + if err != nil { + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} diff --git a/internal/controld/reachability_test.go b/internal/controld/reachability_test.go new file mode 100644 index 00000000..9f39ec75 --- /dev/null +++ b/internal/controld/reachability_test.go @@ -0,0 +1,14 @@ +package controld + +import ( + "context" + "testing" +) + +func TestProbeReachabilityRespectsCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := ProbeReachability(ctx, false); err == nil { + t.Error("expected an error for an already-cancelled context") + } +} diff --git a/internal/controld/split_dns_test.go b/internal/controld/split_dns_test.go new file mode 100644 index 00000000..7c2e1e74 --- /dev/null +++ b/internal/controld/split_dns_test.go @@ -0,0 +1,162 @@ +package controld + +import ( + "encoding/json" + "strings" + "testing" +) + +// The API's utility response carries organization Internal Domains in +// resolver.split_dns. Parsing it must not change anything when the field is +// absent or empty, which is what every resolver without the feature returns. +func TestResolverConfigParsesSplitDNS(t *testing.T) { + for _, tc := range []struct { + name string + body string + want []SplitDNS + }{ + { + name: "absent", + body: `{"success":true,"body":{"resolver":{"doh":"https://dns.controld.com/abc","uid":"abc"}}}`, + }, + { + name: "null", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":null}}}`, + }, + { + name: "empty", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[]}}}`, + want: []SplitDNS{}, + }, + { + name: "explicit resolvers", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[ + {"domain":"aws.example.com","resolvers":["10.0.0.10","10.0.0.11"]}]}}}`, + want: []SplitDNS{{Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}}, + }, + { + name: "os resolver selection", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[ + {"domain":"corp.example.com","resolvers":[]}, + {"domain":"lab.example.com"}]}}}`, + want: []SplitDNS{ + {Domain: "corp.example.com", Resolvers: []string{}}, + {Domain: "lab.example.com"}, + }, + }, + { + name: "coexists with exclude", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a", + "exclude":["corp.example.com"], + "split_dns":[{"domain":"aws.example.com","resolvers":["10.0.0.10"]}]}}}`, + want: []SplitDNS{{Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(tc.body)).Decode(&resp); err != nil { + t.Fatal(err) + } + got := resp.Body.Resolver.SplitDNS + if len(got) != len(tc.want) { + t.Fatalf("split_dns = %+v, want %+v", got, tc.want) + } + for i := range got { + if got[i].Domain != tc.want[i].Domain { + t.Errorf("[%d].Domain = %q, want %q", i, got[i].Domain, tc.want[i].Domain) + } + if len(got[i].Resolvers) != len(tc.want[i].Resolvers) { + t.Fatalf("[%d].Resolvers = %v, want %v", i, got[i].Resolvers, tc.want[i].Resolvers) + } + for j := range got[i].Resolvers { + if got[i].Resolvers[j] != tc.want[i].Resolvers[j] { + t.Errorf("[%d].Resolvers[%d] = %q, want %q", i, j, got[i].Resolvers[j], tc.want[i].Resolvers[j]) + } + } + } + // The rest of the model must be unaffected. + if resp.Body.Resolver.DOH == "" || resp.Body.Resolver.UID == "" { + t.Errorf("resolver = %+v", resp.Body.Resolver) + } + }) + } +} + +// An unknown field in an entry must not fail the whole response: the API can +// add one before ctrld knows about it. +func TestResolverConfigToleratesUnknownSplitDNSFields(t *testing.T) { + body := `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[ + {"domain":"aws.example.com","resolvers":["10.0.0.10"],"future_field":{"a":1}}]}}}` + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil { + t.Fatal(err) + } + if len(resp.Body.Resolver.SplitDNS) != 1 || resp.Body.Resolver.SplitDNS[0].Domain != "aws.example.com" { + t.Fatalf("split_dns = %+v", resp.Body.Resolver.SplitDNS) + } +} + +// The shape the API actually returns, mode field included. +func TestResolverConfigParsesSplitDNSMode(t *testing.T) { + const body = `{ + "success": true, + "body": { + "resolver": { + "split_dns": [ + { + "domain": "corp.example.com", + "mode": "resolvers", + "resolvers": ["10.0.0.53", "10.0.0.54"] + }, + { + "domain": "office.example.com", + "mode": "os", + "resolvers": [] + }, + { + "domain": "vault.example.com", + "mode": "resolvers_only", + "resolvers": ["10.0.0.55"] + } + ] + } + } +}` + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil { + t.Fatal(err) + } + got := resp.Body.Resolver.SplitDNS + if len(got) != 3 { + t.Fatalf("split_dns = %+v", got) + } + if got[0].Domain != "corp.example.com" || got[0].Mode != SplitDNSModeResolvers { + t.Errorf("[0] = %+v", got[0]) + } + if len(got[0].Resolvers) != 2 || got[0].Resolvers[0] != "10.0.0.53" || got[0].Resolvers[1] != "10.0.0.54" { + t.Errorf("[0].Resolvers = %v", got[0].Resolvers) + } + if got[1].Domain != "office.example.com" || got[1].Mode != SplitDNSModeOS { + t.Errorf("[1] = %+v", got[1]) + } + if len(got[1].Resolvers) != 0 { + t.Errorf("[1].Resolvers = %v, want empty", got[1].Resolvers) + } + if got[2].Mode != SplitDNSModeResolversOnly || len(got[2].Resolvers) != 1 { + t.Errorf("[2] = %+v", got[2]) + } +} + +// A response without the field must still decode, so a deployment that predates +// mode keeps working. +func TestResolverConfigToleratesAbsentSplitDNSMode(t *testing.T) { + const body = `{"success":true,"body":{"resolver":{"split_dns":[ + {"domain":"corp.example.com","resolvers":["10.0.0.53"]}]}}}` + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil { + t.Fatal(err) + } + if got := resp.Body.Resolver.SplitDNS; len(got) != 1 || got[0].Mode != "" { + t.Fatalf("split_dns = %+v", got) + } +} diff --git a/internal/dnscache/cache.go b/internal/dnscache/cache.go index af8883e6..90f6bb9f 100644 --- a/internal/dnscache/cache.go +++ b/internal/dnscache/cache.go @@ -1,6 +1,8 @@ package dnscache import ( + "fmt" + "net" "strings" "time" @@ -16,11 +18,18 @@ type Cacher interface { } // Key is the caching key for DNS message. +// +// ECS partitions the cache by EDNS Client Subnet so an answer resolved for one +// subnet is never served to a client in a different subnet. Answer records are +// scoped to the network that generated them (RFC 7871 §7.3), so they must not +// be shared across subnets even when the question is otherwise identical. type Key struct { Qtype uint16 Qclass uint16 Name string Upstream string + ECS string + Variant string } type Value struct { @@ -60,7 +69,66 @@ func NewLRUCache(size int) (*LRUCache, error) { // NewKey creates a new cache key for given DNS message. func NewKey(msg *dns.Msg, upstream string) Key { q := msg.Question[0] - return Key{Qtype: q.Qtype, Qclass: q.Qclass, Name: normalizeQname(q.Name), Upstream: upstream} + return Key{Qtype: q.Qtype, Qclass: q.Qclass, Name: normalizeQname(q.Name), Upstream: upstream, ECS: CanonicalECS(msg)} +} + +// NewVariantKey creates a cache key in a named result variant. Variants keep +// derived answers separate from the upstream's answer to the same question. +func NewVariantKey(msg *dns.Msg, upstream, variant string) Key { + key := NewKey(msg, upstream) + key.Variant = variant + return key +} + +// CanonicalECS returns a canonical string form of the EDNS Client Subnet (ECS, +// EDNS option 8) carried by msg, suitable for partitioning cache and +// singleflight keys. A request with no ECS option returns "", so all ECS-less +// queries share one partition and behave as before. +// +// A request that DOES carry an ECS option is never mapped to "", even at SOURCE +// PREFIX-LENGTH 0: the /0 query is forwarded with an ECS option, may draw +// different upstream data than an ECS-less query, and its answer (with the +// echoed option) must not be served to a client that sent no ECS — RFC 7871 +// §7.3.1 requires /0-cached data to remain distinguishable. The family is part +// of the token, so an IPv4 /0 and an IPv6 /0 stay distinct too. +// +// The address is masked to its source prefix length so only the significant +// subnet bits contribute to the key: two clients in the same subnet share a +// partition, while different subnets (or address families) never do. The +// response-only SCOPE PREFIX-LENGTH is deliberately excluded — it is not part of +// what the client asked for. +func CanonicalECS(msg *dns.Msg) string { + opt := msg.IsEdns0() + if opt == nil { + return "" + } + for _, o := range opt.Option { + e, ok := o.(*dns.EDNS0_SUBNET) + if !ok { + continue + } + bits := int(e.SourceNetmask) + total := 128 + zero := net.IPv6zero + if e.Family == 1 { + total = 32 + zero = net.IPv4zero + } + addr := e.Address + if len(addr) == 0 { + // A /0 request commonly carries an empty address; normalize it to + // the family zero so its token is stable regardless of encoding. + addr = zero + } + masked := addr + if bits <= total { + if m := addr.Mask(net.CIDRMask(bits, total)); m != nil { + masked = m + } + } + return fmt.Sprintf("%d/%d/%s", e.Family, e.SourceNetmask, masked.String()) + } + return "" } // NewValue creates a new cache value for given DNS message. diff --git a/internal/dnscache/cache_test.go b/internal/dnscache/cache_test.go new file mode 100644 index 00000000..449ed1af --- /dev/null +++ b/internal/dnscache/cache_test.go @@ -0,0 +1,186 @@ +package dnscache + +import ( + "net" + "testing" + "time" + + "github.com/miekg/dns" +) + +// msgWithECS builds an A query for name carrying an EDNS Client Subnet option, or none +// when family == 0. +func msgWithECS(name string, family uint16, prefix uint8, addr string) *dns.Msg { + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(name), dns.TypeA) + if family == 0 { + return m + } + m.SetEdns0(4096, true) + m.IsEdns0().Option = append(m.IsEdns0().Option, &dns.EDNS0_SUBNET{ + Code: dns.EDNS0SUBNET, + Family: family, + SourceNetmask: prefix, + Address: net.ParseIP(addr), + }) + return m +} + +// answerWithA builds a cached answer holding a single A record with the given address. +func answerWithA(name, a string) *dns.Msg { + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(name), dns.TypeA) + rr, err := dns.NewRR(dns.Fqdn(name) + " 300 IN A " + a) + if err != nil { + panic(err) + } + m.Answer = []dns.RR{rr} + return m +} + +func firstA(msg *dns.Msg) string { + for _, rr := range msg.Answer { + if a, ok := rr.(*dns.A); ok { + return a.A.String() + } + } + return "" +} + +// TestCanonicalECS covers the key-partitioning helper: only a request with no ECS option +// collapses to the shared empty partition, while a carried /0 keeps its own family-scoped +// token (distinct from no-ECS, per RFC 7871 §7.3.1); host bits below the source prefix do +// not fragment the key, and different subnets / families produce different keys. +func TestCanonicalECS(t *testing.T) { + tests := []struct { + name string + msg *dns.Msg + want string + }{ + {"no ecs", msgWithECS("controld.com", 0, 0, ""), ""}, + // A carried ECS option is never "" (RFC 7871 §7.3.1), and IPv4 /0 vs + // IPv6 /0 stay distinct; the address encoding must not change the token. + {"ipv4 /0", msgWithECS("controld.com", 1, 0, "0.0.0.0"), "1/0/0.0.0.0"}, + {"ipv4 /0 empty addr", msgWithECS("controld.com", 1, 0, ""), "1/0/0.0.0.0"}, + {"ipv6 /0", msgWithECS("controld.com", 2, 0, "::"), "2/0/::"}, + {"ipv6 /0 empty addr", msgWithECS("controld.com", 2, 0, ""), "2/0/::"}, + {"ipv4 /24", msgWithECS("controld.com", 1, 24, "203.0.113.0"), "1/24/203.0.113.0"}, + {"ipv4 host bits masked", msgWithECS("controld.com", 1, 24, "203.0.113.7"), "1/24/203.0.113.0"}, + {"ipv6 /64", msgWithECS("controld.com", 2, 64, "2001:db8:1::"), "2/64/2001:db8:1::"}, + {"ipv6 host bits masked", msgWithECS("controld.com", 2, 64, "2001:db8:1::dead:beef"), "2/64/2001:db8:1::"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CanonicalECS(tt.msg); got != tt.want { + t.Fatalf("CanonicalECS = %q, want %q", got, tt.want) + } + }) + } +} + +// TestNewKey_ECSPartition verifies the cache key distinguishes subnets while collapsing +// same-subnet requests, so the LRU cache cannot serve one subnet's records to another. +func TestNewKey_ECSPartition(t *testing.T) { + const up = "https://dns.example/dns-query" + subnetA := NewKey(msgWithECS("controld.com", 2, 64, "2001:db8:1::"), up) + subnetB := NewKey(msgWithECS("controld.com", 2, 64, "2001:db8:2::"), up) + subnetAHost := NewKey(msgWithECS("controld.com", 2, 64, "2001:db8:1::5"), up) + ipv4 := NewKey(msgWithECS("controld.com", 1, 24, "203.0.113.0"), up) + noECS := NewKey(msgWithECS("controld.com", 0, 0, ""), up) + + if subnetA == subnetB { + t.Fatal("different subnets must not share a cache key") + } + if subnetA != subnetAHost { + t.Fatal("same subnet (different host bits) must share a cache key") + } + if subnetA == ipv4 || subnetA == noECS || ipv4 == noECS { + t.Fatal("different families / no-ECS must not collide") + } +} + +// TestLRUCache_ECSZeroPrefixDistinctFromNoECS is the regression test for the /0-vs-no-ECS +// collision (RFC 7871 §7.3.1). A query carrying an ECS /0 option is forwarded WITH ECS and +// may draw different upstream data, so its cached answer must never be served to a client +// that sent no ECS at all, nor may IPv4 /0 and IPv6 /0 cross-serve each other. +func TestLRUCache_ECSZeroPrefixDistinctFromNoECS(t *testing.T) { + c, err := NewLRUCache(16) + if err != nil { + t.Fatalf("NewLRUCache: %v", err) + } + const up = "https://dns.example/dns-query" + expire := time.Now().Add(time.Minute) + + noECS := msgWithECS("controld.com", 0, 0, "") + zeroV4 := msgWithECS("controld.com", 1, 0, "0.0.0.0") + zeroV6 := msgWithECS("controld.com", 2, 0, "::") + + // Only the IPv4 /0 query's answer is cached. + c.Add(NewKey(zeroV4, up), NewValue(answerWithA("controld.com", "192.0.2.1"), expire)) + + // A no-ECS client must NOT receive the ECS /0 cached answer. + if got := c.Get(NewKey(noECS, up)); got != nil { + t.Fatalf("no-ECS client received the ECS /0 cached record %q", firstA(got.Msg)) + } + // An IPv6 /0 client must NOT receive the IPv4 /0 answer. + if got := c.Get(NewKey(zeroV6, up)); got != nil { + t.Fatalf("IPv6 /0 client received the IPv4 /0 cached record %q", firstA(got.Msg)) + } + // The IPv4 /0 client still hits its own entry. + if got := c.Get(NewKey(zeroV4, up)); got == nil || firstA(got.Msg) != "192.0.2.1" { + t.Fatalf("IPv4 /0 lost its own cached record: %v", got) + } + + // The no-ECS partition is independent and does not corrupt the /0 entry. + c.Add(NewKey(noECS, up), NewValue(answerWithA("controld.com", "198.51.100.1"), expire)) + if got := c.Get(NewKey(noECS, up)); got == nil || firstA(got.Msg) != "198.51.100.1" { + t.Fatalf("no-ECS partition wrong record: %v", got) + } + if got := c.Get(NewKey(zeroV4, up)); got == nil || firstA(got.Msg) != "192.0.2.1" { + t.Fatalf("IPv4 /0 record corrupted by no-ECS insert: %v", got) + } +} + +// TestLRUCache_ECSNoCrossSubnetServe is the real cache-path regression test for #564: +// an entry populated for subnet A must never be returned to a client in subnet B, even +// though the question (name/type/class/upstream) is identical. The two subnets carry +// different A records; the second client must get its own record or a miss, never A's. +func TestLRUCache_ECSNoCrossSubnetServe(t *testing.T) { + c, err := NewLRUCache(16) + if err != nil { + t.Fatalf("NewLRUCache: %v", err) + } + const up = "https://dns.example/dns-query" + expire := time.Now().Add(time.Minute) + + reqA := msgWithECS("controld.com", 2, 64, "2001:db8:1::") + reqB := msgWithECS("controld.com", 2, 64, "2001:db8:2::") + + // Only subnet A's answer (A record 192.0.2.1) is cached. + c.Add(NewKey(reqA, up), NewValue(answerWithA("controld.com", "192.0.2.1"), expire)) + + // A client in subnet B must NOT hit subnet A's entry. + if got := c.Get(NewKey(reqB, up)); got != nil { + t.Fatalf("subnet B received subnet A's cached record %q; cache is not ECS-partitioned", firstA(got.Msg)) + } + + // Subnet A still hits its own entry. + if got := c.Get(NewKey(reqA, up)); got == nil || firstA(got.Msg) != "192.0.2.1" { + t.Fatalf("subnet A lost its own cached record: %+v", got) + } + + // Now cache subnet B's distinct answer and confirm the two never cross. + c.Add(NewKey(reqB, up), NewValue(answerWithA("controld.com", "198.51.100.1"), expire)) + if got := c.Get(NewKey(reqB, up)); got == nil || firstA(got.Msg) != "198.51.100.1" { + t.Fatalf("subnet B got the wrong record: %v", got) + } + if got := c.Get(NewKey(reqA, up)); got == nil || firstA(got.Msg) != "192.0.2.1" { + t.Fatalf("subnet A record corrupted by subnet B insert: %v", got) + } + + // A second host within subnet A shares the partition (cache hit with A's record). + reqAHost := msgWithECS("controld.com", 2, 64, "2001:db8:1::9") + if got := c.Get(NewKey(reqAHost, up)); got == nil || firstA(got.Msg) != "192.0.2.1" { + t.Fatalf("same-subnet host missed the shared cache entry: %v", got) + } +} diff --git a/internal/firewall/allowlist.go b/internal/firewall/allowlist.go new file mode 100644 index 00000000..e6b94276 --- /dev/null +++ b/internal/firewall/allowlist.go @@ -0,0 +1,594 @@ +// Package firewall provides DNS-resolved IP allowlist enforcement for ctrld's +// firewall mode. When enabled, only IPs that were successfully resolved by ctrld +// are allowed outbound connections — blocking hardcoded IPs, direct-IP fallbacks, +// and DNS bypass attempts. +// +// The AllowList is the core data structure: a concurrent map of allowed IPs +// populated by DNS responses, with TTL-based expiry and domain-level invalidation +// for live profile updates. Alongside it are two sets that no DNS response feeds: +// permanent entries (loopback, RFC1918, upstreams — added once, never removed) and +// exceptions, the organization's Allowed Destination IP list, which is replaced as +// a whole set on every configuration refresh (see exceptions.go). +package firewall + +import ( + "context" + "net/netip" + "sync" + "sync/atomic" + "time" +) + +// AllowList tracks IPs that were resolved by ctrld and are allowed for outbound +// connections. It is safe for concurrent use from multiple goroutines. +// +// Architecture: +// - ips: primary map, netip.Addr → *entry (expiry + originating domains) +// - domains: reverse map, domain → []netip.Addr (for bulk invalidation on policy changes) +// - permanent: IPs that are always allowed (loopback, RFC1918, upstream endpoints) +// +// The hot path is Contains(), which must be O(1) with zero allocations. +type AllowList struct { + // ips is the primary allowlist. Keyed by IP address, value is the entry + // containing expiry time and which domains resolved to this IP. + ips sync.Map // netip.Addr → *entry + + // domains is the reverse map for bulk invalidation. When a domain's policy + // changes (e.g., previously-allowed domain gets blocked), we can look up all + // IPs associated with that domain and remove them. + domains sync.Map // string → *domainEntry + + // permanent contains IPs that are always allowed regardless of DNS resolution. + // These include loopback, RFC1918, link-local, ctrld listener IPs, and + // upstream resolver IPs. + permanent sync.Map // netip.Addr → struct{} + + // exceptions holds the administratively allowed destinations — the + // organization's Allowed Destination IP list, delivered by the API on every + // configuration refresh. Unlike permanent entries they are replaced as a whole + // set (see SetExceptions), so an entry removed upstream stops being allowed. + // Held as an immutable index behind an atomic pointer because Contains() + // reads it on the hot path. + exceptions atomic.Pointer[exceptionIndex] + + // exceptionsMu serializes SetExceptions so two concurrent refreshes cannot + // interleave their compare and store steps. It is separate from mu so + // replacing the set never blocks DNS-driven allowlist updates. + exceptionsMu sync.Mutex + + // onChange is called (if non-nil) whenever the allowlist changes. + // The callback receives the IP and whether it was added (true) or removed (false). + // Platform-specific enforcement (pf/WFP) registers a callback here. + onChange func(ip netip.Addr, added bool) + + // onBatchChange is called (if non-nil) with batches of changes. + // When possible, AllowList prefers onBatchChange to reduce platform + // enforcement calls. Single Add() calls still use onChange. + onBatchChange func(added []netip.Addr, removed []netip.Addr) + + // mu protects mutations that need consistency across the ips/domains maps + // and platform callbacks. + mu sync.Mutex + + // stats tracks allowlist metrics without per-operation locks. + totalAdds atomic.Int64 + totalRemoves atomic.Int64 + totalHits atomic.Int64 + totalMisses atomic.Int64 +} + +// entry represents an allowed IP with expiry and provenance tracking. +// Fields are protected by mu since Add() and reap() may access concurrently. +type entry struct { + mu sync.Mutex + expiry time.Time + domains []string // which domains resolved to this IP (for reverse lookup) +} + +// domainEntry tracks all IPs associated with a domain for bulk invalidation. +type domainEntry struct { + ips []netip.Addr +} + +// Stats contains a snapshot of allowlist metrics. +type Stats struct { + // AllowedIPs is the current number of IPs in the allowlist (non-permanent). + AllowedIPs int `json:"allowed_ips"` + // PermanentIPs is the number of permanently allowed IPs. + PermanentIPs int `json:"permanent_ips"` + // ExceptionPrefixes is the number of administratively allowed destination + // prefixes (the organization's Allowed Destination IP list). + ExceptionPrefixes int `json:"exception_prefixes"` + // TrackedDomains is the number of domains with IP associations. + TrackedDomains int `json:"tracked_domains"` + // TotalAdds is the cumulative number of Add() calls. + TotalAdds int64 `json:"total_adds"` + // TotalRemoves is the cumulative number of Remove()/Flush() operations. + TotalRemoves int64 `json:"total_removes"` + // TotalHits is the number of Contains() calls that returned true. + TotalHits int64 `json:"total_hits"` + // TotalMisses is the number of Contains() calls that returned false. + TotalMisses int64 `json:"total_misses"` +} + +// New creates a new empty AllowList. Call AddPermanent() to seed it with +// IPs that should always be allowed (loopback, upstreams, etc.), then +// StartReaper() to begin background TTL expiry. +func New() *AllowList { + return &AllowList{} +} + +// SetOnChange registers a callback that fires on each individual IP change. +// Use SetOnBatchChange instead for platform enforcement (reduces churn). +func (a *AllowList) SetOnChange(fn func(ip netip.Addr, added bool)) { + a.mu.Lock() + defer a.mu.Unlock() + a.onChange = fn +} + +// SetOnBatchChange registers a callback that fires with batched changes. +// The reaper and FlushDomain operations use this to deliver bulk updates. +func (a *AllowList) SetOnBatchChange(fn func(added []netip.Addr, removed []netip.Addr)) { + a.mu.Lock() + defer a.mu.Unlock() + a.onBatchChange = fn +} + +// Add inserts an IP into the allowlist with a TTL and domain association. +// If the IP already exists, its expiry is extended (max of old and new) and the +// domain is added to its provenance list. This is called from the DNS proxy +// response path for every A/AAAA record in a successful resolution. +// +// The domain parameter tracks which DNS query produced this IP, enabling +// FlushDomain() to invalidate IPs when a domain's policy changes. +func (a *AllowList) Add(ip netip.Addr, domain string, ttl time.Duration) { + if !ip.IsValid() { + return + } + + // Normalize IPv4-in-IPv6 to plain IPv4 for consistent lookups. + ip = ip.Unmap() + + expiry := time.Now().Add(ttl) + + a.mu.Lock() + defer a.mu.Unlock() + + // Update or create the IP entry. LoadOrStore avoids overwriting an entry if + // another goroutine won the initialization race. The outer mutex keeps the + // IP and domain reverse maps in step with Flush/FlushDomain callbacks. + entryValue, loaded := a.ips.LoadOrStore(ip, &entry{ + expiry: expiry, + domains: []string{domain}, + }) + if loaded { + e := entryValue.(*entry) + e.mu.Lock() + // Extend expiry if the new one is later. + if expiry.After(e.expiry) { + e.expiry = expiry + } + // Add domain to provenance if not already tracked. + if !containsString(e.domains, domain) { + e.domains = append(e.domains, domain) + } + e.mu.Unlock() + } else { + a.totalAdds.Add(1) + + // Notify platform enforcement of the new IP. + if a.onChange != nil { + a.onChange(ip, true) + } + } + + // Update the domain → IP reverse map. + de, _ := a.domains.LoadOrStore(domain, &domainEntry{}) + d := de.(*domainEntry) + if !containsAddr(d.ips, ip) { + d.ips = append(d.ips, ip) + } +} + +// Contains checks whether an IP is allowed. This is the hot path — called +// per-packet on mobile (via netstack) and per-connection on desktop (via +// pf/WFP). It must be O(1) with zero allocations. +// +// Returns true if the IP is in the allowlist (not expired) or in the +// permanent list. +func (a *AllowList) Contains(ip netip.Addr) bool { + if !ip.IsValid() { + return false + } + + ip = ip.Unmap() + + // Check permanent list first (most common for loopback/private). + if a.containsPermanent(ip) { + a.totalHits.Add(1) + return true + } + + // Administratively allowed destinations are reachable without ctrld having + // resolved them, so they are checked before the DNS-driven map. + if a.containsException(ip) { + a.totalHits.Add(1) + return true + } + + // Check dynamic allowlist. + if existing, ok := a.ips.Load(ip); ok { + e := existing.(*entry) + e.mu.Lock() + expired := time.Now().After(e.expiry) + e.mu.Unlock() + if !expired { + a.totalHits.Add(1) + return true + } + // Expired — will be cleaned up by reaper. Don't remove here to avoid + // per-lookup overhead and lock contention. + } + + a.totalMisses.Add(1) + return false +} + +// Remove deletes a single IP from the allowlist. Does not affect permanent entries. +func (a *AllowList) Remove(ip netip.Addr) { + ip = ip.Unmap() + a.mu.Lock() + defer a.mu.Unlock() + + if existing, ok := a.ips.LoadAndDelete(ip); ok { + e := existing.(*entry) + a.totalRemoves.Add(1) + + e.mu.Lock() + domains := append([]string(nil), e.domains...) + e.mu.Unlock() + + // Clean up domain reverse map entries. + for _, domain := range domains { + a.removeDomainIP(domain, ip) + } + + if a.onChange != nil { + a.onChange(ip, false) + } + } +} + +// FlushDomain removes all IPs associated with a specific domain. This is used +// when a profile/policy change makes a previously-allowed domain blocked. +// IPs that are also associated with OTHER still-allowed domains are kept. +func (a *AllowList) FlushDomain(domain string) { + a.mu.Lock() + defer a.mu.Unlock() + + de, ok := a.domains.LoadAndDelete(domain) + if !ok { + return + } + d := de.(*domainEntry) + ipsToCheck := make([]netip.Addr, len(d.ips)) + copy(ipsToCheck, d.ips) + + var removed []netip.Addr + + for _, ip := range ipsToCheck { + existing, ok := a.ips.Load(ip) + if !ok { + continue + } + e := existing.(*entry) + + e.mu.Lock() + // Remove this domain from the entry's domain list. + e.domains = removeString(e.domains, domain) + empty := len(e.domains) == 0 + e.mu.Unlock() + + // If no other domains reference this IP, remove it entirely. + if empty { + a.ips.Delete(ip) + a.totalRemoves.Add(1) + removed = append(removed, ip) + } + } + + // Batch notify platform enforcement. + if len(removed) > 0 && a.onBatchChange != nil { + a.onBatchChange(nil, removed) + } else if a.onChange != nil { + for _, ip := range removed { + a.onChange(ip, false) + } + } +} + +// Flush removes all non-permanent entries. Used on network state changes +// (WiFi→cellular, interface IP changes) where stale IPs from the old network +// may no longer be valid. DNS cache should also be flushed so that subsequent +// queries repopulate both caches. +func (a *AllowList) Flush() { + a.mu.Lock() + defer a.mu.Unlock() + + var removed []netip.Addr + + a.ips.Range(func(key, value any) bool { + ip := key.(netip.Addr) + a.ips.Delete(ip) + a.totalRemoves.Add(1) + removed = append(removed, ip) + return true + }) + + // Clear all domain reverse mappings. + a.domains.Range(func(key, _ any) bool { + a.domains.Delete(key) + return true + }) + + // Batch notify platform enforcement. + if len(removed) > 0 && a.onBatchChange != nil { + a.onBatchChange(nil, removed) + } else if a.onChange != nil { + for _, ip := range removed { + a.onChange(ip, false) + } + } +} + +// AddPermanent adds an IP to the permanent allowlist. Permanent entries never +// expire and survive Flush(). Use for: +// - Loopback (127.0.0.0/8, ::1) +// - RFC1918 private ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) +// - Link-local (169.254.0.0/16, fe80::/10) +// - ctrld listener IPs +// - DoH/DoT/DoQ upstream IPs +// - ControlD API endpoint IPs +func (a *AllowList) AddPermanent(ip netip.Addr) { + if !ip.IsValid() { + return + } + ip = ip.Unmap() + a.permanent.Store(ip, struct{}{}) +} + +// AddPermanentPrefix adds an entire CIDR prefix to the permanent allowlist. +// Used for ranges like 127.0.0.0/8, 10.0.0.0/8, etc. +// For large prefixes (e.g., /8), this stores the prefix for range-based lookup +// rather than enumerating all IPs. +func (a *AllowList) AddPermanentPrefix(prefix netip.Prefix) { + // For small prefixes, enumerate. For large ones, we'd need a different approach. + // Since we're dealing with a known set of well-defined ranges, and Contains() + // needs to be fast, we check prefixes in a separate path. + // Store the prefix in a separate list for range-based checks. + a.permanent.Store(prefix, struct{}{}) +} + +// containsPermanent checks both individual IPs and prefix ranges in the permanent list. +func (a *AllowList) containsPermanent(ip netip.Addr) bool { + // Direct IP match. + if _, ok := a.permanent.Load(ip); ok { + return true + } + + // Check prefix ranges. We iterate the permanent map looking for Prefix entries. + // This is acceptable because the permanent map is small and rarely changes. + found := false + a.permanent.Range(func(key, _ any) bool { + if prefix, ok := key.(netip.Prefix); ok { + if prefix.Contains(ip) { + found = true + return false // stop iteration + } + } + return true + }) + return found +} + +// PermanentEntries returns snapshots of permanently allowed individual IPs and +// prefixes. Platform enforcers use this to mirror the in-memory permanent +// allowlist into kernel-level permit rules. +func (a *AllowList) PermanentEntries() ([]netip.Addr, []netip.Prefix) { + a.mu.Lock() + defer a.mu.Unlock() + + var addrs []netip.Addr + var prefixes []netip.Prefix + a.permanent.Range(func(key, _ any) bool { + switch v := key.(type) { + case netip.Addr: + addrs = append(addrs, v) + case netip.Prefix: + prefixes = append(prefixes, v) + } + return true + }) + return addrs, prefixes +} + +// RemovePermanent removes an IP from the permanent allowlist. +func (a *AllowList) RemovePermanent(ip netip.Addr) { + ip = ip.Unmap() + a.permanent.Delete(ip) +} + +// Stats returns a snapshot of current allowlist metrics. +func (a *AllowList) Stats() Stats { + a.mu.Lock() + defer a.mu.Unlock() + + var s Stats + now := time.Now() + a.ips.Range(func(_, value any) bool { + e := value.(*entry) + e.mu.Lock() + expired := now.After(e.expiry) + e.mu.Unlock() + if !expired { + s.AllowedIPs++ + } + return true + }) + a.permanent.Range(func(_, _ any) bool { + s.PermanentIPs++ + return true + }) + a.domains.Range(func(_, _ any) bool { + s.TrackedDomains++ + return true + }) + s.ExceptionPrefixes = len(a.exceptionsSnapshot()) + s.TotalAdds = a.totalAdds.Load() + s.TotalRemoves = a.totalRemoves.Load() + s.TotalHits = a.totalHits.Load() + s.TotalMisses = a.totalMisses.Load() + return s +} + +// StartReaper begins a background goroutine that periodically removes expired +// entries from the allowlist. It runs every 30 seconds and batches removals +// for efficient platform enforcement notification. +// +// The reaper is intentionally separate from Contains() to avoid per-lookup +// overhead. Expired entries may linger for up to 30 seconds, which is acceptable +// since DNS TTLs are typically 30s-300s and the reaper interval is a fraction of that. +func (a *AllowList) StartReaper(ctx context.Context) { + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + a.reap() + } + } + }() +} + +// reap removes expired entries and notifies platform enforcement. +func (a *AllowList) reap() { + a.mu.Lock() + defer a.mu.Unlock() + + now := time.Now() + var removed []netip.Addr + + a.ips.Range(func(key, value any) bool { + ip := key.(netip.Addr) + e := value.(*entry) + + e.mu.Lock() + expired := now.After(e.expiry) + var domains []string + if expired { + domains = make([]string, len(e.domains)) + copy(domains, e.domains) + } + e.mu.Unlock() + + if expired { + a.ips.Delete(ip) + a.totalRemoves.Add(1) + removed = append(removed, ip) + + // Clean up domain reverse map. + for _, domain := range domains { + a.removeDomainIP(domain, ip) + } + } + return true + }) + + if len(removed) > 0 && a.onBatchChange != nil { + a.onBatchChange(nil, removed) + } else if a.onChange != nil { + for _, ip := range removed { + a.onChange(ip, false) + } + } +} + +// removeDomainIP removes an IP from a domain's reverse map entry. +func (a *AllowList) removeDomainIP(domain string, ip netip.Addr) { + de, ok := a.domains.Load(domain) + if !ok { + return + } + d := de.(*domainEntry) + d.ips = removeAddr(d.ips, ip) + empty := len(d.ips) == 0 + + // Clean up empty domain entries. + if empty { + a.domains.Delete(domain) + } +} + +// AllowedIPs returns a snapshot of all currently allowed (non-expired, non-permanent) IPs. +// Used by platform enforcement for initial table population. +func (a *AllowList) AllowedIPs() []netip.Addr { + a.mu.Lock() + defer a.mu.Unlock() + + now := time.Now() + var ips []netip.Addr + a.ips.Range(func(key, value any) bool { + ip := key.(netip.Addr) + e := value.(*entry) + e.mu.Lock() + expiry := e.expiry + e.mu.Unlock() + if now.Before(expiry) { + ips = append(ips, ip) + } + return true + }) + return ips +} + +// --- Helpers --- + +func containsString(ss []string, s string) bool { + for _, v := range ss { + if v == s { + return true + } + } + return false +} + +func removeString(ss []string, s string) []string { + for i, v := range ss { + if v == s { + return append(ss[:i], ss[i+1:]...) + } + } + return ss +} + +func containsAddr(addrs []netip.Addr, addr netip.Addr) bool { + for _, a := range addrs { + if a == addr { + return true + } + } + return false +} + +func removeAddr(addrs []netip.Addr, addr netip.Addr) []netip.Addr { + for i, a := range addrs { + if a == addr { + return append(addrs[:i], addrs[i+1:]...) + } + } + return addrs +} diff --git a/internal/firewall/allowlist_test.go b/internal/firewall/allowlist_test.go new file mode 100644 index 00000000..61bdfd98 --- /dev/null +++ b/internal/firewall/allowlist_test.go @@ -0,0 +1,461 @@ +package firewall + +import ( + "context" + "net/netip" + "sync" + "testing" + "time" +) + +func TestAddAndContains(t *testing.T) { + al := New() + ip := netip.MustParseAddr("93.184.216.34") + + // IP should not be allowed before adding. + if al.Contains(ip) { + t.Fatal("expected IP to not be in allowlist before Add") + } + + al.Add(ip, "example.com", 5*time.Minute) + + if !al.Contains(ip) { + t.Fatal("expected IP to be in allowlist after Add") + } +} + +func TestContainsExpired(t *testing.T) { + al := New() + ip := netip.MustParseAddr("93.184.216.34") + + // Add with a TTL that's already past. + al.Add(ip, "example.com", -1*time.Second) + + if al.Contains(ip) { + t.Fatal("expected expired IP to not be allowed") + } +} + +func TestAddExtendsExpiry(t *testing.T) { + al := New() + ip := netip.MustParseAddr("93.184.216.34") + + // Add with short TTL. + al.Add(ip, "example.com", 1*time.Second) + // Extend with longer TTL. + al.Add(ip, "cdn.example.com", 1*time.Hour) + + // Should still be valid. + if !al.Contains(ip) { + t.Fatal("expected IP to be allowed after expiry extension") + } + + // Check domain provenance was tracked. + existing, ok := al.ips.Load(ip) + if !ok { + t.Fatal("expected IP entry to exist") + } + e := existing.(*entry) + e.mu.Lock() + domainCount := len(e.domains) + e.mu.Unlock() + if domainCount != 2 { + t.Fatalf("expected 2 domains, got %d", domainCount) + } +} + +func TestRemove(t *testing.T) { + al := New() + ip := netip.MustParseAddr("93.184.216.34") + + al.Add(ip, "example.com", 5*time.Minute) + al.Remove(ip) + + if al.Contains(ip) { + t.Fatal("expected IP to not be in allowlist after Remove") + } +} + +func TestPermanent(t *testing.T) { + al := New() + ip := netip.MustParseAddr("127.0.0.1") + + al.AddPermanent(ip) + + if !al.Contains(ip) { + t.Fatal("expected permanent IP to be allowed") + } + + // Permanent entries survive Flush. + al.Flush() + + if !al.Contains(ip) { + t.Fatal("expected permanent IP to survive Flush") + } +} + +func TestPermanentPrefix(t *testing.T) { + al := New() + prefix := netip.MustParsePrefix("10.0.0.0/8") + al.AddPermanentPrefix(prefix) + + if !al.Contains(netip.MustParseAddr("10.1.2.3")) { + t.Fatal("expected IP in permanent prefix to be allowed") + } + if !al.Contains(netip.MustParseAddr("10.255.255.255")) { + t.Fatal("expected IP at end of permanent prefix to be allowed") + } + if al.Contains(netip.MustParseAddr("11.0.0.1")) { + t.Fatal("expected IP outside permanent prefix to not be allowed") + } +} + +func TestFlush(t *testing.T) { + al := New() + al.AddPermanent(netip.MustParseAddr("127.0.0.1")) + al.Add(netip.MustParseAddr("93.184.216.34"), "example.com", 5*time.Minute) + al.Add(netip.MustParseAddr("151.101.1.69"), "reddit.com", 5*time.Minute) + + al.Flush() + + // Dynamic entries should be gone. + if al.Contains(netip.MustParseAddr("93.184.216.34")) { + t.Fatal("expected dynamic IP to be removed after Flush") + } + if al.Contains(netip.MustParseAddr("151.101.1.69")) { + t.Fatal("expected dynamic IP to be removed after Flush") + } + + // Permanent should survive. + if !al.Contains(netip.MustParseAddr("127.0.0.1")) { + t.Fatal("expected permanent IP to survive Flush") + } + + // Domain reverse map should be cleared. + stats := al.Stats() + if stats.TrackedDomains != 0 { + t.Fatalf("expected 0 tracked domains after Flush, got %d", stats.TrackedDomains) + } +} + +func TestFlushDomain(t *testing.T) { + al := New() + + // IP shared between two domains. + sharedIP := netip.MustParseAddr("93.184.216.34") + exclusiveIP := netip.MustParseAddr("93.184.216.35") + + al.Add(sharedIP, "example.com", 5*time.Minute) + al.Add(sharedIP, "cdn.example.com", 5*time.Minute) + al.Add(exclusiveIP, "example.com", 5*time.Minute) + + // Flush only example.com. + al.FlushDomain("example.com") + + // Shared IP should remain because cdn.example.com still references it. + if !al.Contains(sharedIP) { + t.Fatal("expected shared IP to remain (still referenced by cdn.example.com)") + } + + // Exclusive IP should be removed (only referenced by example.com). + if al.Contains(exclusiveIP) { + t.Fatal("expected exclusive IP to be removed after FlushDomain") + } +} + +func TestFlushFallsBackToOnChange(t *testing.T) { + al := New() + ip1 := netip.MustParseAddr("93.184.216.34") + ip2 := netip.MustParseAddr("151.101.1.69") + + var removed []netip.Addr + al.SetOnChange(func(ip netip.Addr, added bool) { + if !added { + removed = append(removed, ip) + } + }) + + al.Add(ip1, "example.com", 5*time.Minute) + al.Add(ip2, "reddit.com", 5*time.Minute) + al.Flush() + + if len(removed) != 2 { + t.Fatalf("expected 2 removed IP callbacks, got %d", len(removed)) + } +} + +func TestFlushDomainThenAddKeepsPlatformInSync(t *testing.T) { + al := New() + ip := netip.MustParseAddr("93.184.216.34") + platform := make(map[netip.Addr]bool) + + al.SetOnChange(func(ip netip.Addr, added bool) { + platform[ip] = added + }) + al.SetOnBatchChange(func(added, removed []netip.Addr) { + for _, ip := range added { + platform[ip] = true + } + for _, ip := range removed { + platform[ip] = false + } + }) + + al.Add(ip, "example.com", 5*time.Minute) + al.FlushDomain("example.com") + al.Add(ip, "example.com", 5*time.Minute) + + if !al.Contains(ip) { + t.Fatal("expected IP to be allowed after re-add") + } + if !platform[ip] { + t.Fatal("expected platform callback state to match allowlist after flush/re-add") + } +} + +func TestConcurrentAddFlushDomainAndReap(t *testing.T) { + al := New() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + al.StartReaper(ctx) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for j := 0; j < 200; j++ { + ip := netip.AddrFrom4([4]byte{10, byte(n), byte(j / 255), byte(j % 255)}) + al.Add(ip, "example.com", time.Minute) + al.Contains(ip) + } + }(i) + } + + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + al.FlushDomain("example.com") + al.reap() + } + }() + } + + wg.Wait() +} + +func TestIPv4MappedIPv6(t *testing.T) { + al := New() + + // Add as IPv4. + al.Add(netip.MustParseAddr("93.184.216.34"), "example.com", 5*time.Minute) + + // Look up as IPv4-mapped IPv6 — should still match due to Unmap(). + mapped := netip.AddrFrom16(netip.MustParseAddr("93.184.216.34").As16()) + if !al.Contains(mapped) { + t.Fatal("expected IPv4-mapped IPv6 lookup to match IPv4 entry") + } +} + +func TestIPv6(t *testing.T) { + al := New() + ip := netip.MustParseAddr("2606:2800:220:1:248:1893:25c8:1946") + + al.Add(ip, "example.com", 5*time.Minute) + + if !al.Contains(ip) { + t.Fatal("expected IPv6 address to be in allowlist") + } +} + +func TestReaper(t *testing.T) { + al := New() + ip := netip.MustParseAddr("93.184.216.34") + + // Add with very short TTL that's already expired. + al.Add(ip, "example.com", -1*time.Second) + + // Manually trigger reaper. + al.reap() + + // Should have been cleaned up. + _, ok := al.ips.Load(ip) + if ok { + t.Fatal("expected reaper to remove expired entry from ips map") + } +} + +func TestReaperFallsBackToOnChange(t *testing.T) { + al := New() + + var removed []netip.Addr + al.SetOnChange(func(ip netip.Addr, added bool) { + if !added { + removed = append(removed, ip) + } + }) + + al.Add(netip.MustParseAddr("1.1.1.1"), "one.com", -1*time.Second) + al.Add(netip.MustParseAddr("2.2.2.2"), "two.com", -1*time.Second) + al.reap() + + if len(removed) != 2 { + t.Fatalf("expected 2 removed IP callbacks, got %d", len(removed)) + } +} + +func TestReaperBatchCallback(t *testing.T) { + al := New() + + var removedIPs []netip.Addr + al.SetOnBatchChange(func(added, removed []netip.Addr) { + removedIPs = append(removedIPs, removed...) + }) + + al.Add(netip.MustParseAddr("1.1.1.1"), "one.com", -1*time.Second) + al.Add(netip.MustParseAddr("2.2.2.2"), "two.com", -1*time.Second) + al.Add(netip.MustParseAddr("3.3.3.3"), "three.com", 1*time.Hour) // not expired + + al.reap() + + if len(removedIPs) != 2 { + t.Fatalf("expected 2 removed IPs in batch callback, got %d", len(removedIPs)) + } +} + +func TestStatsExcludesExpiredAllowedIPs(t *testing.T) { + al := New() + + al.Add(netip.MustParseAddr("93.184.216.34"), "example.com", -1*time.Second) + al.Add(netip.MustParseAddr("151.101.1.69"), "reddit.com", 5*time.Minute) + + stats := al.Stats() + if stats.AllowedIPs != 1 { + t.Fatalf("expected 1 non-expired allowed IP, got %d", stats.AllowedIPs) + } +} + +func TestStats(t *testing.T) { + al := New() + + al.AddPermanent(netip.MustParseAddr("127.0.0.1")) + al.AddPermanentPrefix(netip.MustParsePrefix("10.0.0.0/8")) + al.Add(netip.MustParseAddr("93.184.216.34"), "example.com", 5*time.Minute) + al.Add(netip.MustParseAddr("151.101.1.69"), "reddit.com", 5*time.Minute) + + // Generate some hits/misses. + al.Contains(netip.MustParseAddr("93.184.216.34")) // hit + al.Contains(netip.MustParseAddr("127.0.0.1")) // hit (permanent) + al.Contains(netip.MustParseAddr("8.8.8.8")) // miss + + stats := al.Stats() + + if stats.AllowedIPs != 2 { + t.Fatalf("expected 2 allowed IPs, got %d", stats.AllowedIPs) + } + // Permanent count includes both the individual IP and the prefix. + if stats.PermanentIPs != 2 { + t.Fatalf("expected 2 permanent entries, got %d", stats.PermanentIPs) + } + if stats.TrackedDomains != 2 { + t.Fatalf("expected 2 tracked domains, got %d", stats.TrackedDomains) + } + if stats.TotalHits != 2 { + t.Fatalf("expected 2 total hits, got %d", stats.TotalHits) + } + if stats.TotalMisses != 1 { + t.Fatalf("expected 1 total miss, got %d", stats.TotalMisses) + } +} + +func TestConcurrentAccess(t *testing.T) { + al := New() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + al.StartReaper(ctx) + al.AddPermanent(netip.MustParseAddr("127.0.0.1")) + + var wg sync.WaitGroup + // Concurrent writers. + for i := 0; i < 10; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for j := 0; j < 100; j++ { + ip := netip.AddrFrom4([4]byte{10, 0, byte(n), byte(j)}) + al.Add(ip, "test.com", 1*time.Minute) + } + }(i) + } + + // Concurrent readers. + for i := 0; i < 10; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + for j := 0; j < 100; j++ { + ip := netip.AddrFrom4([4]byte{10, 0, byte(n), byte(j)}) + al.Contains(ip) + } + }(i) + } + + // Concurrent flush. + wg.Add(1) + go func() { + defer wg.Done() + time.Sleep(10 * time.Millisecond) + al.Flush() + }() + + wg.Wait() +} + +func TestStartReaperWithContext(t *testing.T) { + // This test verifies the reaper goroutine starts and stops cleanly. + // We test actual reaping behavior via TestReaper (calls reap() directly). + al := New() + ctx, cancel := context.WithCancel(context.Background()) + al.StartReaper(ctx) + cancel() // Should not panic or leak. +} + +func TestOnChangeCallback(t *testing.T) { + al := New() + + var added []netip.Addr + var removed []netip.Addr + al.SetOnChange(func(ip netip.Addr, isAdded bool) { + if isAdded { + added = append(added, ip) + } else { + removed = append(removed, ip) + } + }) + + ip := netip.MustParseAddr("93.184.216.34") + al.Add(ip, "example.com", 5*time.Minute) + al.Remove(ip) + + if len(added) != 1 { + t.Fatalf("expected 1 added callback, got %d", len(added)) + } + if len(removed) != 1 { + t.Fatalf("expected 1 removed callback, got %d", len(removed)) + } +} + +func TestInvalidIP(t *testing.T) { + al := New() + + // Should not panic on invalid IPs. + var invalid netip.Addr + al.Add(invalid, "test.com", time.Minute) + if al.Contains(invalid) { + t.Fatal("expected invalid IP to not be in allowlist") + } + al.AddPermanent(invalid) +} diff --git a/internal/firewall/exceptions.go b/internal/firewall/exceptions.go new file mode 100644 index 00000000..5de5c74d --- /dev/null +++ b/internal/firewall/exceptions.go @@ -0,0 +1,210 @@ +package firewall + +import ( + "net/netip" + "sort" +) + +// Exception handling: the organization's Allowed Destination IP list. +// +// Firewall Mode only permits what ctrld itself resolved, which makes a service +// addressed by literal IP — with no DNS lookup to observe — unreachable. An +// organization can therefore publish a list of destinations that stay reachable +// regardless; the API delivers the effective list (own entries plus any inherited +// from a parent organization) with every configuration refresh. +// +// The list is applied as a *set*, not as individual additions: each refresh +// replaces the previous snapshot, so an entry removed upstream stops bypassing +// Firewall Mode as soon as the refresh lands. Entries never expire in between — +// unlike DNS-resolved IPs they carry no TTL. +// +// This type holds the *desired* set only. Mirroring it into pf/WFP can fail, so +// what platform enforcement has actually accepted is tracked by the caller (see +// prog.reconcileAllowedDestinations), which retries until the two agree. Storing +// "applied" here would make a failed pfctl call look like a success to every +// later refresh. +// +// The same split matters to anything embedding this package: Contains() answers +// from the set the last SetExceptions call installed, which is what ctrld wants +// enforced, not what the kernel is enforcing. On macOS and Windows the platform +// state is the gate and the difference is tracked and retried, so a mirror that +// failed cannot let traffic through. An embedder that gates on Contains() alone +// has no such gate, and inherits the desired set the moment it is set. + +// SetExceptions replaces the allowed-destination set with prefixes. Entries are +// masked and de-duplicated first, so equivalent spellings of the same network +// (e.g. "10.1.2.3/24" and "10.1.2.0/24") do not rebuild the index. +// +// Deliberately reports nothing about whether the set changed. "Nothing changed" +// is not a licence to skip the platform reconcile: enforcement can be behind the +// desired set from an earlier failed mirror, and that retry is driven by +// comparing against what the platform accepted, not against the previous desired +// set. A caller that skipped on "unchanged" would strand exactly the case the +// retry exists for. +func (a *AllowList) SetExceptions(prefixes []netip.Prefix) { + next := normalizeExceptions(prefixes) + + // Serialized so two concurrent refreshes cannot interleave their compare and + // store steps and leave the older set installed. + a.exceptionsMu.Lock() + defer a.exceptionsMu.Unlock() + + if samePrefixes(a.exceptionsSnapshot(), next) { + return + } + a.exceptions.Store(newExceptionIndex(next)) +} + +// Exceptions returns the current allowed-destination set, ordered and masked. +// The result must not be modified — it is the live snapshot shared with the +// Contains() hot path. +func (a *AllowList) Exceptions() []netip.Prefix { + return a.exceptionsSnapshot() +} + +// exceptionsSnapshot returns the stored set, or nil when none was ever applied. +func (a *AllowList) exceptionsSnapshot() []netip.Prefix { + if idx := a.exceptions.Load(); idx != nil { + return idx.prefixes + } + return nil +} + +// containsException reports whether ip falls inside an allowed destination. +// +// Binary search over sorted, merged address ranges: Contains() is the +// per-connection (and, for embedders, per-packet) hot path, and an organization +// list can hold thousands of prefixes, so a linear scan would put its length on +// that path. An empty set — the overwhelmingly common case — costs one nil check. +func (a *AllowList) containsException(ip netip.Addr) bool { + idx := a.exceptions.Load() + if idx == nil { + return false + } + ranges := idx.v6 + if ip.Is4() { + ranges = idx.v4 + } + if len(ranges) == 0 { + return false + } + // Find the last range whose start is <= ip; it is the only one that can + // contain ip, because ranges are sorted and non-overlapping. + i := sort.Search(len(ranges), func(i int) bool { return ranges[i].lo.Compare(ip) > 0 }) + if i == 0 { + return false + } + return ranges[i-1].hi.Compare(ip) >= 0 +} + +// exceptionIndex is an immutable lookup structure over one allowed-destination +// set: the normalized prefixes as applied, plus per-family sorted address ranges +// for lookups. Published as a whole behind an atomic pointer, so a refresh never +// exposes a half-rebuilt index to a concurrent Contains(). +type exceptionIndex struct { + prefixes []netip.Prefix + v4 []addrRange + v6 []addrRange +} + +// addrRange is an inclusive address range, the range form of one prefix (or of +// several that were merged because they overlap or abut). +type addrRange struct { + lo, hi netip.Addr +} + +// newExceptionIndex builds the lookup index for a normalized prefix set. +func newExceptionIndex(prefixes []netip.Prefix) *exceptionIndex { + idx := &exceptionIndex{prefixes: prefixes} + for _, prefix := range prefixes { + r := addrRange{lo: prefix.Addr(), hi: lastAddr(prefix)} + if prefix.Addr().Is4() { + idx.v4 = append(idx.v4, r) + } else { + idx.v6 = append(idx.v6, r) + } + } + idx.v4 = sortAndMerge(idx.v4) + idx.v6 = sortAndMerge(idx.v6) + return idx +} + +// lastAddr returns the highest address in a masked prefix. +func lastAddr(prefix netip.Prefix) netip.Addr { + if prefix.Addr().Is4() { + b := prefix.Addr().As4() + for i := prefix.Bits(); i < 32; i++ { + b[i/8] |= 1 << (7 - i%8) + } + return netip.AddrFrom4(b) + } + b := prefix.Addr().As16() + for i := prefix.Bits(); i < 128; i++ { + b[i/8] |= 1 << (7 - i%8) + } + return netip.AddrFrom16(b) +} + +// sortAndMerge orders ranges by start address and coalesces the ones that +// overlap or abut, so the search invariant (sorted, non-overlapping) holds even +// when an organization lists a network and an address inside it. +func sortAndMerge(ranges []addrRange) []addrRange { + if len(ranges) < 2 { + return ranges + } + sort.Slice(ranges, func(i, j int) bool { return ranges[i].lo.Compare(ranges[j].lo) < 0 }) + + out := ranges[:1] + for _, r := range ranges[1:] { + last := &out[len(out)-1] + // Abutting counts as overlapping: last.hi.Next() == r.lo means the two + // ranges are contiguous with no gap to preserve. + if r.lo.Compare(last.hi) <= 0 || r.lo == last.hi.Next() { + if r.hi.Compare(last.hi) > 0 { + last.hi = r.hi + } + continue + } + out = append(out, r) + } + return out +} + +// normalizeExceptions masks, de-duplicates and orders prefixes so that two sets +// with the same meaning compare equal, and so logged deltas are stable. +func normalizeExceptions(prefixes []netip.Prefix) []netip.Prefix { + if len(prefixes) == 0 { + return nil + } + seen := make(map[netip.Prefix]struct{}, len(prefixes)) + out := make([]netip.Prefix, 0, len(prefixes)) + for _, prefix := range prefixes { + if !prefix.IsValid() { + continue + } + masked := prefix.Masked() + if _, dup := seen[masked]; dup { + continue + } + seen[masked] = struct{}{} + out = append(out, masked) + } + if len(out) == 0 { + return nil + } + sort.Slice(out, func(i, j int) bool { return out[i].String() < out[j].String() }) + return out +} + +// samePrefixes reports whether two normalized sets are identical. +func samePrefixes(a, b []netip.Prefix) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/firewall/exceptions_test.go b/internal/firewall/exceptions_test.go new file mode 100644 index 00000000..f279a071 --- /dev/null +++ b/internal/firewall/exceptions_test.go @@ -0,0 +1,214 @@ +package firewall + +import ( + "fmt" + "net/netip" + "testing" + "time" +) + +func prefixes(t *testing.T, ss ...string) []netip.Prefix { + t.Helper() + out := make([]netip.Prefix, 0, len(ss)) + for _, s := range ss { + out = append(out, netip.MustParsePrefix(s)) + } + return out +} + +func TestSetExceptionsNormalizesAndReplaces(t *testing.T) { + al := New() + + al.SetExceptions(prefixes(t, "203.0.113.10/32", "198.51.100.0/24")) + first := al.Exceptions() + if got := prefixStrings(first); len(got) != 2 { + t.Fatalf("Exceptions() = %v, want 2 entries", got) + } + + // Same set, different order and an unmasked spelling of the same network. The + // index must not be rebuilt: the list is re-applied on every configuration + // refresh, and a rebuild would republish it to the Contains() hot path hourly + // for a set that did not change. + al.SetExceptions(prefixes(t, "198.51.100.77/24", "203.0.113.10/32")) + same := al.Exceptions() + if len(same) != len(first) || &same[0] != &first[0] { + t.Fatalf("an equivalent set replaced the index: %v -> %v", prefixStrings(first), prefixStrings(same)) + } + + al.SetExceptions(prefixes(t, "203.0.113.10/32", "2001:db8::/48")) + changed := al.Exceptions() + if got := prefixStrings(changed); len(got) != 2 { + t.Fatalf("Exceptions() = %v, want 2 entries", got) + } + if &changed[0] == &first[0] { + t.Fatal("a changed set left the previous index installed") + } + + // An empty list clears the set: an organization can withdraw everything. + al.SetExceptions(nil) + if got := al.Exceptions(); len(got) != 0 { + t.Fatalf("Exceptions() after clear = %v, want empty", got) + } + al.SetExceptions(nil) + if got := al.Exceptions(); len(got) != 0 { + t.Fatalf("Exceptions() after clearing twice = %v, want empty", got) + } +} + +func TestExceptionsAllowWithoutDNS(t *testing.T) { + al := New() + inRange := netip.MustParseAddr("198.51.100.7") + host := netip.MustParseAddr("203.0.113.10") + other := netip.MustParseAddr("203.0.113.11") + v6 := netip.MustParseAddr("2001:db8::1") + + for _, ip := range []netip.Addr{inRange, host, other, v6} { + if al.Contains(ip) { + t.Fatalf("%s allowed before any exception was applied", ip) + } + } + + al.SetExceptions(prefixes(t, "198.51.100.0/24", "203.0.113.10/32", "2001:db8::/48")) + + // Allowed with no prior DNS resolution — the point of the list. + for _, ip := range []netip.Addr{inRange, host, v6} { + if !al.Contains(ip) { + t.Fatalf("%s not allowed by the exception set", ip) + } + } + // An unresolved public destination outside the set stays blocked. + if al.Contains(other) { + t.Fatalf("%s allowed although it is outside the exception set", other) + } + + // Removing an entry stops it bypassing, and leaves the rest allowed. + al.SetExceptions(prefixes(t, "203.0.113.10/32")) + if al.Contains(inRange) { + t.Fatalf("%s still allowed after its prefix was removed", inRange) + } + if !al.Contains(host) { + t.Fatalf("%s should still be allowed", host) + } +} + +// TestExceptionsBoundaries pins the range arithmetic the binary-search index +// rests on: the first and last address of a prefix are inside it, the addresses +// on either side are not, and a family never matches the other family's ranges. +func TestExceptionsBoundaries(t *testing.T) { + al := New() + al.SetExceptions(prefixes(t, "198.51.100.0/24", "2001:db8:1::/48")) + + in := []string{ + "198.51.100.0", "198.51.100.255", + "2001:db8:1::", "2001:db8:1:ffff:ffff:ffff:ffff:ffff", + } + out := []string{ + "198.51.99.255", "198.51.101.0", + "2001:db8:0:ffff:ffff:ffff:ffff:ffff", "2001:db8:2::", + } + for _, s := range in { + if !al.Contains(netip.MustParseAddr(s)) { + t.Errorf("%s should be inside the exception set", s) + } + } + for _, s := range out { + if al.Contains(netip.MustParseAddr(s)) { + t.Errorf("%s should be outside the exception set", s) + } + } +} + +// TestExceptionsOverlappingEntries covers an organization listing a network and +// an address within it, plus two adjacent networks - both of which the index +// merges, and neither of which may change what is allowed. +func TestExceptionsOverlappingEntries(t *testing.T) { + al := New() + al.SetExceptions(prefixes(t, + "198.51.100.0/24", "198.51.100.7/32", // contained + "203.0.113.0/25", "203.0.113.128/25", // adjacent halves + )) + + for _, s := range []string{"198.51.100.7", "198.51.100.200", "203.0.113.1", "203.0.113.200"} { + if !al.Contains(netip.MustParseAddr(s)) { + t.Errorf("%s should be allowed", s) + } + } + for _, s := range []string{"198.51.101.7", "203.0.114.1"} { + if al.Contains(netip.MustParseAddr(s)) { + t.Errorf("%s should not be allowed", s) + } + } +} + +func TestExceptionsIndependentOfDNSAllowlist(t *testing.T) { + al := New() + al.SetExceptions(prefixes(t, "203.0.113.0/24")) + + resolved := netip.MustParseAddr("192.0.2.5") + al.Add(resolved, "example.com", time.Minute) + + // Flush discards DNS-resolved IPs; exceptions are administrative and survive. + al.Flush() + if al.Contains(resolved) { + t.Fatalf("%s survived a flush", resolved) + } + if !al.Contains(netip.MustParseAddr("203.0.113.9")) { + t.Fatal("exception did not survive a flush of the DNS-resolved allowlist") + } + + // A removed exception is not resurrected by an unrelated DNS resolution. + al.SetExceptions(nil) + if al.Contains(netip.MustParseAddr("203.0.113.9")) { + t.Fatal("cleared exception still allowed") + } +} + +func TestExceptionsInStats(t *testing.T) { + al := New() + if got := al.Stats().ExceptionPrefixes; got != 0 { + t.Fatalf("ExceptionPrefixes = %d, want 0", got) + } + al.SetExceptions(prefixes(t, "203.0.113.0/24", "2001:db8::/48")) + if got := al.Stats().ExceptionPrefixes; got != 2 { + t.Fatalf("ExceptionPrefixes = %d, want 2", got) + } +} + +func prefixStrings(prefixes []netip.Prefix) []string { + out := make([]string, 0, len(prefixes)) + for _, prefix := range prefixes { + out = append(out, prefix.String()) + } + return out +} + +// benchExceptions builds a set of n distinct /24s plus one /32 the benchmark +// looks up, so lookups traverse the whole index rather than hitting an early +// entry. +func benchExceptions(n int) []netip.Prefix { + out := make([]netip.Prefix, 0, n) + for i := 0; i < n; i++ { + out = append(out, netip.MustParsePrefix(fmt.Sprintf("10.%d.%d.0/24", i/256, i%256))) + } + return out +} + +// BenchmarkContainsWithExceptions guards the hot path: Contains() is called per +// connection (per packet for embedders), so the cost of a large organization +// list must not scale with its length. +func BenchmarkContainsWithExceptions(b *testing.B) { + for _, n := range []int{0, 100, 2000} { + b.Run(fmt.Sprintf("prefixes=%d", n), func(b *testing.B) { + al := New() + if n > 0 { + al.SetExceptions(benchExceptions(n)) + } + ip := netip.MustParseAddr("203.0.113.10") // never in the set: worst case + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + al.Contains(ip) + } + }) + } +} diff --git a/internal/net/net.go b/internal/net/net.go index f4b55860..61a73b69 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -3,7 +3,6 @@ package net import ( "context" "errors" - "io" "net" "os" "os/signal" @@ -12,7 +11,7 @@ import ( "syscall" "time" - "github.com/rs/zerolog" + "go.uber.org/zap" "tailscale.com/logtail/backoff" ) @@ -34,8 +33,8 @@ var Dialer = &net.Dialer{ Dial: func(ctx context.Context, network, address string) (net.Conn, error) { d := ParallelDialer{} d.Timeout = 10 * time.Second - l := zerolog.New(io.Discard) - return d.DialContext(ctx, "udp", []string{v4BootstrapDNS, v6BootstrapDNS}, &l) + l := zap.NewNop() + return d.DialContext(ctx, "udp", []string{v4BootstrapDNS, v6BootstrapDNS}, l) }, }, } @@ -157,54 +156,190 @@ type parallelDialerResult struct { err error } +const ( + // unreachableBackoffBase is the initial suppression window applied to a + // dial address after it returns a network-unreachable error (e.g. + // "connect: no route to host"). The window grows exponentially up to + // unreachableBackoffMax on repeated failures, and is cleared as soon as + // the address dials successfully. + unreachableBackoffBase = 5 * time.Second + // unreachableBackoffMax caps the suppression window so an address is + // always re-probed within a bounded interval, preserving recovery when + // the route comes back. + unreachableBackoffMax = 60 * time.Second +) + +// Windows winsock codes for the unreachable errnos. A failing connect on +// Windows surfaces these raw WSA codes (WSAENETUNREACH/WSAEHOSTUNREACH), whereas +// syscall.ENETUNREACH/EHOSTUNREACH are Go's portable "invented" values +// (APPLICATION_ERROR + iota) that never equal them. Matching these explicitly is +// therefore required for the classifier to detect unreachable errors on Windows; +// errors.Is against the syscall.* constants alone would not. +// +// https://learn.microsoft.com/en-us/windows/win32/winsock/windows-sockets-error-codes-2 +var ( + windowsENETUNREACH = syscall.Errno(10051) + windowsEHOSTUNREACH = syscall.Errno(10065) +) + +// IsUnreachable reports whether err indicates the destination network or host +// has no route (ENETUNREACH/EHOSTUNREACH). These are the errors produced when +// an endpoint's address family is available locally but unroutable, e.g. an +// IPv6 DoH endpoint while the host has IPv6 but no route to it. +func IsUnreachable(err error) bool { + if err == nil { + return false + } + var opErr *net.OpError + if errors.As(err, &opErr) { + return errors.Is(opErr.Err, syscall.ENETUNREACH) || + errors.Is(opErr.Err, syscall.EHOSTUNREACH) || + errors.Is(opErr.Err, windowsENETUNREACH) || + errors.Is(opErr.Err, windowsEHOSTUNREACH) + } + return false +} + +type unreachableEntry struct { + until time.Time + backoff time.Duration +} + +// unreachableTracker records dial addresses that recently failed with a +// network-unreachable error so ParallelDialer can temporarily stop hammering +// them. This prevents an unroutable endpoint from generating a sustained dial +// /health-check storm, while still re-probing each address once its bounded +// backoff window expires so genuine recovery is never permanently blocked. +type unreachableTracker struct { + mu sync.Mutex + entries map[string]unreachableEntry +} + +var unreachable = &unreachableTracker{entries: make(map[string]unreachableEntry)} + +// suppressed reports whether addr is currently within its unreachable backoff +// window and should be skipped. +func (t *unreachableTracker) suppressed(addr string, now time.Time) bool { + t.mu.Lock() + defer t.mu.Unlock() + e, ok := t.entries[addr] + return ok && now.Before(e.until) +} + +// markUnreachable extends the suppression window for addr using a bounded +// exponential backoff. +func (t *unreachableTracker) markUnreachable(addr string, now time.Time) { + t.mu.Lock() + defer t.mu.Unlock() + e := t.entries[addr] + if e.backoff == 0 { + e.backoff = unreachableBackoffBase + } else { + e.backoff *= 2 + if e.backoff > unreachableBackoffMax { + e.backoff = unreachableBackoffMax + } + } + e.until = now.Add(e.backoff) + t.entries[addr] = e +} + +// markReachable clears any suppression for addr after a successful dial. +func (t *unreachableTracker) markReachable(addr string) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.entries, addr) +} + type ParallelDialer struct { net.Dialer } -func (d *ParallelDialer) DialContext(ctx context.Context, network string, addrs []string, logger *zerolog.Logger) (net.Conn, error) { +func (d *ParallelDialer) DialContext(ctx context.Context, network string, addrs []string, logger *zap.Logger) (net.Conn, error) { if len(addrs) == 0 { return nil, errors.New("empty addresses") } + + // Skip addresses that recently returned a network-unreachable error so an + // unroutable endpoint (e.g. an IPv6 DoH address while the host has IPv6 + // but no route to it) does not generate a sustained dial storm. Suppression + // is bounded: once an address's backoff window expires it is re-probed, so + // genuine recovery is preserved. + now := time.Now() + live := make([]string, 0, len(addrs)) + var suppressed int + for _, addr := range addrs { + if unreachable.suppressed(addr, now) { + suppressed++ + continue + } + live = append(live, addr) + } + if len(live) == 0 { + // Every candidate is within its unreachable backoff window. Fail fast + // and quietly instead of re-dialing known-unroutable addresses; the + // windows expire and re-probe shortly, so recovery still happens. + logger.Debug("Skipping unreachable addresses, all in backoff", zap.Int("suppressed", suppressed)) + // TODO: the errno here is hardcoded to EHOSTUNREACH, but the actual + // failure that triggered suppression may have been ENETUNREACH. This is + // harmless today (IsUnreachable treats both the same and nothing else + // inspects the errno), but if these errors are ever recorded/reported we + // should retain the real error in unreachableEntry and surface it here. + return nil, &net.OpError{Op: "dial", Net: network, Err: syscall.EHOSTUNREACH} + } + if suppressed > 0 { + logger.Debug("Skipping unreachable addresses in backoff", zap.Int("suppressed", suppressed)) + } + ctx, cancel := context.WithCancel(ctx) defer cancel() - done := make(chan struct{}) - defer close(done) - ch := make(chan *parallelDialerResult, len(addrs)) + ch := make(chan *parallelDialerResult, len(live)) var wg sync.WaitGroup - wg.Add(len(addrs)) + wg.Add(len(live)) go func() { wg.Wait() close(ch) }() - for _, addr := range addrs { + for _, addr := range live { go func(addr string) { defer wg.Done() - logger.Debug().Msgf("dialing to %s", addr) + logger.Debug("Dialing to", zap.String("address", addr)) conn, err := d.Dialer.DialContext(ctx, network, addr) if err != nil { - logger.Debug().Msgf("failed to dial %s: %v", addr, err) - } - select { - case ch <- ¶llelDialerResult{conn: conn, err: err}: - case <-done: - if conn != nil { - logger.Debug().Msgf("connection closed: %s", conn.RemoteAddr()) - conn.Close() + logger.Debug("Failed to dial", zap.String("address", addr), zap.Error(err)) + if IsUnreachable(err) { + unreachable.markUnreachable(addr, time.Now()) } + } else { + unreachable.markReachable(addr) } + ch <- ¶llelDialerResult{conn: conn, err: err} }(addr) } - errs := make([]error, 0, len(addrs)) + errs := make([]error, 0, len(live)) for res := range ch { if res.err == nil { cancel() - logger.Debug().Msgf("connected to %s", res.conn.RemoteAddr()) - return res.conn, res.err + logger.Debug("Connected to", zap.String("remote_address", res.conn.RemoteAddr().String())) + // Losing dials may already have connected, or may still connect + // after cancel; close whatever they hand back. + go closeLosingConns(ch, logger) + return res.conn, nil } errs = append(errs, res.err) } return nil, errors.Join(errs...) } + +// closeLosingConns drains ch and closes every connection left in it. +func closeLosingConns(ch <-chan *parallelDialerResult, logger *zap.Logger) { + for res := range ch { + if res.conn != nil { + logger.Debug("Connection closed", zap.String("remote_address", res.conn.RemoteAddr().String())) + _ = res.conn.Close() + } + } +} diff --git a/internal/net/net_test.go b/internal/net/net_test.go index 7df3e09c..38187c5c 100644 --- a/internal/net/net_test.go +++ b/internal/net/net_test.go @@ -2,10 +2,77 @@ package net import ( "context" + "net" + "syscall" "testing" "time" ) +func TestIsUnreachable(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"enetunreach", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ENETUNREACH}, true}, + {"ehostunreach", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.EHOSTUNREACH}, true}, + {"windows enetunreach", &net.OpError{Op: "dial", Net: "tcp", Err: windowsENETUNREACH}, true}, + {"windows ehostunreach", &net.OpError{Op: "dial", Net: "tcp", Err: windowsEHOSTUNREACH}, true}, + {"connection refused", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, false}, + {"not an opError", syscall.ENETUNREACH, false}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + if got := IsUnreachable(tc.err); got != tc.want { + t.Errorf("IsUnreachable(%v) = %v, want %v", tc.err, got, tc.want) + } + }) + } +} + +func TestUnreachableTracker(t *testing.T) { + tr := &unreachableTracker{entries: make(map[string]unreachableEntry)} + const addr = "[2606:1a40::22]:443" + now := time.Unix(0, 0) + + // Not suppressed before any failure. + if tr.suppressed(addr, now) { + t.Fatal("addr suppressed before any failure") + } + + // First failure suppresses for the base window. + tr.markUnreachable(addr, now) + if !tr.suppressed(addr, now.Add(unreachableBackoffBase-time.Millisecond)) { + t.Fatal("addr not suppressed within base backoff window") + } + if tr.suppressed(addr, now.Add(unreachableBackoffBase)) { + t.Fatal("addr still suppressed at end of base backoff window") + } + + // Backoff grows exponentially and is capped at the max. + tr.markUnreachable(addr, now) + if got := tr.entries[addr].backoff; got != 2*unreachableBackoffBase { + t.Fatalf("backoff after second failure = %v, want %v", got, 2*unreachableBackoffBase) + } + for i := 0; i < 10; i++ { + tr.markUnreachable(addr, now) + } + if got := tr.entries[addr].backoff; got != unreachableBackoffMax { + t.Fatalf("backoff not capped: got %v, want %v", got, unreachableBackoffMax) + } + + // A successful dial clears suppression entirely. + tr.markReachable(addr) + if tr.suppressed(addr, now) { + t.Fatal("addr still suppressed after markReachable") + } + if _, ok := tr.entries[addr]; ok { + t.Fatal("entry not removed after markReachable") + } +} + func TestProbeStackTimeout(t *testing.T) { done := make(chan struct{}) started := make(chan struct{}) diff --git a/internal/resolvconffile/dns.go b/internal/resolvconffile/dns.go index 0d532eb2..db987504 100644 --- a/internal/resolvconffile/dns.go +++ b/internal/resolvconffile/dns.go @@ -1,5 +1,3 @@ -//go:build !js && !windows - package resolvconffile import ( @@ -11,6 +9,7 @@ import ( const resolvconfPath = "/etc/resolv.conf" +// NameServersWithPort retrieves a list of nameservers with the default DNS port 53 appended to each address. func NameServersWithPort() []string { c, err := resolvconffile.ParseFile(resolvconfPath) if err != nil { @@ -23,16 +22,27 @@ func NameServersWithPort() []string { return ns } +// NameServers retrieves a list of nameservers from the /etc/resolv.conf file +// Returns an empty slice if reading fails. func NameServers() []string { - c, err := resolvconffile.ParseFile(resolvconfPath) + nss, _ := NameserversFromFile(resolvconfPath) + return nss +} + +// NameserversFromFile reads nameserver addresses from the specified resolv.conf file +// and returns them as a slice of strings. +// +// Returns an error if the file cannot be parsed. +func NameserversFromFile(path string) ([]string, error) { + c, err := resolvconffile.ParseFile(path) if err != nil { - return nil + return nil, err } ns := make([]string, 0, len(c.Nameservers)) for _, nameserver := range c.Nameservers { ns = append(ns, nameserver.String()) } - return ns + return ns, nil } // SearchDomains returns the current search domains config in /etc/resolv.conf file. diff --git a/internal/router/ddwrt/ddwrt.go b/internal/router/ddwrt/ddwrt.go deleted file mode 100644 index edd7e6b6..00000000 --- a/internal/router/ddwrt/ddwrt.go +++ /dev/null @@ -1,117 +0,0 @@ -package ddwrt - -import ( - "errors" - "fmt" - "os/exec" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - "github.com/Control-D-Inc/ctrld/internal/router/ntp" - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -const Name = "ddwrt" - -//lint:ignore ST1005 This error is for human. -var errDdwrtJffs2NotEnabled = errors.New(`could not install service without jffs, follow this guide to enable: - -https://wiki.dd-wrt.com/wiki/index.php/Journalling_Flash_File_System -`) - -var nvramKvMap = map[string]string{ - "dns_dnsmasq": "1", // Make dnsmasq running but disable DNS ability, ctrld will replace it. - "dnsmasq_options": "", // Configuration of dnsmasq set by ctrld, filled by setupDDWrt. - "dns_crypt": "0", // Disable DNSCrypt. - "dnssec": "0", // Disable DNSSEC. -} - -type Ddwrt struct { - cfg *ctrld.Config -} - -// New returns a router.Router for configuring/setup/run ctrld on ddwrt routers. -func New(cfg *ctrld.Config) *Ddwrt { - return &Ddwrt{cfg: cfg} -} - -func (d *Ddwrt) ConfigureService(config *service.Config) error { - if !ddwrtJff2Enabled() { - return errDdwrtJffs2NotEnabled - } - return nil -} - -func (d *Ddwrt) Install(_ *service.Config) error { - return nil -} - -func (d *Ddwrt) Uninstall(_ *service.Config) error { - return nil -} - -func (d *Ddwrt) PreRun() error { - _ = d.Cleanup() - return ntp.WaitNvram() -} - -func (d *Ddwrt) Setup() error { - if d.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Already setup. - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" { - return nil - } - - data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, d.cfg) - if err != nil { - return err - } - - nvramKvMap["dnsmasq_options"] = data - if err := nvram.SetKV(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func (d *Ddwrt) Cleanup() error { - if d.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val != "1" { - return nil // was restored, nothing to do. - } - - nvramKvMap["dnsmasq_options"] = "" - // Restore old configs. - if err := nvram.Restore(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func restartDNSMasq() error { - if out, err := exec.Command("restart_dns").CombinedOutput(); err != nil { - return fmt.Errorf("restart_dns: %s, %w", string(out), err) - } - return nil -} - -func ddwrtJff2Enabled() bool { - out, _ := nvram.Run("get", "enable_jffs2") - return out == "1" -} diff --git a/internal/router/dnsmasq/conf.go b/internal/router/dnsmasq/conf.go deleted file mode 100644 index bb81d607..00000000 --- a/internal/router/dnsmasq/conf.go +++ /dev/null @@ -1,90 +0,0 @@ -package dnsmasq - -import ( - "bufio" - "bytes" - "errors" - "io" - "os" - "path/filepath" - "strings" -) - -func InterfaceNameFromConfig(filename string) (string, error) { - buf, err := os.ReadFile(filename) - if err != nil { - return "", err - } - return interfaceNameFromReader(bytes.NewReader(buf)) -} - -func interfaceNameFromReader(r io.Reader) (string, error) { - scanner := bufio.NewScanner(r) - for scanner.Scan() { - line := scanner.Text() - after, found := strings.CutPrefix(line, "interface=") - if found { - return after, nil - } - } - return "", errors.New("not found") -} - -// AdditionalConfigFiles returns a list of Dnsmasq configuration files found in the "/tmp/etc" directory. -func AdditionalConfigFiles() []string { - if paths, err := filepath.Glob("/tmp/etc/dnsmasq-*.conf"); err == nil { - return paths - } - return nil -} - -// AdditionalLeaseFiles returns a list of lease file paths corresponding to the Dnsmasq configuration files. -func AdditionalLeaseFiles() []string { - cfgFiles := AdditionalConfigFiles() - if len(cfgFiles) == 0 { - return nil - } - leaseFiles := make([]string, 0, len(cfgFiles)) - for _, cfgFile := range cfgFiles { - if leaseFile := leaseFileFromConfigFileName(cfgFile); leaseFile != "" { - leaseFiles = append(leaseFiles, leaseFile) - - } else { - leaseFiles = append(leaseFiles, defaultLeaseFileFromConfigPath(cfgFile)) - } - } - return leaseFiles -} - -// leaseFileFromConfigFileName retrieves the DHCP lease file path by reading and parsing the provided configuration file. -func leaseFileFromConfigFileName(cfgFile string) string { - if f, err := os.Open(cfgFile); err == nil { - return leaseFileFromReader(f) - } - return "" -} - -// leaseFileFromReader parses the given io.Reader for the "dhcp-leasefile" configuration and returns its value as a string. -func leaseFileFromReader(r io.Reader) string { - scanner := bufio.NewScanner(r) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "#") { - continue - } - before, after, found := strings.Cut(line, "=") - if !found { - continue - } - if before == "dhcp-leasefile" { - return after - } - } - return "" -} - -// defaultLeaseFileFromConfigPath generates the default lease file path based on the provided configuration file path. -func defaultLeaseFileFromConfigPath(path string) string { - name := filepath.Base(path) - return filepath.Join("/var/lib/misc", strings.TrimSuffix(name, ".conf")+".leases") -} diff --git a/internal/router/dnsmasq/conf_test.go b/internal/router/dnsmasq/conf_test.go deleted file mode 100644 index 9ca672be..00000000 --- a/internal/router/dnsmasq/conf_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package dnsmasq - -import ( - "io" - "strings" - "testing" -) - -func Test_interfaceNameFromReader(t *testing.T) { - tests := []struct { - name string - in string - wantIface string - }{ - { - "good", - `interface=lo`, - "lo", - }, - { - "multiple", - `interface=lo -interface=eth0 -`, - "lo", - }, - { - "no iface", - `cache-size=100`, - "", - }, - } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - ifaceName, err := interfaceNameFromReader(strings.NewReader(tc.in)) - if tc.wantIface != "" && err != nil { - t.Errorf("unexpected error: %v", err) - return - } - if tc.wantIface != ifaceName { - t.Errorf("mismatched, want: %q, got: %q", tc.wantIface, ifaceName) - } - }) - } -} - -func Test_leaseFileFromReader(t *testing.T) { - tests := []struct { - name string - in io.Reader - expected string - }{ - { - "default", - strings.NewReader(` -dhcp-script=/sbin/dhcpc_lease -dhcp-leasefile=/var/lib/misc/dnsmasq-1.leases -script-arp -`), - "/var/lib/misc/dnsmasq-1.leases", - }, - { - "non-default", - strings.NewReader(` -dhcp-script=/sbin/dhcpc_lease -dhcp-leasefile=/tmp/var/lib/misc/dnsmasq-1.leases -script-arp -`), - "/tmp/var/lib/misc/dnsmasq-1.leases", - }, - { - "missing", - strings.NewReader(` -dhcp-script=/sbin/dhcpc_lease -script-arp -`), - "", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - if got := leaseFileFromReader(tc.in); got != tc.expected { - t.Errorf("leaseFileFromReader() = %v, want %v", got, tc.expected) - } - }) - } - -} diff --git a/internal/router/dnsmasq/dnsmasq.go b/internal/router/dnsmasq/dnsmasq.go deleted file mode 100644 index 058b0b59..00000000 --- a/internal/router/dnsmasq/dnsmasq.go +++ /dev/null @@ -1,190 +0,0 @@ -package dnsmasq - -import ( - "errors" - "html/template" - "net" - "os" - "path/filepath" - "strings" - - "github.com/Control-D-Inc/ctrld" -) - -const CtrldMarker = `# GENERATED BY ctrld - DO NOT MODIFY` - -const ConfigContentTmpl = `# GENERATED BY ctrld - DO NOT MODIFY -no-resolv -{{- range .Upstreams}} -server={{ .IP }}#{{ .Port }} -{{- end}} -add-mac -add-subnet=32,128 -{{- if .CacheDisabled}} -cache-size=0 -{{- else}} -max-cache-ttl=0 -{{- end}} -` - -const ( - MerlinConfPath = "/tmp/etc/dnsmasq.conf" - MerlinJffsConfDir = "/jffs/configs" - MerlinJffsConfPath = "/jffs/configs/dnsmasq.conf" - MerlinPostConfPath = "/jffs/scripts/dnsmasq.postconf" -) - -const MerlinPostConfMarker = `# GENERATED BY ctrld - EOF` -const MerlinPostConfTmpl = `# GENERATED BY ctrld - DO NOT MODIFY - -#!/bin/sh - -config_file="$1" -. /usr/sbin/helper.sh - -pid=$(cat /tmp/ctrld.pid 2>/dev/null) -if [ -n "$pid" ] && [ -f "/proc/${pid}/cmdline" ]; then - pc_delete "servers-file" "$config_file" # no WAN DNS settings - pc_append "no-resolv" "$config_file" # do not read /etc/resolv.conf - # use ctrld as upstream - pc_delete "server=" "$config_file" - {{- range .Upstreams}} - pc_append "server={{ .IP }}#{{ .Port }}" "$config_file" - {{- end}} - pc_delete "add-mac" "$config_file" - pc_delete "add-subnet" "$config_file" - pc_append "add-mac" "$config_file" # add client mac - pc_append "add-subnet=32,128" "$config_file" # add client ip - pc_delete "dnssec" "$config_file" # disable DNSSEC - pc_delete "trust-anchor=" "$config_file" # disable DNSSEC - pc_delete "cache-size=" "$config_file" - pc_append "cache-size=0" "$config_file" # disable cache - - # For John fork - pc_delete "resolv-file" "$config_file" # no WAN DNS settings - - # Change /etc/resolv.conf, which may be changed by WAN DNS setup - pc_delete "nameserver" /etc/resolv.conf - pc_append "nameserver 127.0.0.1" /etc/resolv.conf - - exit 0 -fi -` - -type Upstream struct { - IP string - Port int -} - -// ConfTmpl generates dnsmasq configuration from ctrld config. -func ConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) { - return ConfTmplWithCacheDisabled(tmplText, cfg, true) -} - -// ConfTmplWithCacheDisabled is like ConfTmpl, but the caller can control whether -// dnsmasq cache is disabled using cacheDisabled parameter. -// -// Generally, the caller should use ConfTmpl, but on some routers which dnsmasq config may be changed -// after ctrld started (like EdgeOS/Ubios, Firewalla ...), dnsmasq cache should not be disabled because -// the cache-size=0 generated by ctrld will conflict with router's generated config. -func ConfTmplWithCacheDisabled(tmplText string, cfg *ctrld.Config, cacheDisabled bool) (string, error) { - listener := cfg.FirstListener() - if listener == nil { - return "", errors.New("missing listener") - } - ip := listener.IP - if ip == "0.0.0.0" || ip == "::" || ip == "" { - ip = "127.0.0.1" - } - upstreams := []Upstream{{IP: ip, Port: listener.Port}} - return confTmpl(tmplText, upstreams, cacheDisabled) -} - -// FirewallaConfTmpl generates dnsmasq config for Firewalla routers. -func FirewallaConfTmpl(tmplText string, cfg *ctrld.Config) (string, error) { - // If ctrld listen on all interfaces, generating config for all of them. - if lc := cfg.FirstListener(); lc != nil && (lc.IP == "0.0.0.0" || lc.IP == "") { - return confTmpl(tmplText, firewallaUpstreams(lc.Port), false) - } - // Otherwise, generating config for the specific listener from ctrld's config. - return ConfTmplWithCacheDisabled(tmplText, cfg, false) -} - -func confTmpl(tmplText string, upstreams []Upstream, cacheDisabled bool) (string, error) { - tmpl := template.Must(template.New("").Parse(tmplText)) - var to = &struct { - Upstreams []Upstream - CacheDisabled bool - }{ - Upstreams: upstreams, - CacheDisabled: cacheDisabled, - } - var sb strings.Builder - if err := tmpl.Execute(&sb, to); err != nil { - return "", err - } - return sb.String(), nil -} - -func firewallaUpstreams(port int) []Upstream { - ifaces := FirewallaSelfInterfaces() - upstreams := make([]Upstream, 0, len(ifaces)) - for _, netIface := range ifaces { - addrs, _ := netIface.Addrs() - for _, addr := range addrs { - if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { - upstreams = append(upstreams, Upstream{ - IP: netIP.IP.To4().String(), - Port: port, - }) - } - } - } - return upstreams -} - -// firewallaDnsmasqConfFiles returns dnsmasq config files of all firewalla interfaces. -func firewallaDnsmasqConfFiles() ([]string, error) { - return filepath.Glob("/home/pi/firerouter/etc/dnsmasq.dns.*.conf") -} - -// FirewallaSelfInterfaces returns list of interfaces that will be configured with default dnsmasq setup on Firewalla. -func FirewallaSelfInterfaces() []*net.Interface { - matches, err := firewallaDnsmasqConfFiles() - if err != nil { - return nil - } - ifaces := make([]*net.Interface, 0, len(matches)) - for _, match := range matches { - // Trim prefix and suffix to get the iface name only. - ifaceName := strings.TrimSuffix(strings.TrimPrefix(match, "/home/pi/firerouter/etc/dnsmasq.dns."), ".conf") - if netIface, _ := net.InterfaceByName(ifaceName); netIface != nil { - ifaces = append(ifaces, netIface) - } - } - return ifaces -} - -const ( - ubios43ConfPath = "/run/dnsmasq.dhcp.conf.d" - ubios42ConfPath = "/run/dnsmasq.conf.d" - ubios43PidFile = "/run/dnsmasq-main.pid" - ubios42PidFile = "/run/dnsmasq.pid" - UbiosConfName = "zzzctrld.conf" -) - -// UbiosConfPath returns the appropriate configuration path based on the system's directory structure. -func UbiosConfPath() string { - if st, _ := os.Stat(ubios43ConfPath); st != nil && st.IsDir() { - return ubios43ConfPath - } - return ubios42ConfPath -} - -// UbiosPidFile returns the appropriate dnsmasq pid file based on the system's directory structure. -func UbiosPidFile() string { - if st, _ := os.Stat(ubios43PidFile); st != nil && !st.IsDir() { - return ubios43PidFile - } - return ubios42PidFile -} diff --git a/internal/router/edgeos/edgeos.go b/internal/router/edgeos/edgeos.go deleted file mode 100644 index 7364ac11..00000000 --- a/internal/router/edgeos/edgeos.go +++ /dev/null @@ -1,209 +0,0 @@ -package edgeos - -import ( - "bufio" - "bytes" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" -) - -const ( - Name = "edgeos" - edgeOSDNSMasqConfigPath = "/etc/dnsmasq.d/dnsmasq-zzz-ctrld.conf" - usgDNSMasqConfigPath = "/etc/dnsmasq.conf" - usgDNSMasqBackupConfigPath = "/etc/dnsmasq.conf.bak" - toggleContentFilteringLink = "https://community.ui.com/questions/UDM-Pro-disable-enable-DNS-filtering/e2cc4060-e56a-4139-b200-62d7f773ff8f" - toggleDnsShieldLink = "https://community.ui.com/questions/UniFi-OS-3-2-7-DNS-Shield-Missing/d3a85905-4ce0-4fe4-8bf0-6cb04f21371d" -) - -var ErrContentFilteringEnabled = fmt.Errorf(`the "Content Filtering" feature" is enabled, which is conflicted with ctrld.\n -To disable it, folowing instruction here: %s`, toggleContentFilteringLink) - -var ErrDnsShieldEnabled = fmt.Errorf(`the "DNS Shield" feature" is enabled, which is conflicted with ctrld.\n -To disable it, folowing screenshot here: %s`, toggleDnsShieldLink) - -type EdgeOS struct { - cfg *ctrld.Config - isUSG bool -} - -// New returns a router.Router for configuring/setup/run ctrld on EdgeOS routers. -func New(cfg *ctrld.Config) *EdgeOS { - e := &EdgeOS{cfg: cfg} - e.isUSG = checkUSG() - return e -} - -func (e *EdgeOS) ConfigureService(config *service.Config) error { - return nil -} - -func (e *EdgeOS) Install(_ *service.Config) error { - // If "Content Filtering" is enabled, UniFi OS will create firewall rules to intercept all DNS queries - // from outside, and route those queries to separated interfaces (e.g: dnsfilter-2@if79) created by UniFi OS. - // Thus, those queries will never reach ctrld listener. UniFi OS does not provide any mechanism to toggle this - // feature via command line, so there's nothing ctrld can do to disable this feature. For now, reporting an - // error and guiding users to disable the feature using UniFi OS web UI. - if ContentFilteringEnabled() { - return ErrContentFilteringEnabled - } - // If "DNS Shield" is enabled, UniFi OS will spawn dnscrypt-proxy process, and route all DNS queries to it. So - // reporting an error and guiding users to disable the feature using UniFi OS web UI. - if DnsShieldEnabled() { - return ErrDnsShieldEnabled - } - return nil -} - -func (e *EdgeOS) Uninstall(_ *service.Config) error { - return nil -} - -func (e *EdgeOS) PreRun() error { - return nil -} - -func (e *EdgeOS) Setup() error { - if e.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - if e.isUSG { - return e.setupUSG() - } - return e.setupUDM() -} - -func (e *EdgeOS) Cleanup() error { - if e.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - if e.isUSG { - return e.cleanupUSG() - } - return e.cleanupUDM() -} - -func (e *EdgeOS) setupUSG() error { - // On USG, dnsmasq is configured to forward queries to external provider by default. - // So instead of generating config in /etc/dnsmasq.d, we need to create a backup of - // the config, then modify it to forward queries to ctrld listener. - - // Creating a backup. - buf, err := os.ReadFile(usgDNSMasqConfigPath) - if err != nil { - return fmt.Errorf("setupUSG: reading current config: %w", err) - } - if err := os.WriteFile(usgDNSMasqBackupConfigPath, buf, 0600); err != nil { - return fmt.Errorf("setupUSG: backup current config: %w", err) - } - - // Removing all configured upstreams and cache config. - var sb strings.Builder - scanner := bufio.NewScanner(bytes.NewReader(buf)) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "server=") { - continue - } - if strings.HasPrefix(line, "all-servers") { - continue - } - sb.WriteString(line) - } - - data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, e.cfg, false) - if err != nil { - return err - } - sb.WriteString("\n") - sb.WriteString(data) - if err := os.WriteFile(usgDNSMasqConfigPath, []byte(sb.String()), 0644); err != nil { - return fmt.Errorf("setupUSG: writing dnsmasq config: %w", err) - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return fmt.Errorf("setupUSG: restartDNSMasq: %w", err) - } - return nil -} - -func (e *EdgeOS) setupUDM() error { - data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, e.cfg, false) - if err != nil { - return err - } - if err := os.WriteFile(edgeOSDNSMasqConfigPath, []byte(data), 0600); err != nil { - return fmt.Errorf("setupUDM: generating dnsmasq config: %w", err) - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return fmt.Errorf("setupUDM: restartDNSMasq: %w", err) - } - return nil -} - -func (e *EdgeOS) cleanupUSG() error { - if err := os.Rename(usgDNSMasqBackupConfigPath, usgDNSMasqConfigPath); err != nil { - return fmt.Errorf("cleanupUSG: os.Rename: %w", err) - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return fmt.Errorf("cleanupUSG: restartDNSMasq: %w", err) - } - return nil -} - -func (e *EdgeOS) cleanupUDM() error { - // Remove the custom dnsmasq config - if err := os.Remove(edgeOSDNSMasqConfigPath); err != nil { - return fmt.Errorf("cleanupUDM: os.Remove: %w", err) - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return fmt.Errorf("cleanupUDM: restartDNSMasq: %w", err) - } - return nil -} - -func ContentFilteringEnabled() bool { - st, err := os.Stat("/run/dnsfilter/dnsfilter") - return err == nil && !st.IsDir() -} - -// DnsShieldEnabled reports whether DNS Shield is enabled. -// See: https://community.ui.com/releases/UniFi-OS-Dream-Machines-3-2-7/251dfc1e-f4dd-4264-a080-3be9d8b9e02b -func DnsShieldEnabled() bool { - buf, err := os.ReadFile(filepath.Join(dnsmasq.UbiosConfPath(), "dns.conf")) - if err != nil { - return false - } - return bytes.Contains(buf, []byte("server=127.0.0.1#5053")) -} - -func LeaseFileDir() string { - if checkUSG() { - return "" - } - return "/run" -} - -func checkUSG() bool { - out, _ := os.ReadFile("/etc/version") - return bytes.HasPrefix(out, []byte("UniFiSecurityGateway.")) -} - -func restartDNSMasq() error { - if out, err := exec.Command("/etc/init.d/dnsmasq", "restart").CombinedOutput(); err != nil { - return fmt.Errorf("edgeosRestartDNSMasq: %s, %w", string(out), err) - } - return nil -} diff --git a/internal/router/firewalla/firewalla.go b/internal/router/firewalla/firewalla.go deleted file mode 100644 index cdf65864..00000000 --- a/internal/router/firewalla/firewalla.go +++ /dev/null @@ -1,110 +0,0 @@ -package firewalla - -import ( - "fmt" - "os" - "os/exec" - "strings" - - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - - "github.com/Control-D-Inc/ctrld" - "github.com/kardianos/service" -) - -const ( - Name = "firewalla" - - firewallaDNSMasqConfigPath = "/home/pi/.firewalla/config/dnsmasq_local/ctrld" - firewallaConfigPostMainDir = "/home/pi/.firewalla/config/post_main.d" - firewallaCtrldInitScriptPath = "/home/pi/.firewalla/config/post_main.d/start_ctrld.sh" -) - -type Firewalla struct { - cfg *ctrld.Config -} - -// New returns a router.Router for configuring/setup/run ctrld on Firewalla routers. -func New(cfg *ctrld.Config) *Firewalla { - return &Firewalla{cfg: cfg} -} - -func (f *Firewalla) ConfigureService(_ *service.Config) error { - return nil -} - -func (f *Firewalla) Install(_ *service.Config) error { - // Writing startup script. - if err := writeFirewallStartupScript(); err != nil { - return fmt.Errorf("writing startup script: %w", err) - } - return nil -} - -func (f *Firewalla) Uninstall(_ *service.Config) error { - // Removing startup script. - if err := os.Remove(firewallaCtrldInitScriptPath); err != nil { - return fmt.Errorf("removing startup script: %w", err) - } - return nil -} - -func (f *Firewalla) PreRun() error { - return nil -} - -func (f *Firewalla) Setup() error { - if f.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - data, err := dnsmasq.FirewallaConfTmpl(dnsmasq.ConfigContentTmpl, f.cfg) - if err != nil { - return fmt.Errorf("generating dnsmasq config: %w", err) - } - if err := os.WriteFile(firewallaDNSMasqConfigPath, []byte(data), 0600); err != nil { - return fmt.Errorf("writing ctrld config: %w", err) - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return fmt.Errorf("restartDNSMasq: %w", err) - } - - return nil -} - -func (f *Firewalla) Cleanup() error { - if f.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Removing current config. - if err := os.Remove(firewallaDNSMasqConfigPath); err != nil { - return fmt.Errorf("removing ctrld config: %w", err) - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return fmt.Errorf("restartDNSMasq: %w", err) - } - - return nil -} - -func writeFirewallStartupScript() error { - if err := os.MkdirAll(firewallaConfigPostMainDir, 0775); err != nil { - return err - } - exe, err := os.Executable() - if err != nil { - return err - } - // This is called when "ctrld start ..." runs, so recording - // the same command line arguments to use in startup script. - argStr := strings.Join(os.Args[1:], " ") - script := fmt.Sprintf("#!/bin/bash\n\nsudo %q %s\n", exe, argStr) - return os.WriteFile(firewallaCtrldInitScriptPath, []byte(script), 0755) -} - -func restartDNSMasq() error { - return exec.Command("systemctl", "restart", "firerouter_dns").Run() -} diff --git a/internal/router/merlin/merlin.go b/internal/router/merlin/merlin.go deleted file mode 100644 index c1c68210..00000000 --- a/internal/router/merlin/merlin.go +++ /dev/null @@ -1,266 +0,0 @@ -package merlin - -import ( - "bytes" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - "unicode" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - "github.com/Control-D-Inc/ctrld/internal/router/ntp" - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -const Name = "merlin" - -// nvramKvMap is a map of NVRAM key-value pairs used to configure and manage Merlin-specific settings. -var nvramKvMap = map[string]string{ - "dnspriv_enable": "0", // Ensure Merlin native DoT disabled. -} - -// dnsmasqConfig represents configuration paths for dnsmasq operations in Merlin firmware. -type dnsmasqConfig struct { - confPath string - jffsConfPath string -} - -// Merlin represents a configuration handler for setting up and managing ctrld on Merlin routers. -type Merlin struct { - cfg *ctrld.Config -} - -// New returns a router.Router for configuring/setup/run ctrld on Merlin routers. -func New(cfg *ctrld.Config) *Merlin { - return &Merlin{cfg: cfg} -} - -// ConfigureService configures the service based on the provided configuration. It returns an error if the configuration fails. -func (m *Merlin) ConfigureService(config *service.Config) error { - return nil -} - -// Install sets up the necessary configurations and services required for the Merlin instance to function properly. -func (m *Merlin) Install(_ *service.Config) error { - return nil -} - -// Uninstall removes the ctrld-related configurations and services from the Merlin router and reverts to the original state. -func (m *Merlin) Uninstall(_ *service.Config) error { - return nil -} - -// PreRun prepares the Merlin instance for operation by waiting for essential services and directories to become available. -func (m *Merlin) PreRun() error { - // Wait NTP ready. - _ = m.Cleanup() - if err := ntp.WaitNvram(); err != nil { - return err - } - // Wait until directories mounted. - for _, dir := range []string{"/tmp", "/proc"} { - waitDirExists(dir) - } - // Wait dnsmasq started. - for { - out, _ := exec.Command("pidof", "dnsmasq").CombinedOutput() - if len(bytes.TrimSpace(out)) > 0 { - break - } - time.Sleep(time.Second) - } - return nil -} - -// Setup initializes and configures the Merlin instance for use, including setting up dnsmasq and necessary nvram settings. -func (m *Merlin) Setup() error { - if m.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Already setup. - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" { - return nil - } - - if err := m.writeDnsmasqPostconf(); err != nil { - return err - } - - for _, cfg := range getDnsmasqConfigs() { - if err := m.setupDnsmasq(cfg); err != nil { - return fmt.Errorf("failed to setup dnsmasq: config: %s, error: %w", cfg.confPath, err) - } - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - - if err := nvram.SetKV(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - return nil -} - -// Cleanup restores the original dnsmasq and nvram configurations and restarts dnsmasq if necessary. -func (m *Merlin) Cleanup() error { - if m.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val != "1" { - return nil // was restored, nothing to do. - } - - // Restore old configs. - if err := nvram.Restore(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - buf, err := os.ReadFile(dnsmasq.MerlinPostConfPath) - if err != nil && !os.IsNotExist(err) { - return err - } - // Restore dnsmasq post conf file. - if err := os.WriteFile(dnsmasq.MerlinPostConfPath, merlinParsePostConf(buf), 0750); err != nil { - return err - } - - for _, cfg := range getDnsmasqConfigs() { - if err := m.cleanupDnsmasqJffs(cfg); err != nil { - return fmt.Errorf("failed to cleanup jffs dnsmasq: config: %s, error: %w", cfg.confPath, err) - } - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -// setupDnsmasq sets up dnsmasq configuration by writing postconf, copying configuration, and running a postconf script. -func (m *Merlin) setupDnsmasq(cfg *dnsmasqConfig) error { - src, err := os.Open(cfg.confPath) - if os.IsNotExist(err) { - return nil // nothing to do if conf file does not exist. - } - if err != nil { - return fmt.Errorf("failed to open dnsmasq config: %w", err) - } - defer src.Close() - - // Copy current dnsmasq config to cfg.jffsConfPath, - // Then we will run postconf script on this file. - // - // Normally, adding postconf script is enough. However, we see - // reports on some Merlin devices that postconf scripts does not - // work, but manipulating the config directly via /jffs/configs does. - dst, err := os.Create(cfg.jffsConfPath) - if err != nil { - return fmt.Errorf("failed to create %s: %w", cfg.jffsConfPath, err) - } - defer dst.Close() - - if _, err := io.Copy(dst, src); err != nil { - return fmt.Errorf("failed to copy current dnsmasq config: %w", err) - } - if err := dst.Close(); err != nil { - return fmt.Errorf("failed to save %s: %w", cfg.jffsConfPath, err) - } - - // Run postconf script on cfg.jffsConfPath directly. - cmd := exec.Command("/bin/sh", dnsmasq.MerlinPostConfPath, cfg.jffsConfPath) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to run post conf: %s: %w", string(out), err) - } - return nil -} - -// cleanupDnsmasqJffs removes the JFFS configuration file specified in the given dnsmasqConfig, if it exists. -func (m *Merlin) cleanupDnsmasqJffs(cfg *dnsmasqConfig) error { - // Remove cfg.jffsConfPath file. - if err := os.Remove(cfg.jffsConfPath); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// writeDnsmasqPostconf writes the requireddnsmasqConfigs post-configuration for dnsmasq to enable custom DNS settings with ctrld. -func (m *Merlin) writeDnsmasqPostconf() error { - buf, err := os.ReadFile(dnsmasq.MerlinPostConfPath) - // Already setup. - if bytes.Contains(buf, []byte(dnsmasq.MerlinPostConfMarker)) { - return nil - } - if err != nil && !os.IsNotExist(err) { - return err - } - - data, err := dnsmasq.ConfTmpl(dnsmasq.MerlinPostConfTmpl, m.cfg) - if err != nil { - return err - } - data = strings.Join([]string{ - data, - "\n", - dnsmasq.MerlinPostConfMarker, - "\n", - string(buf), - }, "\n") - // Write dnsmasq post conf file. - return os.WriteFile(dnsmasq.MerlinPostConfPath, []byte(data), 0750) -} - -// restartDNSMasq restarts the dnsmasq service by executing the appropriate system command using "service". -// Returns an error if the command fails or if there is an issue processing the command output. -func restartDNSMasq() error { - if out, err := exec.Command("service", "restart_dnsmasq").CombinedOutput(); err != nil { - return fmt.Errorf("restart_dnsmasq: %s, %w", string(out), err) - } - return nil -} - -// getDnsmasqConfigs retrieves a list of dnsmasqConfig containing configuration and JFFS paths for dnsmasq operations. -func getDnsmasqConfigs() []*dnsmasqConfig { - cfgs := []*dnsmasqConfig{ - {dnsmasq.MerlinConfPath, dnsmasq.MerlinJffsConfPath}, - } - for _, path := range dnsmasq.AdditionalConfigFiles() { - jffsConfPath := filepath.Join(dnsmasq.MerlinJffsConfDir, filepath.Base(path)) - cfgs = append(cfgs, &dnsmasqConfig{path, jffsConfPath}) - } - - return cfgs -} - -// merlinParsePostConf parses the dnsmasq post configuration by removing content after the MerlinPostConfMarker, if present. -// If no marker is found, the original buffer is returned unmodified. -// Returns nil if the input buffer is empty. -func merlinParsePostConf(buf []byte) []byte { - if len(buf) == 0 { - return nil - } - parts := bytes.Split(buf, []byte(dnsmasq.MerlinPostConfMarker)) - if len(parts) != 1 { - return bytes.TrimLeftFunc(parts[1], unicode.IsSpace) - } - return buf -} - -// waitDirExists waits until the specified directory exists, polling its existence every second. -func waitDirExists(dir string) { - for { - if _, err := os.Stat(dir); !os.IsNotExist(err) { - return - } - time.Sleep(time.Second) - } -} diff --git a/internal/router/merlin/merlin_test.go b/internal/router/merlin/merlin_test.go deleted file mode 100644 index 057628cd..00000000 --- a/internal/router/merlin/merlin_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package merlin - -import ( - "bytes" - "strings" - "testing" - - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" -) - -func Test_merlinParsePostConf(t *testing.T) { - origContent := "# foo" - data := strings.Join([]string{ - dnsmasq.MerlinPostConfTmpl, - "\n", - dnsmasq.MerlinPostConfMarker, - "\n", - }, "\n") - - tests := []struct { - name string - data string - expected string - }{ - {"empty", "", ""}, - {"no ctrld", origContent, origContent}, - {"ctrld with data", data + origContent, origContent}, - {"ctrld without data", data, ""}, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - //t.Parallel() - if got := merlinParsePostConf([]byte(tc.data)); !bytes.Equal(got, []byte(tc.expected)) { - t.Errorf("unexpected result, want: %q, got: %q", tc.expected, string(got)) - } - }) - } -} diff --git a/internal/router/netgear_orbi_voxel/procd.go b/internal/router/netgear_orbi_voxel/procd.go deleted file mode 100644 index 750a17da..00000000 --- a/internal/router/netgear_orbi_voxel/procd.go +++ /dev/null @@ -1,22 +0,0 @@ -package netgear - -const openWrtScript = `#!/bin/sh /etc/rc.common -USE_PROCD=1 -# After dnsmasq starts -START=61 -# Before network stops -STOP=89 -cmd="{{.Path}}{{range .Arguments}} {{.|cmd}}{{end}}" -name="{{.Name}}" -pid_file="/var/run/${name}.pid" - -start_service() { - echo "Starting ${name}" - procd_open_instance - procd_set_param command ${cmd} - procd_set_param respawn # respawn automatically if something died - procd_set_param pidfile ${pid_file} # write a pid file on instance start and remove it on stop - procd_close_instance - echo "${name} has been started" -} -` diff --git a/internal/router/netgear_orbi_voxel/voxel.go b/internal/router/netgear_orbi_voxel/voxel.go deleted file mode 100644 index 4338f9c6..00000000 --- a/internal/router/netgear_orbi_voxel/voxel.go +++ /dev/null @@ -1,220 +0,0 @@ -package netgear - -import ( - "bufio" - "bytes" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -const ( - Name = "netgear_orbi_voxel" - netgearOrbiVoxelDNSMasqConfigPath = "/etc/dnsmasq.conf" - netgearOrbiVoxelHomedir = "/mnt/bitdefender" - netgearOrbiVoxelStartupScript = "/mnt/bitdefender/rc.user" - netgearOrbiVoxelStartupScriptBackup = "/mnt/bitdefender/rc.user.bak" - netgearOrbiVoxelStartupScriptMarker = "\n# GENERATED BY ctrld" -) - -var nvramKvMap = map[string]string{ - "dns_hijack": "0", // Disable dns hijacking -} - -type NetgearOrbiVoxel struct { - cfg *ctrld.Config -} - -// New returns a router.Router for configuring/setup/run ctrld on ddwrt routers. -func New(cfg *ctrld.Config) *NetgearOrbiVoxel { - return &NetgearOrbiVoxel{cfg: cfg} -} - -func (d *NetgearOrbiVoxel) ConfigureService(svc *service.Config) error { - if err := d.checkInstalledDir(); err != nil { - return err - } - svc.Option["SysvScript"] = openWrtScript - return nil -} - -func (d *NetgearOrbiVoxel) Install(_ *service.Config) error { - // Ignoring error here at this moment is ok, since everything will be wiped out on reboot. - _ = exec.Command("/etc/init.d/ctrld", "enable").Run() - if err := d.checkInstalledDir(); err != nil { - return err - } - if err := backupVoxelStartupScript(); err != nil { - return fmt.Errorf("backup startup script: %w", err) - } - if err := writeVoxelStartupScript(); err != nil { - return fmt.Errorf("writing startup script: %w", err) - } - return nil -} - -func (d *NetgearOrbiVoxel) Uninstall(_ *service.Config) error { - if err := os.Remove(netgearOrbiVoxelStartupScript); err != nil && !os.IsNotExist(err) { - return err - } - err := os.Rename(netgearOrbiVoxelStartupScriptBackup, netgearOrbiVoxelStartupScript) - if err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -func (d *NetgearOrbiVoxel) PreRun() error { - return nil -} - -func (d *NetgearOrbiVoxel) Setup() error { - if d.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Already setup. - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" { - return nil - } - - data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, d.cfg, false) - if err != nil { - return err - } - currentConfig, _ := os.ReadFile(netgearOrbiVoxelDNSMasqConfigPath) - configContent := append(currentConfig, data...) - if err := os.WriteFile(netgearOrbiVoxelDNSMasqConfigPath, configContent, 0600); err != nil { - return err - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - - if err := nvram.SetKV(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - return nil -} - -func (d *NetgearOrbiVoxel) Cleanup() error { - if d.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val != "1" { - return nil // was restored, nothing to do. - } - - // Restore old configs. - if err := nvram.Restore(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - // Restore dnsmasq config. - if err := restoreDnsmasqConf(); err != nil { - return err - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -// checkInstalledDir checks that ctrld binary was installed in the correct directory. -func (d *NetgearOrbiVoxel) checkInstalledDir() error { - exePath, err := os.Executable() - if err != nil { - return fmt.Errorf("checkHomeDir: failed to get binary path %w", err) - } - if !strings.HasSuffix(filepath.Dir(exePath), netgearOrbiVoxelHomedir) { - return fmt.Errorf("checkHomeDir: could not install service outside %s", netgearOrbiVoxelHomedir) - } - return nil -} - -// backupVoxelStartupScript creates a backup of original startup script if existed. -func backupVoxelStartupScript() error { - // Do nothing if the startup script was modified by ctrld. - script, _ := os.ReadFile(netgearOrbiVoxelStartupScript) - if bytes.Contains(script, []byte(netgearOrbiVoxelStartupScriptMarker)) { - return nil - } - err := os.Rename(netgearOrbiVoxelStartupScript, netgearOrbiVoxelStartupScriptBackup) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("backupVoxelStartupScript: %w", err) - } - return nil -} - -// writeVoxelStartupScript writes startup script to re-install ctrld upon reboot. -// See: https://github.com/SVoxel/ORBI-RBK50/pull/7 -func writeVoxelStartupScript() error { - exe, err := os.Executable() - if err != nil { - return fmt.Errorf("configure service: failed to get binary path %w", err) - } - // This is called when "ctrld start ..." runs, so recording - // the same command line arguments to use in startup script. - argStr := strings.Join(os.Args[1:], " ") - script, _ := os.ReadFile(netgearOrbiVoxelStartupScriptBackup) - script = append(script, fmt.Sprintf("%s\n%q %s\n", netgearOrbiVoxelStartupScriptMarker, exe, argStr)...) - f, err := os.Create(netgearOrbiVoxelStartupScript) - if err != nil { - return fmt.Errorf("failed to create startup script: %w", err) - } - defer f.Close() - - if _, err := f.Write(script); err != nil { - return fmt.Errorf("failed to write startup script: %w", err) - } - if err := f.Close(); err != nil { - return fmt.Errorf("failed to save startup script: %w", err) - } - return nil -} - -// restoreDnsmasqConf restores original dnsmasq configuration. -func restoreDnsmasqConf() error { - f, err := os.Open(netgearOrbiVoxelDNSMasqConfigPath) - if err != nil { - return err - } - defer f.Close() - - var bs []byte - buf := bytes.NewBuffer(bs) - - removed := false - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - if line == dnsmasq.CtrldMarker { - removed = true - } - if !removed { - _, err := buf.WriteString(line + "\n") - if err != nil { - return err - } - } - } - return os.WriteFile(netgearOrbiVoxelDNSMasqConfigPath, buf.Bytes(), 0644) -} - -func restartDNSMasq() error { - if out, err := exec.Command("/etc/init.d/dnsmasq", "restart").CombinedOutput(); err != nil { - return fmt.Errorf("restartDNSMasq: %s, %w", string(out), err) - } - return nil -} diff --git a/internal/router/ntp/ntp.go b/internal/router/ntp/ntp.go deleted file mode 100644 index 5c04a36d..00000000 --- a/internal/router/ntp/ntp.go +++ /dev/null @@ -1,49 +0,0 @@ -package ntp - -import ( - "bytes" - "context" - "errors" - "fmt" - "os/exec" - "time" - - "tailscale.com/logtail/backoff" - - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -// WaitNvram waits NTP synced by checking "ntp_ready" value using nvram. -func WaitNvram() error { - // Wait until `ntp_ready=1` set. - b := backoff.NewBackoff("ntp.Wait", func(format string, args ...any) {}, 10*time.Second) - for { - // ddwrt use "ntp_done": https://github.com/mirror/dd-wrt/blob/a08c693527ab3204bf7bebd408a7c9a83b6ede47/src/router/rc/ntp.c#L100 - for _, key := range []string{"ntp_ready", "ntp_done"} { - out, err := nvram.Run("get", key) - if err != nil { - return fmt.Errorf("PreStart: nvram: %w", err) - } - if out == "1" { - return nil - } - } - b.BackOff(context.Background(), errors.New("ntp not ready")) - } -} - -// WaitUpstart waits NTP synced by checking upstart task "ntpsync" is in "stop/waiting" state. -func WaitUpstart() error { - // Wait until `initctl status ntpsync` returns stop state. - b := backoff.NewBackoff("ntp.WaitUpstart", func(format string, args ...any) {}, 10*time.Second) - for { - out, err := exec.Command("initctl", "status", "ntpsync").CombinedOutput() - if err != nil { - return fmt.Errorf("exec.Command: %w", err) - } - if bytes.Contains(out, []byte("stop/waiting")) { - return nil - } - b.BackOff(context.Background(), errors.New("ntp not ready")) - } -} diff --git a/internal/router/nvram/nvram.go b/internal/router/nvram/nvram.go deleted file mode 100644 index e76c0171..00000000 --- a/internal/router/nvram/nvram.go +++ /dev/null @@ -1,89 +0,0 @@ -package nvram - -import ( - "bytes" - "fmt" - "os/exec" - "strings" -) - -const ( - CtrldKeyPrefix = "ctrld_" - CtrldSetupKey = "ctrld_setup" - CtrldInstallKey = "ctrld_install" - RCStartupKey = "rc_startup" -) - -// Run runs the given nvram command. -func Run(args ...string) (string, error) { - cmd := exec.Command("nvram", args...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - return "", fmt.Errorf("%s:%w", stderr.String(), err) - } - return strings.TrimSpace(stdout.String()), nil -} - -/* -NOTE: - - For Openwrt, DNSSEC is not included in default dnsmasq (require dnsmasq-full). - - For Merlin, DNSSEC is configured during postconf script (see merlinDNSMasqPostConfTmpl). - - For Ubios UDM Pro/Dream Machine, DNSSEC is not included in their dnsmasq package: - +https://community.ui.com/questions/Implement-DNSSEC-into-UniFi/951c72b0-4d88-4c86-9174-45417bd2f9ca - +https://community.ui.com/questions/Enable-DNSSEC-for-Unifi-Dream-Machine-FW-updates/e68e367c-d09b-4459-9444-18908f7c1ea1 -*/ - -// SetKV writes the given key/value from map to nvram. -// The given setupKey is set to 1 to indicates key/value set. -func SetKV(m map[string]string, setupKey string) error { - // Backup current value, store ctrld's configs. - for key, value := range m { - old, err := Run("get", key) - if err != nil { - return fmt.Errorf("%s: %w", old, err) - } - if out, err := Run("set", CtrldKeyPrefix+key+"="+old); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - if out, err := Run("set", key+"="+value); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - } - - if out, err := Run("set", setupKey+"=1"); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - // Commit. - if out, err := Run("commit"); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - return nil -} - -// Restore restores the old value of given key from map m. -// The given setupKey is set to 0 to indicates key/value restored. -func Restore(m map[string]string, setupKey string) error { - // Restore old configs. - for key := range m { - ctrldKey := CtrldKeyPrefix + key - old, err := Run("get", ctrldKey) - if err != nil { - return fmt.Errorf("%s: %w", old, err) - } - _, _ = Run("unset", ctrldKey) - if out, err := Run("set", key+"="+old); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - } - - if out, err := Run("unset", setupKey); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - // Commit. - if out, err := Run("commit"); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - return nil -} diff --git a/internal/router/openwrt/openwrt.go b/internal/router/openwrt/openwrt.go deleted file mode 100644 index 73f5a06f..00000000 --- a/internal/router/openwrt/openwrt.go +++ /dev/null @@ -1,191 +0,0 @@ -package openwrt - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" -) - -const ( - Name = "openwrt" - openwrtDNSMasqConfigName = "ctrld.conf" - openwrtDNSMasqDefaultConfigDir = "/tmp/dnsmasq.d" -) - -var openwrtDnsmasqDefaultConfigPath = filepath.Join(openwrtDNSMasqDefaultConfigDir, openwrtDNSMasqConfigName) - -type Openwrt struct { - cfg *ctrld.Config - dnsmasqCacheSize string -} - -// New returns a router.Router for configuring/setup/run ctrld on Openwrt routers. -func New(cfg *ctrld.Config) *Openwrt { - return &Openwrt{cfg: cfg} -} - -func (o *Openwrt) ConfigureService(svc *service.Config) error { - svc.Option["SysvScript"] = openWrtScript - return nil -} - -func (o *Openwrt) Install(config *service.Config) error { - return exec.Command("/etc/init.d/ctrld", "enable").Run() -} - -func (o *Openwrt) Uninstall(config *service.Config) error { - return nil -} - -func (o *Openwrt) PreRun() error { - return nil -} - -func (o *Openwrt) Setup() error { - if o.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - - // Save current dnsmasq config cache size if present. - if cs, err := uci("get", "dhcp.@dnsmasq[0].cachesize"); err == nil { - o.dnsmasqCacheSize = cs - if _, err := uci("delete", "dhcp.@dnsmasq[0].cachesize"); err != nil { - return err - } - // Commit. - if _, err := uci("commit", "dhcp"); err != nil { - return err - } - } - - data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, o.cfg) - if err != nil { - return err - } - if err := os.WriteFile(dnsmasqConfPathFromUbus(), []byte(data), 0600); err != nil { - return err - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func (o *Openwrt) Cleanup() error { - if o.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Remove the custom dnsmasq config - if err := os.Remove(dnsmasqConfPathFromUbus()); err != nil { - return err - } - - // Restore original value if present. - if o.dnsmasqCacheSize != "" { - if _, err := uci("set", fmt.Sprintf("dhcp.@dnsmasq[0].cachesize=%s", o.dnsmasqCacheSize)); err != nil { - return err - } - // Commit. - if _, err := uci("commit", "dhcp"); err != nil { - return err - } - } - - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func restartDNSMasq() error { - if out, err := exec.Command("/etc/init.d/dnsmasq", "restart").CombinedOutput(); err != nil { - return fmt.Errorf("%s: %w", string(out), err) - } - return nil -} - -var errUCIEntryNotFound = errors.New("uci: Entry not found") - -func uci(args ...string) (string, error) { - cmd := exec.Command("uci", args...) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - if strings.HasPrefix(stderr.String(), errUCIEntryNotFound.Error()) { - return "", errUCIEntryNotFound - } - return "", fmt.Errorf("%s:%w", stderr.String(), err) - } - return strings.TrimSpace(stdout.String()), nil -} - -// openwrtServiceList represents openwrt services config. -type openwrtServiceList struct { - Dnsmasq dnsmasqConf `json:"dnsmasq"` -} - -// dnsmasqConf represents dnsmasq config. -type dnsmasqConf struct { - Instances map[string]confInstances `json:"instances"` -} - -// confInstances represents an instance config of a service. -type confInstances struct { - Mount map[string]string `json:"mount"` -} - -// dnsmasqConfPath returns the dnsmasq config path. -// -// Since version 24.10, openwrt makes some changes to dnsmasq to support -// multiple instances of dnsmasq. This change causes breaking changes to -// software which depends on the default dnsmasq path. -// -// There are some discussion/PRs in openwrt repo to address this: -// -// - https://github.com/openwrt/openwrt/pull/16806 -// - https://github.com/openwrt/openwrt/pull/16890 -// -// In the meantime, workaround this problem by querying the actual config path -// by querying ubus service list. -func dnsmasqConfPath(r io.Reader) string { - var svc openwrtServiceList - if err := json.NewDecoder(r).Decode(&svc); err != nil { - return openwrtDnsmasqDefaultConfigPath - } - for _, inst := range svc.Dnsmasq.Instances { - for mount := range inst.Mount { - dirName := filepath.Base(mount) - parts := strings.Split(dirName, ".") - if len(parts) < 2 { - continue - } - if parts[0] == "dnsmasq" && parts[len(parts)-1] == "d" { - return filepath.Join(mount, openwrtDNSMasqConfigName) - } - } - } - return openwrtDnsmasqDefaultConfigPath -} - -// dnsmasqConfPathFromUbus get dnsmasq config path from ubus service list. -func dnsmasqConfPathFromUbus() string { - output, err := exec.Command("ubus", "call", "service", "list").Output() - if err != nil { - return openwrtDnsmasqDefaultConfigPath - } - return dnsmasqConfPath(bytes.NewReader(output)) -} diff --git a/internal/router/openwrt/openwrt_test.go b/internal/router/openwrt/openwrt_test.go deleted file mode 100644 index 8b260e88..00000000 --- a/internal/router/openwrt/openwrt_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package openwrt - -import ( - "io" - "path/filepath" - "strings" - "testing" -) - -// Sample output from https://github.com/openwrt/openwrt/pull/16806#issuecomment-2448255734 -const ubusDnsmasqBefore2410 = `{ - "dnsmasq": { - "instances": { - "guest_dns": { - "mount": { - "/tmp/dnsmasq.d": "0", - "/var/run/dnsmasq/": "1" - } - } - } - } -}` - -const ubusDnsmasq2410 = `{ - "dnsmasq": { - "instances": { - "guest_dns": { - "mount": { - "/tmp/dnsmasq.guest_dns.d": "0", - "/var/run/dnsmasq/": "1" - } - } - } - } -}` - -func Test_dnsmasqConfPath(t *testing.T) { - var dnsmasq2410expected = filepath.Join("/tmp/dnsmasq.guest_dns.d", openwrtDNSMasqConfigName) - tests := []struct { - name string - in io.Reader - expected string - }{ - {"empty", strings.NewReader(""), openwrtDnsmasqDefaultConfigPath}, - {"invalid", strings.NewReader("}}"), openwrtDnsmasqDefaultConfigPath}, - {"before 24.10", strings.NewReader(ubusDnsmasqBefore2410), openwrtDnsmasqDefaultConfigPath}, - {"24.10", strings.NewReader(ubusDnsmasq2410), dnsmasq2410expected}, - } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - if got := dnsmasqConfPath(tc.in); got != tc.expected { - t.Errorf("dnsmasqConfPath() = %v, want %v", got, tc.expected) - } - }) - } -} diff --git a/internal/router/openwrt/procd.go b/internal/router/openwrt/procd.go deleted file mode 100644 index bf7253e6..00000000 --- a/internal/router/openwrt/procd.go +++ /dev/null @@ -1,25 +0,0 @@ -package openwrt - -const openWrtScript = `#!/bin/sh /etc/rc.common -USE_PROCD=1 -# After network starts -START=21 -# Before network stops -STOP=89 -cmd="{{.Path}}{{range .Arguments}} {{.|cmd}}{{end}}" -name="{{.Name}}" -pid_file="/var/run/${name}.pid" - -start_service() { - echo "Starting ${name}" - procd_open_instance - procd_set_param command ${cmd} - procd_set_param respawn # respawn automatically if something died - procd_set_param stdout 1 # forward stdout of the command to logd - procd_set_param stderr 1 # same for stderr - procd_set_param pidfile ${pid_file} # write a pid file on instance start and remove it on stop - procd_set_param term_timeout 10 - procd_close_instance - echo "${name} has been started" -} -` diff --git a/internal/router/os_config_freebsd.go b/internal/router/os_config_freebsd.go deleted file mode 100644 index 9066191e..00000000 --- a/internal/router/os_config_freebsd.go +++ /dev/null @@ -1,40 +0,0 @@ -package router - -import ( - "encoding/xml" - "os" -) - -// Config represents /conf/config.xml file found on pfsense/opnsense. -type Config struct { - PfsenseUnbound *string `xml:"unbound>enable,omitempty"` - OPNsenseUnbound *string `xml:"OPNsense>unboundplus>general>enabled,omitempty"` - Dnsmasq *string `xml:"dnsmasq>enable,omitempty"` -} - -// DnsmasqEnabled reports whether dnsmasq is enabled. -func (c *Config) DnsmasqEnabled() bool { - if isPfsense() { // pfsense only set the attribute if dnsmasq is enabled. - return c.Dnsmasq != nil - } - return c.Dnsmasq != nil && *c.Dnsmasq == "1" -} - -// UnboundEnabled reports whether unbound is enabled. -func (c *Config) UnboundEnabled() bool { - if isPfsense() { // pfsense only set the attribute if unbound is enabled. - return c.PfsenseUnbound != nil - } - return c.OPNsenseUnbound != nil && *c.OPNsenseUnbound == "1" -} - -// currentConfig does unmarshalling /conf/config.xml file, -// return the corresponding *Config represent it. -func currentConfig() (*Config, error) { - buf, _ := os.ReadFile("/conf/config.xml") - c := Config{} - if err := xml.Unmarshal(buf, &c); err != nil { - return nil, err - } - return &c, nil -} diff --git a/internal/router/os_freebsd.go b/internal/router/os_freebsd.go deleted file mode 100644 index 9a79188f..00000000 --- a/internal/router/os_freebsd.go +++ /dev/null @@ -1,157 +0,0 @@ -package router - -import ( - "bytes" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "text/template" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" -) - -const ( - osName = "freebsd" - rcPath = "/usr/local/etc/rc.d" - rcConfPath = "/etc/rc.conf.d/" - unboundRcPath = rcPath + "/unbound" - dnsmasqRcPath = rcPath + "/dnsmasq" -) - -func newOsRouter(cfg *ctrld.Config, cdMode bool) Router { - return &osRouter{cfg: cfg, cdMode: cdMode} -} - -type osRouter struct { - cfg *ctrld.Config - svcName string - // cdMode indicates whether the router will configure ctrld in cd mode (aka --cd=). - // When ctrld is running on freebsd-like routers, and there's process running on port 53 - // in cd mode, ctrld will attempt to kill the process and become direct listener. - // See details implemenation in osRouter.PreRun method. - cdMode bool -} - -func (or *osRouter) ConfigureService(svc *service.Config) error { - svc.Option["SysvScript"] = bsdInitScript - or.svcName = svc.Name - rcFile := filepath.Join(rcConfPath, or.svcName) - var to = &struct { - Name string - }{ - or.svcName, - } - - f, err := os.Create(rcFile) - if err != nil { - return fmt.Errorf("os.Create: %w", err) - } - defer f.Close() - if err := template.Must(template.New("").Parse(rcConfTmpl)).Execute(f, to); err != nil { - return err - } - return f.Close() -} - -func (or *osRouter) Install(_ *service.Config) error { - if isPfsense() { - // pfsense need ".sh" extension for script to be run at boot. - // See: https://docs.netgate.com/pfsense/en/latest/development/boot-commands.html#shell-script-option - oldname := filepath.Join(rcPath, or.svcName) - newname := filepath.Join(rcPath, or.svcName+".sh") - _ = os.Remove(newname) - if err := os.Symlink(oldname, newname); err != nil { - return fmt.Errorf("os.Symlink: %w", err) - } - } - return nil -} - -func (or *osRouter) Uninstall(_ *service.Config) error { - rcFiles := []string{filepath.Join(rcConfPath, or.svcName)} - if isPfsense() { - rcFiles = append(rcFiles, filepath.Join(rcPath, or.svcName+".sh")) - } - for _, filename := range rcFiles { - if err := os.Remove(filename); err != nil { - return fmt.Errorf("os.Remove: %w", err) - } - } - - return nil -} - -func (or *osRouter) PreRun() error { - if or.cdMode { - addr := "0.0.0.0:53" - udpLn, udpErr := net.ListenPacket("udp", addr) - if udpLn != nil { - udpLn.Close() - } - tcpLn, tcpErr := net.Listen("tcp", addr) - if tcpLn != nil { - tcpLn.Close() - } - // If we could not listen on :53 for any reason, try killing unbound/dnsmasq, become direct listener - if udpErr != nil || tcpErr != nil { - _ = exec.Command("killall", "unbound").Run() - _ = exec.Command("killall", "dnsmasq").Run() - } - } - return nil -} - -func (or *osRouter) Setup() error { - return nil -} - -func (or *osRouter) Cleanup() error { - if or.cdMode { - c, err := currentConfig() - if err != nil { - return err - } - if c.UnboundEnabled() { - _ = exec.Command(unboundRcPath, "onerestart").Run() - } - if c.DnsmasqEnabled() { - _ = exec.Command(dnsmasqRcPath, "onerestart").Run() - } - } - return nil -} - -func isPfsense() bool { - b, err := os.ReadFile("/etc/platform") - return err == nil && bytes.HasPrefix(b, []byte("pfSense")) -} - -const bsdInitScript = `#!/bin/sh - -# PROVIDE: {{.Name}} -# REQUIRE: SERVERS -# REQUIRE: unbound dnsmasq securelevel -# KEYWORD: shutdown - -. /etc/rc.subr - -name="{{.Name}}" -rcvar="${name}_enable" -{{.Name}}_env="IS_DAEMON=1" -pidfile="/var/run/${name}.pid" -child_pidfile="/var/run/${name}_child.pid" -command="/usr/sbin/daemon" -daemon_args="-r -P ${pidfile} -p ${child_pidfile} -t \"${name}: daemon\"{{if .WorkingDirectory}} -c {{.WorkingDirectory}}{{end}}" -command_args="${daemon_args} {{.Path}}{{range .Arguments}} {{.}}{{end}}" - -load_rc_config "${name}" -run_rc_command "$1" -` - -var rcConfTmpl = `# {{.Name}} -{{.Name}}_enable="YES" -` diff --git a/internal/router/os_others.go b/internal/router/os_others.go deleted file mode 100644 index 52b41e4b..00000000 --- a/internal/router/os_others.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build !freebsd - -package router - -import ( - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" -) - -const osName = "" - -func newOsRouter(cfg *ctrld.Config, cdMode bool) Router { - return &osRouter{} -} - -type osRouter struct{} - -func (d *osRouter) ConfigureService(_ *service.Config) error { - return nil -} - -func (d *osRouter) Install(_ *service.Config) error { - return nil -} - -func (d *osRouter) Uninstall(_ *service.Config) error { - return nil -} - -func (d *osRouter) PreRun() error { - return nil -} - -func (d *osRouter) Setup() error { - return nil -} - -func (d *osRouter) Cleanup() error { - return nil -} diff --git a/internal/router/router.go b/internal/router/router.go deleted file mode 100644 index 2d8c462d..00000000 --- a/internal/router/router.go +++ /dev/null @@ -1,288 +0,0 @@ -package router - -import ( - "bytes" - "crypto/x509" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "sync/atomic" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/certs" - "github.com/Control-D-Inc/ctrld/internal/router/ddwrt" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - "github.com/Control-D-Inc/ctrld/internal/router/edgeos" - "github.com/Control-D-Inc/ctrld/internal/router/firewalla" - "github.com/Control-D-Inc/ctrld/internal/router/merlin" - netgear "github.com/Control-D-Inc/ctrld/internal/router/netgear_orbi_voxel" - "github.com/Control-D-Inc/ctrld/internal/router/openwrt" - "github.com/Control-D-Inc/ctrld/internal/router/synology" - "github.com/Control-D-Inc/ctrld/internal/router/tomato" - "github.com/Control-D-Inc/ctrld/internal/router/ubios" -) - -// Service is the interface to manage ctrld service on router. -type Service interface { - // ConfigureService performs works for installing ctrla as a service on router. - ConfigureService(*service.Config) error - // Install performs necessary works after service.Install done. - Install(*service.Config) error - // Uninstall performs necessary works after service.Uninstallation done. - Uninstall(*service.Config) error -} - -// Router is the interface for managing ctrld running on router. -type Router interface { - Service - - // PreRun performs works need to be done before ctrld being run on router. - // Implementation should only return if the pre-condition was met (e.g: ntp synced). - PreRun() error - // Setup configures ctrld to be run on the router. - Setup() error - // Cleanup cleans up works setup on router by ctrld. - Cleanup() error -} - -// New returns new Router interface. -func New(cfg *ctrld.Config, cdMode bool) Router { - switch Name() { - case ddwrt.Name: - return ddwrt.New(cfg) - case merlin.Name: - return merlin.New(cfg) - case openwrt.Name: - return openwrt.New(cfg) - case edgeos.Name: - return edgeos.New(cfg) - case ubios.Name: - return ubios.New(cfg) - case synology.Name: - return synology.New(cfg) - case tomato.Name: - return tomato.New(cfg) - case firewalla.Name: - return firewalla.New(cfg) - case netgear.Name: - return netgear.New(cfg) - } - return newOsRouter(cfg, cdMode) -} - -// IsNetGearOrbi reports whether the router is a Netgear Orbi router. -func IsNetGearOrbi() bool { - return Name() == netgear.Name -} - -// IsGLiNet reports whether the router is an GL.iNet router. -func IsGLiNet() bool { - if Name() != openwrt.Name { - return false - } - buf, _ := os.ReadFile("/proc/version") - // The output of /proc/version contains "(glinet@glinet)". - return bytes.Contains(buf, []byte(" (glinet")) -} - -// IsOldOpenwrt reports whether the router is an "old" version of Openwrt, -// aka versions which don't have "service" command. -func IsOldOpenwrt() bool { - if Name() != openwrt.Name { - return false - } - cmd, _ := exec.LookPath("service") - return cmd == "" -} - -// WaitProcessExited reports whether the "ctrld stop" command have to wait until ctrld process exited. -func WaitProcessExited() bool { - return Name() == openwrt.Name -} - -var routerPlatform atomic.Pointer[router] - -type router struct { - name string -} - -// Name returns name of the router platform. -func Name() string { - if r := routerPlatform.Load(); r != nil { - return r.name - } - r := &router{} - r.name = distroName() - routerPlatform.Store(r) - return r.name -} - -// DefaultInterfaceName returns the default interface name of the current router. -func DefaultInterfaceName() string { - switch Name() { - case ubios.Name: - return "lo" - } - return "" -} - -// LocalResolverIP returns the IP that could be used as nameserver in /etc/resolv.conf file. -func LocalResolverIP() string { - var iface string - switch Name() { - case edgeos.Name: - // On EdgeOS, dnsmasq is run with "--local-service", so we need to get - // the proper interface from dnsmasq config. - if name, _ := dnsmasq.InterfaceNameFromConfig("/etc/dnsmasq.conf"); name != "" { - iface = name - } - case firewalla.Name: - // On Firewalla, the lo interface is excluded in all dnsmasq settings of all interfaces. - // Thus, we use "br0" as the nameserver in /etc/resolv.conf file. - iface = "br0" - } - if netIface, _ := net.InterfaceByName(iface); netIface != nil { - addrs, _ := netIface.Addrs() - for _, addr := range addrs { - if netIP, ok := addr.(*net.IPNet); ok && netIP.IP.To4() != nil { - return netIP.IP.To4().String() - } - } - } - return "" -} - -// HomeDir returns the home directory of ctrld on current router. -func HomeDir() (string, error) { - switch Name() { - case ddwrt.Name, firewalla.Name, merlin.Name, netgear.Name, tomato.Name: - exe, err := os.Executable() - if err != nil { - return "", err - } - return filepath.Dir(exe), nil - case edgeos.Name: - exe, err := os.Executable() - if err != nil { - return "", err - } - // Using binary directory as home dir if it is located in /config. - // Otherwise, fallback to old behavior for compatibility. - if strings.HasPrefix(exe, "/config/") { - return filepath.Dir(exe), nil - } - } - return "", nil -} - -// CertPool returns the system certificate pool of the current router. -func CertPool() *x509.CertPool { - if Name() == ddwrt.Name { - return certs.CACertPool() - } - return nil -} - -// CanListenLocalhost reports whether the ctrld can listen on localhost with current host. -func CanListenLocalhost() bool { - switch { - case Name() == firewalla.Name: - return false - default: - return true - } -} - -// SelfInterfaces return list of *net.Interface that will be source of requests from router itself. -func SelfInterfaces() []*net.Interface { - switch Name() { - case firewalla.Name: - return dnsmasq.FirewallaSelfInterfaces() - default: - return nil - } -} - -// LeaseFilesDir is the directory which contains lease files. -func LeaseFilesDir() string { - if Name() == edgeos.Name { - edgeos.LeaseFileDir() - } - return "" -} - -// ServiceDependencies returns list of dependencies that ctrld services needs on this router. -// See https://pkg.go.dev/github.com/kardianos/service#Config for list format. -func ServiceDependencies() []string { - if Name() == ubios.Name { - // On Ubios, ctrld needs to start after unifi-mongodb, - // so it can query custom client info mapping. - return []string{ - "Wants=unifi-mongodb.service", - "After=unifi-mongodb.service", - } - } - return nil -} - -func distroName() string { - switch { - case bytes.HasPrefix(unameO(), []byte("DD-WRT")): - return ddwrt.Name - case bytes.HasPrefix(unameO(), []byte("ASUSWRT-Merlin")): - return merlin.Name - case haveFile("/etc/openwrt_version"): - if haveFile("/bin/config") { // TODO: is there any more reliable way? - return netgear.Name - } - return openwrt.Name - case isUbios(): - return ubios.Name - case bytes.HasPrefix(unameU(), []byte("synology")): - return synology.Name - case bytes.HasPrefix(unameO(), []byte("Tomato")): - return tomato.Name - case haveDir("/config/scripts/post-config.d"): - return edgeos.Name - case haveFile("/etc/ubnt/init/vyatta-router"): - return edgeos.Name // For 2.x - case haveFile("/etc/firewalla_release"): - return firewalla.Name - } - return osName -} - -func haveFile(file string) bool { - _, err := os.Stat(file) - return err == nil -} - -func haveDir(dir string) bool { - fi, _ := os.Stat(dir) - return fi != nil && fi.IsDir() -} - -func unameO() []byte { - out, _ := exec.Command("uname", "-o").Output() - return out -} - -func unameU() []byte { - out, _ := exec.Command("uname", "-u").Output() - return out -} - -// isUbios reports whether the current machine is running on Ubios. -func isUbios() bool { - if haveDir("/data/unifi") { - return true - } - if err := exec.Command("ubnt-device-info", "firmware").Run(); err == nil { - return true - } - return false -} diff --git a/internal/router/service.go b/internal/router/service.go deleted file mode 100644 index 33339646..00000000 --- a/internal/router/service.go +++ /dev/null @@ -1,96 +0,0 @@ -package router - -import ( - "bytes" - "os" - "os/exec" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld/internal/router/ddwrt" - "github.com/Control-D-Inc/ctrld/internal/router/merlin" - "github.com/Control-D-Inc/ctrld/internal/router/tomato" - "github.com/Control-D-Inc/ctrld/internal/router/ubios" -) - -func init() { - systems := []service.System{ - &linuxSystemService{ - name: "ddwrt", - detect: func() bool { return Name() == ddwrt.Name }, - interactive: func() bool { - is, _ := isInteractive() - return is - }, - new: newddwrtService, - }, - &linuxSystemService{ - name: "merlin", - detect: func() bool { return Name() == merlin.Name }, - interactive: func() bool { - is, _ := isInteractive() - return is - }, - new: newMerlinService, - }, - &linuxSystemService{ - name: "ubios", - detect: func() bool { - if Name() != ubios.Name { - return false - } - out, err := exec.Command("ubnt-device-info", "firmware").CombinedOutput() - if err == nil { - // For v2/v3, UbiOS use a Debian base with systemd, so it is not - // necessary to use custom implementation for supporting init system. - return bytes.HasPrefix(out, []byte("1.")) - } - return true - }, - interactive: func() bool { - is, _ := isInteractive() - return is - }, - new: newUbiosService, - }, - &linuxSystemService{ - name: "tomato", - detect: func() bool { return Name() == tomato.Name }, - interactive: func() bool { - is, _ := isInteractive() - return is - }, - new: newTomatoService, - }, - } - systems = append(systems, service.AvailableSystems()...) - service.ChooseSystem(systems...) -} - -type linuxSystemService struct { - name string - detect func() bool - interactive func() bool - new func(i service.Interface, platform string, c *service.Config) (service.Service, error) -} - -func (sc linuxSystemService) String() string { - return sc.name -} -func (sc linuxSystemService) Detect() bool { - return sc.detect() -} -func (sc linuxSystemService) Interactive() bool { - return sc.interactive() -} -func (sc linuxSystemService) New(i service.Interface, c *service.Config) (service.Service, error) { - return sc.new(i, sc.String(), c) -} - -func isInteractive() (bool, error) { - ppid := os.Getppid() - if ppid == 1 { - return false, nil - } - return true, nil -} diff --git a/internal/router/service_ddwrt.go b/internal/router/service_ddwrt.go deleted file mode 100644 index 3217f8a4..00000000 --- a/internal/router/service_ddwrt.go +++ /dev/null @@ -1,294 +0,0 @@ -package router - -import ( - "bytes" - "errors" - "fmt" - "os" - "os/exec" - "os/signal" - "strings" - "syscall" - "text/template" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -type ddwrtSvc struct { - i service.Interface - platform string - *service.Config - rcStartup string -} - -func newddwrtService(i service.Interface, platform string, c *service.Config) (service.Service, error) { - s := &ddwrtSvc{ - i: i, - platform: platform, - Config: c, - } - if err := os.MkdirAll("/jffs/etc/config", 0644); err != nil { - return nil, err - } - return s, nil -} - -func (s *ddwrtSvc) String() string { - if len(s.DisplayName) > 0 { - return s.DisplayName - } - return s.Name -} - -func (s *ddwrtSvc) Platform() string { - return s.platform -} - -func (s *ddwrtSvc) configPath() string { - return fmt.Sprintf("/jffs/etc/config/%s.startup", s.Config.Name) -} - -func (s *ddwrtSvc) template() *template.Template { - return template.Must(template.New("").Parse(ddwrtSvcScript)) -} - -func (s *ddwrtSvc) Install() error { - confPath := s.configPath() - if _, err := os.Stat(confPath); err == nil { - return fmt.Errorf("already installed: %s", confPath) - } - - path, err := os.Executable() - if err != nil { - return err - } - - if !strings.HasPrefix(path, "/jffs/") { - return errors.New("could not install service outside /jffs") - } - - var to = &struct { - *service.Config - Path string - }{ - s.Config, - path, - } - - f, err := os.Create(confPath) - if err != nil { - return err - } - defer f.Close() - - if err := s.template().Execute(f, to); err != nil { - return err - } - - if err = os.Chmod(confPath, 0755); err != nil { - return err - } - - var sb strings.Builder - if err := template.Must(template.New("").Parse(ddwrtStartupCmd)).Execute(&sb, to); err != nil { - return err - } - s.rcStartup = sb.String() - curVal, err := nvram.Run("get", nvram.RCStartupKey) - if err != nil { - return err - } - if _, err := nvram.Run("set", nvram.CtrldKeyPrefix+nvram.RCStartupKey+"="+curVal); err != nil { - return err - } - val := strings.Join([]string{curVal, s.rcStartup + " &", fmt.Sprintf(`echo $! > "/tmp/%s.pid"`, s.Config.Name)}, "\n") - - if _, err := nvram.Run("set", nvram.RCStartupKey+"="+val); err != nil { - return err - } - if out, err := nvram.Run("commit"); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - - return nil -} - -func (s *ddwrtSvc) Uninstall() error { - if err := os.Remove(s.configPath()); err != nil { - return err - } - - ctrldStartupKey := nvram.CtrldKeyPrefix + nvram.RCStartupKey - rcStartup, err := nvram.Run("get", ctrldStartupKey) - if err != nil { - return err - } - _, _ = nvram.Run("unset", ctrldStartupKey) - if _, err := nvram.Run("set", nvram.RCStartupKey+"="+rcStartup); err != nil { - return err - } - if out, err := nvram.Run("commit"); err != nil { - return fmt.Errorf("%s: %w", out, err) - } - - return nil -} - -func (s *ddwrtSvc) Logger(errs chan<- error) (service.Logger, error) { - if service.Interactive() { - return service.ConsoleLogger, nil - } - return s.SystemLogger(errs) -} - -func (s *ddwrtSvc) SystemLogger(errs chan<- error) (service.Logger, error) { - // TODO(cuonglm): detect syslog enable and return proper logger? - // this at least works with default configuration. - if service.Interactive() { - return service.ConsoleLogger, nil - - } - return &noopLogger{}, nil -} - -func (s *ddwrtSvc) Run() (err error) { - err = s.i.Start(s) - if err != nil { - return err - } - - if interactice, _ := isInteractive(); !interactice { - signal.Ignore(syscall.SIGHUP) - } - var sigChan = make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGTERM, os.Interrupt) - <-sigChan - - return s.i.Stop(s) -} - -func (s *ddwrtSvc) Status() (service.Status, error) { - if _, err := os.Stat(s.configPath()); os.IsNotExist(err) { - return service.StatusUnknown, service.ErrNotInstalled - } - out, err := exec.Command(s.configPath(), "status").CombinedOutput() - if err != nil { - return service.StatusUnknown, err - } - switch string(bytes.TrimSpace(out)) { - case "running": - return service.StatusRunning, nil - default: - return service.StatusStopped, nil - } -} - -func (s *ddwrtSvc) Start() error { - return exec.Command(s.configPath(), "start").Run() -} - -func (s *ddwrtSvc) Stop() error { - return exec.Command(s.configPath(), "stop").Run() -} - -func (s *ddwrtSvc) Restart() error { - err := s.Stop() - if err != nil { - return err - } - return s.Start() -} - -type noopLogger struct { -} - -func (c noopLogger) Error(v ...interface{}) error { - return nil -} -func (c noopLogger) Warning(v ...interface{}) error { - return nil -} -func (c noopLogger) Info(v ...interface{}) error { - return nil -} -func (c noopLogger) Errorf(format string, a ...interface{}) error { - return nil -} -func (c noopLogger) Warningf(format string, a ...interface{}) error { - return nil -} -func (c noopLogger) Infof(format string, a ...interface{}) error { - return nil -} - -const ddwrtStartupCmd = `{{.Path}}{{range .Arguments}} {{.}}{{end}}` -const ddwrtSvcScript = `#!/bin/sh - -name="{{.Name}}" -cmd="{{.Path}}{{range .Arguments}} {{.}}{{end}}" -pid_file="/tmp/$name.pid" - -get_pid() { - cat "$pid_file" -} - -is_running() { - [ -f "$pid_file" ] && ps | grep -q "^ *$(get_pid) " -} - -case "$1" in - start) - if is_running; then - echo "Already started" - else - echo "Starting $name" - $cmd & - echo $! > "$pid_file" - chmod 600 "$pid_file" - if ! is_running; then - echo "Failed to start $name" - exit 1 - fi - fi - ;; - stop) - if is_running; then - echo -n "Stopping $name..." - kill "$(get_pid)" - for _ in 1 2 3 4 5; do - if ! is_running; then - echo "stopped" - if [ -f "$pid_file" ]; then - rm "$pid_file" - fi - exit 0 - fi - printf "." - sleep 2 - done - echo "failed to stop $name" - exit 1 - fi - exit 0 - ;; - restart) - $0 stop - $0 start - ;; - status) - if is_running; then - echo "running" - else - echo "stopped" - exit 1 - fi - ;; - *) - echo "Usage: $0 {start|stop|restart|status}" - exit 1 - ;; -esac -exit 0 -` diff --git a/internal/router/service_merlin.go b/internal/router/service_merlin.go deleted file mode 100644 index 8ab6d6a7..00000000 --- a/internal/router/service_merlin.go +++ /dev/null @@ -1,360 +0,0 @@ -package router - -import ( - "bytes" - "errors" - "fmt" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strings" - "syscall" - "text/template" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -const ( - merlinJFFSScriptPath = "/jffs/scripts/services-start" - merlinJFFSServiceEventScriptPath = "/jffs/scripts/service-event" -) - -type merlinSvc struct { - i service.Interface - platform string - *service.Config -} - -func newMerlinService(i service.Interface, platform string, c *service.Config) (service.Service, error) { - s := &merlinSvc{ - i: i, - platform: platform, - Config: c, - } - return s, nil -} - -func (s *merlinSvc) String() string { - if len(s.DisplayName) > 0 { - return s.DisplayName - } - return s.Name -} - -func (s *merlinSvc) Platform() string { - return s.platform -} - -func (s *merlinSvc) configPath() string { - bin := s.Config.Executable - if bin == "" { - path, err := os.Executable() - if err != nil { - return "" - } - bin = path - } - return bin + ".startup" -} - -func (s *merlinSvc) template() *template.Template { - return template.Must(template.New("").Parse(merlinSvcScript)) -} - -func (s *merlinSvc) Install() error { - exePath, err := os.Executable() - if err != nil { - return err - } - - if !strings.HasPrefix(exePath, "/jffs/") { - return errors.New("could not install service outside /jffs") - } - if _, err := nvram.Run("set", "jffs2_scripts=1"); err != nil { - return err - } - if _, err := nvram.Run("commit"); err != nil { - return err - } - - confPath := s.configPath() - if _, err := os.Stat(confPath); err == nil { - return fmt.Errorf("already installed: %s", confPath) - } - - var to = &struct { - *service.Config - Path string - }{ - s.Config, - exePath, - } - - f, err := os.Create(confPath) - if err != nil { - return fmt.Errorf("os.Create: %w", err) - } - defer f.Close() - - if err := s.template().Execute(f, to); err != nil { - return fmt.Errorf("s.template.Execute: %w", err) - } - - if err = os.Chmod(confPath, 0755); err != nil { - return fmt.Errorf("os.Chmod: startup script: %w", err) - } - - if err := os.MkdirAll(filepath.Dir(merlinJFFSScriptPath), 0755); err != nil { - return fmt.Errorf("os.MkdirAll: %w", err) - } - - tmpScript, err := os.CreateTemp("", "ctrld_install") - if err != nil { - return fmt.Errorf("os.CreateTemp: %w", err) - } - defer os.Remove(tmpScript.Name()) - defer tmpScript.Close() - - if _, err := tmpScript.WriteString(merlinAddLineToScript); err != nil { - return fmt.Errorf("tmpScript.WriteString: %w", err) - } - if err := tmpScript.Close(); err != nil { - return fmt.Errorf("tmpScript.Close: %w", err) - } - addLineToScript := func(line, script string) error { - if _, err := os.Stat(script); os.IsNotExist(err) { - if err := os.WriteFile(script, []byte("#!/bin/sh\n"), 0755); err != nil { - return err - } - } - if err := os.Chmod(script, 0755); err != nil { - return fmt.Errorf("os.Chmod: jffs script: %w", err) - } - - if err := exec.Command("sh", tmpScript.Name(), line, script).Run(); err != nil { - return fmt.Errorf("exec.Command: add startup script: %w", err) - } - return nil - } - - for script, line := range map[string]string{ - merlinJFFSScriptPath: s.configPath() + " start", - merlinJFFSServiceEventScriptPath: s.configPath() + ` service_event "$1" "$2"`, - } { - if err := addLineToScript(line, script); err != nil { - return err - } - } - - return nil -} - -func (s *merlinSvc) Uninstall() error { - if err := os.Remove(s.configPath()); err != nil { - return fmt.Errorf("os.Remove: %w", err) - } - tmpScript, err := os.CreateTemp("", "ctrld_uninstall") - if err != nil { - return fmt.Errorf("os.CreateTemp: %w", err) - } - defer os.Remove(tmpScript.Name()) - defer tmpScript.Close() - - if _, err := tmpScript.WriteString(merlinRemoveLineFromScript); err != nil { - return fmt.Errorf("tmpScript.WriteString: %w", err) - } - if err := tmpScript.Close(); err != nil { - return fmt.Errorf("tmpScript.Close: %w", err) - } - removeLineFromScript := func(line, script string) error { - if _, err := os.Stat(script); os.IsNotExist(err) { - if err := os.WriteFile(script, []byte("#!/bin/sh\n"), 0755); err != nil { - return err - } - } - if err := os.Chmod(script, 0755); err != nil { - return fmt.Errorf("os.Chmod: jffs script: %w", err) - } - - if err := exec.Command("sh", tmpScript.Name(), line, script).Run(); err != nil { - return fmt.Errorf("exec.Command: add startup script: %w", err) - } - return nil - } - - for script, line := range map[string]string{ - merlinJFFSScriptPath: s.configPath() + " start", - merlinJFFSServiceEventScriptPath: s.configPath() + ` service_event "$1" "$2"`, - } { - if err := removeLineFromScript(line, script); err != nil { - return err - } - } - - return nil -} - -func (s *merlinSvc) Logger(errs chan<- error) (service.Logger, error) { - if service.Interactive() { - return service.ConsoleLogger, nil - } - return s.SystemLogger(errs) -} - -func (s *merlinSvc) SystemLogger(errs chan<- error) (service.Logger, error) { - return newSysLogger(s.Name, errs) -} - -func (s *merlinSvc) Run() (err error) { - err = s.i.Start(s) - if err != nil { - return err - } - - if interactice, _ := isInteractive(); !interactice { - signal.Ignore(syscall.SIGHUP) - } - - var sigChan = make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGTERM, os.Interrupt) - <-sigChan - - return s.i.Stop(s) -} - -func (s *merlinSvc) Status() (service.Status, error) { - if _, err := os.Stat(s.configPath()); os.IsNotExist(err) { - return service.StatusUnknown, service.ErrNotInstalled - } - out, err := exec.Command(s.configPath(), "status").CombinedOutput() - if err != nil { - return service.StatusUnknown, err - } - switch string(bytes.TrimSpace(out)) { - case "running": - return service.StatusRunning, nil - default: - return service.StatusStopped, nil - } -} - -func (s *merlinSvc) Start() error { - return exec.Command(s.configPath(), "start").Run() -} - -func (s *merlinSvc) Stop() error { - return exec.Command(s.configPath(), "stop").Run() -} - -func (s *merlinSvc) Restart() error { - err := s.Stop() - if err != nil { - return err - } - return s.Start() -} - -const merlinSvcScript = `#!/bin/sh - -name="{{.Name}}" -cmd="{{.Path}}{{range .Arguments}} {{.}}{{end}}" -pid_file="/tmp/$name.pid" - -get_pid() { - cat "$pid_file" -} - -is_running() { - [ -f "$pid_file" ] && ps | grep -q "^ *$(get_pid) " -} - -case "$1" in - start) - if is_running; then - logger -c "Already started" - else - logger -c "Starting $name" - if [ -f /rom/ca-bundle.crt ]; then - # For John’s fork - export SSL_CERT_FILE=/rom/ca-bundle.crt - fi - $cmd & - echo $! > "$pid_file" - chmod 600 "$pid_file" - if ! is_running; then - logger -c "Failed to start $name" - exit 1 - fi - fi - ;; - stop) - if is_running; then - logger -c "Stopping $name..." - kill "$(get_pid)" - for _ in 1 2 3 4 5; do - if ! is_running; then - logger -c "stopped" - if [ -f "$pid_file" ]; then - rm "$pid_file" - fi - exit 0 - fi - printf "." - sleep 2 - done - logger -c "failed to stop $name" - exit 1 - fi - exit 0 - ;; - restart) - $0 stop - $0 start - ;; - status) - if is_running; then - echo "running" - else - echo "stopped" - exit 1 - fi - ;; - service_event) - event=$2 - svc=$3 - dnsmasq_pid_file=$(sed -n '/pid-file=/s///p' /etc/dnsmasq.conf) - - if [ "$event" = "restart" ] && [ "$svc" = "diskmon" ]; then - kill "$(cat "$dnsmasq_pid_file")" >/dev/null 2>&1 - fi - ;; - *) - echo "Usage: $0 {start|stop|restart|status}" - exit 1 - ;; -esac -exit 0 -` - -const merlinAddLineToScript = `#!/bin/sh - -line=$1 -file=$2 - -. /usr/sbin/helper.sh - -pc_append "$line" "$file" -` - -const merlinRemoveLineFromScript = `#!/bin/sh - -line=$1 -file=$2 - -. /usr/sbin/helper.sh - -pc_delete "$line" "$file" -` diff --git a/internal/router/service_tomato.go b/internal/router/service_tomato.go deleted file mode 100644 index 2cf59391..00000000 --- a/internal/router/service_tomato.go +++ /dev/null @@ -1,289 +0,0 @@ -package router - -import ( - "bytes" - "errors" - "fmt" - "os" - "os/exec" - "os/signal" - "strings" - "syscall" - "text/template" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld/internal/router/nvram" -) - -const tomatoNvramScriptWanupKey = "script_wanup" - -type tomatoSvc struct { - i service.Interface - platform string - *service.Config -} - -func newTomatoService(i service.Interface, platform string, c *service.Config) (service.Service, error) { - s := &tomatoSvc{ - i: i, - platform: platform, - Config: c, - } - return s, nil -} - -func (s *tomatoSvc) String() string { - if len(s.DisplayName) > 0 { - return s.DisplayName - } - return s.Name -} - -func (s *tomatoSvc) Platform() string { - return s.platform -} - -func (s *tomatoSvc) configPath() string { - bin := s.Config.Executable - if bin == "" { - path, err := os.Executable() - if err != nil { - return "" - } - bin = path - } - return bin + ".startup" -} - -func (s *tomatoSvc) template() *template.Template { - return template.Must(template.New("").Parse(tomatoSvcScript)) -} - -func (s *tomatoSvc) Install() error { - exePath, err := os.Executable() - if err != nil { - return err - } - - if !strings.HasPrefix(exePath, "/jffs/") { - return errors.New("could not install service outside /jffs") - } - if _, err := nvram.Run("set", "jffs2_on=1"); err != nil { - return err - } - if _, err := nvram.Run("commit"); err != nil { - return err - } - - confPath := s.configPath() - if _, err := os.Stat(confPath); err == nil { - return fmt.Errorf("already installed: %s", confPath) - } - - var to = &struct { - *service.Config - Path string - }{ - s.Config, - exePath, - } - - f, err := os.Create(confPath) - if err != nil { - return fmt.Errorf("os.Create: %w", err) - } - defer f.Close() - - if err := s.template().Execute(f, to); err != nil { - return fmt.Errorf("s.template.Execute: %w", err) - } - - if err = os.Chmod(confPath, 0755); err != nil { - return fmt.Errorf("os.Chmod: startup script: %w", err) - } - - nvramKvMap := map[string]string{ - tomatoNvramScriptWanupKey: "", // script to start ctrld, filled by tomatoSvc.Install method. - } - old, err := nvram.Run("get", tomatoNvramScriptWanupKey) - if err != nil { - return fmt.Errorf("nvram: %w", err) - } - nvramKvMap[tomatoNvramScriptWanupKey] = strings.Join([]string{old, s.configPath() + " start"}, "\n") - if err := nvram.SetKV(nvramKvMap, nvram.CtrldInstallKey); err != nil { - return err - } - return nil -} - -func (s *tomatoSvc) Uninstall() error { - if err := os.Remove(s.configPath()); err != nil { - return fmt.Errorf("os.Remove: %w", err) - } - nvramKvMap := map[string]string{ - tomatoNvramScriptWanupKey: "", // script to start ctrld, filled by tomatoSvc.Install method. - } - // Restore old configs. - if err := nvram.Restore(nvramKvMap, nvram.CtrldInstallKey); err != nil { - return err - } - return nil -} - -func (s *tomatoSvc) Logger(errs chan<- error) (service.Logger, error) { - if service.Interactive() { - return service.ConsoleLogger, nil - } - return s.SystemLogger(errs) -} - -func (s *tomatoSvc) SystemLogger(errs chan<- error) (service.Logger, error) { - return newSysLogger(s.Name, errs) -} - -func (s *tomatoSvc) Run() (err error) { - err = s.i.Start(s) - if err != nil { - return err - } - - if interactice, _ := isInteractive(); !interactice { - signal.Ignore(syscall.SIGHUP) - } - - var sigChan = make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGTERM, os.Interrupt) - <-sigChan - - return s.i.Stop(s) -} - -func (s *tomatoSvc) Status() (service.Status, error) { - if _, err := os.Stat(s.configPath()); os.IsNotExist(err) { - return service.StatusUnknown, service.ErrNotInstalled - } - out, err := exec.Command(s.configPath(), "status").CombinedOutput() - if err != nil { - return service.StatusUnknown, err - } - switch string(bytes.TrimSpace(out)) { - case "running": - return service.StatusRunning, nil - default: - return service.StatusStopped, nil - } -} - -func (s *tomatoSvc) Start() error { - return exec.Command(s.configPath(), "start").Run() -} - -func (s *tomatoSvc) Stop() error { - return exec.Command(s.configPath(), "stop").Run() -} - -func (s *tomatoSvc) Restart() error { - return exec.Command(s.configPath(), "restart").Run() -} - -// https://wiki.freshtomato.org/doku.php/freshtomato_zerotier?s[]=%2Aservice%2A -const tomatoSvcScript = `#!/bin/sh - - -NAME="{{.Name}}" -CMD="{{.Path}}{{range .Arguments}} {{.}}{{end}}" -LOG_FILE="/var/log/${NAME}.log" -PID_FILE="/tmp/$NAME.pid" - - -alias elog="logger -t $NAME -s" - - -COND=$1 -[ $# -eq 0 ] && COND="start" - -get_pid() { - cat "$PID_FILE" -} - -is_running() { - [ -f "$PID_FILE" ] && ps | grep -q "^ *$(get_pid) " -} - -start() { - if is_running; then - elog "$NAME is already running." - exit 1 - fi - elog "Starting $NAME Services: " - $CMD & - echo $! > "$PID_FILE" - chmod 600 "$PID_FILE" - if is_running; then - elog "succeeded." - else - elog "failed." - fi -} - - -stop() { - if ! is_running; then - elog "$NAME is not running." - exit 0 - fi - elog "Shutting down $NAME Services: " - kill -SIGTERM "$(get_pid)" - for _ in 1 2 3 4 5; do - if ! is_running; then - if [ -f "$pid_file" ]; then - rm "$pid_file" - fi - return 0 - fi - printf "." - sleep 2 - done - if ! is_running; then - elog "succeeded." - else - elog "failed." - fi -} - - -do_restart() { - stop - start -} - - -do_status() { - if ! is_running; then - echo "stopped" - else - echo "running" - fi -} - - -case "$COND" in -start) - start - ;; -stop) - stop - ;; -restart) - do_restart - ;; -status) - do_status - ;; -*) - elog "Usage: $0 (start|stop|restart|status)" - ;; -esac -exit 0 -` diff --git a/internal/router/service_ubios.go b/internal/router/service_ubios.go deleted file mode 100644 index 9ad971d2..00000000 --- a/internal/router/service_ubios.go +++ /dev/null @@ -1,340 +0,0 @@ -package router - -import ( - "bytes" - "fmt" - "os" - "os/exec" - "os/signal" - "path/filepath" - "strings" - "syscall" - "text/template" - "time" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" -) - -// This is a copy of https://github.com/kardianos/service/blob/v1.2.1/service_sysv_linux.go, -// with modification for supporting ubios v1 init system. - -type ubiosSvc struct { - i service.Interface - platform string - *service.Config -} - -func newUbiosService(i service.Interface, platform string, c *service.Config) (service.Service, error) { - s := &ubiosSvc{ - i: i, - platform: platform, - Config: c, - } - return s, nil -} - -func (s *ubiosSvc) String() string { - if len(s.DisplayName) > 0 { - return s.DisplayName - } - return s.Name -} - -func (s *ubiosSvc) Platform() string { - return s.platform -} - -func (s *ubiosSvc) configPath() string { - return "/etc/init.d/" + s.Config.Name -} - -func (s *ubiosSvc) execPath() (string, error) { - if len(s.Executable) != 0 { - return filepath.Abs(s.Executable) - } - return os.Executable() -} - -func (s *ubiosSvc) template() *template.Template { - return template.Must(template.New("").Funcs(tf).Parse(ubiosSvcScript)) -} - -func (s *ubiosSvc) Install() error { - confPath := s.configPath() - if _, err := os.Stat(confPath); err == nil { - return fmt.Errorf("init already exists: %s", confPath) - } - - f, err := os.Create(confPath) - if err != nil { - return fmt.Errorf("failed to create config path: %w", err) - } - defer f.Close() - - path, err := s.execPath() - if err != nil { - return fmt.Errorf("failed to get exec path: %w", err) - } - - var to = &struct { - *service.Config - Path string - DnsMasqConfPath string - }{ - s.Config, - path, - filepath.Join(dnsmasq.UbiosConfPath(), dnsmasq.UbiosConfName), - } - - if err := s.template().Execute(f, to); err != nil { - return fmt.Errorf("failed to create init script: %w", err) - } - - if err := f.Close(); err != nil { - return fmt.Errorf("failed to save init script: %w", err) - } - - if err = os.Chmod(confPath, 0755); err != nil { - return fmt.Errorf("failed to set init script executable: %w", err) - } - - // Enable on boot - script, err := os.CreateTemp("", "ctrld_boot.service") - if err != nil { - return fmt.Errorf("failed to create boot service tmp file: %w", err) - } - defer script.Close() - - svcConfig := *to.Config - svcConfig.Arguments = os.Args[1:] - to.Config = &svcConfig - if err := template.Must(template.New("").Funcs(tf).Parse(ubiosBootSystemdService)).Execute(script, &to); err != nil { - return fmt.Errorf("failed to create boot service file: %w", err) - } - if err := script.Close(); err != nil { - return fmt.Errorf("failed to save boot service file: %w", err) - } - - // Copy the boot script to container and start. - cmd := exec.Command("podman", "cp", "--pause=false", script.Name(), "unifi-os:/lib/systemd/system/ctrld-boot.service") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to copy boot script, out: %s, err: %v", string(out), err) - } - cmd = exec.Command("podman", "exec", "unifi-os", "systemctl", "enable", "--now", "ctrld-boot.service") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to start ctrld boot script, out: %s, err: %v", string(out), err) - } - return nil -} - -func (s *ubiosSvc) Uninstall() error { - if err := os.Remove(s.configPath()); err != nil { - return err - } - // Remove ctrld-boot service inside unifi-os container. - cmd := exec.Command("podman", "exec", "unifi-os", "systemctl", "disable", "ctrld-boot.service") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to disable ctrld-boot service, out: %s, err: %v", string(out), err) - } - cmd = exec.Command("podman", "exec", "unifi-os", "rm", "/lib/systemd/system/ctrld-boot.service") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to remove ctrld-boot service file, out: %s, err: %v", string(out), err) - } - cmd = exec.Command("podman", "exec", "unifi-os", "systemctl", "daemon-reload") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to reload systemd service, out: %s, err: %v", string(out), err) - } - cmd = exec.Command("podman", "exec", "unifi-os", "systemctl", "reset-failed") - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to reset-failed systemd service, out: %s, err: %v", string(out), err) - } - return nil -} - -func (s *ubiosSvc) Logger(errs chan<- error) (service.Logger, error) { - if service.Interactive() { - return service.ConsoleLogger, nil - } - return s.SystemLogger(errs) -} - -func (s *ubiosSvc) SystemLogger(errs chan<- error) (service.Logger, error) { - return newSysLogger(s.Name, errs) -} - -func (s *ubiosSvc) Run() (err error) { - err = s.i.Start(s) - if err != nil { - return err - } - - if interactice, _ := isInteractive(); !interactice { - signal.Ignore(syscall.SIGHUP) - } - - var sigChan = make(chan os.Signal, 3) - signal.Notify(sigChan, syscall.SIGTERM, os.Interrupt) - <-sigChan - - return s.i.Stop(s) -} - -func (s *ubiosSvc) Status() (service.Status, error) { - if _, err := os.Stat(s.configPath()); os.IsNotExist(err) { - return service.StatusUnknown, service.ErrNotInstalled - } - out, err := exec.Command(s.configPath(), "status").CombinedOutput() - if err != nil { - return service.StatusUnknown, err - } - switch string(bytes.TrimSpace(out)) { - case "Running": - return service.StatusRunning, nil - default: - return service.StatusStopped, nil - } -} - -func (s *ubiosSvc) Start() error { - return exec.Command(s.configPath(), "start").Run() -} - -func (s *ubiosSvc) Stop() error { - return exec.Command(s.configPath(), "stop").Run() -} - -func (s *ubiosSvc) Restart() error { - err := s.Stop() - if err != nil { - return err - } - time.Sleep(50 * time.Millisecond) - return s.Start() -} - -const ubiosBootSystemdService = `[Unit] -Description=Run ctrld On Startup UDM -Wants=network-online.target -After=network-online.target -Wants=unifi-mongodb -After=unifi-mongodb -StartLimitIntervalSec=500 -StartLimitBurst=5 - -[Service] -Restart=on-failure -RestartSec=5s -ExecStart=/sbin/ssh-proxy '[ -f "{{.DnsMasqConfPath}}" ] || {{.Path}}{{range .Arguments}} {{.|cmd}}{{end}}' -RemainAfterExit=true -[Install] -WantedBy=multi-user.target -` - -const ubiosSvcScript = `#!/bin/sh -# For RedHat and cousins: -# chkconfig: - 99 01 -# description: {{.Description}} -# processname: {{.Path}} - -### BEGIN INIT INFO -# Provides: {{.Path}} -# Required-Start: -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: {{.DisplayName}} -# Description: {{.Description}} -### END INIT INFO - -cmd="{{.Path}}{{range .Arguments}} {{.|cmd}}{{end}}" - -name=$(basename $(readlink -f $0)) -pid_file="/var/run/$name.pid" -stdout_log="/var/log/$name.log" -stderr_log="/var/log/$name.err" - -[ -e /etc/sysconfig/$name ] && . /etc/sysconfig/$name - -get_pid() { - cat "$pid_file" -} - -is_running() { - [ -f "$pid_file" ] && cat /proc/$(get_pid)/stat > /dev/null 2>&1 -} - -case "$1" in - start) - if is_running; then - echo "Already started" - else - echo "Starting $name" - {{if .WorkingDirectory}}cd '{{.WorkingDirectory}}'{{end}} - $cmd >> "$stdout_log" 2>> "$stderr_log" & - echo $! > "$pid_file" - if ! is_running; then - echo "Unable to start, see $stdout_log and $stderr_log" - exit 1 - fi - fi - ;; - stop) - if is_running; then - echo -n "Stopping $name.." - kill $(get_pid) - for i in $(seq 1 10) - do - if ! is_running; then - break - fi - echo -n "." - sleep 1 - done - echo - if is_running; then - echo "Not stopped; may still be shutting down or shutdown may have failed" - exit 1 - else - echo "Stopped" - if [ -f "$pid_file" ]; then - rm "$pid_file" - fi - fi - else - echo "Not running" - fi - ;; - restart) - $0 stop - if is_running; then - echo "Unable to stop, will not attempt to start" - exit 1 - fi - $0 start - ;; - status) - if is_running; then - echo "Running" - else - echo "Stopped" - exit 1 - fi - ;; - *) - echo "Usage: $0 {start|stop|restart|status}" - exit 1 - ;; -esac -exit 0 -` - -var tf = map[string]interface{}{ - "cmd": func(s string) string { - return `"` + strings.Replace(s, `"`, `\"`, -1) + `"` - }, - "cmdEscape": func(s string) string { - return strings.Replace(s, " ", `\x20`, -1) - }, -} diff --git a/internal/router/synology/synology.go b/internal/router/synology/synology.go deleted file mode 100644 index 79339430..00000000 --- a/internal/router/synology/synology.go +++ /dev/null @@ -1,125 +0,0 @@ -package synology - -import ( - "bytes" - "context" - "errors" - "fmt" - "os" - "os/exec" - "strings" - "time" - - "github.com/kardianos/service" - "tailscale.com/logtail/backoff" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - "github.com/Control-D-Inc/ctrld/internal/router/ntp" -) - -const ( - Name = "synology" - - synologyDNSMasqConfigPath = "/etc/dhcpd/dhcpd-zzz-ctrld.conf" - synologyDhcpdInfoPath = "/etc/dhcpd/dhcpd-zzz-ctrld.info" -) - -type Synology struct { - cfg *ctrld.Config - useUpstart bool -} - -// New returns a router.Router for configuring/setup/run ctrld on Ubios routers. -func New(cfg *ctrld.Config) *Synology { - return &Synology{ - cfg: cfg, - useUpstart: service.Platform() == "linux-upstart", - } -} - -func (s *Synology) ConfigureService(svc *service.Config) error { - svc.Option["LogOutput"] = true - return nil -} - -func (s *Synology) Install(_ *service.Config) error { - return nil -} - -func (s *Synology) Uninstall(_ *service.Config) error { - return nil -} - -func (s *Synology) PreRun() error { - if s.useUpstart { - if err := ntp.WaitUpstart(); err != nil { - return err - } - return waitDhcpServer() - } - return nil -} - -func (s *Synology) Setup() error { - if s.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, s.cfg) - if err != nil { - return err - } - if err := os.WriteFile(synologyDNSMasqConfigPath, []byte(data), 0600); err != nil { - return err - } - if err := os.WriteFile(synologyDhcpdInfoPath, []byte(`enable="yes"`), 0600); err != nil { - return err - } - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func (s *Synology) Cleanup() error { - if s.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Remove the custom config files. - for _, f := range []string{synologyDNSMasqConfigPath, synologyDhcpdInfoPath} { - if err := os.Remove(f); err != nil { - return err - } - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func restartDNSMasq() error { - if out, err := exec.Command("/etc/rc.network", "nat-restart-dhcp").CombinedOutput(); err != nil { - return fmt.Errorf("synologyRestartDNSMasq: %s - %w", string(out), err) - } - return nil -} - -func waitDhcpServer() error { - // Wait until `initctl status dhcpserver` returns running state. - b := backoff.NewBackoff("waitDhcpServer", func(format string, args ...any) {}, 10*time.Second) - for { - out, err := exec.Command("initctl", "status", "dhcpserver").CombinedOutput() - if err != nil { - if strings.Contains(err.Error(), "Unknown job") { - // dhcpserver service does not exist. - return nil - } - return fmt.Errorf("exec.Command: %w", err) - } - if bytes.Contains(out, []byte("start/running")) { - return nil - } - b.BackOff(context.Background(), errors.New("ntp not ready")) - } -} diff --git a/internal/router/syslog.go b/internal/router/syslog.go deleted file mode 100644 index 008bbeb7..00000000 --- a/internal/router/syslog.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build linux || darwin || freebsd - -package router - -import ( - "fmt" - "log/syslog" - - "github.com/kardianos/service" -) - -func newSysLogger(name string, errs chan<- error) (service.Logger, error) { - w, err := syslog.New(syslog.LOG_INFO, name) - if err != nil { - return nil, err - } - return sysLogger{w, errs}, nil -} - -type sysLogger struct { - *syslog.Writer - errs chan<- error -} - -func (s sysLogger) send(err error) error { - if err != nil && s.errs != nil { - s.errs <- err - } - return err -} - -func (s sysLogger) Error(v ...interface{}) error { - return s.send(s.Writer.Err(fmt.Sprint(v...))) -} -func (s sysLogger) Warning(v ...interface{}) error { - return s.send(s.Writer.Warning(fmt.Sprint(v...))) -} -func (s sysLogger) Info(v ...interface{}) error { - return s.send(s.Writer.Info(fmt.Sprint(v...))) -} -func (s sysLogger) Errorf(format string, a ...interface{}) error { - return s.send(s.Writer.Err(fmt.Sprintf(format, a...))) -} -func (s sysLogger) Warningf(format string, a ...interface{}) error { - return s.send(s.Writer.Warning(fmt.Sprintf(format, a...))) -} -func (s sysLogger) Infof(format string, a ...interface{}) error { - return s.send(s.Writer.Info(fmt.Sprintf(format, a...))) -} diff --git a/internal/router/syslog_windows.go b/internal/router/syslog_windows.go deleted file mode 100644 index ecac969f..00000000 --- a/internal/router/syslog_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -package router - -import "github.com/kardianos/service" - -func newSysLogger(name string, errs chan<- error) (service.Logger, error) { - return service.ConsoleLogger, nil -} diff --git a/internal/router/tomato/tomato.go b/internal/router/tomato/tomato.go deleted file mode 100644 index ee5f09b8..00000000 --- a/internal/router/tomato/tomato.go +++ /dev/null @@ -1,133 +0,0 @@ -package tomato - -import ( - "fmt" - "os/exec" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - "github.com/Control-D-Inc/ctrld/internal/router/ntp" - "github.com/Control-D-Inc/ctrld/internal/router/nvram" - "github.com/kardianos/service" -) - -const ( - Name = "freshtomato" - - tomatoDnsCryptProxySvcName = "dnscrypt-proxy" - tomatoStubbySvcName = "stubby" - tomatoDNSMasqSvcName = "dnsmasq" -) - -var nvramKvMap = map[string]string{ - "dnsmasq_custom": "", // Configuration of dnsmasq set by ctrld, filled by setupTomato. - "dnscrypt_proxy": "0", // Disable DNSCrypt. - "dnssec_enable": "0", // Disable DNSSEC. - "stubby_proxy": "0", // Disable Stubby -} - -type FreshTomato struct { - cfg *ctrld.Config -} - -// New returns a router.Router for configuring/setup/run ctrld on Ubios routers. -func New(cfg *ctrld.Config) *FreshTomato { - return &FreshTomato{cfg: cfg} -} - -func (f *FreshTomato) ConfigureService(config *service.Config) error { - return nil -} - -func (f *FreshTomato) Install(_ *service.Config) error { - return nil -} - -func (f *FreshTomato) Uninstall(_ *service.Config) error { - return nil -} - -func (f *FreshTomato) PreRun() error { - _ = f.Cleanup() - return ntp.WaitNvram() -} - -func (f *FreshTomato) Setup() error { - if f.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Already setup. - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val == "1" { - return nil - } - - data, err := dnsmasq.ConfTmpl(dnsmasq.ConfigContentTmpl, f.cfg) - if err != nil { - return err - } - nvramKvMap["dnsmasq_custom"] = data - if err := nvram.SetKV(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - // Restart dnscrypt-proxy service. - if err := tomatoRestartServiceWithKill(tomatoDnsCryptProxySvcName, true); err != nil { - return err - } - // Restart stubby service. - if err := tomatoRestartService(tomatoStubbySvcName); err != nil { - return err - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func (f *FreshTomato) Cleanup() error { - if f.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - if val, _ := nvram.Run("get", nvram.CtrldSetupKey); val != "1" { - return nil // was restored, nothing to do. - } - - nvramKvMap["dnsmasq_custom"] = "" - // Restore old configs. - if err := nvram.Restore(nvramKvMap, nvram.CtrldSetupKey); err != nil { - return err - } - - // Restart dnscrypt-proxy service. - if err := tomatoRestartServiceWithKill(tomatoDnsCryptProxySvcName, true); err != nil { - return err - } - // Restart stubby service. - if err := tomatoRestartService(tomatoStubbySvcName); err != nil { - return err - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func tomatoRestartService(name string) error { - return tomatoRestartServiceWithKill(name, false) -} - -func tomatoRestartServiceWithKill(name string, killBeforeRestart bool) error { - if killBeforeRestart { - _, _ = exec.Command("killall", name).CombinedOutput() - } - if out, err := exec.Command("service", name, "restart").CombinedOutput(); err != nil { - return fmt.Errorf("service restart %s: %s, %w", name, string(out), err) - } - return nil -} - -func restartDNSMasq() error { - return tomatoRestartService(tomatoDNSMasqSvcName) -} diff --git a/internal/router/ubios/ubios.go b/internal/router/ubios/ubios.go deleted file mode 100644 index cba68426..00000000 --- a/internal/router/ubios/ubios.go +++ /dev/null @@ -1,102 +0,0 @@ -package ubios - -import ( - "bytes" - "os" - "path/filepath" - "strconv" - - "github.com/kardianos/service" - - "github.com/Control-D-Inc/ctrld" - "github.com/Control-D-Inc/ctrld/internal/router/dnsmasq" - "github.com/Control-D-Inc/ctrld/internal/router/edgeos" -) - -const Name = "ubios" - -type Ubios struct { - cfg *ctrld.Config - dnsmasqConfPath string -} - -// New returns a router.Router for configuring/setup/run ctrld on Ubios routers. -func New(cfg *ctrld.Config) *Ubios { - return &Ubios{ - cfg: cfg, - dnsmasqConfPath: filepath.Join(dnsmasq.UbiosConfPath(), dnsmasq.UbiosConfName), - } -} - -func (u *Ubios) ConfigureService(config *service.Config) error { - return nil -} - -func (u *Ubios) Install(config *service.Config) error { - // See comment in (*edgeos.EdgeOS).Install method. - if edgeos.ContentFilteringEnabled() { - return edgeos.ErrContentFilteringEnabled - } - // See comment in (*edgeos.EdgeOS).Install method. - if edgeos.DnsShieldEnabled() { - return edgeos.ErrDnsShieldEnabled - } - return nil -} - -func (u *Ubios) Uninstall(_ *service.Config) error { - return nil -} - -func (u *Ubios) PreRun() error { - return nil -} - -func (u *Ubios) Setup() error { - if u.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - data, err := dnsmasq.ConfTmplWithCacheDisabled(dnsmasq.ConfigContentTmpl, u.cfg, false) - if err != nil { - return err - } - if err := os.WriteFile(u.dnsmasqConfPath, []byte(data), 0600); err != nil { - return err - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func (u *Ubios) Cleanup() error { - if u.cfg.FirstListener().IsDirectDnsListener() { - return nil - } - // Remove the custom dnsmasq config - if err := os.Remove(u.dnsmasqConfPath); err != nil { - return err - } - // Restart dnsmasq service. - if err := restartDNSMasq(); err != nil { - return err - } - return nil -} - -func restartDNSMasq() error { - buf, err := os.ReadFile(dnsmasq.UbiosPidFile()) - if err != nil { - return err - } - pid, err := strconv.ParseUint(string(bytes.TrimSpace(buf)), 10, 64) - if err != nil { - return err - } - proc, err := os.FindProcess(int(pid)) - if err != nil { - return err - } - return proc.Kill() -} diff --git a/internal/rulematcher/domain.go b/internal/rulematcher/domain.go new file mode 100644 index 00000000..e70ea583 --- /dev/null +++ b/internal/rulematcher/domain.go @@ -0,0 +1,31 @@ +package rulematcher + +import ( + "context" +) + +// DomainRuleMatcher handles matching of domain-based rules +type DomainRuleMatcher struct{} + +// Match evaluates domain rules against the requested domain +func (d *DomainRuleMatcher) Match(ctx context.Context, req *MatchRequest) *MatchResult { + if req.Policy == nil || len(req.Policy.Rules) == 0 { + return &MatchResult{Matched: false, RuleType: RuleTypeDomain} + } + + for _, rule := range req.Policy.Rules { + // There's only one entry per rule, config validation ensures this. + for source, targets := range rule { + if source == req.Domain || wildcardMatches(source, req.Domain) { + return &MatchResult{ + Matched: true, + Targets: targets, + MatchedRule: source, + RuleType: RuleTypeDomain, + } + } + } + } + + return &MatchResult{Matched: false, RuleType: RuleTypeDomain} +} diff --git a/internal/rulematcher/engine.go b/internal/rulematcher/engine.go new file mode 100644 index 00000000..8a5b9513 --- /dev/null +++ b/internal/rulematcher/engine.go @@ -0,0 +1,148 @@ +// Package rulematcher provides a flexible rule matching engine for DNS request routing. +// +// The rulematcher package implements a policy-based DNS routing system that allows +// configuring different types of rules to determine which upstream DNS servers should +// handle specific requests. It supports three types of rules: +// +// - Network rules: Match requests based on source IP address ranges +// - MAC rules: Match requests based on source MAC addresses +// - Domain rules: Match requests based on requested domain names +// +// The matching engine uses a configurable priority order to determine which rules +// take precedence when multiple rules match. By default, the priority order is: +// Network -> MAC -> Domain, with Domain rules having the highest priority and +// overriding all other matches. +// +// Example usage: +// +// config := &MatchingConfig{ +// Order: []RuleType{RuleTypeNetwork, RuleTypeMac, RuleTypeDomain}, +// } +// engine := NewMatchingEngine(config) +// +// request := &MatchRequest{ +// SourceIP: net.ParseIP("192.168.1.100"), +// SourceMac: "aa:bb:cc:dd:ee:ff", +// Domain: "example.com", +// Policy: policyConfig, +// Config: appConfig, +// } +// +// result := engine.FindUpstreams(ctx, request) +// if result.Matched { +// // Use result.Upstreams to route the request +// } +// +// The package maintains backward compatibility with existing behavior while +// providing a clean, extensible interface for adding new rule types. +package rulematcher + +import ( + "context" +) + +// MatchingEngine orchestrates rule matching based on configurable order +type MatchingEngine struct { + config *MatchingConfig + matchers map[RuleType]RuleMatcher +} + +// NewMatchingEngine creates a new matching engine with the given configuration +func NewMatchingEngine(config *MatchingConfig) *MatchingEngine { + if config == nil { + config = DefaultMatchingConfig() + } + + engine := &MatchingEngine{ + config: config, + matchers: map[RuleType]RuleMatcher{ + RuleTypeNetwork: &NetworkRuleMatcher{}, + RuleTypeMac: &MacRuleMatcher{}, + RuleTypeDomain: &DomainRuleMatcher{}, + }, + } + + return engine +} + +// FindUpstreams determines which upstreams should handle a request based on policy rules +// It implements the original behavior where MAC and domain rules can override network rules +func (e *MatchingEngine) FindUpstreams(ctx context.Context, req *MatchRequest) *MatchingResult { + result := &MatchingResult{ + Upstreams: []string{}, + MatchedPolicy: "no policy", + MatchedNetwork: "no network", + MatchedRule: "no rule", + Matched: false, + SrcAddr: req.SourceIP.String(), + MatchedRuleType: "", + MatchingOrder: e.config.Order, + } + + if req.Policy == nil { + return result + } + + result.MatchedPolicy = req.Policy.Name + + var networkMatch *MatchResult + var macMatch *MatchResult + var domainMatch *MatchResult + + // Check all rule types and store matches + for _, ruleType := range e.config.Order { + matcher, exists := e.matchers[ruleType] + if !exists { + continue + } + + matchResult := matcher.Match(ctx, req) + if matchResult.Matched { + switch matchResult.RuleType { + case RuleTypeNetwork: + networkMatch = matchResult + case RuleTypeMac: + macMatch = matchResult + case RuleTypeDomain: + domainMatch = matchResult + } + } + } + + // Determine the final match based on original logic: + // Domain rules override everything, MAC rules override network rules + if domainMatch != nil { + result.Upstreams = domainMatch.Targets + result.Matched = true + result.MatchedRuleType = string(domainMatch.RuleType) + result.MatchedRule = domainMatch.MatchedRule + // Special case: domain rules override network rules + if networkMatch != nil { + result.MatchedNetwork = networkMatch.MatchedRule + " (unenforced)" + } + } else if macMatch != nil { + result.Upstreams = macMatch.Targets + result.Matched = true + result.MatchedRuleType = string(macMatch.RuleType) + result.MatchedNetwork = macMatch.MatchedRule + } else if networkMatch != nil { + result.Upstreams = networkMatch.Targets + result.Matched = true + result.MatchedRuleType = string(networkMatch.RuleType) + result.MatchedNetwork = networkMatch.MatchedRule + } + + return result +} + +// MatchingResult represents the result of the matching engine +type MatchingResult struct { + Upstreams []string + MatchedPolicy string + MatchedNetwork string + MatchedRule string + Matched bool + SrcAddr string + MatchedRuleType string + MatchingOrder []RuleType +} diff --git a/internal/rulematcher/engine_test.go b/internal/rulematcher/engine_test.go new file mode 100644 index 00000000..1c388dc4 --- /dev/null +++ b/internal/rulematcher/engine_test.go @@ -0,0 +1,216 @@ +package rulematcher + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/Control-D-Inc/ctrld/testhelper" +) + +func TestMatchingEngine(t *testing.T) { + cfg := testhelper.SampleConfig(t) + // Convert Cidrs to IPNets like in the original test + for _, nc := range cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatal(err) + } + nc.IPNets = append(nc.IPNets, ipNet) + } + } + + tests := []struct { + name string + config *MatchingConfig + request *MatchRequest + expected *MatchingResult + }{ + { + name: "Default config - network match first", + config: DefaultMatchingConfig(), + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + SourceMac: "14:45:A0:67:83:0A", + Domain: "example.ru", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchingResult{ + Upstreams: []string{"upstream.1"}, + MatchedPolicy: "My Policy", + MatchedNetwork: "network.0 (unenforced)", + MatchedRule: "*.ru", + Matched: true, + SrcAddr: "192.168.0.1", + MatchedRuleType: "domain", + MatchingOrder: []RuleType{RuleTypeNetwork, RuleTypeMac, RuleTypeDomain}, + }, + }, + { + name: "Custom order - domain first", + config: &MatchingConfig{ + Order: []RuleType{RuleTypeDomain, RuleTypeNetwork, RuleTypeMac}, + }, + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + SourceMac: "14:45:A0:67:83:0A", + Domain: "example.ru", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchingResult{ + Upstreams: []string{"upstream.1"}, + MatchedPolicy: "My Policy", + MatchedNetwork: "network.0 (unenforced)", + MatchedRule: "*.ru", + Matched: true, + SrcAddr: "192.168.0.1", + MatchedRuleType: "domain", + MatchingOrder: []RuleType{RuleTypeDomain, RuleTypeNetwork, RuleTypeMac}, + }, + }, + { + name: "Custom order - MAC first", + config: &MatchingConfig{ + Order: []RuleType{RuleTypeMac, RuleTypeNetwork, RuleTypeDomain}, + }, + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + SourceMac: "14:45:A0:67:83:0A", + Domain: "example.ru", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchingResult{ + Upstreams: []string{"upstream.1"}, + MatchedPolicy: "My Policy", + MatchedNetwork: "network.0 (unenforced)", + MatchedRule: "*.ru", + Matched: true, + SrcAddr: "192.168.0.1", + MatchedRuleType: "domain", + MatchingOrder: []RuleType{RuleTypeMac, RuleTypeNetwork, RuleTypeDomain}, + }, + }, + { + name: "No policy", + config: DefaultMatchingConfig(), + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + SourceMac: "14:45:A0:67:83:0A", + Domain: "example.ru", + Policy: nil, + Config: cfg, + }, + expected: &MatchingResult{ + Upstreams: []string{}, + MatchedPolicy: "no policy", + MatchedNetwork: "no network", + MatchedRule: "no rule", + Matched: false, + SrcAddr: "192.168.0.1", + MatchedRuleType: "", + MatchingOrder: []RuleType{RuleTypeNetwork, RuleTypeMac, RuleTypeDomain}, + }, + }, + { + name: "No matches", + config: DefaultMatchingConfig(), + request: &MatchRequest{ + SourceIP: net.ParseIP("10.0.0.1"), + SourceMac: "00:11:22:33:44:55", + Domain: "example.com", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchingResult{ + Upstreams: []string{}, + MatchedPolicy: "My Policy", + MatchedNetwork: "no network", + MatchedRule: "no rule", + Matched: false, + SrcAddr: "10.0.0.1", + MatchedRuleType: "", + MatchingOrder: []RuleType{RuleTypeNetwork, RuleTypeMac, RuleTypeDomain}, + }, + }, + { + name: "MAC rule overrides network rule", + config: DefaultMatchingConfig(), + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + SourceMac: "14:45:A0:67:83:0A", + Domain: "example.com", // This domain doesn't match any domain rules + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchingResult{ + Upstreams: []string{"upstream.2"}, + MatchedPolicy: "My Policy", + MatchedNetwork: "14:45:a0:67:83:0a", + MatchedRule: "no rule", + Matched: true, + SrcAddr: "192.168.0.1", + MatchedRuleType: "mac", + MatchingOrder: []RuleType{RuleTypeNetwork, RuleTypeMac, RuleTypeDomain}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + engine := NewMatchingEngine(tc.config) + result := engine.FindUpstreams(context.Background(), tc.request) + + assert.Equal(t, tc.expected.Upstreams, result.Upstreams) + assert.Equal(t, tc.expected.MatchedPolicy, result.MatchedPolicy) + assert.Equal(t, tc.expected.MatchedNetwork, result.MatchedNetwork) + assert.Equal(t, tc.expected.MatchedRule, result.MatchedRule) + assert.Equal(t, tc.expected.Matched, result.Matched) + assert.Equal(t, tc.expected.SrcAddr, result.SrcAddr) + assert.Equal(t, tc.expected.MatchedRuleType, result.MatchedRuleType) + assert.Equal(t, tc.expected.MatchingOrder, result.MatchingOrder) + }) + } +} + +func TestDefaultMatchingConfig(t *testing.T) { + config := DefaultMatchingConfig() + + assert.Equal(t, []RuleType{RuleTypeNetwork, RuleTypeMac, RuleTypeDomain}, config.Order) +} + +func TestMatchingEngineWithInvalidRuleType(t *testing.T) { + cfg := testhelper.SampleConfig(t) + // Convert Cidrs to IPNets like in the original test + for _, nc := range cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatal(err) + } + nc.IPNets = append(nc.IPNets, ipNet) + } + } + + config := &MatchingConfig{ + Order: []RuleType{RuleType("invalid"), RuleTypeNetwork}, + } + + engine := NewMatchingEngine(config) + request := &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + Policy: cfg.Listener["0"].Policy, + Config: cfg, + } + + result := engine.FindUpstreams(context.Background(), request) + + // Should still work, just skip the invalid rule type + assert.True(t, result.Matched) + assert.Equal(t, "network", result.MatchedRuleType) +} diff --git a/internal/rulematcher/mac.go b/internal/rulematcher/mac.go new file mode 100644 index 00000000..ff20e814 --- /dev/null +++ b/internal/rulematcher/mac.go @@ -0,0 +1,62 @@ +package rulematcher + +import ( + "context" + "strings" +) + +// MacRuleMatcher handles matching of MAC address-based rules +type MacRuleMatcher struct{} + +// Match evaluates MAC address rules against the source MAC address +func (m *MacRuleMatcher) Match(ctx context.Context, req *MatchRequest) *MatchResult { + if req.Policy == nil || len(req.Policy.Macs) == 0 { + return &MatchResult{Matched: false, RuleType: RuleTypeMac} + } + + for _, rule := range req.Policy.Macs { + for source, targets := range rule { + if source != "" && (strings.EqualFold(source, req.SourceMac) || wildcardMatches(strings.ToLower(source), strings.ToLower(req.SourceMac))) { + return &MatchResult{ + Matched: true, + Targets: targets, + MatchedRule: source, // Return the original source from the rule + RuleType: RuleTypeMac, + } + } + } + } + + return &MatchResult{Matched: false, RuleType: RuleTypeMac} +} + +// wildcardMatches checks if a wildcard pattern matches a string +// This is copied from the original implementation to maintain compatibility +func wildcardMatches(wildcard, str string) bool { + if wildcard == "" { + return false + } + if wildcard == "*" { + return true + } + if !strings.Contains(wildcard, "*") { + return wildcard == str + } + + parts := strings.Split(wildcard, "*") + if len(parts) != 2 { + return false + } + + prefix := parts[0] + suffix := parts[1] + + if prefix != "" && !strings.HasPrefix(str, prefix) { + return false + } + if suffix != "" && !strings.HasSuffix(str, suffix) { + return false + } + + return true +} diff --git a/internal/rulematcher/network.go b/internal/rulematcher/network.go new file mode 100644 index 00000000..1c20406a --- /dev/null +++ b/internal/rulematcher/network.go @@ -0,0 +1,38 @@ +package rulematcher + +import ( + "context" + "strings" +) + +// NetworkRuleMatcher handles matching of network-based rules +type NetworkRuleMatcher struct{} + +// Match evaluates network rules against the source IP address +func (n *NetworkRuleMatcher) Match(ctx context.Context, req *MatchRequest) *MatchResult { + if req.Policy == nil || len(req.Policy.Networks) == 0 { + return &MatchResult{Matched: false, RuleType: RuleTypeNetwork} + } + + for _, rule := range req.Policy.Networks { + for source, targets := range rule { + networkNum := strings.TrimPrefix(source, "network.") + nc := req.Config.Network[networkNum] + if nc == nil { + continue + } + for _, ipNet := range nc.IPNets { + if ipNet.Contains(req.SourceIP) { + return &MatchResult{ + Matched: true, + Targets: targets, + MatchedRule: source, + RuleType: RuleTypeNetwork, + } + } + } + } + } + + return &MatchResult{Matched: false, RuleType: RuleTypeNetwork} +} diff --git a/internal/rulematcher/rulematcher_test.go b/internal/rulematcher/rulematcher_test.go new file mode 100644 index 00000000..d4eb2356 --- /dev/null +++ b/internal/rulematcher/rulematcher_test.go @@ -0,0 +1,248 @@ +package rulematcher + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// Test NetworkRuleMatcher +func TestNetworkRuleMatcher(t *testing.T) { + cfg := testhelper.SampleConfig(t) + // Convert Cidrs to IPNets like in the original test + for _, nc := range cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatal(err) + } + nc.IPNets = append(nc.IPNets, ipNet) + } + } + matcher := &NetworkRuleMatcher{} + + tests := []struct { + name string + request *MatchRequest + expected *MatchResult + }{ + { + name: "No policy", + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + Policy: nil, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeNetwork}, + }, + { + name: "No network rules", + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + Policy: &ctrld.ListenerPolicyConfig{}, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeNetwork}, + }, + { + name: "Match network rule", + request: &MatchRequest{ + SourceIP: net.ParseIP("192.168.0.1"), + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{ + Matched: true, + Targets: []string{"upstream.1", "upstream.0"}, + MatchedRule: "network.0", + RuleType: RuleTypeNetwork, + }, + }, + { + name: "No match for IP", + request: &MatchRequest{ + SourceIP: net.ParseIP("10.0.0.1"), + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeNetwork}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := matcher.Match(context.Background(), tc.request) + assert.Equal(t, tc.expected.Matched, result.Matched) + assert.Equal(t, tc.expected.RuleType, result.RuleType) + if tc.expected.Matched { + assert.Equal(t, tc.expected.Targets, result.Targets) + assert.Equal(t, tc.expected.MatchedRule, result.MatchedRule) + } + }) + } +} + +// Test MacRuleMatcher +func TestMacRuleMatcher(t *testing.T) { + cfg := testhelper.SampleConfig(t) + matcher := &MacRuleMatcher{} + + tests := []struct { + name string + request *MatchRequest + expected *MatchResult + }{ + { + name: "No policy", + request: &MatchRequest{ + SourceMac: "14:45:A0:67:83:0A", + Policy: nil, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeMac}, + }, + { + name: "No MAC rules", + request: &MatchRequest{ + SourceMac: "14:45:A0:67:83:0A", + Policy: &ctrld.ListenerPolicyConfig{}, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeMac}, + }, + { + name: "Match MAC rule - exact", + request: &MatchRequest{ + SourceMac: "14:45:A0:67:83:0A", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{ + Matched: true, + Targets: []string{"upstream.2"}, + MatchedRule: "14:45:a0:67:83:0a", // Config loading normalizes MAC addresses to lowercase + RuleType: RuleTypeMac, + }, + }, + { + name: "Match MAC rule - case insensitive", + request: &MatchRequest{ + SourceMac: "14:54:4a:8e:08:2d", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{ + Matched: true, + Targets: []string{"upstream.2"}, + MatchedRule: "14:54:4a:8e:08:2d", + RuleType: RuleTypeMac, + }, + }, + { + name: "No match for MAC", + request: &MatchRequest{ + SourceMac: "00:11:22:33:44:55", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeMac}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := matcher.Match(context.Background(), tc.request) + assert.Equal(t, tc.expected.Matched, result.Matched) + assert.Equal(t, tc.expected.RuleType, result.RuleType) + if tc.expected.Matched { + assert.Equal(t, tc.expected.Targets, result.Targets) + assert.Equal(t, tc.expected.MatchedRule, result.MatchedRule) + } + }) + } +} + +// Test DomainRuleMatcher +func TestDomainRuleMatcher(t *testing.T) { + cfg := testhelper.SampleConfig(t) + matcher := &DomainRuleMatcher{} + + tests := []struct { + name string + request *MatchRequest + expected *MatchResult + }{ + { + name: "No policy", + request: &MatchRequest{ + Domain: "example.com", + Policy: nil, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeDomain}, + }, + { + name: "No domain rules", + request: &MatchRequest{ + Domain: "example.com", + Policy: &ctrld.ListenerPolicyConfig{}, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeDomain}, + }, + { + name: "Match domain rule - exact", + request: &MatchRequest{ + Domain: "example.ru", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{ + Matched: true, + Targets: []string{"upstream.1"}, + MatchedRule: "*.ru", + RuleType: RuleTypeDomain, + }, + }, + { + name: "Match domain rule - wildcard", + request: &MatchRequest{ + Domain: "test.ru", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{ + Matched: true, + Targets: []string{"upstream.1"}, + MatchedRule: "*.ru", + RuleType: RuleTypeDomain, + }, + }, + { + name: "No match for domain", + request: &MatchRequest{ + Domain: "example.com", + Policy: cfg.Listener["0"].Policy, + Config: cfg, + }, + expected: &MatchResult{Matched: false, RuleType: RuleTypeDomain}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := matcher.Match(context.Background(), tc.request) + assert.Equal(t, tc.expected.Matched, result.Matched) + assert.Equal(t, tc.expected.RuleType, result.RuleType) + if tc.expected.Matched { + assert.Equal(t, tc.expected.Targets, result.Targets) + assert.Equal(t, tc.expected.MatchedRule, result.MatchedRule) + } + }) + } +} diff --git a/internal/rulematcher/types.go b/internal/rulematcher/types.go new file mode 100644 index 00000000..073830e1 --- /dev/null +++ b/internal/rulematcher/types.go @@ -0,0 +1,52 @@ +package rulematcher + +import ( + "context" + "net" + + "github.com/Control-D-Inc/ctrld" +) + +// RuleType represents the type of rule being matched +type RuleType string + +const ( + RuleTypeNetwork RuleType = "network" + RuleTypeMac RuleType = "mac" + RuleTypeDomain RuleType = "domain" +) + +// RuleMatcher defines the interface for matching different types of rules +type RuleMatcher interface { + Match(ctx context.Context, request *MatchRequest) *MatchResult +} + +// MatchRequest contains all the information needed for rule matching +type MatchRequest struct { + SourceIP net.IP + SourceMac string + Domain string + Policy *ctrld.ListenerPolicyConfig + Config *ctrld.Config +} + +// MatchResult represents the result of a rule matching operation +type MatchResult struct { + Matched bool + Targets []string + MatchedRule string + RuleType RuleType +} + +// MatchingConfig defines the configuration for rule matching behavior +type MatchingConfig struct { + Order []RuleType `json:"order" yaml:"order"` +} + +// DefaultMatchingConfig returns the default matching configuration +// This maintains backward compatibility with the current behavior +func DefaultMatchingConfig() *MatchingConfig { + return &MatchingConfig{ + Order: []RuleType{RuleTypeNetwork, RuleTypeMac, RuleTypeDomain}, + } +} diff --git a/internal/system/chassis_darwin.go b/internal/system/chassis_darwin.go new file mode 100644 index 00000000..49a7317b --- /dev/null +++ b/internal/system/chassis_darwin.go @@ -0,0 +1,25 @@ +package system + +import ( + "errors" + "fmt" + + "github.com/brunogui0812/sysprofiler" +) + +// GetChassisInfo retrieves hardware information including machine model type and vendor from the system profiler. +func GetChassisInfo() (*ChassisInfo, error) { + hardwares, err := sysprofiler.Hardware() + if err != nil { + return nil, fmt.Errorf("failed to get hardware info: %w", err) + } + if len(hardwares) == 0 { + return nil, errors.New("no hardware info found") + } + hardware := hardwares[0] + info := &ChassisInfo{ + Type: hardware.MachineModel, + Vendor: "Apple Inc.", + } + return info, nil +} diff --git a/internal/system/chassis_others.go b/internal/system/chassis_others.go new file mode 100644 index 00000000..49e38aaa --- /dev/null +++ b/internal/system/chassis_others.go @@ -0,0 +1,20 @@ +//go:build !darwin + +package system + +import "github.com/jaypipes/ghw" + +// GetChassisInfo retrieves hardware information including machine model type and vendor from the system profiler. +func GetChassisInfo() (*ChassisInfo, error) { + // Disable warnings from ghw, since these are undesirable but recoverable errors. + // With warnings enabled, ghw will emit unnecessary log messages. + chassis, err := ghw.Chassis(ghw.WithDisableWarnings()) + if err != nil { + return nil, err + } + info := &ChassisInfo{ + Type: chassis.TypeDescription, + Vendor: chassis.Vendor, + } + return info, nil +} diff --git a/internal/system/metadata.go b/internal/system/metadata.go new file mode 100644 index 00000000..cfe02e3c --- /dev/null +++ b/internal/system/metadata.go @@ -0,0 +1,7 @@ +package system + +// ChassisInfo represents the structural framework of a device, specifying its type and manufacturer information. +type ChassisInfo struct { + Type string + Vendor string +} diff --git a/internal/system/metadata_others.go b/internal/system/metadata_others.go new file mode 100644 index 00000000..f20a1508 --- /dev/null +++ b/internal/system/metadata_others.go @@ -0,0 +1,8 @@ +//go:build !windows + +package system + +// GetActiveDirectoryDomain returns AD domain name of this computer. +func GetActiveDirectoryDomain() (string, error) { + return "", nil +} diff --git a/internal/system/metadata_windows.go b/internal/system/metadata_windows.go new file mode 100644 index 00000000..40f137fb --- /dev/null +++ b/internal/system/metadata_windows.go @@ -0,0 +1,74 @@ +package system + +import ( + "errors" + "fmt" + "io" + "log" + "os" + "strings" + "unsafe" + + "github.com/microsoft/wmi/pkg/base/host" + hh "github.com/microsoft/wmi/pkg/hardware/host" + "golang.org/x/sys/windows" +) + +// GetActiveDirectoryDomain returns AD domain name of this computer. +func GetActiveDirectoryDomain() (string, error) { + log.SetOutput(io.Discard) + defer log.SetOutput(os.Stderr) + + // 1) Check environment variable + envDomain := os.Getenv("USERDNSDOMAIN") + if envDomain != "" { + return strings.TrimSpace(envDomain), nil + } + + // 2) Query WMI via the microsoft/wmi library + whost := host.NewWmiLocalHost() + cs, err := hh.GetComputerSystem(whost) + if cs != nil { + defer cs.Close() + } + if err != nil { + return "", err + } + pod, err := cs.GetPropertyPartOfDomain() + if err != nil { + return "", err + } + if pod { + domainVal, err := cs.GetPropertyDomain() + if err != nil { + return "", fmt.Errorf("failed to get domain property: %w", err) + } + domainName := strings.TrimSpace(fmt.Sprintf("%v", domainVal)) + if domainName == "" { + return "", errors.New("machine does not appear to have a domain set") + } + return domainName, nil + } + return "", nil +} + +// DomainJoinedStatus returns the domain joined status of the current computer. +// +// NETSETUP_JOIN_STATUS constants from Microsoft Windows API +// See: https://learn.microsoft.com/en-us/windows/win32/api/lmjoin/ne-lmjoin-netsetup_join_status +// +// NetSetupUnknownStatus uint32 = 0 // The status is unknown +// NetSetupUnjoined uint32 = 1 // The computer is not joined to a domain or workgroup +// NetSetupWorkgroupName uint32 = 2 // The computer is joined to a workgroup +// NetSetupDomainName uint32 = 3 // The computer is joined to a domain +func DomainJoinedStatus() (uint32, error) { + var domain *uint16 + var status uint32 + + if err := windows.NetGetJoinInformation(nil, &domain, &status); err != nil { + return 0, fmt.Errorf("failed to get domain join status: %w", err) + } + defer windows.NetApiBufferFree((*byte)(unsafe.Pointer(domain))) + + return status, nil +} diff --git a/log.go b/log.go index 14c82e8a..e02a0c0c 100644 --- a/log.go +++ b/log.go @@ -4,25 +4,131 @@ import ( "context" "fmt" "io" - "sync/atomic" + "os" + "time" - "github.com/rs/zerolog" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) -// ProxyLog emits the log record for proxy operations. -// The caller should set it only once. -// DEPRECATED: use ProxyLogger instead. -var ProxyLog = zerolog.New(io.Discard) +// Custom log level for NOTICE (between INFO and WARN) +// DEBUG = -1, INFO = 0, WARN = 1, ERROR = 2, FATAL = 3 +// Since there's no integer between INFO (0) and WARN (1), we'll use the same value as WARN +// but handle NOTICE specially in the encoder to display it differently. +// Note: NOTICE and WARN share the same numeric value (1), so they will both display as "NOTICE" +// when using the custom encoder. This is the intended behavior for visual distinction. +const NoticeLevel = zapcore.Level(zapcore.WarnLevel) // Same value as WARN, but handled specially -// ProxyLogger emits the log record for proxy operations. -var ProxyLogger atomic.Pointer[zerolog.Logger] +// LoggerCtxKey is the context.Context key for a logger. +type LoggerCtxKey struct{} + +// LoggerCtx returns a context.Context with LoggerCtxKey set. +func LoggerCtx(ctx context.Context, l *Logger) context.Context { + return context.WithValue(ctx, LoggerCtxKey{}, l) +} + +// A Logger provides fast, leveled, structured logging. +type Logger struct { + *zap.Logger +} + +var noOpZapLogger = zap.NewNop() + +// NopLogger returns a logger which all operation are no-op. +var NopLogger = &Logger{noOpZapLogger} + +// LoggerFromCtx returns the logger associated with given ctx. +// +// If there's no logger, a no-op logger will be returned. +func LoggerFromCtx(ctx context.Context) *Logger { + if logger, ok := ctx.Value(LoggerCtxKey{}).(*Logger); ok && logger != nil { + return logger + } + return NopLogger +} // ReqIdCtxKey is the context.Context key for a request id. type ReqIdCtxKey struct{} -// Log emits the logs for a particular zerolog event. +// LogEvent represents a logging event with structured fields +type LogEvent struct { + logger *zap.Logger + level zapcore.Level + fields []zap.Field +} + +// Msg logs the message with the collected fields +func (e *LogEvent) Msg(msg string) { + e.logger.Check(e.level, msg).Write(e.fields...) +} + +// Msgf logs a formatted message with the collected fields +func (e *LogEvent) Msgf(format string, v ...any) { + e.Msg(fmt.Sprintf(format, v...)) +} + +// MsgFunc logs a message from a function with the collected fields +func (e *LogEvent) MsgFunc(fn func() string) { + e.Msg(fn()) +} + +// Str adds a string field to the event +func (e *LogEvent) Str(key, val string) *LogEvent { + e.fields = append(e.fields, zap.String(key, val)) + return e +} + +// Int adds an integer field to the event +func (e *LogEvent) Int(key string, val int) *LogEvent { + e.fields = append(e.fields, zap.Int(key, val)) + return e +} + +// Int64 adds an int64 field to the event +func (e *LogEvent) Int64(key string, val int64) *LogEvent { + e.fields = append(e.fields, zap.Int64(key, val)) + return e +} + +func (e *LogEvent) Uint64(key string, val uint64) *LogEvent { + e.fields = append(e.fields, zap.Uint64(key, val)) + return e +} + +// Err adds an error field to the event +func (e *LogEvent) Err(err error) *LogEvent { + if err != nil { + e.fields = append(e.fields, zap.Error(err)) + } + return e +} + +// Bool adds a boolean field to the event +func (e *LogEvent) Bool(key string, val bool) *LogEvent { + e.fields = append(e.fields, zap.Bool(key, val)) + return e +} + +// Interface adds an interface field to the event +func (e *LogEvent) Interface(key string, val interface{}) *LogEvent { + e.fields = append(e.fields, zap.Any(key, val)) + return e +} + +// Any adds an interface field to the event (alias for Interface) +func (e *LogEvent) Any(key string, val interface{}) *LogEvent { + return e.Interface(key, val) +} + +// Strs adds a string slice field to the event +func (e *LogEvent) Strs(key string, vals []string) *LogEvent { + e.fields = append(e.fields, zap.Strings(key, vals)) + return e +} + +// Log emits the logs for a particular logging event. // The request id associated with the context will be included if presents. -func Log(ctx context.Context, e *zerolog.Event, format string, v ...any) { +func Log(ctx context.Context, e *LogEvent, format string, v ...any) { id, ok := ctx.Value(ReqIdCtxKey{}).(string) if !ok { e.Msgf(format, v...) @@ -32,3 +138,209 @@ func Log(ctx context.Context, e *zerolog.Event, format string, v ...any) { return fmt.Sprintf("[%s] %s", id, fmt.Sprintf(format, v...)) }) } + +// Logger methods that mimic zerolog API +func (l *Logger) Debug() *LogEvent { + return &LogEvent{ + logger: l.Logger, + level: zapcore.DebugLevel, + fields: []zap.Field{}, + } +} + +func (l *Logger) Info() *LogEvent { + return &LogEvent{ + logger: l.Logger, + level: zapcore.InfoLevel, + fields: []zap.Field{}, + } +} + +func (l *Logger) Warn() *LogEvent { + return &LogEvent{ + logger: l.Logger, + level: zapcore.WarnLevel, + fields: []zap.Field{}, + } +} + +func (l *Logger) Error() *LogEvent { + return &LogEvent{ + logger: l.Logger, + level: zapcore.ErrorLevel, + fields: []zap.Field{}, + } +} + +func (l *Logger) Fatal() *LogEvent { + return &LogEvent{ + logger: l.Logger, + level: zapcore.FatalLevel, + fields: []zap.Field{}, + } +} + +func (l *Logger) Notice() *LogEvent { + return &LogEvent{ + logger: l.Logger, + level: NoticeLevel, // Custom NOTICE level between INFO and WARN + fields: []zap.Field{}, + } +} + +// With returns a logger with additional fields +func (l *Logger) With() *Logger { + return l +} + +// Str adds a string field to the logger +func (l *Logger) Str(key, val string) *Logger { + // Create a new logger with the field added + newLogger := l.Logger.With(zap.String(key, val)) + return &Logger{newLogger} +} + +// Err adds an error field to the logger +func (l *Logger) Err(err error) *Logger { + // Create a new logger with the error field added + newLogger := l.Logger.With(zap.Error(err)) + return &Logger{newLogger} +} + +// Any adds an interface field to the logger +func (l *Logger) Any(key string, val interface{}) *Logger { + // Create a new logger with the field added + newLogger := l.Logger.With(zap.Any(key, val)) + return &Logger{newLogger} +} + +// Bool adds a boolean field to the logger +func (l *Logger) Bool(key string, val bool) *Logger { + // Create a new logger with the field added + newLogger := l.Logger.With(zap.Bool(key, val)) + return &Logger{newLogger} +} + +// Msgf logs a formatted message at info level +func (l *Logger) Msgf(format string, v ...any) { + l.Info().Msgf(format, v...) +} + +// Msg logs a message at info level +func (l *Logger) Msg(msg string) { + l.Info().Msg(msg) +} + +// Output returns a logger with the specified output +func (l *Logger) Output(w io.Writer) *Logger { + // Create a new zap logger with the writer + encoderConfig := zap.NewDevelopmentEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout(time.RFC3339) + encoder := zapcore.NewConsoleEncoder(encoderConfig) + core := zapcore.NewCore(encoder, zapcore.AddSync(w), zapcore.InfoLevel) + newLogger := zap.New(core) + return &Logger{newLogger} +} + +// GetLogger returns the underlying logger +func (l *Logger) GetLogger() *Logger { + return l +} + +// Write implements io.Writer to allow direct writing to the logger +func (l *Logger) Write(p []byte) (n int, err error) { + stdoutSyncer := zapcore.AddSync(os.Stdout) + stdoutSyncer.Write(p) + return len(p), nil +} + +// Printf logs a formatted message at info level +func (l *Logger) Printf(format string, v ...any) { + l.Info().Msgf(format, v...) +} + +// JournalField marks an event for the retained journal stream. +const JournalField = "journal" + +// Journal marks an event for the retained journal stream. The journal core +// keeps every line that carries the field, whatever the level of the line is. +func Journal(e *LogEvent) *LogEvent { + return e.Bool(JournalField, true) +} + +// Time adds a time field to the event +func (e *LogEvent) Time(key string, val time.Time) *LogEvent { + e.fields = append(e.fields, zap.Time(key, val)) + return e +} + +// Float64 adds a float64 field to the event +func (e *LogEvent) Float64(key string, val float64) *LogEvent { + e.fields = append(e.fields, zap.Float64(key, val)) + return e +} + +// Dict starts a nested object. Dict on an event adds it under a key. +func Dict() *LogEvent { + return &LogEvent{} +} + +// Dict adds the fields of dict as a nested object under key +func (e *LogEvent) Dict(key string, dict *LogEvent) *LogEvent { + e.fields = append(e.fields, zap.Object(key, logFields(dict.fields))) + return e +} + +// Array adds an array field to the event +func (e *LogEvent) Array(key string, arr *LogArray) *LogEvent { + e.fields = append(e.fields, zap.Array(key, arr)) + return e +} + +// logFields renders a field list as one nested object +type logFields []zap.Field + +// MarshalLogObject implements zapcore.ObjectMarshaler +func (f logFields) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, field := range f { + field.AddTo(enc) + } + return nil +} + +// LogArray holds the items of an array field. Arr starts one. +type LogArray struct { + items []func(zapcore.ArrayEncoder) error +} + +// Arr starts an array field. Array on an event adds it under a key. +func Arr() *LogArray { + return &LogArray{} +} + +// Dict appends a nested object to the array +func (a *LogArray) Dict(dict *LogEvent) *LogArray { + fields := logFields(dict.fields) + a.items = append(a.items, func(enc zapcore.ArrayEncoder) error { return enc.AppendObject(fields) }) + return a +} + +// Str appends a string to the array +func (a *LogArray) Str(val string) *LogArray { + a.items = append(a.items, func(enc zapcore.ArrayEncoder) error { + enc.AppendString(val) + return nil + }) + return a +} + +// MarshalLogArray implements zapcore.ArrayMarshaler +func (a *LogArray) MarshalLogArray(enc zapcore.ArrayEncoder) error { + for _, item := range a.items { + if err := item(enc); err != nil { + return err + } + } + return nil +} diff --git a/metadata.go b/metadata.go new file mode 100644 index 00000000..dbf2e797 --- /dev/null +++ b/metadata.go @@ -0,0 +1,90 @@ +package ctrld + +import ( + "context" + + "github.com/cuonglm/osinfo" + + "github.com/Control-D-Inc/ctrld/internal/system" +) + +const ( + metadataOsKey = "os" + metadataChassisTypeKey = "chassis_type" + metadataChassisVendorKey = "chassis_vendor" + metadataUsernameKey = "username" + metadataDomainOrWorkgroupKey = "domain_or_workgroup" + metadataDomainKey = "domain" +) + +var ( + chassisType string + chassisVendor string +) + +// username discovery modes for systemMetadata. +const ( + usernameNone = iota // omit username + usernameSession // username only from an active login session + usernameFull // username with account-list fallbacks, or "unknown" +) + +var ( + discoverMainUserFn = DiscoverMainUser + discoverSessionUserFn = discoverSessionUser +) + +// SystemMetadata collects full system metadata including username discovery. +// Use for initial provisioning, where full device identification is needed. +func SystemMetadata(ctx context.Context) map[string]string { + return systemMetadata(ctx, usernameFull) +} + +// SystemMetadataStartup collects the metadata for the first managed check-in of +// each daemon start. The daemon usually starts at boot, before anyone logs in, +// so username is sent only when an active login session provides it. Otherwise +// the key is omitted and the API keeps the stored value instead of replacing it +// with an account-list guess or "unknown". +func SystemMetadataStartup(ctx context.Context) map[string]string { + return systemMetadata(ctx, usernameSession) +} + +// SystemMetadataRuntime collects system metadata without username discovery. +// Use for runtime API calls (config reload, self-uninstall check, deactivation +// pin refresh) to avoid repeated user enumeration that can trigger EDR alerts. +func SystemMetadataRuntime(ctx context.Context) map[string]string { + return systemMetadata(ctx, usernameNone) +} + +func systemMetadata(ctx context.Context, usernameMode int) map[string]string { + logger := LoggerFromCtx(ctx) + m := make(map[string]string) + oi := osinfo.New() + m[metadataOsKey] = oi.String() + if chassisType == "" && chassisVendor == "" { + ci, err := system.GetChassisInfo() + if err != nil { + logger.Debug().Err(err).Msg("Failed to get chassis info") + } else { + chassisType, chassisVendor = ci.Type, ci.Vendor + } + } + m[metadataChassisTypeKey] = chassisType + m[metadataChassisVendorKey] = chassisVendor + switch usernameMode { + case usernameFull: + m[metadataUsernameKey] = discoverMainUserFn(ctx) + case usernameSession: + if user := discoverSessionUserFn(ctx); user != "" { + m[metadataUsernameKey] = user + } + } + m[metadataDomainOrWorkgroupKey] = partOfDomainOrWorkgroup(ctx) + domain, err := system.GetActiveDirectoryDomain() + if err != nil { + logger.Debug().Err(err).Msg("Failed to get active directory domain name") + } + m[metadataDomainKey] = domain + + return m +} diff --git a/metadata_others.go b/metadata_others.go new file mode 100644 index 00000000..2b060ac3 --- /dev/null +++ b/metadata_others.go @@ -0,0 +1,10 @@ +//go:build !windows + +package ctrld + +import "context" + +// partOfDomainOrWorkgroup checks if the computer is part of a domain or workgroup and returns "true" or "false". +func partOfDomainOrWorkgroup(ctx context.Context) string { + return "false" +} diff --git a/metadata_test.go b/metadata_test.go new file mode 100644 index 00000000..ab0df76c --- /dev/null +++ b/metadata_test.go @@ -0,0 +1,67 @@ +package ctrld + +import ( + "context" + "reflect" + "testing" +) + +func Test_metadata(t *testing.T) { + m := SystemMetadata(context.Background()) + t.Logf("metadata: %v", m) +} + +func TestSystemMetadataUsernameScope(t *testing.T) { + oldMain, oldSession := discoverMainUserFn, discoverSessionUserFn + t.Cleanup(func() { discoverMainUserFn, discoverSessionUserFn = oldMain, oldSession }) + + var mainCalls int + discoverMainUserFn = func(context.Context) string { + mainCalls++ + return "lowest-uid-guess" + } + + t.Run("startup with no login session omits username", func(t *testing.T) { + mainCalls = 0 + discoverSessionUserFn = func(context.Context) string { return "" } + m := SystemMetadataStartup(context.Background()) + if v, ok := m[metadataUsernameKey]; ok { + t.Errorf("startup metadata sent username %q with no login session; the API would overwrite the stored value", v) + } + if mainCalls != 0 { + t.Error("startup ran the account-list fallbacks") + } + }) + + t.Run("startup with a login session sends that user", func(t *testing.T) { + discoverSessionUserFn = func(context.Context) string { return "session-user" } + if got := SystemMetadataStartup(context.Background())[metadataUsernameKey]; got != "session-user" { + t.Errorf("startup username = %q, want %q", got, "session-user") + } + }) + + t.Run("provisioning keeps the full discovery chain", func(t *testing.T) { + discoverSessionUserFn = func(context.Context) string { return "" } + if got := SystemMetadata(context.Background())[metadataUsernameKey]; got != "lowest-uid-guess" { + t.Errorf("provisioning username = %q, want the full-discovery result", got) + } + }) + + t.Run("runtime omits username", func(t *testing.T) { + mainCalls = 0 + discoverSessionUserFn = func(context.Context) string { + t.Error("runtime ran session discovery") + return "session-user" + } + if _, ok := SystemMetadataRuntime(context.Background())[metadataUsernameKey]; ok || mainCalls != 0 { + t.Error("runtime metadata included or discovered username") + } + }) + + t.Run("startup carries the same non-username keys as runtime", func(t *testing.T) { + discoverSessionUserFn = func(context.Context) string { return "" } + if s, r := SystemMetadataStartup(context.Background()), SystemMetadataRuntime(context.Background()); !reflect.DeepEqual(s, r) { + t.Errorf("startup %v and runtime %v differ beyond username", s, r) + } + }) +} diff --git a/metadata_windows.go b/metadata_windows.go new file mode 100644 index 00000000..21d63249 --- /dev/null +++ b/metadata_windows.go @@ -0,0 +1,23 @@ +package ctrld + +import ( + "context" + + "github.com/Control-D-Inc/ctrld/internal/system" +) + +// partOfDomainOrWorkgroup checks if the computer is part of a domain or workgroup and returns "true" or "false". +func partOfDomainOrWorkgroup(ctx context.Context) string { + status, err := system.DomainJoinedStatus() + if err != nil { + logger := LoggerFromCtx(ctx) + logger.Debug().Err(err).Msg("Failed to get domain join status") + return "false" + } + switch status { + case 2, 3: + return "true" + default: + return "false" + } +} diff --git a/nameservers.go b/nameservers.go index 0aebf9e1..da573e67 100644 --- a/nameservers.go +++ b/nameservers.go @@ -1,9 +1,15 @@ package ctrld -type dnsFn func() []string +import ( + "context" + + "github.com/Control-D-Inc/ctrld/internal/resolvconffile" +) + +type dnsFn func(ctx context.Context) []string // nameservers returns DNS nameservers from system settings. -func nameservers() []string { +func nameservers(ctx context.Context) []string { var dns []string seen := make(map[string]bool) ch := make(chan []string) @@ -11,7 +17,7 @@ func nameservers() []string { for _, fn := range fns { go func(fn dnsFn) { - ch <- fn() + ch <- fn(ctx) }(fn) } for range fns { @@ -26,3 +32,8 @@ func nameservers() []string { return dns } + +// CurrentNameserversFromResolvconf returns the current nameservers set from /etc/resolv.conf file. +func CurrentNameserversFromResolvconf() []string { + return resolvconffile.NameServers() +} diff --git a/nameservers_bsd.go b/nameservers_bsd.go index 09c9516d..15c30c94 100644 --- a/nameservers_bsd.go +++ b/nameservers_bsd.go @@ -3,6 +3,7 @@ package ctrld import ( + "context" "net" "syscall" @@ -13,7 +14,7 @@ func dnsFns() []dnsFn { return []dnsFn{dnsFromResolvConf, dnsFromRIB} } -func dnsFromRIB() []string { +func dnsFromRIB(_ context.Context) []string { var dns []string rib, err := route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeRoute, 0) if err != nil { diff --git a/nameservers_darwin.go b/nameservers_darwin.go index 1bf45746..a6516921 100644 --- a/nameservers_darwin.go +++ b/nameservers_darwin.go @@ -6,11 +6,9 @@ import ( "bufio" "bytes" "context" - "fmt" "io" "net" "os/exec" - "regexp" "slices" "strings" "time" @@ -22,15 +20,17 @@ func dnsFns() []dnsFn { return []dnsFn{dnsFromResolvConf, getDNSFromScutil, getAllDHCPNameservers} } -func getDNSFromScutil() []string { - logger := *ProxyLogger.Load() +var scutilLocalAddresses = netmon.LocalAddresses + +func getDNSFromScutil(ctx context.Context) []string { + logger := LoggerFromCtx(ctx) const ( maxRetries = 10 retryInterval = 100 * time.Millisecond ) - regularIPs, loopbackIPs, _ := netmon.LocalAddresses() + regularIPs, loopbackIPs, _ := scutilLocalAddresses() var nameservers []string for attempt := 0; attempt < maxRetries; attempt++ { @@ -41,41 +41,13 @@ func getDNSFromScutil() []string { cmd := exec.Command("scutil", "--dns") output, err := cmd.Output() if err != nil { - Log(context.Background(), logger.Error(), "failed to execute scutil --dns (attempt %d/%d): %v", attempt+1, maxRetries, err) + Log(context.Background(), logger.Error(), "Failed to execute scutil --dns (attempt %d/%d): %v", attempt+1, maxRetries, err) continue } - var localDNS []string - seen := make(map[string]bool) - - scanner := bufio.NewScanner(bytes.NewReader(output)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if strings.HasPrefix(line, "nameserver[") { - parts := strings.Split(line, ":") - if len(parts) == 2 { - ns := strings.TrimSpace(parts[1]) - if ip := net.ParseIP(ns); ip != nil { - // skip loopback IPs - isLocal := false - for _, v := range slices.Concat(regularIPs, loopbackIPs) { - ipStr := v.String() - if ip.String() == ipStr { - isLocal = true - break - } - } - if !isLocal && !seen[ip.String()] { - seen[ip.String()] = true - localDNS = append(localDNS, ip.String()) - } - } - } - } - } - - if err := scanner.Err(); err != nil { - Log(context.Background(), logger.Error(), "error scanning scutil output (attempt %d/%d): %v", attempt+1, maxRetries, err) + localDNS, err := parseScutilNameservers(output, slices.Concat(regularIPs, loopbackIPs)) + if err != nil { + Log(context.Background(), logger.Error(), "Error scanning scutil output (attempt %d/%d): %v", attempt+1, maxRetries, err) continue } @@ -89,28 +61,26 @@ func getDNSFromScutil() []string { } func getDHCPNameservers(iface string) ([]string, error) { - // Run the ipconfig command for the given interface. - cmd := exec.Command("ipconfig", "getpacket", iface) - output, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("error running ipconfig: %v", err) - } + return dhcpNameserversFromCommands(context.Background(), iface, func(_ context.Context, args ...string) ([]byte, error) { + return exec.Command("ipconfig", args...).Output() + }) +} - // Look for a line like: - // domain_name_servers = 192.168.1.1 8.8.8.8; - re := regexp.MustCompile(`domain_name_servers\s*=\s*(.*);`) - matches := re.FindStringSubmatch(string(output)) - if len(matches) < 2 { - return nil, fmt.Errorf("no DHCP nameservers found") - } +// DHCPNameserversForInterfaceContext bounds both commands with the caller's +// context. Other system-discovery callers retain their existing behavior. +func DHCPNameserversForInterfaceContext(ctx context.Context, iface string) ([]string, error) { + return dhcpNameserversFromCommands(ctx, iface, func(ctx context.Context, args ...string) ([]byte, error) { + return dhcpCommandOutput(ctx, "/usr/sbin/ipconfig", args...) + }) +} - // Split the nameservers by whitespace. - nameservers := strings.Fields(matches[1]) - return nameservers, nil +// DHCPNameserversForInterface returns DHCP option 6 for exactly iface. +func DHCPNameserversForInterface(iface string) ([]string, error) { + return getDHCPNameservers(iface) } -func getAllDHCPNameservers() []string { - logger := *ProxyLogger.Load() +func getAllDHCPNameservers(ctx context.Context) []string { + logger := LoggerFromCtx(ctx) interfaces, err := net.Interfaces() if err != nil { @@ -172,7 +142,7 @@ func getAllDHCPNameservers() []string { // if we have static DNS servers saved for the current default route, we should add them to the list drIfaceName, err := netmon.DefaultRouteInterface() - Log(context.Background(), logger.Debug(), "checking for static DNS servers for default route interface: %s", drIfaceName) + Log(context.Background(), logger.Debug(), "Checking for static DNS servers for default route interface: %s", drIfaceName) if err != nil { Log(context.Background(), logger.Debug(), "Failed to get default route interface: %v", err) @@ -186,7 +156,7 @@ func getAllDHCPNameservers() []string { Log(context.Background(), logger.Debug(), "Failed to patch interface name %s: %v", drIfaceName, err) } - staticNs, file := SavedStaticNameservers(drIface) + staticNs, file := SavedStaticNameserversAndPath(drIface) Log(context.Background(), logger.Debug(), "static dns servers from %s: %v", file, staticNs) if len(staticNs) > 0 { diff --git a/nameservers_darwin_deadline_test.go b/nameservers_darwin_deadline_test.go new file mode 100644 index 00000000..32caee37 --- /dev/null +++ b/nameservers_darwin_deadline_test.go @@ -0,0 +1,18 @@ +//go:build darwin + +package ctrld + +import ( + "context" + "errors" + "testing" +) + +func TestDHCPInterfaceContextCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + got, err := DHCPNameserversForInterfaceContext(ctx, "unused-test-interface") + if got != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("canceled discovery: got=%v err=%v", got, err) + } +} diff --git a/nameservers_darwin_parser.go b/nameservers_darwin_parser.go new file mode 100644 index 00000000..f7d942d0 --- /dev/null +++ b/nameservers_darwin_parser.go @@ -0,0 +1,44 @@ +package ctrld + +import ( + "net" + "strings" + "unicode" +) + +func parseDHCPOptionNameservers(output []byte) []string { + return parseIPv4Nameservers(string(output)) +} + +func parseDHCPPacketNameservers(output []byte) []string { + for _, line := range strings.Split(string(output), "\n") { + field := strings.TrimSpace(line) + if strings.HasPrefix(field, "domain_name_server ") || + strings.HasPrefix(field, "domain_name_server:") || + strings.HasPrefix(field, "domain_name_servers ") || + strings.HasPrefix(field, "domain_name_servers:") { + return parseIPv4Nameservers(field) + } + } + return nil +} + +func parseIPv4Nameservers(value string) []string { + seen := make(map[string]struct{}) + var nameservers []string + for _, token := range strings.FieldsFunc(value, func(r rune) bool { + return r != '.' && !unicode.IsDigit(r) + }) { + ip := net.ParseIP(token) + if ip == nil || ip.To4() == nil { + continue + } + ns := ip.String() + if _, ok := seen[ns]; ok { + continue + } + seen[ns] = struct{}{} + nameservers = append(nameservers, ns) + } + return nameservers +} diff --git a/nameservers_darwin_parser_test.go b/nameservers_darwin_parser_test.go new file mode 100644 index 00000000..6ec8b72b --- /dev/null +++ b/nameservers_darwin_parser_test.go @@ -0,0 +1,64 @@ +package ctrld + +import ( + "slices" + "testing" +) + +func TestParseDHCPOptionNameservers(t *testing.T) { + tests := []struct { + name string + output string + want []string + }{ + {"single", "192.168.10.1\n", []string{"192.168.10.1"}}, + {"multiple", "192.168.10.1\n1.1.1.1\n", []string{"192.168.10.1", "1.1.1.1"}}, + {"deduplicate and reject invalid", "192.168.10.1 999.1.1.1 192.168.10.1", []string{"192.168.10.1"}}, + {"empty", "", nil}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := parseDHCPOptionNameservers([]byte(tc.output)); !slices.Equal(got, tc.want) { + t.Fatalf("parseDHCPOptionNameservers() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestParseDHCPPacketNameservers(t *testing.T) { + tests := []struct { + name string + output string + want []string + }{ + { + name: "macos singular ip_mult", + output: `op = BOOTREPLY +` + + `yiaddr = 192.168.10.155 +` + + `domain_name_server (ip_mult): {192.168.10.1, 1.1.1.1} +` + + `server_identifier (ip): 192.168.10.1 +`, + want: []string{"192.168.10.1", "1.1.1.1"}, + }, + { + name: "legacy plural equals", + output: "domain_name_servers = 192.168.1.1 8.8.8.8;\n", + want: []string{"192.168.1.1", "8.8.8.8"}, + }, + { + name: "packet addresses without option are ignored", + output: "yiaddr = 192.168.10.155\nserver_identifier (ip): 192.168.10.1\n", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := parseDHCPPacketNameservers([]byte(tc.output)); !slices.Equal(got, tc.want) { + t.Fatalf("parseDHCPPacketNameservers() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/nameservers_dhcp_commands.go b/nameservers_dhcp_commands.go new file mode 100644 index 00000000..493d4990 --- /dev/null +++ b/nameservers_dhcp_commands.go @@ -0,0 +1,41 @@ +package ctrld + +import ( + "context" + "fmt" + "os/exec" + "time" +) + +// dhcpNameserversFromCommands shares one context across both discovery attempts. +// Cancellation is unknown state, never a successful empty DNS result. +func dhcpNameserversFromCommands(ctx context.Context, iface string, run func(context.Context, ...string) ([]byte, error)) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + output, optionErr := run(ctx, "getoption", iface, "domain_name_server") + if err := ctx.Err(); err != nil { + return nil, err + } + if optionErr == nil { + return parseDHCPOptionNameservers(output), nil + } + output, packetErr := run(ctx, "getpacket", iface) + if err := ctx.Err(); err != nil { + return nil, err + } + if packetErr != nil { + return nil, fmt.Errorf("error reading DHCP DNS option: getoption: %w; getpacket: %w", optionErr, packetErr) + } + return parseDHCPPacketNameservers(output), nil +} + +func dhcpCommandOutput(ctx context.Context, path string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, path, args...) + cmd.WaitDelay = 100 * time.Millisecond + output, err := cmd.Output() + if ctx.Err() != nil { + return nil, ctx.Err() + } + return output, err +} diff --git a/nameservers_dhcp_commands_test.go b/nameservers_dhcp_commands_test.go new file mode 100644 index 00000000..acc81595 --- /dev/null +++ b/nameservers_dhcp_commands_test.go @@ -0,0 +1,112 @@ +package ctrld + +import ( + "context" + "errors" + "os" + "os/exec" + "slices" + "testing" + "time" +) + +func TestDHCPNameserversCommandFallback(t *testing.T) { + for _, tc := range []struct { + name, option, packet string + optionErr, packetErr error + want []string + calls int + wantErr bool + }{ + {name: "option", option: "192.0.2.53\n", want: []string{"192.0.2.53"}, calls: 1}, + {name: "empty option", calls: 1}, + {name: "packet", optionErr: errors.New("exit 1"), packet: "domain_name_server (ip_mult): {192.0.2.53, 192.0.2.54}", want: []string{"192.0.2.53", "192.0.2.54"}, calls: 2}, + {name: "empty packet", optionErr: errors.New("exit 1"), calls: 2}, + {name: "both fail", optionErr: errors.New("exit 1"), packetErr: errors.New("exit 1"), calls: 2, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + got, err := dhcpNameserversFromCommands(context.Background(), "en0", func(_ context.Context, args ...string) ([]byte, error) { + calls++ + if calls == 1 { + if !slices.Equal(args, []string{"getoption", "en0", "domain_name_server"}) { + t.Fatal(args) + } + return []byte(tc.option), tc.optionErr + } + if !slices.Equal(args, []string{"getpacket", "en0"}) { + t.Fatal(args) + } + return []byte(tc.packet), tc.packetErr + }) + if calls != tc.calls || (err != nil) != tc.wantErr || !slices.Equal(got, tc.want) { + t.Fatalf("calls=%d got=%v err=%v", calls, got, err) + } + }) + } +} + +func TestDHCPNameserversRetainsCommandCauses(t *testing.T) { + option, packet := &exec.ExitError{}, &exec.ExitError{} + calls := 0 + _, err := dhcpNameserversFromCommands(context.Background(), "en0", func(context.Context, ...string) ([]byte, error) { + calls++ + if calls == 1 { + return nil, option + } + return nil, packet + }) + var exit *exec.ExitError + if !errors.As(err, &exit) || !errors.Is(err, option) || !errors.Is(err, packet) { + t.Fatalf("DHCP command causes were flattened: %T: %v", err, err) + } +} + +func TestDHCPNameserversCancellation(t *testing.T) { + for _, cancelAt := range []int{0, 1, 2} { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + if cancelAt == 0 { + cancel() + } + got, err := dhcpNameserversFromCommands(ctx, "en0", func(received context.Context, _ ...string) ([]byte, error) { + if received != ctx { + t.Fatal("discovery renewed the context") + } + calls++ + if calls == cancelAt { + cancel() + } + return nil, errors.New("unavailable") + }) + cancel() + if !errors.Is(err, context.Canceled) || got != nil || calls != cancelAt { + t.Fatalf("cancel at %d: got=%v err=%v calls=%d", cancelAt, got, err, calls) + } + } +} + +func TestDHCPCommandDeadlineHelper(t *testing.T) { + if os.Getenv("CTRLD_DHCP_DEADLINE_HELPER") != "1" { + return + } + time.Sleep(4 * time.Second) +} + +func TestDHCPCommandDeadline(t *testing.T) { + t.Setenv("CTRLD_DHCP_DEADLINE_HELPER", "1") + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 750*time.Millisecond) + defer cancel() + started := time.Now() + _, err = dhcpCommandOutput(ctx, exe, "-test.run=^TestDHCPCommandDeadlineHelper$") + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err=%v", err) + } + if elapsed := time.Since(started); elapsed >= 3*time.Second { + t.Fatalf("command ignored deadline: %v", elapsed) + } +} diff --git a/nameservers_linux.go b/nameservers_linux.go index 37a9ed24..7a0406df 100644 --- a/nameservers_linux.go +++ b/nameservers_linux.go @@ -3,6 +3,7 @@ package ctrld import ( "bufio" "bytes" + "context" "encoding/hex" "net" "net/netip" @@ -23,7 +24,7 @@ func dnsFns() []dnsFn { return []dnsFn{dnsFromResolvConf, dns4, dns6, dnsFromSystemdResolver} } -func dns4() []string { +func dns4(ctx context.Context) []string { f, err := os.Open(v4RouteFile) if err != nil { return nil @@ -32,7 +33,7 @@ func dns4() []string { var dns []string seen := make(map[string]bool) - vis := virtualInterfaces() + vis := virtualInterfaces(ctx) s := bufio.NewScanner(f) first := true for s.Scan() { @@ -45,7 +46,7 @@ func dns4() []string { continue } // Skip virtual interfaces. - if vis.contains(string(bytes.TrimSpace(fields[0]))) { + if _, ok := vis[string(bytes.TrimSpace(fields[0]))]; ok { continue } gw := make([]byte, net.IPv4len) @@ -63,7 +64,7 @@ func dns4() []string { return dns } -func dns6() []string { +func dns6(ctx context.Context) []string { f, err := os.Open(v6RouteFile) if err != nil { return nil @@ -71,7 +72,7 @@ func dns6() []string { defer f.Close() var dns []string - vis := virtualInterfaces() + vis := virtualInterfaces(ctx) s := bufio.NewScanner(f) for s.Scan() { fields := bytes.Fields(s.Bytes()) @@ -79,7 +80,7 @@ func dns6() []string { continue } // Skip virtual interfaces. - if vis.contains(string(bytes.TrimSpace(fields[len(fields)-1]))) { + if _, ok := vis[string(bytes.TrimSpace(fields[len(fields)-1]))]; ok { continue } @@ -97,7 +98,7 @@ func dns6() []string { return dns } -func dnsFromSystemdResolver() []string { +func dnsFromSystemdResolver(_ context.Context) []string { c, err := resolvconffile.ParseFile("/run/systemd/resolve/resolv.conf") if err != nil { return nil @@ -109,34 +110,29 @@ func dnsFromSystemdResolver() []string { return ns } -type set map[string]struct{} - -func (s *set) add(e string) { - (*s)[e] = struct{}{} -} - -func (s *set) contains(e string) bool { - _, ok := (*s)[e] - return ok -} - -// virtualInterfaces returns a set of virtual interfaces on current machine. -func virtualInterfaces() set { - s := make(set) - entries, _ := os.ReadDir("/sys/devices/virtual/net") +// virtualInterfaces returns a map of virtual interfaces on the current machine. +// This reads from /sys/devices/virtual/net to identify virtual network interfaces +// Virtual interfaces should not have DNS configured as they don't represent physical network connections +func virtualInterfaces(ctx context.Context) map[string]struct{} { + logger := LoggerFromCtx(ctx) + s := make(map[string]struct{}) + entries, err := os.ReadDir("/sys/devices/virtual/net") + if err != nil { + logger.Error().Err(err).Msg("Failed to read /sys/devices/virtual/net") + return nil + } for _, entry := range entries { if entry.IsDir() { - s.add(strings.TrimSpace(entry.Name())) + s[strings.TrimSpace(entry.Name())] = struct{}{} } } return s } -// validInterfacesMap returns a set containing non virtual interfaces. -// TODO: deduplicated with cmd/cli/net_linux.go in v2. -func validInterfaces() set { +// ValidInterfaces returns a set containing non virtual interfaces. +func ValidInterfaces(ctx context.Context) map[string]struct{} { m := make(map[string]struct{}) - vis := virtualInterfaces() + vis := virtualInterfaces(ctx) netmon.ForeachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { if _, existed := vis[i.Name]; existed { return diff --git a/nameservers_linux_test.go b/nameservers_linux_test.go index 23f15441..dddd377e 100644 --- a/nameservers_linux_test.go +++ b/nameservers_linux_test.go @@ -1,10 +1,11 @@ package ctrld import ( + "context" "testing" ) func Test_virtualInterfaces(t *testing.T) { - vis := virtualInterfaces() + vis := virtualInterfaces(context.Background()) t.Log(vis) } diff --git a/nameservers_scutil.go b/nameservers_scutil.go new file mode 100644 index 00000000..cf745915 --- /dev/null +++ b/nameservers_scutil.go @@ -0,0 +1,45 @@ +package ctrld + +import ( + "bufio" + "bytes" + "net/netip" + "strings" +) + +// Link-local addresses identify a host only within their interface's zone. +// In particular, an unzoned local fe80::1 must not exclude fe80::1%en0. +func scutilLocalAddressKey(addr netip.Addr) netip.Addr { + addr = addr.Unmap() + if addr.IsLinkLocalUnicast() { + return addr + } + return addr.WithZone("") +} + +// parseScutilNameservers retains IPv6 zones for the eventual socket dial. +func parseScutilNameservers(output []byte, local []netip.Addr) ([]string, error) { + excluded := make(map[netip.Addr]bool, len(local)) + for _, addr := range local { + excluded[scutilLocalAddressKey(addr)] = true + } + var servers []string + seen := make(map[netip.Addr]bool) + scanner := bufio.NewScanner(bytes.NewReader(output)) + for scanner.Scan() { + key, value, ok := strings.Cut(strings.TrimSpace(scanner.Text()), ":") + if !ok || !strings.HasPrefix(key, "nameserver[") { + continue + } + addr, err := netip.ParseAddr(strings.TrimSpace(value)) + if err != nil || excluded[scutilLocalAddressKey(addr)] || seen[addr] { + continue + } + seen[addr] = true + servers = append(servers, addr.String()) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return servers, nil +} diff --git a/nameservers_scutil_darwin_test.go b/nameservers_scutil_darwin_test.go new file mode 100644 index 00000000..9679f8f9 --- /dev/null +++ b/nameservers_scutil_darwin_test.go @@ -0,0 +1,55 @@ +//go:build darwin + +package ctrld + +import ( + "context" + "net" + "net/netip" + "os" + "path/filepath" + "slices" + "testing" +) + +func TestScopedOSResolverDoesNotBindDefaultSource(t *testing.T) { + old4, old6 := GetDefaultLocalIPv4(), GetDefaultLocalIPv6() + SetDefaultLocalIPv4(context.Background(), net.ParseIP("192.0.2.10")) + SetDefaultLocalIPv6(context.Background(), net.ParseIP("2001:db8::10")) + t.Cleanup(func() { + SetDefaultLocalIPv4(context.Background(), old4) + SetDefaultLocalIPv6(context.Background(), old6) + }) + if got := defaultLocalIPForServer("[fe80::1%en0]:53"); got != nil { + t.Fatalf("scoped resolver bound unrelated source %v", got) + } + if got := defaultLocalIPForServer("[2001:db8::53]:53"); !got.Equal(net.ParseIP("2001:db8::10")) { + t.Fatalf("global IPv6 source changed: %v", got) + } + if got := defaultLocalIPForServer("192.0.2.53:53"); !got.Equal(net.ParseIP("192.0.2.10")) { + t.Fatalf("IPv4 source changed: %v", got) + } +} + +func TestScutilIPv6ProductionReader(t *testing.T) { + old := scutilLocalAddresses + t.Cleanup(func() { scutilLocalAddresses = old }) + localReads := 0 + scutilLocalAddresses = func() ([]netip.Addr, []netip.Addr, error) { + localReads++ + return []netip.Addr{netip.MustParseAddr("2001:db8::10"), netip.MustParseAddr("fe80::2%en0")}, + []netip.Addr{netip.MustParseAddr("::1"), netip.MustParseAddr("fe80::1")}, nil + } + dir := t.TempDir() + script := "#!/bin/sh\nprintf '%s\\n' 'nameserver[0] : 2001:db8::53' 'nameserver[1] : fe80::1%en0' 'nameserver[2] : fe80::2%en0' 'nameserver[3] : ::1' 'nameserver[4] : 2001:db8::10'\n" + if err := os.WriteFile(filepath.Join(dir, "scutil"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + if got := getDNSFromScutil(context.Background()); !slices.Equal(got, []string{"2001:db8::53", "fe80::1%en0"}) { + t.Fatalf("production reader lost IPv6: %v", got) + } + if localReads != 1 { + t.Fatalf("local-address reads=%d, want 1", localReads) + } +} diff --git a/nameservers_scutil_test.go b/nameservers_scutil_test.go new file mode 100644 index 00000000..7c2b0904 --- /dev/null +++ b/nameservers_scutil_test.go @@ -0,0 +1,80 @@ +package ctrld + +import ( + "net/netip" + "slices" + "strings" + "testing" +) + +func TestScutilNameserversIPv6AndZones(t *testing.T) { + output := []byte(`DNS configuration +resolver #1 + nameserver[0] : 192.0.2.53 + nameserver[1] : 2001:db8::53 + nameserver[2] : fe80::1%en0 + nameserver[3] : 2001:db8::53 + nameserver[4] : fe80::1%en1 + nameserver[5] : 127.0.0.1 + nameserver[6] : ::1 + nameserver[7] : fe80::2%en0 + nameserver[8] : not-an-address + search domain[0] : internal.example +`) + local := []netip.Addr{netip.MustParseAddr("127.0.0.1"), netip.MustParseAddr("::1"), netip.MustParseAddr("fe80::2%en0")} + got, err := parseScutilNameservers(output, local) + want := []string{"192.0.2.53", "2001:db8::53", "fe80::1%en0", "fe80::1%en1"} + if err != nil || !slices.Equal(got, want) { + t.Fatalf("nameservers=%v err=%v", got, err) + } + effective := initializeOsResolver(got) + r := newResolverWithNameserver(effective) + if !slices.Contains(*r.lanServers.Load(), "[fe80::1%en0]:53") || !slices.Contains(*r.lanServers.Load(), "[fe80::1%en1]:53") { + t.Fatalf("zones lost in effective resolver: %v", effective) + } + if !slices.Contains(*r.publicServers.Load(), "[2001:db8::53]:53") { + t.Fatalf("global IPv6 lost: %v", effective) + } +} + +func TestScutilNameserversLocalExclusion(t *testing.T) { + for _, tt := range []struct { + name, local, server string + excluded bool + }{ + {"unzoned link-local does not exclude scoped router", "fe80::1", "fe80::1%en0", false}, + {"loopback link-local does not exclude router", "fe80::1%lo0", "fe80::1%en0", false}, + {"other link does not exclude router", "fe80::1%en1", "fe80::1%en0", false}, + {"same link excludes local address", "fe80::1%en0", "fe80::1%en0", true}, + {"unzoned link-local matches unzoned local", "fe80::1", "fe80::1", true}, + {"scoped local does not exclude unzoned candidate", "fe80::1%en0", "fe80::1", false}, + {"global ignores zones", "2001:db8::1%en0", "2001:db8::1%en1", true}, + {"loopback ignores zones", "::1%lo0", "::1", true}, + {"IPv4 mapped local", "::ffff:192.0.2.1", "192.0.2.1", true}, + {"IPv4 mapped candidate", "192.0.2.1", "::ffff:192.0.2.1", true}, + {"IPv4 link-local mapped candidate", "169.254.1.1", "::ffff:169.254.1.1", true}, + {"other global is retained", "2001:db8::1", "2001:db8::53", false}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := parseScutilNameservers([]byte("nameserver[0] : "+tt.server+"\n"), []netip.Addr{netip.MustParseAddr(tt.local)}) + if err != nil { + t.Fatal(err) + } + var want []string + if !tt.excluded { + want = []string{tt.server} + } + if !slices.Equal(got, want) { + t.Fatalf("nameservers=%v, want %v", got, want) + } + }) + } +} + +func TestScutilNameserversRejectsPartialScan(t *testing.T) { + output := []byte("nameserver[0] : 192.0.2.53\n" + strings.Repeat("x", 128*1024)) + got, err := parseScutilNameservers(output, nil) + if err == nil || got != nil { + t.Fatalf("partial scan accepted: %v, %v", got, err) + } +} diff --git a/nameservers_test.go b/nameservers_test.go index 166cced6..e2e2bace 100644 --- a/nameservers_test.go +++ b/nameservers_test.go @@ -1,9 +1,12 @@ package ctrld -import "testing" +import ( + "context" + "testing" +) func TestNameservers(t *testing.T) { - ns := nameservers() + ns := nameservers(context.Background()) if len(ns) == 0 { t.Fatal("failed to get nameservers") } diff --git a/nameservers_unix.go b/nameservers_unix.go index d8e6035e..d813bf44 100644 --- a/nameservers_unix.go +++ b/nameservers_unix.go @@ -3,24 +3,44 @@ package ctrld import ( + "context" "net" + "net/netip" "slices" "time" "tailscale.com/net/netmon" - - "github.com/Control-D-Inc/ctrld/internal/resolvconffile" ) -// currentNameserversFromResolvconf returns the current nameservers set from /etc/resolv.conf file. -func currentNameserversFromResolvconf() []string { - return resolvconffile.NameServers() +// localNameservers filters a list of nameserver strings, returning only those +// that are not loopback or local machine IP addresses. +func localNameservers(nss []string, regularIPs, loopbackIPs []netip.Addr) []string { + var result []string + seen := make(map[string]bool) + + for _, ns := range nss { + if ip := net.ParseIP(ns); ip != nil { + // skip loopback and local IPs + isLocal := false + for _, v := range slices.Concat(regularIPs, loopbackIPs) { + if ip.String() == v.String() { + isLocal = true + break + } + } + if !isLocal && !seen[ip.String()] { + seen[ip.String()] = true + result = append(result, ip.String()) + } + } + } + return result } // dnsFromResolvConf reads usable nameservers from /etc/resolv.conf file. // A nameserver is usable if it's not one of current machine's IP addresses // and loopback IP addresses. -func dnsFromResolvConf() []string { +func dnsFromResolvConf(_ context.Context) []string { const ( maxRetries = 10 retryInterval = 100 * time.Millisecond @@ -34,25 +54,8 @@ func dnsFromResolvConf() []string { time.Sleep(retryInterval) } - nss := resolvconffile.NameServers() - var localDNS []string - seen := make(map[string]bool) - - for _, ns := range nss { - if ip := net.ParseIP(ns); ip != nil { - // skip loopback IPs - for _, v := range slices.Concat(regularIPs, loopbackIPs) { - ipStr := v.String() - if ip.String() == ipStr { - continue - } - } - if !seen[ip.String()] { - seen[ip.String()] = true - localDNS = append(localDNS, ip.String()) - } - } - } + nss := CurrentNameserversFromResolvconf() + localDNS := localNameservers(nss, regularIPs, loopbackIPs) // If we successfully read the file and found nameservers, return them if len(localDNS) > 0 { diff --git a/nameservers_unix_test.go b/nameservers_unix_test.go new file mode 100644 index 00000000..a771dc12 --- /dev/null +++ b/nameservers_unix_test.go @@ -0,0 +1,105 @@ +//go:build unix + +package ctrld + +import ( + "net/netip" + "testing" +) + +func Test_localNameservers(t *testing.T) { + loopbackIPs := []netip.Addr{ + netip.MustParseAddr("127.0.0.1"), + netip.MustParseAddr("::1"), + } + regularIPs := []netip.Addr{ + netip.MustParseAddr("192.168.1.100"), + netip.MustParseAddr("10.0.0.5"), + } + + tests := []struct { + name string + nss []string + regularIPs []netip.Addr + loopbackIPs []netip.Addr + want []string + }{ + { + name: "filters loopback IPv4", + nss: []string{"127.0.0.1", "8.8.8.8"}, + regularIPs: nil, + loopbackIPs: loopbackIPs, + want: []string{"8.8.8.8"}, + }, + { + name: "filters loopback IPv6", + nss: []string{"::1", "1.1.1.1"}, + regularIPs: nil, + loopbackIPs: loopbackIPs, + want: []string{"1.1.1.1"}, + }, + { + name: "filters local machine IPs", + nss: []string{"192.168.1.100", "8.8.4.4"}, + regularIPs: regularIPs, + loopbackIPs: nil, + want: []string{"8.8.4.4"}, + }, + { + name: "filters both loopback and local IPs", + nss: []string{"127.0.0.1", "192.168.1.100", "8.8.8.8"}, + regularIPs: regularIPs, + loopbackIPs: loopbackIPs, + want: []string{"8.8.8.8"}, + }, + { + name: "deduplicates results", + nss: []string{"8.8.8.8", "8.8.8.8", "1.1.1.1"}, + regularIPs: regularIPs, + loopbackIPs: loopbackIPs, + want: []string{"8.8.8.8", "1.1.1.1"}, + }, + { + name: "all filtered returns nil", + nss: []string{"127.0.0.1", "::1", "192.168.1.100"}, + regularIPs: regularIPs, + loopbackIPs: loopbackIPs, + want: nil, + }, + { + name: "empty input returns nil", + nss: nil, + regularIPs: regularIPs, + loopbackIPs: loopbackIPs, + want: nil, + }, + { + name: "skips unparseable entries", + nss: []string{"not-an-ip", "8.8.8.8"}, + regularIPs: regularIPs, + loopbackIPs: loopbackIPs, + want: []string{"8.8.8.8"}, + }, + { + name: "no local IPs filters nothing", + nss: []string{"8.8.8.8", "1.1.1.1"}, + regularIPs: nil, + loopbackIPs: nil, + want: []string{"8.8.8.8", "1.1.1.1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := localNameservers(tt.nss, tt.regularIPs, tt.loopbackIPs) + if len(got) != len(tt.want) { + t.Fatalf("localNameservers() = %v, want %v", got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("localNameservers()[%d] = %q, want %q", i, got[i], tt.want[i]) + } + } + }) + } +} diff --git a/nameservers_windows.go b/nameservers_windows.go index 7b16e8e1..ea92aaa6 100644 --- a/nameservers_windows.go +++ b/nameservers_windows.go @@ -8,6 +8,7 @@ import ( "net" "os" "strings" + "sync" "syscall" "time" "unsafe" @@ -17,9 +18,12 @@ import ( "github.com/microsoft/wmi/pkg/base/query" "github.com/microsoft/wmi/pkg/constant" "github.com/microsoft/wmi/pkg/hardware/network/netadapter" + "github.com/miekg/dns" "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld/internal/system" ) const ( @@ -34,6 +38,30 @@ const ( DS_IP_REQUIRED = 0x00000200 DS_IS_DNS_NAME = 0x00020000 DS_RETURN_DNS_NAME = 0x40000000 + + // AD DC retry constants. + dcRetryInitialDelay = 1 * time.Second + dcRetryMaxDelay = 30 * time.Second + dcRetryMaxAttempts = 10 + + // DsGetDcName error codes. + errNoSuchDomain uintptr = 1355 + errNoLogonServers uintptr = 1311 + errDCNotFound uintptr = 1004 + errRPCUnavailable uintptr = 1722 + errConnReset uintptr = 10054 + errNetUnreachable uintptr = 1231 +) + +var ( + dcRetryMu sync.Mutex + dcRetryCancel context.CancelFunc + dcRetryDomain string + dcRetryID uint64 + + // Lazy-loaded netapi32 for DsGetDcNameW calls. + netapi32DLL = windows.NewLazySystemDLL("netapi32.dll") + dsGetDcNameW = netapi32DLL.NewProc("DsGetDcNameW") ) type DomainControllerInfo struct { @@ -52,28 +80,25 @@ func dnsFns() []dnsFn { return []dnsFn{dnsFromAdapter} } -func dnsFromAdapter() []string { - ctx, cancel := context.WithTimeout(context.Background(), defaultDNSAdapterTimeout) +func dnsFromAdapter(ctx context.Context) []string { + ctx, cancel := context.WithTimeout(ctx, defaultDNSAdapterTimeout) defer cancel() var ns []string var err error - logger := *ProxyLogger.Load() + logger := LoggerFromCtx(ctx) for i := 0; i < maxDNSAdapterRetries; i++ { if ctx.Err() != nil { - Log(context.Background(), logger.Debug(), - "dnsFromAdapter lookup cancelled or timed out, attempt %d", i) + logger.Debug().Msgf("dnsFromAdapter lookup cancelled or timed out, attempt %d", i) return nil } ns, err = getDNSServers(ctx) if err == nil && len(ns) >= minDNSServers { if i > 0 { - Log(context.Background(), logger.Debug(), - "Successfully got DNS servers after %d attempts, found %d servers", - i+1, len(ns)) + logger.Debug().Msgf("Successfully got DNS servers after %d attempts, found %d servers", i+1, len(ns)) } return ns } @@ -85,11 +110,9 @@ func dnsFromAdapter() []string { } if err != nil { - Log(context.Background(), logger.Debug(), - "Failed to get DNS servers, attempt %d: %v", i+1, err) + logger.Debug().Msgf("Failed to get DNS servers, attempt %d: %v", i+1, err) } else { - Log(context.Background(), logger.Debug(), - "Got insufficient DNS servers, retrying, found %d servers", len(ns)) + logger.Debug().Msgf("Got insufficient DNS servers, retrying, found %d servers", len(ns)) } select { @@ -99,14 +122,12 @@ func dnsFromAdapter() []string { } } - Log(context.Background(), logger.Debug(), - "Failed to get sufficient DNS servers after all attempts, max_retries=%d", maxDNSAdapterRetries) + logger.Debug().Msgf("Failed to get sufficient DNS servers after all attempts, max_retries=%d", maxDNSAdapterRetries) + return ns } func getDNSServers(ctx context.Context) ([]string, error) { - logger := *ProxyLogger.Load() - // Check context before making the call if ctx.Err() != nil { return nil, ctx.Err() @@ -121,35 +142,35 @@ func getDNSServers(ctx context.Context) ([]string, error) { return nil, fmt.Errorf("getting adapters: %w", err) } - Log(context.Background(), logger.Debug(), - "Found network adapters, count=%d", len(aas)) + logger := LoggerFromCtx(ctx) + logger.Debug().Msgf("Found network adapters, count=%d", len(aas)) // Try to get domain controller info if domain-joined var dcServers []string - isDomain := checkDomainJoined() + var adDomain string + isDomain := checkDomainJoined(ctx) + if !isDomain { + cancelDCRetry() + } if isDomain { - domainName, err := getLocalADDomain() + domainName, err := system.GetActiveDirectoryDomain() if err != nil { - Log(context.Background(), logger.Debug(), - "Failed to get local AD domain: %v", err) + logger.Debug().Msgf("Failed to get local AD domain: %v", err) } else { - // Load netapi32.dll - netapi32 := windows.NewLazySystemDLL("netapi32.dll") - dsDcName := netapi32.NewProc("DsGetDcNameW") + adDomain = domainName + cancelDCRetryForOtherDomain(domainName) var info *DomainControllerInfo flags := uint32(DS_RETURN_DNS_NAME | DS_IP_REQUIRED | DS_IS_DNS_NAME) domainUTF16, err := windows.UTF16PtrFromString(domainName) if err != nil { - Log(context.Background(), logger.Debug(), - "Failed to convert domain name to UTF16: %v", err) + logger.Debug().Msgf("Failed to convert domain name to UTF16: %v", err) } else { - Log(context.Background(), logger.Debug(), - "Attempting to get DC for domain: %s with flags: 0x%x", domainName, flags) + logger.Debug().Msgf("Attempting to get DC for domain: %s with flags: 0x%x", domainName, flags) // Call DsGetDcNameW with domain name - ret, _, err := dsDcName.Call( + ret, _, err := dsGetDcNameW.Call( 0, // ComputerName - can be NULL uintptr(unsafe.Pointer(domainUTF16)), // DomainName 0, // DomainGuid - not needed @@ -159,39 +180,39 @@ func getDNSServers(ctx context.Context) ([]string, error) { if ret != 0 { switch ret { - case 1355: // ERROR_NO_SUCH_DOMAIN - Log(context.Background(), logger.Debug(), - "Domain not found: %s (%d)", domainName, ret) - case 1311: // ERROR_NO_LOGON_SERVERS - Log(context.Background(), logger.Debug(), - "No logon servers available for domain: %s (%d)", domainName, ret) - case 1004: // ERROR_DC_NOT_FOUND - Log(context.Background(), logger.Debug(), - "Domain controller not found for domain: %s (%d)", domainName, ret) - case 1722: // RPC_S_SERVER_UNAVAILABLE - Log(context.Background(), logger.Debug(), - "RPC server unavailable for domain: %s (%d)", domainName, ret) + case errNoSuchDomain: + logger.Debug().Msgf("Domain not found: %s (%d)", domainName, ret) + case errNoLogonServers: + logger.Debug().Msgf("No logon servers available for domain: %s (%d)", domainName, ret) + case errDCNotFound: + logger.Debug().Msgf("Domain controller not found for domain: %s (%d)", domainName, ret) + case errRPCUnavailable: + logger.Debug().Msgf("RPC server unavailable for domain: %s (%d)", domainName, ret) default: - Log(context.Background(), logger.Debug(), - "Failed to get domain controller info for domain %s: %d, %v", domainName, ret, err) + logger.Debug().Msgf("Failed to get domain controller info for domain %s: %d, %v", domainName, ret, err) + } + // Start background retry for transient DC errors. + if isTransientDCError(ret) { + logger.Info().Msgf("AD DC detection failed with retryable error %d for %s, ensuring background retry", ret, domainName) + startDCRetry(domainName) } } else if info != nil { defer windows.NetApiBufferFree((*byte)(unsafe.Pointer(info))) if info.DomainControllerAddress != nil { dcAddr := windows.UTF16PtrToString(info.DomainControllerAddress) + // Remove "\\" prefix from domain controller address + // Windows domain controller addresses are returned with "\\" prefix, + // but we need just the IP address for DNS resolution dcAddr = strings.TrimPrefix(dcAddr, "\\\\") - Log(context.Background(), logger.Debug(), - "Found domain controller address: %s", dcAddr) - + logger.Debug().Msgf("Found domain controller address: %s", dcAddr) if ip := net.ParseIP(dcAddr); ip != nil { dcServers = append(dcServers, ip.String()) - Log(context.Background(), logger.Debug(), - "Added domain controller DNS servers: %v", dcServers) + cancelDCRetry() + logger.Debug().Msgf("Added domain controller DNS servers: %v", dcServers) } } else { - Log(context.Background(), logger.Debug(), - "No domain controller address found") + logger.Debug().Msg("No domain controller address found") } } } @@ -206,32 +227,32 @@ func getDNSServers(ctx context.Context) ([]string, error) { // Collect all local IPs for _, aa := range aas { if aa.OperStatus != winipcfg.IfOperStatusUp { - Log(context.Background(), logger.Debug(), - "Skipping adapter %s - not up, status: %d", aa.FriendlyName(), aa.OperStatus) + logger.Debug().Msgf("Skipping adapter %s - not up, status: %d", aa.FriendlyName(), aa.OperStatus) continue } // Skip if software loopback or other non-physical types // This is to avoid the "Loopback Pseudo-Interface 1" issue we see on windows if aa.IfType == winipcfg.IfTypeSoftwareLoopback { - Log(context.Background(), logger.Debug(), - "Skipping %s (software loopback)", aa.FriendlyName()) + logger.Debug().Msgf("Skipping %s (software loopback)", aa.FriendlyName()) continue } - Log(context.Background(), logger.Debug(), - "Processing adapter %s", aa.FriendlyName()) + logger.Debug().Msgf("Processing adapter %s", aa.FriendlyName()) for a := aa.FirstUnicastAddress; a != nil; a = a.Next { ip := a.Address.IP().String() addressMap[ip] = struct{}{} - Log(context.Background(), logger.Debug(), - "Added local IP %s from adapter %s", ip, aa.FriendlyName()) + logger.Debug().Msgf("Added local IP %s from adapter %s", ip, aa.FriendlyName()) } } - validInterfacesMap := validInterfaces() + validInterfacesMap := ValidInterfaces(ctx) + if isDomain && adDomain == "" { + logger.Warn().Msg("The machine is joined domain, but domain name is empty") + } + checkDnsSuffix := isDomain && adDomain != "" // Collect DNS servers for _, aa := range aas { if aa.OperStatus != winipcfg.IfOperStatusUp { @@ -241,23 +262,33 @@ func getDNSServers(ctx context.Context) ([]string, error) { // Skip if software loopback or other non-physical types // This is to avoid the "Loopback Pseudo-Interface 1" issue we see on windows if aa.IfType == winipcfg.IfTypeSoftwareLoopback { - Log(context.Background(), logger.Debug(), - "Skipping %s (software loopback)", aa.FriendlyName()) + logger.Debug().Msgf("Skipping %s (software loopback)", aa.FriendlyName()) continue } - // if not in the validInterfacesMap, skip - if _, ok := validInterfacesMap[aa.FriendlyName()]; !ok { - Log(context.Background(), logger.Debug(), - "Skipping %s (not in validInterfacesMap)", aa.FriendlyName()) + _, valid := validInterfacesMap[aa.FriendlyName()] + if !valid && checkDnsSuffix { + for suffix := aa.FirstDNSSuffix; suffix != nil; suffix = suffix.Next { + // For non-physical adapters, if the DNS suffix matches the domain name, + // (or vice versa) consider it valid. This can happen on remote VPN machines. + ds := strings.TrimSpace(suffix.String()) + if dns.IsSubDomain(adDomain, ds) || dns.IsSubDomain(ds, adDomain) { + logger.Debug().Msgf("Found valid interface %s with DNS suffix %s", aa.FriendlyName(), suffix.String()) + valid = true + break + } + } + } + // if not a valid interface, skip it + if !valid { + logger.Debug().Msgf("Skipping %s (not in validInterfacesMap)", aa.FriendlyName()) continue } for dns := aa.FirstDNSServerAddress; dns != nil; dns = dns.Next { ip := dns.Address.IP() if ip == nil { - Log(context.Background(), logger.Debug(), - "Skipping nil IP from adapter %s", aa.FriendlyName()) + logger.Debug().Msgf("Skipping nil IP from adapter %s", aa.FriendlyName()) continue } @@ -290,28 +321,23 @@ func getDNSServers(ctx context.Context) ([]string, error) { if !seen[dcServer] { seen[dcServer] = true ns = append(ns, dcServer) - Log(context.Background(), logger.Debug(), - "Added additional domain controller DNS server: %s", dcServer) + logger.Debug().Msgf("Added additional domain controller DNS server: %s", dcServer) } } // if we have static DNS servers saved for the current default route, we should add them to the list drIfaceName, err := netmon.DefaultRouteInterface() if err != nil { - Log(context.Background(), logger.Debug(), - "Failed to get default route interface: %v", err) + logger.Debug().Msgf("Failed to get default route interface: %v", err) } else { drIface, err := net.InterfaceByName(drIfaceName) if err != nil { - Log(context.Background(), logger.Debug(), - "Failed to get interface by name %s: %v", drIfaceName, err) + logger.Debug().Msgf("Failed to get interface by name %s: %v", drIfaceName, err) } else { - staticNs, file := SavedStaticNameservers(drIface) - Log(context.Background(), logger.Debug(), - "static dns servers from %s: %v", file, staticNs) + staticNs, file := SavedStaticNameserversAndPath(drIface) + logger.Debug().Msgf("Static dns servers from %s: %v", file, staticNs) if len(staticNs) > 0 { - Log(context.Background(), logger.Debug(), - "Adding static DNS servers from %s: %v", drIfaceName, staticNs) + logger.Debug().Msgf("Adding static DNS servers from %s: %v", drIfaceName, staticNs) ns = append(ns, staticNs...) } } @@ -321,100 +347,202 @@ func getDNSServers(ctx context.Context) ([]string, error) { return nil, fmt.Errorf("no valid DNS servers found") } - Log(context.Background(), logger.Debug(), - "DNS server discovery completed, count=%d, servers=%v (including %d DC servers)", - len(ns), ns, len(dcServers)) + logger.Debug().Msgf("DNS server discovery completed, count=%d, servers=%v (including %d DC servers)", len(ns), ns, len(dcServers)) return ns, nil } -// currentNameserversFromResolvconf returns a nil slice of strings. -func currentNameserversFromResolvconf() []string { - return nil -} - // checkDomainJoined checks if the machine is joined to an Active Directory domain // Returns whether it's domain joined and the domain name if available -func checkDomainJoined() bool { - logger := *ProxyLogger.Load() +func checkDomainJoined(ctx context.Context) bool { + logger := LoggerFromCtx(ctx) - var domain *uint16 - var status uint32 - - if err := windows.NetGetJoinInformation(nil, &domain, &status); err != nil { - Log(context.Background(), logger.Debug(), "Failed to get domain join status: %v", err) + status, err := system.DomainJoinedStatus() + if err != nil { + logger.Debug().Msgf("Failed to get domain joined status: %v", err) return false } - defer windows.NetApiBufferFree((*byte)(unsafe.Pointer(domain))) - - // NETSETUP_JOIN_STATUS constants from Microsoft Windows API - // See: https://learn.microsoft.com/en-us/windows/win32/api/lmjoin/ne-lmjoin-netsetup_join_status - // - // NetSetupUnknownStatus uint32 = 0 // The status is unknown - // NetSetupUnjoined uint32 = 1 // The computer is not joined to a domain or workgroup - // NetSetupWorkgroupName uint32 = 2 // The computer is joined to a workgroup - // NetSetupDomainName uint32 = 3 // The computer is joined to a domain - // - // We only care about NetSetupDomainName. - domainName := windows.UTF16PtrToString(domain) - Log(context.Background(), logger.Debug(), - "Domain join status: domain=%s status=%d (UnknownStatus=0, Unjoined=1, WorkgroupName=2, DomainName=3)", - domainName, status) - isDomain := status == syscall.NetSetupDomainName - Log(context.Background(), logger.Debug(), "Is domain joined? status=%d, result=%v", status, isDomain) + logger.Debug().Msg("Domain join status: (UnknownStatus=0, Unjoined=1, WorkgroupName=2, DomainName=3)") + logger.Debug().Msgf("Is domain joined? status=%d, result=%v", status, isDomain) return isDomain } -// getLocalADDomain uses Microsoft's WMI wrappers (github.com/microsoft/wmi/pkg/*) -// to query the Domain field from Win32_ComputerSystem instead of a direct go-ole call. -func getLocalADDomain() (string, error) { - log.SetOutput(io.Discard) - defer log.SetOutput(os.Stderr) - // 1) Check environment variable - envDomain := os.Getenv("USERDNSDOMAIN") - if envDomain != "" { - return strings.TrimSpace(envDomain), nil +// isTransientDCError returns true if the DsGetDcName error code indicates +// a transient failure that may succeed on retry. ERROR_NO_SUCH_DOMAIN is +// retryable here because we only call this path after Windows already reported +// the machine is domain joined and returned a local AD domain name; during VPN +// DNS churn, DC locator can temporarily fail to resolve that known domain. +func isTransientDCError(code uintptr) bool { + switch code { + case errNoSuchDomain, errConnReset, errRPCUnavailable, errNoLogonServers, errDCNotFound, errNetUnreachable: + return true + default: + return false } +} - // 2) Query WMI via the microsoft/wmi library - whost := host.NewWmiLocalHost() - q := query.NewWmiQuery("Win32_ComputerSystem") - instances, err := instance.GetWmiInstancesFromHost(whost, string(constant.CimV2), q) - if instances != nil { - defer instances.Close() +// cancelDCRetry cancels any in-flight DC retry goroutine. +func cancelDCRetry() { + dcRetryMu.Lock() + defer dcRetryMu.Unlock() + if dcRetryCancel != nil { + dcRetryCancel() + dcRetryCancel = nil + dcRetryDomain = "" + dcRetryID = 0 } - if err != nil { - return "", fmt.Errorf("WMI query failed: %v", err) +} + +// cancelDCRetryForOtherDomain keeps an existing retry alive during noisy +// network-change refreshes, but stops it if Windows reports a different AD +// domain. This avoids the start/cancel storm seen when DsGetDcName briefly +// returns ERROR_NO_SUCH_DOMAIN while VPN DNS is still settling. +func cancelDCRetryForOtherDomain(domainName string) { + dcRetryMu.Lock() + defer dcRetryMu.Unlock() + if dcRetryCancel == nil || dcRetryDomain == "" || strings.EqualFold(dcRetryDomain, domainName) { + return } + dcRetryCancel() + dcRetryCancel = nil + dcRetryDomain = "" + dcRetryID = 0 +} - // If no results, return an error - if len(instances) == 0 { - return "", fmt.Errorf("no rows returned from Win32_ComputerSystem") +// startDCRetry spawns a background goroutine that retries DsGetDcName with +// exponential backoff. On success it appends the DC IP to the OS resolver. +func startDCRetry(domainName string) { + dcRetryMu.Lock() + if dcRetryCancel != nil && strings.EqualFold(dcRetryDomain, domainName) { + LoggerFromCtx(context.Background()).Debug().Msgf("AD DC retry already running for domain %s", domainName) + dcRetryMu.Unlock() + return + } + if dcRetryCancel != nil { + dcRetryCancel() } + ctx, cancel := context.WithCancel(context.Background()) + dcRetryID++ + retryID := dcRetryID + dcRetryCancel = cancel + dcRetryDomain = domainName + dcRetryMu.Unlock() + + go func(retryID uint64) { + logger := LoggerFromCtx(context.Background()) + defer clearDCRetryIfCurrent(domainName, retryID) + delay := dcRetryInitialDelay + + for attempt := 1; attempt <= dcRetryMaxAttempts; attempt++ { + select { + case <-ctx.Done(): + logger.Debug().Msgf("AD DC retry cancelled for domain %s", domainName) + return + case <-time.After(delay): + } + + logger.Debug().Msgf("AD DC retry attempt %d/%d for domain %s (delay was %v)", + attempt, dcRetryMaxAttempts, domainName, delay) + + dcIP, errCode := tryGetDCAddress(domainName) + if dcIP != "" { + logger.Info().Msgf("AD DC retry succeeded: found DC at %s for domain %s (attempt %d)", + dcIP, domainName, attempt) + if AppendOsResolverNameservers([]string{dcIP}) { + logger.Info().Msgf("Added DC %s to OS resolver nameservers", dcIP) + } else { + logger.Warn().Msgf("AD DC retry: OS resolver not initialized, DC IP %s was not added", dcIP) + } + return + } + + // Permanent error or unexpected empty result — stop retrying. + if errCode != 0 && !isTransientDCError(errCode) { + logger.Debug().Msgf("AD DC retry stopping: permanent error %d for domain %s", errCode, domainName) + return + } + if errCode == 0 { + // DsGetDcName returned success but no usable address — don't retry. + logger.Debug().Msgf("AD DC retry stopping: DsGetDcName returned no address for domain %s", domainName) + return + } + + // Exponential backoff. + delay *= 2 + if delay > dcRetryMaxDelay { + delay = dcRetryMaxDelay + } + } - // We only care about the first row - domainVal, err := instances[0].GetProperty("Domain") + logger.Warn().Msgf("AD DC retry exhausted %d attempts for domain %s", dcRetryMaxAttempts, domainName) + }(retryID) +} + +func clearDCRetryIfCurrent(domainName string, retryID uint64) { + dcRetryMu.Lock() + defer dcRetryMu.Unlock() + if dcRetryCancel != nil && dcRetryID == retryID && strings.EqualFold(dcRetryDomain, domainName) { + dcRetryCancel = nil + dcRetryDomain = "" + dcRetryID = 0 + } +} + +// tryGetDCAddress attempts a single DsGetDcName call and returns the DC IP on success, +// or empty string and the error code on failure. +func tryGetDCAddress(domainName string) (string, uintptr) { + logger := LoggerFromCtx(context.Background()) + + var info *DomainControllerInfo + // Use DS_FORCE_REDISCOVERY on retries to bypass the DC locator cache, + // which may have cached the initial transient failure. + flags := uint32(DS_RETURN_DNS_NAME | DS_IP_REQUIRED | DS_IS_DNS_NAME | DS_FORCE_REDISCOVERY) + + domainUTF16, err := windows.UTF16PtrFromString(domainName) if err != nil { - return "", fmt.Errorf("machine does not appear to have a domain set: %v", err) + logger.Debug().Msgf("Failed to convert domain name to UTF16: %v", err) + return "", 0 + } + + ret, _, _ := dsGetDcNameW.Call( + 0, + uintptr(unsafe.Pointer(domainUTF16)), + 0, + 0, + uintptr(flags), + uintptr(unsafe.Pointer(&info))) + + if ret != 0 { + logger.Debug().Msgf("DsGetDcName retry failed for %s: error %d", domainName, ret) + return "", ret + } + + if info == nil { + return "", 0 + } + defer windows.NetApiBufferFree((*byte)(unsafe.Pointer(info))) + + if info.DomainControllerAddress == nil { + return "", 0 } - domainName := strings.TrimSpace(fmt.Sprintf("%v", domainVal)) - if domainName == "" { - return "", fmt.Errorf("machine does not appear to have a domain set") + dcAddr := windows.UTF16PtrToString(info.DomainControllerAddress) + dcAddr = strings.TrimPrefix(dcAddr, "\\\\") + if ip := net.ParseIP(dcAddr); ip != nil { + return ip.String(), 0 } - return domainName, nil + return "", 0 } -// validInterfaces returns a list of all physical interfaces. -// this is a duplicate of what is in net_windows.go, we should -// clean this up so there is only one version -func validInterfaces() map[string]struct{} { +// ValidInterfaces returns a map of valid network interface names as keys with empty struct values. +// It filters interfaces to include only physical, hardware-based adapters using WMI queries. +func ValidInterfaces(ctx context.Context) map[string]struct{} { log.SetOutput(io.Discard) defer log.SetOutput(os.Stderr) //load the logger - logger := *ProxyLogger.Load() + logger := LoggerFromCtx(ctx) whost := host.NewWmiLocalHost() q := query.NewWmiQuery("MSFT_NetAdapter") @@ -423,23 +551,20 @@ func validInterfaces() map[string]struct{} { defer instances.Close() } if err != nil { - Log(context.Background(), logger.Warn(), - "failed to get wmi network adapter: %v", err) + logger.Warn().Msgf("Failed to get wmi network adapter: %v", err) return nil } var adapters []string for _, i := range instances { adapter, err := netadapter.NewNetworkAdapter(i) if err != nil { - Log(context.Background(), logger.Warn(), - "failed to get network adapter: %v", err) + logger.Warn().Msgf("Failed to get network adapter: %v", err) continue } name, err := adapter.GetPropertyName() if err != nil { - Log(context.Background(), logger.Warn(), - "failed to get interface name: %v", err) + logger.Warn().Msgf("Failed to get interface name: %v", err) continue } @@ -449,13 +574,11 @@ func validInterfaces() map[string]struct{} { // if this is a physical adapter or FALSE if this is not a physical adapter." physical, err := adapter.GetPropertyConnectorPresent() if err != nil { - Log(context.Background(), logger.Debug(), - "failed to get network adapter connector present property: %v", err) + logger.Debug().Msgf("Failed to get network adapter connector present property: %v", err) continue } if !physical { - Log(context.Background(), logger.Debug(), - "skipping non-physical adapter: %s", name) + logger.Debug().Msgf("Skipping non-physical adapter: %s", name) continue } @@ -463,13 +586,11 @@ func validInterfaces() map[string]struct{} { // because some interfaces are not physical but have a connector. hardware, err := adapter.GetPropertyHardwareInterface() if err != nil { - Log(context.Background(), logger.Debug(), - "failed to get network adapter hardware interface property: %v", err) + logger.Debug().Msgf("Failed to get network adapter hardware interface property: %v", err) continue } if !hardware { - Log(context.Background(), logger.Debug(), - "skipping non-hardware interface: %s", name) + logger.Debug().Msgf("Skipping non-hardware interface: %s", name) continue } diff --git a/net.go b/net.go index 7bbf54bb..b20d85ab 100644 --- a/net.go +++ b/net.go @@ -6,8 +6,6 @@ import ( "sync/atomic" "time" - "tailscale.com/net/netmon" - ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) @@ -17,37 +15,37 @@ var ( ) // HasIPv6 reports whether the current network stack has IPv6 available. -func HasIPv6() bool { +// +// Keep the first call cheap and bounded. It runs on the DNS path (upstream +// transport setup) behind a sync.Once, so every query waits for it, and under +// Windows NRPT it can run after DNS already points at ctrld. v1.5.7 started a +// netmon monitor here; its first interface state runs WinHTTP proxy discovery, +// which needs DNS through this same ctrld, and a --config start stalled for +// about 100 s (docs/known-issues.md). Network monitoring belongs to the one +// monitor that reports through SetIPv6Available. +func HasIPv6(ctx context.Context) bool { hasIPv6Once.Do(func() { - ProxyLogger.Load().Debug().Msg("checking for IPv6 availability once") + logger := LoggerFromCtx(ctx) + logger.Debug().Msg("Checking for ipv6 availability once") ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() val := ctrldnet.IPv6Available(ctx) ipv6Available.Store(val) - ProxyLogger.Load().Debug().Msgf("ipv6 availability: %v", val) - mon, err := netmon.New(func(format string, args ...any) {}) - if err != nil { - ProxyLogger.Load().Debug().Err(err).Msg("failed to monitor IPv6 state") - return - } - mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { - old := ipv6Available.Load() - cur := delta.Monitor.InterfaceState().HaveV6 - if old != cur { - ProxyLogger.Load().Warn().Msgf("ipv6 availability changed, old: %v, new: %v", old, cur) - } else { - ProxyLogger.Load().Debug().Msg("ipv6 availability does not changed") - } - ipv6Available.Store(cur) - }) - mon.Start() + logger.Debug().Msgf("ipv6 availability: %v", val) }) return ipv6Available.Load() } +// SetIPv6Available stores the IPv6 state that the network monitor reports, so +// one monitor serves the whole process. +func SetIPv6Available(v bool) { + ipv6Available.Store(v) +} + // DisableIPv6 marks IPv6 as unavailable if enabled. -func DisableIPv6() { +func DisableIPv6(ctx context.Context) { if ipv6Available.CompareAndSwap(true, false) { - ProxyLogger.Load().Debug().Msg("turned off IPv6 availability") + logger := LoggerFromCtx(ctx) + logger.Debug().Msg("Turned off ipv6 availability") } } diff --git a/net_darwin.go b/net_darwin.go index 5b01e9f2..42c26a2c 100644 --- a/net_darwin.go +++ b/net_darwin.go @@ -3,14 +3,14 @@ package ctrld import ( "bufio" "bytes" + "context" "io" "os/exec" "strings" ) -// validInterfaces returns a set of all valid hardware ports. -// TODO: deduplicated with cmd/cli/net_darwin.go in v2. -func validInterfaces() map[string]struct{} { +// ValidInterfaces returns a set of all valid hardware ports. +func ValidInterfaces(_ context.Context) map[string]struct{} { b, err := exec.Command("networksetup", "-listallhardwareports").Output() if err != nil { return nil diff --git a/net_ipv6_test.go b/net_ipv6_test.go new file mode 100644 index 00000000..b37d01f1 --- /dev/null +++ b/net_ipv6_test.go @@ -0,0 +1,38 @@ +package ctrld + +import ( + "context" + "sync" + "testing" +) + +// resetIPv6Probe clears the one-time IPv6 probe. Test only: the process runs +// the probe once, so a test cannot see it without this reset. +func resetIPv6Probe(t *testing.T) { + t.Helper() + reset := func() { + hasIPv6Once = sync.Once{} + ipv6Available.Store(false) + } + reset() + t.Cleanup(reset) +} + +func TestHasIPv6KeepsTheProbeThenTakesTheStoredFlag(t *testing.T) { + resetIPv6Probe(t) + + probed := HasIPv6(context.Background()) + if got := HasIPv6(context.Background()); got != probed { + t.Fatalf("HasIPv6 = %v on the second call, want the probe value %v", got, probed) + } + + SetIPv6Available(!probed) + if got := HasIPv6(context.Background()); got == probed { + t.Fatalf("HasIPv6 = %v after SetIPv6Available(%v), want the stored value", got, !probed) + } + + SetIPv6Available(false) + if HasIPv6(context.Background()) { + t.Fatal("HasIPv6 = true after SetIPv6Available(false), want false") + } +} diff --git a/net_others.go b/net_others.go index ae7ab8e2..fef1e7d6 100644 --- a/net_others.go +++ b/net_others.go @@ -2,11 +2,14 @@ package ctrld -import "tailscale.com/net/netmon" +import ( + "context" -// validInterfaces returns a set containing only default route interfaces. -// TODO: deuplicated with cmd/cli/net_others.go in v2. -func validInterfaces() map[string]struct{} { + "tailscale.com/net/netmon" +) + +// ValidInterfaces returns a set containing only default route interfaces. +func ValidInterfaces(_ context.Context) map[string]struct{} { defaultRoute, err := netmon.DefaultRoute() if err != nil { return nil diff --git a/resolver.go b/resolver.go index 3aeddd0d..1a232326 100644 --- a/resolver.go +++ b/resolver.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "net" "net/netip" "runtime" @@ -15,10 +14,12 @@ import ( "time" "github.com/miekg/dns" - "github.com/rs/zerolog" "golang.org/x/sync/singleflight" "tailscale.com/net/netmon" "tailscale.com/net/tsaddr" + + "github.com/Control-D-Inc/ctrld/internal/dnscache" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) const ( @@ -36,8 +37,6 @@ const ( ResolverTypeLegacy = "legacy" // ResolverTypePrivate is like ResolverTypeOS, but use for private resolver only. ResolverTypePrivate = "private" - // ResolverTypeLocal is like ResolverTypeOS, but use for local resolver only. - ResolverTypeLocal = "local" // ResolverTypeSDNS specifies resolver with information encoded using DNS Stamps. // See: https://dnscrypt.info/stamps-specifications/ ResolverTypeSDNS = "sdns" @@ -45,33 +44,71 @@ const ( const controldPublicDns = "76.76.2.0" -var controldPublicDnsWithPort = net.JoinHostPort(controldPublicDns, "53") - -var localResolver Resolver +const maxConcurrentOSResolverExchanges = 128 -func init() { - // Initializing ProxyLogger here, so other places don't have to do nil check. - l := zerolog.New(io.Discard) - ProxyLogger.Store(&l) +var controldPublicDnsWithPort = net.JoinHostPort(controldPublicDns, "53") - localResolver = newLocalResolver() -} +var osResolverExchangeSem = make(chan struct{}, maxConcurrentOSResolverExchanges) var ( - resolverMutex sync.Mutex - or *osResolver + resolverMutex sync.Mutex + or *osResolver + // osResolverPtr holds the same resolver as or, for readers alone. An + // initialization holds resolverMutex across a scutil read of several + // seconds, and a log header render must not wait for it. + osResolverPtr atomic.Pointer[osResolver] defaultLocalIPv4 atomic.Value // holds net.IP (IPv4) defaultLocalIPv6 atomic.Value // holds net.IP (IPv6) ) -func newLocalResolver() Resolver { - var nss []string - for _, addr := range Rfc1918Addresses() { - nss = append(nss, net.JoinHostPort(addr, "53")) +// storeOsResolver keeps the resolver of the process and the copy that readers +// take. The caller holds resolverMutex. +func storeOsResolver(resolver *osResolver) { + or = resolver + osResolverPtr.Store(resolver) +} + +// NameserversFn reads the nameservers of the system. A test of this package +// or of a package that drives the OS resolver replaces it to stay off the host. +var NameserversFn = nameservers + +// osResolverReasonUnspecified marks a resolver read that no caller explained. +const osResolverReasonUnspecified = "unspecified" + +// nameserverReads remembers the last nameserver list of one read. The system +// reports the same list many times, and only a change is news. +type nameserverReads struct { + seen bool + last []string + repeats int +} + +// record stores a new read. It reports whether the list changed, how many +// reads repeated the list before it, and the list that this read replaces. +func (n *nameserverReads) record(list []string) (changed bool, repeats int, before []string) { + if n.seen && slices.Equal(n.last, list) { + n.repeats++ + return false, n.repeats, n.last } - return NewResolverWithNameserver(nss) + before, repeats = n.last, n.repeats + n.seen = true + n.last = slices.Clone(list) + n.repeats = 0 + return true, repeats, before +} + +// osResolverLogState keeps the last nameserver reads and the reason of the +// initialization that runs. The read sits deep in the call chain, so the +// reason cannot travel as a parameter. +type osResolverLogState struct { + mu sync.Mutex + system nameserverReads + final nameserverReads + reason string } +var osResolverLog = osResolverLogState{reason: osResolverReasonUnspecified} + // LanQueryCtxKey is the context.Context key to indicate that the request is for LAN network. type LanQueryCtxKey struct{} @@ -80,9 +117,41 @@ func LanQueryCtx(ctx context.Context) context.Context { return context.WithValue(ctx, LanQueryCtxKey{}, true) } +type privateResolverCtxKey struct{} + +// PrivateResolverCtx keeps legacy resolver transport errors, which include +// private endpoint addresses, at debug level. The caller reports a sanitized +// failure classification at its normal level. +func PrivateResolverCtx(ctx context.Context) context.Context { + return context.WithValue(ctx, privateResolverCtxKey{}, true) +} + +// LanOnlyQueryCtxKey is the context.Context key that limits an OS resolver +// query to its LAN nameservers. +type LanOnlyQueryCtxKey struct{} + +// LanOnlyQueryCtx returns a context.Context with LanOnlyQueryCtxKey set. +// +// An OS resolver query made with it goes only to LAN-classified nameservers: +// private, loopback, link-local and CGNAT addresses. Every public nameserver +// is left out, the ones DHCP or a VPN supplied as well as ctrld's own public +// fallback, and with no LAN nameserver the query fails without being sent. It +// is for names that must not reach public DNS, such as an organization's +// Internal Domains. LanQueryCtx, by contrast, only drops ctrld's own public +// fallback. +func LanOnlyQueryCtx(ctx context.Context) context.Context { + return context.WithValue(ctx, LanOnlyQueryCtxKey{}, true) +} + +// isLanOnlyQuery reports whether ctx was made by LanOnlyQueryCtx. +func isLanOnlyQuery(ctx context.Context) bool { + lanOnly, _ := ctx.Value(LanOnlyQueryCtxKey{}).(bool) + return lanOnly +} + // defaultNameservers is like nameservers with each element formed "ip:53". -func defaultNameservers() []string { - ns := nameservers() +func defaultNameservers(ctx context.Context) []string { + ns := nameservers(ctx) nss := make([]string, len(ns)) for i := range ns { nss[i] = net.JoinHostPort(ns[i], "53") @@ -91,61 +160,234 @@ func defaultNameservers() []string { } // availableNameservers returns list of current available DNS servers of the system. -func availableNameservers() []string { +func availableNameservers(ctx context.Context) []string { var nss []string // Ignore local addresses to prevent loop. regularIPs, loopbackIPs, _ := netmon.LocalAddresses() machineIPsMap := make(map[string]struct{}, len(regularIPs)) - //load the logger - logger := *ProxyLogger.Load() - - Log(context.Background(), logger.Debug(), - "Got local addresses - regular IPs: %v, loopback IPs: %v", regularIPs, loopbackIPs) + // Load the logger. + logger := LoggerFromCtx(ctx) + logger.Debug().Msgf("Got local addresses - regular IPs: %v, loopback IPs: %v", regularIPs, loopbackIPs) for _, v := range slices.Concat(regularIPs, loopbackIPs) { ipStr := v.String() machineIPsMap[ipStr] = struct{}{} - Log(context.Background(), logger.Debug(), - "Added local IP to OS resolverexclusion map: %s", ipStr) + logger.Debug().Msgf("Added local IP to OS resolverexclusion map: %s", ipStr) } - systemNameservers := nameservers() - Log(context.Background(), logger.Debug(), - "Got system nameservers: %v", systemNameservers) + systemNameservers := NameserversFn(ctx) + logSystemNameservers(ctx, systemNameservers) for _, ns := range systemNameservers { if _, ok := machineIPsMap[ns]; ok { - Log(context.Background(), logger.Debug(), - "Skipping local nameserver: %s", ns) + logger.Debug().Msgf("Skipping local nameserver: %s", ns) continue } nss = append(nss, ns) - Log(context.Background(), logger.Debug(), - "Added non-local nameserver: %s", ns) + logger.Debug().Msgf("Added non-local nameserver: %s", ns) } - Log(context.Background(), logger.Debug(), - "Final available nameservers: %v", nss) + logFinalNameservers(ctx, nss) return nss } +// logSystemNameservers logs the system read at debug level on change only. +func logSystemNameservers(ctx context.Context, list []string) { + osResolverLog.mu.Lock() + changed, repeats, _ := osResolverLog.system.record(list) + osResolverLog.mu.Unlock() + if !changed { + return + } + Log(ctx, LoggerFromCtx(ctx).Debug().Int("repeats", repeats), + "Got system nameservers: %v", list) +} + +// logFinalNameservers logs the effective read at debug level on change only, +// and sends each change to the journal, where an outage report needs it. +func logFinalNameservers(ctx context.Context, list []string) { + osResolverLog.mu.Lock() + changed, repeats, before := osResolverLog.final.record(list) + reason := osResolverLog.reason + osResolverLog.mu.Unlock() + if !changed { + return + } + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug().Int("repeats", repeats), + "Final available nameservers: %v", list) + Journal(logger.Info()). + Str("source", osResolverSource()). + Strs("before", before). + Strs("after", list). + Str("default_route", defaultRouteInterfaceName()). + Str("reason", reason). + Msg("OS resolver set changed") +} + +// osResolverSource names the place where the platform holds the nameservers. +func osResolverSource() string { + switch runtime.GOOS { + case "darwin": + return "scutil" + case "windows": + return "dhcp" + case "linux", "dragonfly", "freebsd", "netbsd", "openbsd": + return "resolv.conf" + } + return "unknown" +} + +// defaultRouteInterfaceName returns the interface of the default route, and an +// empty string when the route table does not answer. +func defaultRouteInterfaceName() string { + name, err := netmon.DefaultRouteInterface() + if err != nil { + return "" + } + return name +} + +// setOsResolverReason keeps the reason of the initialization that runs. +func setOsResolverReason(reason string) { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.reason = reason +} + // InitializeOsResolver initializes OS resolver using the current system DNS settings. // It returns the nameservers that is going to be used by the OS resolver. // // It's the caller's responsibility to ensure the system DNS is in a clean state before // calling this function. -func InitializeOsResolver(guardAgainstNoNameservers bool) []string { - nameservers := availableNameservers() - // if no nameservers, return empty slice so we dont remove all nameservers - if len(nameservers) == 0 && guardAgainstNoNameservers { - return []string{} +func InitializeOsResolver(ctx context.Context, guardAgainstNoNameservers bool) []string { + return InitializeOsResolverWithReason(ctx, guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithReason is InitializeOsResolver with the reason that +// the journal reports when the resolver set changes. +func InitializeOsResolverWithReason(ctx context.Context, guardAgainstNoNameservers bool, reason string) []string { + effective, _ := InitializeOsResolverWithSystemNameserversReason(ctx, guardAgainstNoNameservers, reason) + return effective +} + +// InitializeOsResolverWithSystemNameservers initializes the OS resolver and +// returns both the effective resolver list and the unmodified nameservers +// discovered from the system. The latter deliberately excludes synthetic +// fallbacks added by initializeOsResolver. +func InitializeOsResolverWithSystemNameservers(ctx context.Context, guardAgainstNoNameservers bool) (effective, system []string) { + return InitializeOsResolverWithSystemNameserversReason(ctx, guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithSystemNameserversReason is +// InitializeOsResolverWithSystemNameservers with the reason that the journal +// reports when the resolver set changes. +func InitializeOsResolverWithSystemNameserversReason(ctx context.Context, guardAgainstNoNameservers bool, reason string) (effective, system []string) { + resolverMutex.Lock() + defer resolverMutex.Unlock() + + setOsResolverReason(reason) + defer setOsResolverReason(osResolverReasonUnspecified) + + system = availableNameservers(ctx) + if system == nil { + // A non-nil empty slice means discovery completed and found no DNS. + // Callers use nil to mean that discovery was not attempted. + system = []string{} + } + effective, system, skip := osResolverNameserverSets(system, guardAgainstNoNameservers) + if skip { + return effective, system + } + storeOsResolver(newResolverWithNameserver(effective)) + return effective, system +} + +func osResolverNameserverSets(system []string, guardAgainstNoNameservers bool) (effective, discovered []string, skip bool) { + if len(system) == 0 && guardAgainstNoNameservers { + return []string{}, system, true + } + return initializeOsResolver(system), system, false +} + +// OsResolverNameservers returns the current OS resolver nameservers (host:port format). +// Returns nil if the OS resolver has not been initialized. +// The read takes no lock, so a log header render never waits for a resolver +// initialization. +func OsResolverNameservers() []string { + r := osResolverPtr.Load() + if r == nil { + return nil + } + var nss []string + if lan := r.lanServers.Load(); lan != nil { + nss = append(nss, *lan...) + } + if pub := r.publicServers.Load(); pub != nil { + nss = append(nss, *pub...) + } + return nss +} + +// AppendOsResolverNameservers adds additional nameservers to the existing OS resolver +// without reinitializing it. This is used for late-arriving nameservers such as AD +// domain controller IPs discovered via background retry. +// Returns true if nameservers were actually added. +func AppendOsResolverNameservers(servers []string) bool { + if len(servers) == 0 { + return false } - ns := initializeOsResolver(nameservers) resolverMutex.Lock() defer resolverMutex.Unlock() - or = newResolverWithNameserver(ns) - return ns + if or == nil { + return false + } + + // Collect existing nameservers to avoid duplicates. + existing := make(map[string]bool) + if lan := or.lanServers.Load(); lan != nil { + for _, server := range *lan { + existing[server] = true + } + } + if pub := or.publicServers.Load(); pub != nil { + for _, server := range *pub { + existing[server] = true + } + } + + var added bool + for _, server := range servers { + // Normalize to host:port format. + if _, _, err := net.SplitHostPort(server); err != nil { + server = net.JoinHostPort(server, "53") + } + if existing[server] { + continue + } + existing[server] = true + added = true + + ip, _, _ := net.SplitHostPort(server) + addr, _ := netip.ParseAddr(ip) + if isLanAddr(addr) { + var newLan []string + if lan := or.lanServers.Load(); lan != nil { + newLan = append(newLan, (*lan)...) + } + newLan = append(newLan, server) + or.lanServers.Store(&newLan) + } else { + var newPub []string + if pub := or.publicServers.Load(); pub != nil { + newPub = append(newPub, (*pub)...) + } + newPub = append(newPub, server) + or.publicServers.Store(&newPub) + } + } + return added } // initializeOsResolver performs logic for choosing OS resolver nameserver. @@ -154,10 +396,11 @@ func InitializeOsResolver(guardAgainstNoNameservers bool) []string { // - First available LAN servers are saved and store. // - Later calls, if no LAN servers available, the saved servers above will be used. func initializeOsResolver(servers []string) []string { - var lanNss, publicNss []string - // First categorize servers + // Categorize DNS servers into LAN and public servers + // This is needed because LAN servers should be tried first for better performance, + // while public servers serve as fallback for external queries for _, ns := range servers { addr, err := netip.ParseAddr(ns) if err != nil { @@ -171,6 +414,8 @@ func initializeOsResolver(servers []string) []string { } } + // Ensure we have at least one public DNS server as fallback + // This prevents DNS resolution failures when no public servers are configured if len(publicNss) == 0 { publicNss = []string{controldPublicDnsWithPort} } @@ -188,7 +433,7 @@ type Resolver interface { var errUnknownResolver = errors.New("unknown resolver") // NewResolver creates a Resolver based on the given upstream config. -func NewResolver(uc *UpstreamConfig) (Resolver, error) { +func NewResolver(ctx context.Context, uc *UpstreamConfig) (Resolver, error) { typ := uc.Type switch typ { case ResolverTypeDOH, ResolverTypeDOH3: @@ -200,17 +445,16 @@ func NewResolver(uc *UpstreamConfig) (Resolver, error) { case ResolverTypeOS: resolverMutex.Lock() if or == nil { - ProxyLogger.Load().Debug().Msgf("Initialize new OS resolver") - or = newResolverWithNameserver(defaultNameservers()) + logger := LoggerFromCtx(ctx) + logger.Debug().Msgf("Initialize new OS resolver") + storeOsResolver(newResolverWithNameserver(defaultNameservers(ctx))) } resolverMutex.Unlock() return or, nil case ResolverTypeLegacy: return &legacyResolver{uc: uc}, nil case ResolverTypePrivate: - return NewPrivateResolver(), nil - case ResolverTypeLocal: - return localResolver, nil + return NewPrivateResolver(ctx), nil } return nil, fmt.Errorf("%w: %s", errUnknownResolver, typ) } @@ -220,6 +464,10 @@ type osResolver struct { publicServers atomic.Pointer[[]string] group *singleflight.Group cache *sync.Map + // Per-resolver seams let tests exercise the production Resolve path without + // mutating process-wide resolver state. + exchangeDNS dnsExchangeFunc + localIP func(string) net.IP } type osResolverResult struct { @@ -235,14 +483,16 @@ type publicResponse struct { } // SetDefaultLocalIPv4 updates the stored local IPv4. -func SetDefaultLocalIPv4(ip net.IP) { - Log(context.Background(), ProxyLogger.Load().Debug(), "SetDefaultLocalIPv4: %s", ip) +func SetDefaultLocalIPv4(ctx context.Context, ip net.IP) { + logger := LoggerFromCtx(ctx) + logger.Debug().Msgf("SetDefaultLocalIPv4: %s", ip) defaultLocalIPv4.Store(ip) } // SetDefaultLocalIPv6 updates the stored local IPv6. -func SetDefaultLocalIPv6(ip net.IP) { - Log(context.Background(), ProxyLogger.Load().Debug(), "SetDefaultLocalIPv6: %s", ip) +func SetDefaultLocalIPv6(ctx context.Context, ip net.IP) { + logger := LoggerFromCtx(ctx) + logger.Debug().Msgf("SetDefaultLocalIPv6: %s", ip) defaultLocalIPv6.Store(ip) } @@ -262,21 +512,70 @@ func GetDefaultLocalIPv6() net.IP { return nil } -// customDNSExchange wraps the DNS exchange to use our debug dialer. -// It uses dns.ExchangeWithConn so that our custom dialer is used directly. -func customDNSExchange(ctx context.Context, msg *dns.Msg, server string, desiredLocalIP net.IP) (*dns.Msg, time.Duration, error) { +type dnsExchangeFunc func(context.Context, *dns.Msg, string, net.IP) (*dns.Msg, time.Duration, error) + +func exchangeDNS(ctx context.Context, msg *dns.Msg, server string, localIP net.IP) (*dns.Msg, time.Duration, error) { baseDialer := &net.Dialer{ Timeout: 3 * time.Second, Resolver: &net.Resolver{PreferGo: true}, } - if desiredLocalIP != nil { - baseDialer.LocalAddr = &net.UDPAddr{IP: desiredLocalIP, Port: 0} + if localIP != nil { + baseDialer.LocalAddr = &net.UDPAddr{IP: localIP, Port: 0} } dnsClient := &dns.Client{Net: "udp"} dnsClient.Dialer = baseDialer return dnsClient.ExchangeContext(ctx, msg, server) } +func defaultLocalIPForServer(server string) net.IP { + if runtime.GOOS != "darwin" { + return nil + } + host, _, err := net.SplitHostPort(server) + if err != nil { + return nil + } + addr, err := netip.ParseAddr(host) + if err == nil && addr.Zone() != "" { + // Scoped resolvers select their interface via the destination zone. + // A cached default-interface source can have the wrong family or scope. + return nil + } + if err == nil && addr.Is6() && !addr.Is4In6() { + return GetDefaultLocalIPv6() + } + return GetDefaultLocalIPv4() +} + +func preSendUnreachable(err error) bool { + var opErr *net.OpError + if !errors.As(err, &opErr) || (opErr.Op != "dial" && opErr.Op != "write") { + return false + } + return ctrldnet.IsUnreachable(err) +} + +// customDNSExchangeWith preserves the preferred source first. A route-selected +// retry is allowed only when the caller knows the server is an OS-selected resolver, +// not ctrld's synthetic public fallback. This includes public DNS pushed by a VPN: +// unbinding changes the source route, not the recipient. +func customDNSExchangeWith(ctx context.Context, msg *dns.Msg, server string, desiredLocalIP net.IP, allowRouteSelectedRetry bool, exchange dnsExchangeFunc) (*dns.Msg, time.Duration, error) { + answer, rtt, err := exchange(ctx, msg, server, desiredLocalIP) + if answer != nil || err == nil || ctx.Err() != nil || desiredLocalIP == nil || !allowRouteSelectedRetry || !preSendUnreachable(err) { + return answer, rtt, err + } + + LoggerFromCtx(ctx).Debug().Msg("OS resolver source binding is unreachable; retrying with route-selected source") + return exchange(ctx, msg.Copy(), server, nil) +} + +// allowRouteSelectedRetryForOSServer excludes only ctrld's synthetic public +// fallback. System-provided resolvers remain eligible even when their addresses +// are public, as with VPNs that push public DNS servers. +func allowRouteSelectedRetryForOSServer(server string) bool { + return server != controldPublicDnsWithPort +} + const hotCacheTTL = time.Second // Resolve resolves DNS queries using pre-configured nameservers. @@ -291,19 +590,33 @@ const hotCacheTTL = time.Second // for a short period (currently 1 second), reducing unnecessary traffics // sent to upstreams. func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + if err := validateMsg(msg); err != nil { + return nil, err + } if len(msg.Question) == 0 { return nil, errors.New("no question found") } domain := strings.TrimSuffix(msg.Question[0].Name, ".") qtype := msg.Question[0].Qtype - // Unique key for the singleflight group. - key := fmt.Sprintf("%s:%d:", domain, qtype) + // Unique key for the singleflight group. The EDNS Client Subnet is part of + // the key so subnet-specific answers are neither coalesced nor hot-cached + // across different subnets (RFC 7871 §7.3). + key := fmt.Sprintf("%s:%d:%s", domain, qtype, dnscache.CanonicalECS(msg)) + // A LAN-only query must not join, or be answered from the hot cache of, an + // ordinary query for the same name: that answer may have come from a public + // nameserver. + if isLanOnlyQuery(ctx) { + key += ":lan-only" + } + + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "OS resolver query started: %s - %s", domain, dns.TypeToString[qtype]) // Checking the cache first. if val, ok := o.cache.Load(key); ok { if val, ok := val.(*dns.Msg); ok { - Log(ctx, ProxyLogger.Load().Debug(), "hit hot cached result: %s - %s", domain, dns.TypeToString[qtype]) + Log(ctx, logger.Debug(), "Hit hot cached result: %s - %s", domain, dns.TypeToString[qtype]) res := val.Copy() SetCacheReply(res, msg, val.Rcode) return res, nil @@ -312,8 +625,13 @@ func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error // Ensure only one DNS query is in flight for the key. v, err, shared := o.group.Do(key, func() (interface{}, error) { + Log(ctx, logger.Debug(), "Resolving query: %s - %s", domain, dns.TypeToString[qtype]) msg, err := o.resolve(ctx, msg) if err != nil { + // One line for each failed query. The caller reports the failure at + // error level through its sampler, so this line stays at debug and + // out of the retained journal. + Log(ctx, logger.Debug().Err(err), "OS resolver query failed: %s - %s", domain, dns.TypeToString[qtype]) return nil, err } // If we got an answer, storing it to the hot cache for hotCacheTTL @@ -325,6 +643,7 @@ func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error time.AfterFunc(hotCacheTTL, func() { o.removeCache(key) }) + Log(ctx, logger.Debug(), "OS resolver query successful: %s - %s", domain, dns.TypeToString[qtype]) return msg, nil }) if err != nil { @@ -338,7 +657,7 @@ func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error res := sharedMsg.Copy() SetCacheReply(res, msg, sharedMsg.Rcode) if shared { - Log(ctx, ProxyLogger.Load().Debug(), "shared result: %s - %s", domain, dns.TypeToString[qtype]) + Log(ctx, logger.Debug(), "Shared result: %s - %s", domain, dns.TypeToString[qtype]) } return res, nil @@ -351,6 +670,10 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error if p := o.lanServers.Load(); p != nil { nss = append(nss, (*p)...) } + // A LAN-only query is never sent to a public nameserver. + if isLanOnlyQuery(ctx) { + publicServers = nil + } numServers := len(nss) + len(publicServers) // If this is a LAN query, skip public DNS. @@ -368,7 +691,8 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error if msg != nil && len(msg.Question) > 0 { question = msg.Question[0].Name } - Log(ctx, ProxyLogger.Load().Debug(), "os resolver query for %s with nameservers: %v public: %v", question, nss, publicServers) + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "OS resolver query for %s with nameservers: %v public: %v", question, nss, publicServers) // New check: If no resolvers are available, return an error. if numServers == 0 { @@ -380,6 +704,14 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error ch := make(chan *osResolverResult, numServers) wg := &sync.WaitGroup{} + exchange := o.exchangeDNS + if exchange == nil { + exchange = exchangeDNS + } + localIPForServer := o.localIP + if localIPForServer == nil { + localIPForServer = defaultLocalIPForServer + } wg.Add(numServers) go func() { wg.Wait() @@ -390,22 +722,15 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error for _, server := range servers { go func(server string) { defer wg.Done() - var answer *dns.Msg - var err error - var localOSResolverIP net.IP - if runtime.GOOS == "darwin" { - host, _, err := net.SplitHostPort(server) - if err == nil { - ip := net.ParseIP(host) - if ip != nil && ip.To4() == nil { - // IPv6 nameserver; use default IPv6 address (if set) - localOSResolverIP = GetDefaultLocalIPv6() - } else { - localOSResolverIP = GetDefaultLocalIPv4() - } - } + release, ok := acquireOSResolverExchangeSlot(ctx) + if !ok { + ch <- &osResolverResult{err: ctx.Err(), server: server, lan: isLan} + return } - answer, _, err = customDNSExchange(ctx, msg.Copy(), server, localOSResolverIP) + defer release() + + localOSResolverIP := localIPForServer(server) + answer, _, err := customDNSExchangeWith(ctx, msg.Copy(), server, localOSResolverIP, allowRouteSelectedRetryForOSServer(server), exchange) ch <- &osResolverResult{answer: answer, err: err, server: server, lan: isLan} }(server) } @@ -417,7 +742,7 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error // If splitting fails, fallback to the original server string host = server } - Log(ctx, ProxyLogger.Load().Debug(), "got answer from nameserver: %s", host) + Log(ctx, logger.Debug(), "Got answer from nameserver: %s", host) } // try local nameservers @@ -444,7 +769,7 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error switch { case res.lan: // Always prefer LAN responses immediately - Log(ctx, ProxyLogger.Load().Debug(), "using LAN answer from: %s", res.server) + Log(ctx, logger.Debug(), "Using LAN answer from: %s", res.server) cancel() logAnswer(res.server) return res.answer, nil @@ -454,7 +779,7 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error // if there are no LAN nameservers, we should not wait // just use the first response if len(nss) == 0 { - Log(ctx, ProxyLogger.Load().Debug(), "using public answer from: %s", res.server) + Log(ctx, logger.Debug(), "Using public answer from: %s", res.server) cancel() logAnswer(res.server) return res.answer, nil @@ -465,12 +790,12 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error }) } case res.answer != nil: - Log(ctx, ProxyLogger.Load().Debug(), "got non-success answer from: %s with code: %d", + Log(ctx, logger.Debug(), "Got non-success answer from: %s with code: %d", res.server, res.answer.Rcode) // When there are no LAN nameservers, we should not wait // for other nameservers to respond. if len(nss) == 0 { - Log(ctx, ProxyLogger.Load().Debug(), "no lan nameservers using public non success answer") + Log(ctx, logger.Debug(), "No lan nameservers using public non success answer") cancel() logAnswer(res.server) return res.answer, nil @@ -483,23 +808,32 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error if len(publicResponses) > 0 { resp := publicResponses[0] - Log(ctx, ProxyLogger.Load().Debug(), "using public answer from: %s", resp.server) + Log(ctx, logger.Debug(), "Using public answer from: %s", resp.server) logAnswer(resp.server) return resp.answer, nil } if controldSuccessAnswer != nil { - Log(ctx, ProxyLogger.Load().Debug(), "using ControlD answer from: %s", controldPublicDnsWithPort) + Log(ctx, logger.Debug(), "Using ControlD answer from: %s", controldPublicDnsWithPort) logAnswer(controldPublicDnsWithPort) return controldSuccessAnswer, nil } if nonSuccessAnswer != nil { - Log(ctx, ProxyLogger.Load().Debug(), "using non-success answer from: %s", nonSuccessServer) + Log(ctx, logger.Debug(), "Using non-success answer from: %s", nonSuccessServer) logAnswer(nonSuccessServer) return nonSuccessAnswer, nil } return nil, errors.Join(errs...) } +func acquireOSResolverExchangeSlot(ctx context.Context) (func(), bool) { + select { + case osResolverExchangeSem <- struct{}{}: + return func() { <-osResolverExchangeSem }, true + case <-ctx.Done(): + return nil, false + } +} + func (o *osResolver) removeCache(key string) { o.cache.Delete(key) } @@ -509,13 +843,19 @@ type legacyResolver struct { } func (r *legacyResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + if err := validateMsg(msg); err != nil { + return nil, err + } + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "Legacy resolver query started") + // See comment in (*dotResolver).resolve method. dialer := newDialer(net.JoinHostPort(controldPublicDns, "53")) dnsTyp := uint16(0) if msg != nil && len(msg.Question) > 0 { dnsTyp = msg.Question[0].Qtype } - _, udpNet := r.uc.netForDNSType(dnsTyp) + _, udpNet := r.uc.netForDNSType(ctx, dnsTyp) dnsClient := &dns.Client{ Net: udpNet, Dialer: dialer, @@ -527,13 +867,26 @@ func (r *legacyResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, e endpoint = net.JoinHostPort(r.uc.BootstrapIP, port) } + Log(ctx, logger.Debug(), "Sending legacy request to: %s", endpoint) answer, _, err := dnsClient.ExchangeContext(ctx, msg, endpoint) + if err != nil { + event := logger.Error() + if private, _ := ctx.Value(privateResolverCtxKey{}).(bool); private { + event = logger.Debug() + } + Log(ctx, event.Err(err), "Legacy request failed") + } else { + Log(ctx, logger.Debug(), "Legacy resolver query successful") + } return answer, err } type dummyResolver struct{} func (d dummyResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + if err := validateMsg(msg); err != nil { + return nil, err + } ans := new(dns.Msg) ans.SetReply(msg) return ans, nil @@ -541,39 +894,43 @@ func (d dummyResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, err // LookupIP looks up domain using current system nameservers settings. // It returns a slice of that host's IPv4 and IPv6 addresses. -func LookupIP(domain string) []string { - nss := initDefaultOsResolver() - return lookupIP(domain, -1, nss) +func LookupIP(ctx context.Context, domain string) []string { + nss := initDefaultOsResolver(ctx) + return lookupIP(ctx, domain, -1, nss) } // initDefaultOsResolver initializes the default OS resolver with system's default nameservers if it hasn't been initialized yet. // It returns the combined list of LAN and public nameservers currently held by the resolver. -func initDefaultOsResolver() []string { +func initDefaultOsResolver(ctx context.Context) []string { + logger := LoggerFromCtx(ctx) resolverMutex.Lock() defer resolverMutex.Unlock() if or == nil { - ProxyLogger.Load().Debug().Msgf("Initialize new OS resolver with default nameservers") - or = newResolverWithNameserver(defaultNameservers()) + logger.Debug().Msgf("Initialize new OS resolver with default nameservers") + storeOsResolver(newResolverWithNameserver(defaultNameservers(ctx))) } nss := *or.lanServers.Load() nss = append(nss, *or.publicServers.Load()...) return nss + } // lookupIP looks up domain with given timeout and bootstrapDNS. // If the timeout is negative, default timeout 2000 ms will be used. // It returns nil if bootstrapDNS is nil or empty. -func lookupIP(domain string, timeout int, bootstrapDNS []string) (ips []string) { +func lookupIP(ctx context.Context, domain string, timeout int, bootstrapDNS []string) (ips []string) { if net.ParseIP(domain) != nil { return []string{domain} } + logger := LoggerFromCtx(ctx) if bootstrapDNS == nil { - ProxyLogger.Load().Debug().Msgf("empty bootstrap DNS") + logger.Debug().Msgf("Empty bootstrap dns") return nil } resolver := newResolverWithNameserver(bootstrapDNS) - ProxyLogger.Load().Debug().Msgf("resolving %q using bootstrap DNS %q", domain, bootstrapDNS) + logger.Debug().Msgf("Resolving %q using bootstrap dns %q", domain, bootstrapDNS) + timeoutMs := 2000 if timeout > 0 && timeout < timeoutMs { timeoutMs = timeout @@ -616,15 +973,15 @@ func lookupIP(domain string, timeout int, bootstrapDNS []string) (ips []string) r, err := resolver.Resolve(ctx, m) if err != nil { - ProxyLogger.Load().Error().Err(err).Msgf("could not lookup %q record for domain %q", dns.TypeToString[dnsType], domain) + logger.Error().Err(err).Msgf("Could not lookup %q record for domain %q", dns.TypeToString[dnsType], domain) return } if r.Rcode != dns.RcodeSuccess { - ProxyLogger.Load().Error().Msgf("could not resolve domain %q, return code: %s", domain, dns.RcodeToString[r.Rcode]) + logger.Error().Msgf("Could not resolve domain %q, return code: %s", domain, dns.RcodeToString[r.Rcode]) return } if len(r.Answer) == 0 { - ProxyLogger.Load().Error().Msg("no answer from OS resolver") + logger.Error().Msg("No answer from os resolver") return } target := targetDomain(r.Answer) @@ -641,22 +998,6 @@ func lookupIP(domain string, timeout int, bootstrapDNS []string) (ips []string) return ips } -// NewBootstrapResolver returns an OS resolver, which use following nameservers: -// -// - Gateway IP address (depends on OS). -// - Input servers. -func NewBootstrapResolver(servers ...string) Resolver { - logger := *ProxyLogger.Load() - - Log(context.Background(), logger.Debug(), "NewBootstrapResolver called with servers: %v", servers) - nss := defaultNameservers() - nss = append([]string{controldPublicDnsWithPort}, nss...) - for _, ns := range servers { - nss = append([]string{net.JoinHostPort(ns, "53")}, nss...) - } - return NewResolverWithNameserver(nss) -} - // NewPrivateResolver returns an OS resolver, which includes only private DNS servers, // excluding: // @@ -664,9 +1005,9 @@ func NewBootstrapResolver(servers ...string) Resolver { // - Nameservers which is local RFC1918 addresses. // // This is useful for doing PTR lookup in LAN network. -func NewPrivateResolver() Resolver { - nss := initDefaultOsResolver() - resolveConfNss := currentNameserversFromResolvconf() +func NewPrivateResolver(ctx context.Context) Resolver { + nss := initDefaultOsResolver(ctx) + resolveConfNss := CurrentNameserversFromResolvconf() localRfc1918Addrs := Rfc1918Addresses() n := 0 for _, ns := range nss { @@ -731,7 +1072,7 @@ func newResolverWithNameserver(nameservers []string) *osResolver { // Rfc1918Addresses returns the list of local physical interfaces private IP addresses func Rfc1918Addresses() []string { - vis := validInterfaces() + vis := ValidInterfaces(context.Background()) var res []string netmon.ForeachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { // Skip virtual interfaces. @@ -769,3 +1110,13 @@ func isLanAddr(addr netip.Addr) bool { addr.IsLinkLocalUnicast() || tsaddr.CGNATRange().Contains(addr) } + +func validateMsg(msg *dns.Msg) error { + if msg == nil { + return errors.New("nil DNS message") + } + if len(msg.Question) == 0 { + return errors.New("no question found") + } + return nil +} diff --git a/resolver_test.go b/resolver_test.go index f030739e..ce108629 100644 --- a/resolver_test.go +++ b/resolver_test.go @@ -4,13 +4,22 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" + "errors" "net" + "os" + "runtime" + "slices" + "strings" "sync" "sync/atomic" + "syscall" "testing" "time" "github.com/miekg/dns" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) func Test_osResolver_Resolve(t *testing.T) { @@ -70,6 +79,270 @@ func Test_osResolver_ResolveLanHostname(t *testing.T) { } } +func Test_customDNSExchangeWith_RetriesUnboundOnUnreachableSource(t *testing.T) { + tests := []struct { + name string + boundIP net.IP + server string + errno syscall.Errno + }{ + {"ipv4 network unreachable", net.ParseIP("192.0.2.10"), "192.0.2.53:53", syscall.ENETUNREACH}, + {"ipv4 host unreachable", net.ParseIP("192.0.2.10"), "192.0.2.53:53", syscall.EHOSTUNREACH}, + {"ipv6 network unreachable", net.ParseIP("2001:db8::10"), "[2001:db8::53]:53", syscall.ENETUNREACH}, + {"ipv6 host unreachable", net.ParseIP("2001:db8::10"), "[2001:db8::53]:53", syscall.EHOSTUNREACH}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + var localIPs []net.IP + var servers []string + exchange := func(_ context.Context, msg *dns.Msg, server string, localIP net.IP) (*dns.Msg, time.Duration, error) { + localIPs = append(localIPs, append(net.IP(nil), localIP...)) + servers = append(servers, server) + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: &os.SyscallError{Syscall: "write", Err: tt.errno}} + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + answer, _, err := customDNSExchangeWith(context.Background(), msg, tt.server, tt.boundIP, true, exchange) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from route-selected retry") + } + if len(localIPs) != 2 { + t.Fatalf("exchange calls: got %d, want 2", len(localIPs)) + } + if !localIPs[0].Equal(tt.boundIP) { + t.Fatalf("first source: got %v, want %v", localIPs[0], tt.boundIP) + } + if localIPs[1] != nil { + t.Fatalf("retry source: got %v, want route-selected nil", localIPs[1]) + } + if len(servers) != 2 || servers[0] != tt.server || servers[1] != tt.server { + t.Fatalf("exchange servers: got %v, want two attempts to %s", servers, tt.server) + } + }) + } +} + +func Test_customDNSExchangeWith_PreservesReachableBoundSource(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + boundIP := net.ParseIP("192.0.2.10") + calls := 0 + exchange := func(_ context.Context, msg *dns.Msg, _ string, localIP net.IP) (*dns.Msg, time.Duration, error) { + calls++ + if !localIP.Equal(boundIP) { + t.Fatalf("source: got %v, want %v", localIP, boundIP) + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + answer, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", boundIP, true, exchange) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from bound exchange") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryOtherFailures(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, context.DeadlineExceeded + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryWithoutBoundSource(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", nil, true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryCanceledContext(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + _, _, err := customDNSExchangeWith(ctx, msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_customDNSExchangeWith_ReturnsUnboundRetryFailure(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + retryErr := errors.New("route-selected exchange failed") + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, localIP net.IP) (*dns.Msg, time.Duration, error) { + calls++ + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + return nil, 0, retryErr + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if !errors.Is(err, retryErr) { + t.Fatalf("exchange error: got %v, want retry error %v", err, retryErr) + } + if calls != 2 { + t.Fatalf("exchange calls: got %d, want 2", calls) + } +} + +func Test_customDNSExchangeWith_DoesNotRetryReadSideUnreachable(t *testing.T) { + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + calls := 0 + exchange := func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "read", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + _, _, err := customDNSExchangeWith(context.Background(), msg, "192.0.2.53:53", net.ParseIP("192.0.2.10"), true, exchange) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want 1", calls) + } +} + +func Test_osResolver_ResolveUsesRouteSelectedFallbackForLANServer(t *testing.T) { + const server = "10.0.0.53:53" + boundIP := net.ParseIP("192.0.2.10") + resolver := newResolverWithNameserver([]string{server}) + resolver.localIP = func(string) net.IP { return boundIP } + + var localIPs []net.IP + var servers []string + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, gotServer string, localIP net.IP) (*dns.Msg, time.Duration, error) { + servers = append(servers, gotServer) + localIPs = append(localIPs, append(net.IP(nil), localIP...)) + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + answer, err := resolver.Resolve(context.Background(), msg) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from route-selected retry") + } + if len(localIPs) != 2 || !localIPs[0].Equal(boundIP) || localIPs[1] != nil { + t.Fatalf("exchange sources: got %v, want [%v ]", localIPs, boundIP) + } + if len(servers) != 2 || servers[0] != server || servers[1] != server { + t.Fatalf("exchange servers: got %v, want two attempts to %s", servers, server) + } +} + +// A VPN-pushed public DNS address is categorized as public by IP, but it is +// still a system-selected resolver and must get the same route-compatible retry. +func Test_osResolver_ResolveUsesRouteSelectedFallbackForPublicVPNServer(t *testing.T) { + const server = "192.0.2.53:53" + boundIP := net.ParseIP("198.51.100.10") + resolver := newResolverWithNameserver([]string{server}) + resolver.localIP = func(string) net.IP { return boundIP } + + var localIPs []net.IP + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, localIP net.IP) (*dns.Msg, time.Duration, error) { + localIPs = append(localIPs, append(net.IP(nil), localIP...)) + if localIP != nil { + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + answer, err := resolver.Resolve(context.Background(), msg) + if err != nil { + t.Fatal(err) + } + if answer == nil { + t.Fatal("expected answer from route-selected retry") + } + if len(localIPs) != 2 || !localIPs[0].Equal(boundIP) || localIPs[1] != nil { + t.Fatalf("exchange sources: got %v, want [%v ]", localIPs, boundIP) + } +} + +func Test_osResolver_ResolveDoesNotRetrySyntheticControlDFallbackUnbound(t *testing.T) { + resolver := newResolverWithNameserver([]string{controldPublicDnsWithPort}) + resolver.localIP = func(string) net.IP { return net.ParseIP("198.51.100.10") } + calls := 0 + resolver.exchangeDNS = func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + return nil, 0, &net.OpError{Op: "write", Net: "udp", Err: syscall.EHOSTUNREACH} + } + + msg := new(dns.Msg) + msg.SetQuestion("internal.example.", dns.TypeA) + _, err := resolver.Resolve(context.Background(), msg) + if err == nil { + t.Fatal("expected exchange failure") + } + if calls != 1 { + t.Fatalf("exchange calls: got %d, want one bound synthetic fallback attempt", calls) + } +} + func Test_osResolver_ResolveWithNonSuccessAnswer(t *testing.T) { // Set up a LAN nameserver that returns a success response. lanPC, err := net.ListenPacket("udp", "127.0.0.1:0") // 127.0.0.1 is considered LAN (loopback) @@ -125,6 +398,31 @@ func Test_osResolver_ResolveWithNonSuccessAnswer(t *testing.T) { } } +func TestOSResolverNameserverSetsKeepsSyntheticFallbackOutOfSystemDiscovery(t *testing.T) { + system := []string{"fe80::1"} + effective, discovered, skip := osResolverNameserverSets(system, false) + if skip { + t.Fatal("non-empty discovery unexpectedly skipped resolver replacement") + } + + if len(discovered) != 1 || discovered[0] != system[0] { + t.Fatalf("discovered nameservers = %v, want raw system list %v", discovered, system) + } + if len(effective) != 2 || effective[0] != "[fe80::1]:53" || effective[1] != controldPublicDnsWithPort { + t.Fatalf("effective nameservers = %v, want IPv6 system resolver plus synthetic fallback", effective) + } +} + +func TestOSResolverNameserverSetsHonorsEmptyGuard(t *testing.T) { + effective, discovered, skip := osResolverNameserverSets(nil, true) + if len(effective) != 0 || len(discovered) != 0 { + t.Fatalf("guarded empty discovery returned effective=%v discovered=%v", effective, discovered) + } + if !skip { + t.Fatal("guarded empty discovery did not return the skip decision") + } +} + func Test_osResolver_InitializationRace(t *testing.T) { var wg sync.WaitGroup n := 10 @@ -132,7 +430,7 @@ func Test_osResolver_InitializationRace(t *testing.T) { for range n { go func() { defer wg.Done() - InitializeOsResolver(false) + InitializeOsResolver(LoggerCtx(context.Background(), nil), false) }() } wg.Wait() @@ -143,6 +441,8 @@ func Test_osResolver_Singleflight(t *testing.T) { if err != nil { t.Fatalf("failed to listen on LAN address: %v", err) } + defer lanPC.Close() + call := &atomic.Int64{} lanServer, lanAddr, err := runLocalPacketConnTestServer(t, lanPC, countHandler(call)) if err != nil { @@ -153,7 +453,13 @@ func Test_osResolver_Singleflight(t *testing.T) { or := newResolverWithNameserver([]string{lanAddr}) domain := "controld.com" n := 10 + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var wg sync.WaitGroup + errs := make(chan error, n) + wg.Add(n) for i := 0; i < n; i++ { go func() { @@ -161,25 +467,40 @@ func Test_osResolver_Singleflight(t *testing.T) { m := new(dns.Msg) m.SetQuestion(dns.Fqdn(domain), dns.TypeA) m.RecursionDesired = true - _, err := or.Resolve(context.Background(), m) + _, err := or.Resolve(ctx, m) if err != nil { - t.Error(err) + errs <- err } }() } wg.Wait() + close(errs) + + // Collect any errors that occurred + for err := range errs { + t.Errorf("resolver error: %v", err) + } // All above queries should only make 1 call to server. - if call.Load() != 1 { - t.Fatalf("expected 1 result from singleflight lookup, got %d", call) + if got := call.Load(); got != 1 { + t.Fatalf("expected 1 result from singleflight lookup, got %d", got) } } func Test_osResolver_HotCache(t *testing.T) { + const ( + testIterations = 2 + cacheCheckTimeout = 5 * time.Second + pollInterval = 10 * time.Millisecond + ) + + // Setup test server lanPC, err := net.ListenPacket("udp", "127.0.0.1:0") if err != nil { t.Fatalf("failed to listen on LAN address: %v", err) } + defer lanPC.Close() + call := &atomic.Int64{} lanServer, lanAddr, err := runLocalPacketConnTestServer(t, lanPC, countHandler(call)) if err != nil { @@ -187,58 +508,81 @@ func Test_osResolver_HotCache(t *testing.T) { } defer lanServer.Shutdown() + // Initialize resolver or := newResolverWithNameserver([]string{lanAddr}) domain := "controld.com" m := new(dns.Msg) m.SetQuestion(dns.Fqdn(domain), dns.TypeA) m.RecursionDesired = true - // Make 2 repeated queries to server, should hit hot cache. - for i := 0; i < 2; i++ { - if _, err := or.Resolve(context.Background(), m.Copy()); err != nil { + // Setup context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Make repeated queries to server, should hit hot cache + for i := 0; i < testIterations; i++ { + resp, err := or.Resolve(ctx, m.Copy()) + if err != nil { t.Fatal(err) } + // Verify response content + if resp.Rcode != dns.RcodeSuccess { + t.Errorf("expected success response, got %v", resp.Rcode) + } } + if call.Load() != 1 { t.Fatalf("cache not hit, server was called: %d", call.Load()) } + // Wait for cache to be cleaned timeoutChan := make(chan struct{}) - time.AfterFunc(5*time.Second, func() { + time.AfterFunc(cacheCheckTimeout, func() { close(timeoutChan) }) + // Check cache with proper polling interval +waitLoop: for { select { case <-timeoutChan: t.Fatal("timed out waiting for cache cleaned") - default: + case <-time.After(pollInterval): count := 0 or.cache.Range(func(key, value interface{}) bool { count++ return true }) - if count != 0 { - t.Logf("hot cache is not empty: %d elements", count) - continue + if count == 0 { + break waitLoop } + t.Logf("hot cache is not empty: %d elements", count) } - break } - if _, err := or.Resolve(context.Background(), m.Copy()); err != nil { + // Verify cache miss after cleanup + resp, err := or.Resolve(ctx, m.Copy()) + if err != nil { t.Fatal(err) } + if resp.Rcode != dns.RcodeSuccess { + t.Errorf("expected success response after cache cleanup, got %v", resp.Rcode) + } if call.Load() != 2 { t.Fatal("cache hit unexpectedly") } } func Test_Edns0_CacheReply(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + lanPC, err := net.ListenPacket("udp", "127.0.0.1:0") if err != nil { t.Fatalf("failed to listen on LAN address: %v", err) } + defer lanPC.Close() + call := &atomic.Int64{} lanServer, lanAddr, err := runLocalPacketConnTestServer(t, lanPC, countHandler(call)) if err != nil { @@ -252,33 +596,132 @@ func Test_Edns0_CacheReply(t *testing.T) { m.SetQuestion(dns.Fqdn(domain), dns.TypeA) m.RecursionDesired = true - do := func() *dns.Msg { + do := func() (*dns.Msg, error) { msg := m.Copy() msg.SetEdns0(4096, true) cookieOption := new(dns.EDNS0_COOKIE) cookieOption.Code = dns.EDNS0COOKIE cookieOption.Cookie = generateEdns0ClientCookie() msg.IsEdns0().Option = append(msg.IsEdns0().Option, cookieOption) + return or.Resolve(ctx, msg) + } - answer, err := or.Resolve(context.Background(), msg) - if err != nil { - t.Fatal(err) - } - return answer + answer1, err := do() + if err != nil { + t.Fatalf("first resolve failed: %v", err) } - answer1 := do() - answer2 := do() - // Ensure the cache was hit, so we can check that edns0 cookie must be modified. - if call.Load() != 1 { - t.Fatalf("cache not hit, server was called: %d", call.Load()) + + answer2, err := do() + if err != nil { + t.Fatalf("second resolve failed: %v", err) } + + // Ensure the cache was hit + if got := call.Load(); got != 1 { + t.Fatalf("expected 1 server call, got: %d", got) + } + cookie1 := getEdns0Cookie(answer1.IsEdns0()) cookie2 := getEdns0Cookie(answer2.IsEdns0()) + if cookie1 == nil || cookie2 == nil { - t.Fatalf("unexpected nil cookie value (cookie1: %v, cookie2: %v)", cookie1, cookie2) + t.Fatalf("unexpected nil cookie (cookie1: %v, cookie2: %v)", cookie1, cookie2) } + if cookie1.Cookie == cookie2.Cookie { - t.Fatalf("edns0 cookie is not modified: %v", cookie1) + t.Fatalf("edns0 cookie was not modified (cookie: %v)", cookie1.Cookie) + } + + // Validate response code + if answer1.Rcode != dns.RcodeSuccess || answer2.Rcode != dns.RcodeSuccess { + t.Errorf("expected success response code, got: %v, %v", answer1.Rcode, answer2.Rcode) + } +} + +// ecsAnswerHandler returns a distinct A record per EDNS Client Subnet, so a test can +// prove one subnet never receives another subnet's cached record. It counts upstream +// calls to confirm the hot cache/singleflight is partitioned by ECS rather than shared. +func ecsAnswerHandler(call *atomic.Int64) dns.HandlerFunc { + return func(w dns.ResponseWriter, msg *dns.Msg) { + call.Add(1) + a := "203.0.113.1" // no/other subnet + if opt := msg.IsEdns0(); opt != nil { + for _, o := range opt.Option { + if e, ok := o.(*dns.EDNS0_SUBNET); ok { + switch { + case e.Address.Equal(net.ParseIP("2001:db8:1::")): + a = "192.0.2.1" + case e.Address.Equal(net.ParseIP("2001:db8:2::")): + a = "198.51.100.1" + } + } + } + } + m := new(dns.Msg) + m.SetReply(msg) + rr, _ := dns.NewRR(msg.Question[0].Name + " 300 IN A " + a) + m.Answer = []dns.RR{rr} + w.WriteMsg(m) + } +} + +// Test_osResolver_HotCache_ECSPartition is the real cache-path regression test for #564 on +// the osResolver hot cache / singleflight path: the upstream returns a different A record +// per subnet, and a client in subnet B must never be served subnet A's hot-cached record. +func Test_osResolver_HotCache_ECSPartition(t *testing.T) { + lanPC, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen on LAN address: %v", err) + } + call := &atomic.Int64{} + lanServer, lanAddr, err := runLocalPacketConnTestServer(t, lanPC, ecsAnswerHandler(call)) + if err != nil { + t.Fatalf("failed to run LAN test server: %v", err) + } + defer lanServer.Shutdown() + + or := newResolverWithNameserver([]string{lanAddr}) + query := func(subnet string) string { + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn("controld.com"), dns.TypeA) + m.RecursionDesired = true + m.SetEdns0(4096, true) + m.IsEdns0().Option = append(m.IsEdns0().Option, &dns.EDNS0_SUBNET{ + Code: dns.EDNS0SUBNET, + Family: 2, + SourceNetmask: 64, + Address: net.ParseIP(subnet), + }) + answer, err := or.Resolve(context.Background(), m) + if err != nil { + t.Fatal(err) + } + for _, rr := range answer.Answer { + if a, ok := rr.(*dns.A); ok { + return a.A.String() + } + } + return "" + } + + // Subnet A populates the hot cache; a repeat hits it (upstream called once). + if got := query("2001:db8:1::"); got != "192.0.2.1" { + t.Fatalf("subnet A: got %q, want 192.0.2.1", got) + } + if got := query("2001:db8:1::"); got != "192.0.2.1" { + t.Fatalf("subnet A repeat: got %q, want 192.0.2.1", got) + } + if call.Load() != 1 { + t.Fatalf("subnet A repeat did not hit the hot cache: %d upstream calls", call.Load()) + } + + // Subnet B must get ITS OWN record, not subnet A's hot-cached one, and this + // requires a fresh upstream call (the cache is partitioned, not shared). + if got := query("2001:db8:2::"); got != "198.51.100.1" { + t.Fatalf("subnet B was served the wrong record %q (want 198.51.100.1); hot cache is not ECS-partitioned", got) + } + if call.Load() != 2 { + t.Fatalf("subnet B unexpectedly served from subnet A's cache: %d upstream calls, want 2", call.Load()) } } @@ -299,8 +742,9 @@ func Test_legacyResolverWithBigExtraSection(t *testing.T) { Type: ResolverTypeLegacy, Endpoint: lanAddr, } - uc.Init() - r, err := NewResolver(uc) + ctx := context.Background() + uc.Init(ctx) + r, err := NewResolver(ctx, uc) if err != nil { t.Fatal(err) } @@ -383,6 +827,11 @@ func nonSuccessHandlerWithRcode(rcode int) dns.HandlerFunc { func countHandler(call *atomic.Int64) dns.HandlerFunc { return func(w dns.ResponseWriter, msg *dns.Msg) { + // Count the call before writing the reply. The client returns as soon + // as it receives the response, so a caller that reads this counter right + // after Resolve returns would race an increment done after WriteMsg and + // could observe a stale zero. + call.Add(1) m := new(dns.Msg) m.SetRcode(msg, dns.RcodeSuccess) if cookie := getEdns0Cookie(msg.IsEdns0()); cookie != nil { @@ -395,7 +844,6 @@ func countHandler(call *atomic.Int64) dns.HandlerFunc { m.IsEdns0().Option = append(m.IsEdns0().Option, cookieOption) } w.WriteMsg(m) - call.Add(1) } } @@ -476,3 +924,330 @@ func generateEdns0ServerCookie(clientCookie string) string { } return clientCookie + hex.EncodeToString(cookie) } + +// syncLogBuffer collects the log lines that any goroutine writes. +type syncLogBuffer struct { + mu sync.Mutex + sb strings.Builder +} + +func (b *syncLogBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.Write(p) +} + +func (b *syncLogBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.String() +} + +// captureProxyLog returns a context whose logger writes one JSON object for +// each line into the returned buffer, at debug level. The resolver code logs +// through the logger of its context. +func captureProxyLog(t *testing.T) (context.Context, *syncLogBuffer) { + t.Helper() + buf := &syncLogBuffer{} + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.TimeKey = "time" + encoderConfig.LevelKey = "level" + encoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder + encoderConfig.MessageKey = "message" + core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderConfig), zapcore.AddSync(buf), zapcore.DebugLevel) + return LoggerCtx(context.Background(), &Logger{Logger: zap.New(core)}), buf +} + +// logEventsWithPrefix returns the log events whose message starts with prefix. +func logEventsWithPrefix(t *testing.T, logs, prefix string) []map[string]any { + t.Helper() + var events []map[string]any + for _, line := range strings.Split(logs, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + message, _ := event["message"].(string) + if !strings.HasPrefix(message, prefix) { + continue + } + events = append(events, event) + } + return events +} + +// wantLogField fails the test when an event misses a field or holds another value. +func wantLogField(t *testing.T, event map[string]any, field string, want any) { + t.Helper() + got, ok := event[field] + if !ok { + t.Fatalf("log event has no field %q: %v", field, event) + } + if got != want { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// wantLogStrings fails the test when a list field does not hold want. +func wantLogStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("log event field %q is not a list: %v", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, ok := value.(string) + if !ok { + t.Fatalf("log event field %q holds a value that is not a string: %v", field, value) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// stubNameservers answers each system nameserver read from lists, in order. +// The last list answers every read after it. +func stubNameservers(t *testing.T, lists ...[]string) { + t.Helper() + previous := NameserversFn + reads := 0 + NameserversFn = func(context.Context) []string { + list := lists[min(reads, len(lists)-1)] + reads++ + return slices.Clone(list) + } + t.Cleanup(func() { NameserversFn = previous }) +} + +// resetOsResolverLog clears the change-only state, so one test does not see +// the nameserver reads of another. +func resetOsResolverLog(t *testing.T) { + t.Helper() + reset := func() { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.system = nameserverReads{} + osResolverLog.final = nameserverReads{} + osResolverLog.reason = osResolverReasonUnspecified + } + reset() + t.Cleanup(reset) +} + +// keepOsResolver puts back the OS resolver that the process had before the test. +func keepOsResolver(t *testing.T) { + t.Helper() + resolverMutex.Lock() + previous := or + resolverMutex.Unlock() + t.Cleanup(func() { + resolverMutex.Lock() + storeOsResolver(previous) + resolverMutex.Unlock() + }) +} + +// TestOsResolverNameserversReadsWithoutTheResolverLock covers a log header +// render during a resolver initialization. That initialization holds +// resolverMutex across a scutil read of several seconds, and the header must +// not wait for it. +func TestOsResolverNameserversReadsWithoutTheResolverLock(t *testing.T) { + keepOsResolver(t) + resolverMutex.Lock() + storeOsResolver(newResolverWithNameserver([]string{"192.0.2.1:53"})) + + read := make(chan []string, 1) + go func() { read <- OsResolverNameservers() }() + + select { + case nameservers := <-read: + resolverMutex.Unlock() + if want := []string{"192.0.2.1:53"}; !slices.Equal(nameservers, want) { + t.Fatalf("nameservers = %v, want %v", nameservers, want) + } + case <-time.After(5 * time.Second): + resolverMutex.Unlock() + t.Fatal("OsResolverNameservers waited for resolverMutex") + } +} + +func TestJournalMarksTheEventAndKeepsItsLevel(t *testing.T) { + ctx, logs := captureProxyLog(t) + + Journal(LoggerFromCtx(ctx).Info()).Msg("Journal test event") + + events := logEventsWithPrefix(t, logs.String(), "Journal test event") + if len(events) != 1 { + t.Fatalf("got %d events, want 1: %s", len(events), logs.String()) + } + wantLogField(t, events[0], JournalField, true) + wantLogField(t, events[0], "level", "info") +} + +func TestOsResolverNameserverReadsLogOnChange(t *testing.T) { + ctx, logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + first := []string{"192.0.2.1", "192.0.2.2"} + second := []string{"192.0.2.3"} + stubNameservers(t, first, first, second) + + for range 3 { + InitializeOsResolverWithReason(ctx, false, "transition") + } + + systemReads := logEventsWithPrefix(t, logs.String(), "Got system nameservers") + if len(systemReads) != 2 { + t.Fatalf("got %d system nameserver lines, want 2: %s", len(systemReads), logs.String()) + } + wantLogField(t, systemReads[0], "repeats", float64(0)) + wantLogField(t, systemReads[1], "repeats", float64(1)) + wantLogField(t, systemReads[1], "message", "Got system nameservers: [192.0.2.3]") + + finalReads := logEventsWithPrefix(t, logs.String(), "Final available nameservers") + if len(finalReads) != 2 { + t.Fatalf("got %d final nameserver lines, want 2: %s", len(finalReads), logs.String()) + } + wantLogField(t, finalReads[1], "repeats", float64(1)) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 2 { + t.Fatalf("got %d resolver change events, want one for each changed list: %s", len(changes), logs.String()) + } + last := changes[1] + wantLogStrings(t, last, "before", first) + wantLogStrings(t, last, "after", second) + wantLogField(t, last, JournalField, true) + wantLogField(t, last, "level", "info") + wantLogField(t, last, "reason", "transition") + if _, ok := last["default_route"]; !ok { + t.Fatalf("resolver change event has no default_route field: %v", last) + } + wantSource := map[string]string{"darwin": "scutil", "windows": "dhcp", "linux": "resolv.conf"}[runtime.GOOS] + if wantSource != "" { + wantLogField(t, last, "source", wantSource) + } +} + +func TestInitializeOsResolverReportsTheUnspecifiedReason(t *testing.T) { + ctx, logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + stubNameservers(t, []string{"192.0.2.10"}) + + InitializeOsResolver(ctx, false) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 1 { + t.Fatalf("got %d resolver change events, want 1: %s", len(changes), logs.String()) + } + wantLogField(t, changes[0], "reason", "unspecified") + wantLogStrings(t, changes[0], "after", []string{"192.0.2.10"}) +} + +// TestOsResolverQueryFailureLogsAtDebug guards the level of the per-query +// failure line. The journal keeps every error line, and a broken network +// fails every query, so this line must not be an error. +func TestOsResolverQueryFailureLogsAtDebug(t *testing.T) { + ctx, logs := captureProxyLog(t) + resolver := newResolverWithNameserver([]string{"127.0.0.1:1"}) + resolver.exchangeDNS = func(_ context.Context, _ *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + return nil, 0, errors.New("dial udp: connection refused") + } + msg := new(dns.Msg) + msg.SetQuestion("private.example.", dns.TypeA) + + if _, err := resolver.Resolve(ctx, msg); err == nil { + t.Fatal("expected the query to fail") + } + + events := logEventsWithPrefix(t, logs.String(), "OS resolver query failed") + if len(events) == 0 { + t.Fatalf("no failure line: %s", logs.String()) + } + for _, event := range events { + if event["level"] != "debug" { + t.Fatalf("failure line at level %v, want debug: %v", event["level"], event) + } + } +} + +// A LAN-only query must reach the LAN nameservers and nothing else. A public +// nameserver in the same pool, as DHCP hands out next to the router, is not +// asked at all, so the name is never sent to it. +func Test_osResolver_LanOnlyQuerySkipsPublicNameservers(t *testing.T) { + const lan, public = "10.0.0.53:53", "192.0.2.53:53" + resolver := newResolverWithNameserver([]string{lan, public, controldPublicDnsWithPort}) + + var mu sync.Mutex + var asked []string + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, server string, _ net.IP) (*dns.Msg, time.Duration, error) { + mu.Lock() + asked = append(asked, server) + mu.Unlock() + answer := new(dns.Msg) + answer.SetRcode(msg, dns.RcodeNameError) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + answer, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg) + if err != nil { + t.Fatal(err) + } + if answer.Rcode != dns.RcodeNameError { + t.Errorf("rcode = %s, want the LAN nameserver's NXDOMAIN", dns.RcodeToString[answer.Rcode]) + } + if len(asked) != 1 || asked[0] != lan { + t.Fatalf("nameservers asked = %v, want only %s", asked, lan) + } +} + +// With no LAN nameserver at all, a LAN-only query fails without sending +// anything, rather than falling through to the public ones. +func Test_osResolver_LanOnlyQueryWithoutLanNameserverSendsNothing(t *testing.T) { + resolver := newResolverWithNameserver([]string{"192.0.2.53:53", controldPublicDnsWithPort}) + calls := 0 + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + if _, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg); err == nil { + t.Fatal("a LAN-only query with no LAN nameserver returned an answer") + } + if calls != 0 { + t.Fatalf("exchange calls = %d, want none", calls) + } +} + +// The hot cache is keyed separately for LAN-only queries: an answer an +// ordinary query got from a public nameserver must not serve a LAN-only one. +func Test_osResolver_LanOnlyQueryDoesNotShareOrdinaryHotCache(t *testing.T) { + resolver := newResolverWithNameserver([]string{"192.0.2.53:53"}) + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + if _, err := resolver.Resolve(context.Background(), msg); err != nil { + t.Fatalf("ordinary query: %v", err) + } + if _, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg); err == nil { + t.Fatal("a LAN-only query was served the ordinary query's public answer from the hot cache") + } +} diff --git a/scripts/build.sh b/scripts/build.sh index 2faeddc8..aff8166f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,7 +1,7 @@ #!/bin/bash go=${GOBIN:-go} -executable_name="ctrld" +executable_name="ctrld-client" os_archs=( darwin/arm64 darwin/amd64 @@ -44,11 +44,11 @@ compress() { return 0 ;; *-linux-armv*) - echo >&2 "upx does not work on arm routers" + echo >&2 "upx does not work on arm platforms" return 0 ;; *-linux-mips*) - echo >&2 "upx does not work on mips routers" + echo >&2 "upx does not work on mips platforms" return 0 ;; esac diff --git a/scripts/docker-smoke.sh b/scripts/docker-smoke.sh new file mode 100755 index 00000000..a5005bcf --- /dev/null +++ b/scripts/docker-smoke.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Build a Docker image and prove the binary its ENTRYPOINT names is actually in it. +# +# The container path has no compile-time link to the build: the image copies an +# artifact whose name comes from scripts/build.sh and starts it by a name spelled +# out in the Dockerfile. Renaming the binary once left both Dockerfiles copying a +# glob that matched nothing and starting a file that did not exist - the image +# built clean and only failed at "docker run". Nothing in the test suite covers +# that, so this does. +# +# Usage: docker-smoke.sh [dockerfile ...] (default: both docker/Dockerfile*) + +set -euo pipefail + +dockerfiles=("$@") +if [ ${#dockerfiles[@]} -eq 0 ]; then + dockerfiles=(docker/Dockerfile docker/Dockerfile.debug) +fi + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +cd -- "$repo_root" + +status=0 +for dockerfile in "${dockerfiles[@]}"; do + tag="ctrld-client-smoke:$(basename "$dockerfile" | tr '[:upper:]' '[:lower:]')" + echo ">>> building $dockerfile as $tag" + docker build -q -t "$tag" -f "$dockerfile" . + + # Override the entrypoint rather than appending an argument to it: the image's + # ENTRYPOINT is " run", so a bare "docker run --version" asks the + # run subcommand for a version and starts the proxy instead of exiting. Naming + # the binary explicitly is also the stronger check - it fails if the file is + # absent or not executable, which is the regression this guards. + echo ">>> running $tag entrypoint binary with --version" + if ! out=$(docker run --rm --entrypoint ./ctrld-client "$tag" --version 2>&1); then + echo >&2 "FAIL: $dockerfile: could not run the image's entrypoint: $out" + status=1 + docker image rm -f "$tag" >/dev/null 2>&1 || : + continue + fi + + echo " $out" + case "$out" in + "ctrld-client version "*) echo " OK: $dockerfile" ;; + *) + echo >&2 "FAIL: $dockerfile: unexpected --version output: $out" + status=1 + ;; + esac + docker image rm -f "$tag" >/dev/null 2>&1 || : +done + +exit "$status" diff --git a/scripts/macos-533-net-check.sh b/scripts/macos-533-net-check.sh new file mode 100755 index 00000000..630be78a --- /dev/null +++ b/scripts/macos-533-net-check.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu + +iface=$(/sbin/route -n get default 2>/dev/null | /usr/bin/awk '/interface:/{print $2; exit}') + +printf 'default_interface=%s\n' "$iface" + +printf 'ipv4_address=' +/usr/sbin/ipconfig getifaddr "$iface" 2>/dev/null || printf '\n' + +printf 'dhcp_ipv4_dns=' +/usr/sbin/ipconfig getoption "$iface" domain_name_server 2>/dev/null | + /usr/bin/awk ' + { + for (i = 1; i <= NF; i++) { + value = $i + gsub(/[{},;]/, "", value) + if (value ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}$/ && !seen[value]++) { + if (found) { + printf "," + } + printf "%s", value + found = 1 + } + } + } + END { + if (!found) { + printf "" + } + printf "\n" + } + ' + +printf 'effective_ipv4_dns=' +/usr/sbin/scutil --dns | + /usr/bin/awk ' + /nameserver\[[0-9]+\] : [0-9]+\./ { + if (!seen[$3]++) { + if (found) { + printf "," + } + printf "%s", $3 + found = 1 + } + } + END { + if (!found) { + printf "" + } + printf "\n" + } + ' diff --git a/scripts/nrpt-diag.ps1 b/scripts/nrpt-diag.ps1 new file mode 100644 index 00000000..230ae94f --- /dev/null +++ b/scripts/nrpt-diag.ps1 @@ -0,0 +1,132 @@ +#Requires -RunAsAdministrator +<# +.SYNOPSIS + NRPT diagnostic script for ctrld DNS intercept troubleshooting. +.DESCRIPTION + Captures the full NRPT state: registry keys (both GP and direct paths), + effective policy, active rules, DNS Client service status, and resolver + config. Run as Administrator. +.EXAMPLE + .\nrpt-diag.ps1 + .\nrpt-diag.ps1 | Out-File nrpt-diag-output.txt +#> + +$ErrorActionPreference = 'SilentlyContinue' + +Write-Host "=== NRPT Diagnostic Report ===" -ForegroundColor Cyan +Write-Host "Date: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" +Write-Host "Computer: $env:COMPUTERNAME" +Write-Host "OS: $((Get-CimInstance Win32_OperatingSystem).Caption) $((Get-CimInstance Win32_OperatingSystem).BuildNumber)" +Write-Host "" + +# --- 1. DNS Client Service --- +Write-Host "=== 1. DNS Client (Dnscache) Service ===" -ForegroundColor Yellow +$svc = Get-Service Dnscache +Write-Host "Status: $($svc.Status) StartType: $($svc.StartType)" +Write-Host "" + +# --- 2. GP Path (Policy store) --- +$gpPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig" +Write-Host "=== 2. GP Path: $gpPath ===" -ForegroundColor Yellow +$gpKey = Get-Item $gpPath 2>$null +if ($gpKey) { + Write-Host "Key EXISTS" + $subkeys = Get-ChildItem $gpPath 2>$null + if ($subkeys) { + foreach ($sk in $subkeys) { + Write-Host "" + Write-Host " Subkey: $($sk.PSChildName)" -ForegroundColor Green + foreach ($prop in $sk.Property) { + $val = $sk.GetValue($prop) + $kind = $sk.GetValueKind($prop) + Write-Host " $prop ($kind) = $val" + } + } + } else { + Write-Host " ** EMPTY (no subkeys) — this blocks NRPT activation! **" -ForegroundColor Red + } +} else { + Write-Host "Key does NOT exist (clean state)" +} +Write-Host "" + +# --- 3. Direct Path (Service store) --- +$directPath = "HKLM:\SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig" +Write-Host "=== 3. Direct Path: $directPath ===" -ForegroundColor Yellow +$directKey = Get-Item $directPath 2>$null +if ($directKey) { + Write-Host "Key EXISTS" + $subkeys = Get-ChildItem $directPath 2>$null + if ($subkeys) { + foreach ($sk in $subkeys) { + Write-Host "" + Write-Host " Subkey: $($sk.PSChildName)" -ForegroundColor Green + foreach ($prop in $sk.Property) { + $val = $sk.GetValue($prop) + $kind = $sk.GetValueKind($prop) + Write-Host " $prop ($kind) = $val" + } + } + } else { + Write-Host " ** EMPTY (no subkeys) **" -ForegroundColor Red + } +} else { + Write-Host "Key does NOT exist" +} +Write-Host "" + +# --- 4. Effective NRPT Rules (what Windows sees) --- +Write-Host "=== 4. Get-DnsClientNrptRule ===" -ForegroundColor Yellow +$rules = Get-DnsClientNrptRule 2>$null +if ($rules) { + $rules | Format-List Name, Version, Namespace, NameServers, NameEncoding, DnsSecEnabled +} else { + Write-Host "(none)" +} +Write-Host "" + +# --- 5. Effective NRPT Policy (what DNS Client actually applies) --- +Write-Host "=== 5. Get-DnsClientNrptPolicy ===" -ForegroundColor Yellow +$policy = Get-DnsClientNrptPolicy 2>$null +if ($policy) { + $policy | Format-List Namespace, NameServers, NameEncoding, QueryPolicy +} else { + Write-Host "(none — DNS Client is NOT honoring any NRPT rules)" -ForegroundColor Red +} +Write-Host "" + +# --- 6. Interface DNS servers --- +Write-Host "=== 6. Interface DNS Configuration ===" -ForegroundColor Yellow +Get-DnsClientServerAddress -AddressFamily IPv4 | Where-Object { $_.ServerAddresses } | + Format-Table InterfaceAlias, InterfaceIndex, ServerAddresses -AutoSize +Write-Host "" + +# --- 7. DNS resolution test --- +Write-Host "=== 7. DNS Resolution Test ===" -ForegroundColor Yellow +Write-Host "Resolve-DnsName example.com (uses DNS Client / NRPT):" +try { + $result = Resolve-DnsName example.com -Type A -DnsOnly -ErrorAction Stop + $result | Format-Table Name, Type, IPAddress -AutoSize +} catch { + Write-Host " FAILED: $_" -ForegroundColor Red +} +Write-Host "" +Write-Host "nslookup example.com 127.0.0.1 (direct to ctrld, bypasses NRPT):" +$ns = nslookup example.com 127.0.0.1 2>&1 +$ns | ForEach-Object { Write-Host " $_" } +Write-Host "" + +# --- 8. Domain join status --- +Write-Host "=== 8. Domain Status ===" -ForegroundColor Yellow +$cs = Get-CimInstance Win32_ComputerSystem +Write-Host "Domain: $($cs.Domain) PartOfDomain: $($cs.PartOfDomain)" +Write-Host "" + +# --- 9. Group Policy NRPT --- +Write-Host "=== 9. GP Result (NRPT section) ===" -ForegroundColor Yellow +Write-Host "(Running gpresult — may take a few seconds...)" +$gp = gpresult /r 2>&1 +$gp | Select-String -Pattern "DNS|NRPT|Policy" | ForEach-Object { Write-Host " $_" } +Write-Host "" + +Write-Host "=== End of Diagnostic Report ===" -ForegroundColor Cyan diff --git a/staticdns.go b/staticdns.go index 1bfd5562..5ca20a02 100644 --- a/staticdns.go +++ b/staticdns.go @@ -8,14 +8,9 @@ import ( "strings" ) -var homedir string - -// absHomeDir returns the absolute path to given filename using home directory as root dir. -func absHomeDir(filename string) string { - if homedir != "" { - return filepath.Join(homedir, filename) - } - dir, err := userHomeDir() +// AbsHomeDir returns the absolute path to given filename using home directory as root dir. +func AbsHomeDir(filename string) string { + dir, err := UserHomeDir() if err != nil { return filename } @@ -31,8 +26,11 @@ func dirWritable(dir string) (bool, error) { return true, f.Close() } -func userHomeDir() (string, error) { - // viper will expand for us. +// ServiceHomeDir returns the directory where a ctrld service with root or +// administrator rights keeps its files. It has no fallback to the home +// directory of the current user. A caller without root that only reads can +// thus look where the service wrote. +func ServiceHomeDir() (string, error) { if runtime.GOOS == "windows" { // If we're on windows, use the install path for this. exePath, err := os.Executable() @@ -42,7 +40,16 @@ func userHomeDir() (string, error) { return filepath.Dir(exePath), nil } - dir := "/etc/controld" + return "/etc/controld", nil +} + +// UserHomeDir returns the home directory for user who is running ctrld. +func UserHomeDir() (string, error) { + // viper will expand for us. + dir, err := ServiceHomeDir() + if err != nil || runtime.GOOS == "windows" { + return dir, err + } if err := os.MkdirAll(dir, 0750); err != nil { return os.UserHomeDir() // fallback to user home directory } @@ -54,13 +61,18 @@ func userHomeDir() (string, error) { // SavedStaticDnsSettingsFilePath returns the file path where the static DNS settings // for the provided interface are saved. +// +// The caller must ensure iface is non-nil. func SavedStaticDnsSettingsFilePath(iface *net.Interface) string { // The file is stored in the user home directory under a hidden file. - return absHomeDir(".dns_" + iface.Name) + return AbsHomeDir(".dns_" + iface.Name) } -// SavedStaticNameservers returns the stored static nameservers for the given interface. -func SavedStaticNameservers(iface *net.Interface) ([]string, string) { +// SavedStaticNameserversAndPath returns the stored static nameservers for the given interface, +// and the absolute path to file that stored the settings. +// +// The caller must ensure iface is non-nil. +func SavedStaticNameserversAndPath(iface *net.Interface) ([]string, string) { file := SavedStaticDnsSettingsFilePath(iface) data, err := os.ReadFile(file) if err != nil || len(data) == 0 { @@ -77,3 +89,9 @@ func SavedStaticNameservers(iface *net.Interface) ([]string, string) { } return ns, file } + +// SavedStaticNameservers is like SavedStaticNameserversAndPath, but only returns the static nameservers. +func SavedStaticNameservers(iface *net.Interface) []string { + nss, _ := SavedStaticNameserversAndPath(iface) + return nss +} diff --git a/test-scripts/README.md b/test-scripts/README.md new file mode 100644 index 00000000..7ae5fd67 --- /dev/null +++ b/test-scripts/README.md @@ -0,0 +1,44 @@ +# DNS Intercept Test Scripts + +Manual test scripts for verifying DNS intercept mode behavior. These require root/admin privileges and a running ctrld instance. + +## Structure + +``` +test-scripts/ +├── darwin/ +│ ├── test-recovery-bypass.sh # Captive portal recovery simulation +│ ├── test-dns-intercept.sh # Basic pf intercept verification +│ ├── test-pf-group-exemption.sh # Group-based pf exemption test +│ └── validate-pf-rules.sh # Dry-run pf rule validation +└── windows/ + ├── test-recovery-bypass.ps1 # Captive portal recovery simulation + └── test-dns-intercept.ps1 # Basic WFP intercept verification +``` + +## Prerequisites + +- ctrld running with `--intercept-mode dns` (or `--intercept-mode hard`) +- Verbose logging: `-v 1 --log /tmp/dns.log` (macOS) or `--log C:\temp\dns.log` (Windows) +- Root (macOS) or Administrator (Windows) +- For recovery tests: disconnect VPNs (e.g., Tailscale) that provide alternative routes + +## Recovery Bypass Test + +Simulates a captive portal by blackholing ctrld's upstream DoH IPs and cycling wifi. Verifies that ctrld's recovery bypass activates, discovers DHCP nameservers, and forwards queries to them until the upstream recovers. + +### macOS +```bash +sudo bash test-scripts/darwin/test-recovery-bypass.sh en0 +``` + +### Windows (PowerShell as Administrator) +```powershell +.\test-scripts\windows\test-recovery-bypass.ps1 -WifiAdapter "Wi-Fi" +``` + +## Safety + +All scripts clean up on exit (including Ctrl+C): +- **macOS**: Removes route blackholes, re-enables wifi +- **Windows**: Removes firewall rules, re-enables adapter diff --git a/test-scripts/darwin/test-dns-intercept.sh b/test-scripts/darwin/test-dns-intercept.sh new file mode 100755 index 00000000..b54e9c15 --- /dev/null +++ b/test-scripts/darwin/test-dns-intercept.sh @@ -0,0 +1,556 @@ +#!/bin/bash +# ============================================================================= +# DNS Intercept Mode Test Script — macOS (pf) +# ============================================================================= +# Run as root: sudo bash test-dns-intercept-mac.sh +# +# Tests the dns-intercept feature end-to-end with validation at each step. +# Logs are read from /tmp/dns.log (ctrld log location on test machine). +# +# Manual steps marked with [MANUAL] require human interaction. +# ============================================================================= + +set -euo pipefail + +CTRLD_LOG="/tmp/dns.log" +PF_ANCHOR="com.controld.ctrld" +PASS=0 +FAIL=0 +WARN=0 +RESULTS=() + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +header() { echo -e "\n${CYAN}${BOLD}━━━ $1 ━━━${NC}"; } +info() { echo -e " ${BOLD}ℹ${NC} $1"; } +pass() { echo -e " ${GREEN}✅ PASS${NC}: $1"; PASS=$((PASS+1)); RESULTS+=("PASS: $1"); } +fail() { echo -e " ${RED}❌ FAIL${NC}: $1"; FAIL=$((FAIL+1)); RESULTS+=("FAIL: $1"); } +warn() { echo -e " ${YELLOW}⚠️ WARN${NC}: $1"; WARN=$((WARN+1)); RESULTS+=("WARN: $1"); } +manual() { echo -e " ${YELLOW}[MANUAL]${NC} $1"; } +separator() { echo -e "${CYAN}─────────────────────────────────────────────────────${NC}"; } + +check_root() { + if [[ $EUID -ne 0 ]]; then + echo -e "${RED}This script must be run as root (sudo).${NC}" + exit 1 + fi +} + +wait_for_key() { + echo -e "\n Press ${BOLD}Enter${NC} to continue..." + read -r +} + +# Grep recent log entries (last N lines) +log_grep() { + local pattern="$1" + local lines="${2:-200}" + tail -n "$lines" "$CTRLD_LOG" 2>/dev/null | grep -i "$pattern" 2>/dev/null || true +} + +log_grep_count() { + local pattern="$1" + local lines="${2:-200}" + tail -n "$lines" "$CTRLD_LOG" 2>/dev/null | grep -ci "$pattern" 2>/dev/null || echo "0" +} + +# ============================================================================= +# TEST SECTIONS +# ============================================================================= + +test_prereqs() { + header "0. Prerequisites" + + if command -v pfctl &>/dev/null; then + pass "pfctl available" + else + fail "pfctl not found" + exit 1 + fi + + if [[ -f "$CTRLD_LOG" ]]; then + pass "ctrld log exists at $CTRLD_LOG" + else + warn "ctrld log not found at $CTRLD_LOG — log checks will be skipped" + fi + + if command -v dig &>/dev/null; then + pass "dig available" + else + fail "dig not found — install bind tools" + exit 1 + fi + + info "Default route interface: $(route -n get default 2>/dev/null | grep interface | awk '{print $2}' || echo 'unknown')" + info "Current DNS servers:" + scutil --dns | grep "nameserver\[" | head -5 | sed 's/^/ /' +} + +test_pf_state() { + header "1. PF State Validation" + + # Is pf enabled? + local pf_status + pf_status=$(pfctl -si 2>&1 | grep "Status:" || true) + if echo "$pf_status" | grep -q "Enabled"; then + pass "pf is enabled" + else + fail "pf is NOT enabled (status: $pf_status)" + fi + + # Is our anchor referenced in the running ruleset? + local sr_match sn_match + sr_match=$(pfctl -sr 2>&1 | grep "$PF_ANCHOR" || true) + sn_match=$(pfctl -sn 2>&1 | grep "$PF_ANCHOR" || true) + + if [[ -n "$sr_match" ]]; then + pass "anchor '$PF_ANCHOR' found in filter rules (pfctl -sr)" + info " $sr_match" + else + fail "anchor '$PF_ANCHOR' NOT in filter rules — main ruleset doesn't reference it" + fi + + if [[ -n "$sn_match" ]]; then + pass "rdr-anchor '$PF_ANCHOR' found in NAT rules (pfctl -sn)" + info " $sn_match" + else + fail "rdr-anchor '$PF_ANCHOR' NOT in NAT rules — redirect won't work" + fi + + # Check anchor rules + separator + info "Anchor filter rules (pfctl -a '$PF_ANCHOR' -sr):" + local anchor_sr + anchor_sr=$(pfctl -a "$PF_ANCHOR" -sr 2>&1 | grep -v "ALTQ" || true) + if [[ -n "$anchor_sr" ]]; then + echo "$anchor_sr" | sed 's/^/ /' + # Check for route-to rules + if echo "$anchor_sr" | grep -q "route-to"; then + pass "route-to lo0 rules present (needed for local traffic interception)" + else + warn "No route-to rules found — local DNS may not be intercepted" + fi + else + fail "No filter rules in anchor" + fi + + info "Anchor redirect rules (pfctl -a '$PF_ANCHOR' -sn):" + local anchor_sn + anchor_sn=$(pfctl -a "$PF_ANCHOR" -sn 2>&1 | grep -v "ALTQ" || true) + if [[ -n "$anchor_sn" ]]; then + echo "$anchor_sn" | sed 's/^/ /' + if echo "$anchor_sn" | grep -q "rdr.*lo0.*port = 53"; then + pass "rdr rules on lo0 present (redirect DNS to ctrld)" + else + warn "rdr rules don't match expected pattern" + fi + else + fail "No redirect rules in anchor" + fi + + # Check anchor file exists + if [[ -f "/etc/pf.anchors/$PF_ANCHOR" ]]; then + pass "Anchor file exists: /etc/pf.anchors/$PF_ANCHOR" + else + fail "Anchor file missing: /etc/pf.anchors/$PF_ANCHOR" + fi + + # Check pf.conf was NOT modified + if grep -q "$PF_ANCHOR" /etc/pf.conf 2>/dev/null; then + warn "pf.conf contains '$PF_ANCHOR' reference — should NOT be modified on disk" + else + pass "pf.conf NOT modified on disk (anchor injected at runtime only)" + fi +} + +test_dns_interception() { + header "2. DNS Interception Tests" + + # Mark position in log + local log_lines_before=0 + if [[ -f "$CTRLD_LOG" ]]; then + log_lines_before=$(wc -l < "$CTRLD_LOG") + fi + + # Test 1: Query to external resolver should be intercepted + info "Test: dig @8.8.8.8 example.com (should be intercepted by ctrld)" + local dig_result + dig_result=$(dig @8.8.8.8 example.com +short +timeout=5 2>&1 || true) + + if [[ -n "$dig_result" ]] && ! echo "$dig_result" | grep -q "timed out"; then + pass "dig @8.8.8.8 returned result: $dig_result" + else + fail "dig @8.8.8.8 failed or timed out" + fi + + # Check if ctrld logged the query + sleep 1 + if [[ -f "$CTRLD_LOG" ]]; then + local intercepted + intercepted=$(tail -n +$((log_lines_before+1)) "$CTRLD_LOG" | grep -c "example.com" || echo "0") + if [[ "$intercepted" -gt 0 ]]; then + pass "ctrld logged the intercepted query for example.com" + else + fail "ctrld did NOT log query for example.com — interception may not be working" + fi + fi + + # Check dig reports ctrld answered (not 8.8.8.8) + local full_dig + full_dig=$(dig @8.8.8.8 example.com +timeout=5 2>&1 || true) + local server_line + server_line=$(echo "$full_dig" | grep "SERVER:" || true) + info "dig SERVER line: $server_line" + if echo "$server_line" | grep -q "127.0.0.1"; then + pass "Response came from 127.0.0.1 (ctrld intercepted)" + elif echo "$server_line" | grep -q "8.8.8.8"; then + fail "Response came from 8.8.8.8 directly — NOT intercepted" + else + warn "Could not determine response server from dig output" + fi + + separator + + # Test 2: Query to another external resolver + info "Test: dig @1.1.1.1 cloudflare.com (should also be intercepted)" + local dig2 + dig2=$(dig @1.1.1.1 cloudflare.com +short +timeout=5 2>&1 || true) + if [[ -n "$dig2" ]] && ! echo "$dig2" | grep -q "timed out"; then + pass "dig @1.1.1.1 returned result" + else + fail "dig @1.1.1.1 failed or timed out" + fi + + separator + + # Test 3: Query to localhost should work (not double-redirected) + info "Test: dig @127.0.0.1 example.org (direct to ctrld, should NOT be redirected)" + local dig3 + dig3=$(dig @127.0.0.1 example.org +short +timeout=5 2>&1 || true) + if [[ -n "$dig3" ]] && ! echo "$dig3" | grep -q "timed out"; then + pass "dig @127.0.0.1 works (no loop)" + else + fail "dig @127.0.0.1 failed — possible redirect loop" + fi + + separator + + # Test 4: System DNS resolution + info "Test: host example.net (system resolver, should go through ctrld)" + local host_result + host_result=$(host example.net 2>&1 || true) + if echo "$host_result" | grep -q "has address"; then + pass "System DNS resolution works via host command" + else + fail "System DNS resolution failed" + fi + + separator + + # Test 5: TCP DNS query + info "Test: dig @9.9.9.9 example.com +tcp (TCP DNS should also be intercepted)" + local dig_tcp + dig_tcp=$(dig @9.9.9.9 example.com +tcp +short +timeout=5 2>&1 || true) + if [[ -n "$dig_tcp" ]] && ! echo "$dig_tcp" | grep -q "timed out"; then + pass "TCP DNS query intercepted and resolved" + else + warn "TCP DNS query failed (may not be critical if UDP works)" + fi +} + +test_non_dns_unaffected() { + header "3. Non-DNS Traffic Unaffected" + + # HTTPS should work fine + info "Test: curl https://example.com (HTTPS port 443 should NOT be affected)" + local curl_result + curl_result=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 https://example.com 2>&1 || echo "000") + if [[ "$curl_result" == "200" ]] || [[ "$curl_result" == "301" ]] || [[ "$curl_result" == "302" ]]; then + pass "HTTPS works (HTTP $curl_result)" + else + fail "HTTPS failed (HTTP $curl_result) — pf may be affecting non-DNS traffic" + fi + + # SSH-style connection test (port 22 should be unaffected) + info "Test: nc -z -w5 github.com 22 (SSH port should NOT be affected)" + if nc -z -w5 github.com 22 2>/dev/null; then + pass "SSH port reachable (non-DNS traffic unaffected)" + else + warn "SSH port unreachable (may be firewall, not necessarily our fault)" + fi +} + +test_ctrld_log_health() { + header "4. ctrld Log Health Check" + + if [[ ! -f "$CTRLD_LOG" ]]; then + warn "Skipping log checks — $CTRLD_LOG not found" + return + fi + + # Check for intercept initialization + if log_grep "DNS intercept.*initializing" 500 | grep -q "."; then + pass "DNS intercept initialization logged" + else + fail "No DNS intercept initialization in recent logs" + fi + + # Check for successful anchor load + if log_grep "pf anchor.*active" 500 | grep -q "."; then + pass "PF anchor reported as active" + else + fail "PF anchor not reported as active" + fi + + # Check for anchor reference injection + if log_grep "anchor reference active" 500 | grep -q "."; then + pass "Anchor reference injected into running ruleset" + else + fail "Anchor reference NOT injected — this is the critical step" + fi + + # Check for errors + separator + info "Recent errors/warnings in ctrld log:" + local errors + errors=$(log_grep '"level":"error"' 500) + if [[ -n "$errors" ]]; then + echo "$errors" | tail -5 | sed 's/^/ /' + warn "Errors found in recent logs (see above)" + else + pass "No errors in recent logs" + fi + + local warnings + warnings=$(log_grep '"level":"warn"' 500 | grep -v "skipping self-upgrade" || true) + if [[ -n "$warnings" ]]; then + echo "$warnings" | tail -5 | sed 's/^/ /' + info "(warnings above may be expected)" + fi + + # Check for recovery bypass state + if log_grep "recoveryBypass\|recovery bypass\|prepareForRecovery" 500 | grep -q "."; then + info "Recovery bypass activity detected in logs" + log_grep "recovery" 500 | tail -3 | sed 's/^/ /' + fi + + # Check for VPN DNS detection + if log_grep "VPN DNS" 500 | grep -q "."; then + info "VPN DNS activity in logs:" + log_grep "VPN DNS" 500 | tail -5 | sed 's/^/ /' + else + info "No VPN DNS activity (expected if no VPN is connected)" + fi +} + +test_pf_counters() { + header "5. PF Statistics & Counters" + + info "PF info (pfctl -si):" + pfctl -si 2>&1 | grep -v "ALTQ" | head -15 | sed 's/^/ /' + + info "PF state table entries:" + pfctl -ss 2>&1 | grep -c "." | sed 's/^/ States: /' + + # Count evaluations of our anchor + info "Anchor-specific stats (if available):" + local anchor_info + anchor_info=$(pfctl -a "$PF_ANCHOR" -si 2>&1 | grep -v "ALTQ" || true) + if [[ -n "$anchor_info" ]]; then + echo "$anchor_info" | head -10 | sed 's/^/ /' + else + info " (no per-anchor stats available)" + fi +} + +test_cleanup_on_stop() { + header "6. Cleanup Validation (After ctrld Stop)" + + manual "Stop ctrld now (Ctrl+C or 'ctrld stop'), then press Enter" + wait_for_key + + # Check anchor is flushed + local anchor_rules_after + anchor_rules_after=$(pfctl -a "$PF_ANCHOR" -sr 2>&1 | grep -v "ALTQ" | grep -v "^$" || true) + if [[ -z "$anchor_rules_after" ]]; then + pass "Anchor filter rules flushed after stop" + else + fail "Anchor filter rules still present after stop" + echo "$anchor_rules_after" | sed 's/^/ /' + fi + + local anchor_rdr_after + anchor_rdr_after=$(pfctl -a "$PF_ANCHOR" -sn 2>&1 | grep -v "ALTQ" | grep -v "^$" || true) + if [[ -z "$anchor_rdr_after" ]]; then + pass "Anchor redirect rules flushed after stop" + else + fail "Anchor redirect rules still present after stop" + fi + + # Check anchor file removed + if [[ ! -f "/etc/pf.anchors/$PF_ANCHOR" ]]; then + pass "Anchor file removed after stop" + else + fail "Anchor file still exists: /etc/pf.anchors/$PF_ANCHOR" + fi + + # Check pf.conf is clean + if ! grep -q "$PF_ANCHOR" /etc/pf.conf 2>/dev/null; then + pass "pf.conf is clean (no ctrld references)" + else + fail "pf.conf still has ctrld references after stop" + fi + + # DNS should work normally without ctrld + info "Test: dig example.com (should resolve via system DNS)" + local dig_after + dig_after=$(dig example.com +short +timeout=5 2>&1 || true) + if [[ -n "$dig_after" ]] && ! echo "$dig_after" | grep -q "timed out"; then + pass "DNS works after ctrld stop" + else + fail "DNS broken after ctrld stop — cleanup may have failed" + fi +} + +test_restart_resilience() { + header "7. Restart Resilience" + + manual "Start ctrld again with --dns-intercept, then press Enter" + wait_for_key + + sleep 3 + + # Re-run pf state checks + local sr_match sn_match + sr_match=$(pfctl -sr 2>&1 | grep "$PF_ANCHOR" || true) + sn_match=$(pfctl -sn 2>&1 | grep "$PF_ANCHOR" || true) + + if [[ -n "$sr_match" ]] && [[ -n "$sn_match" ]]; then + pass "Anchor references restored after restart" + else + fail "Anchor references NOT restored after restart" + fi + + # Quick interception test + local dig_after_restart + dig_after_restart=$(dig @8.8.8.8 example.com +short +timeout=5 2>&1 || true) + if [[ -n "$dig_after_restart" ]] && ! echo "$dig_after_restart" | grep -q "timed out"; then + pass "DNS interception works after restart" + else + fail "DNS interception broken after restart" + fi +} + +test_network_change() { + header "8. Network Change Recovery" + + info "This test verifies recovery after network changes." + manual "Switch Wi-Fi networks (or disconnect/reconnect Ethernet), then press Enter" + wait_for_key + + sleep 5 + + # Check pf rules still active + local sr_after sn_after + sr_after=$(pfctl -sr 2>&1 | grep "$PF_ANCHOR" || true) + sn_after=$(pfctl -sn 2>&1 | grep "$PF_ANCHOR" || true) + + if [[ -n "$sr_after" ]] && [[ -n "$sn_after" ]]; then + pass "Anchor references survived network change" + else + fail "Anchor references lost after network change" + fi + + # Check interception still works + local dig_after_net + dig_after_net=$(dig @8.8.8.8 example.com +short +timeout=10 2>&1 || true) + if [[ -n "$dig_after_net" ]] && ! echo "$dig_after_net" | grep -q "timed out"; then + pass "DNS interception works after network change" + else + fail "DNS interception broken after network change" + fi + + # Check logs for recovery bypass activity + if [[ -f "$CTRLD_LOG" ]]; then + local recovery_logs + recovery_logs=$(log_grep "recovery\|network change\|network monitor" 100) + if [[ -n "$recovery_logs" ]]; then + info "Recovery/network change log entries:" + echo "$recovery_logs" | tail -5 | sed 's/^/ /' + fi + fi +} + +# ============================================================================= +# SUMMARY +# ============================================================================= + +print_summary() { + header "TEST SUMMARY" + echo "" + for r in "${RESULTS[@]}"; do + if [[ "$r" == PASS* ]]; then + echo -e " ${GREEN}✅${NC} ${r#PASS: }" + elif [[ "$r" == FAIL* ]]; then + echo -e " ${RED}❌${NC} ${r#FAIL: }" + elif [[ "$r" == WARN* ]]; then + echo -e " ${YELLOW}⚠️${NC} ${r#WARN: }" + fi + done + echo "" + separator + echo -e " ${GREEN}Passed: $PASS${NC} | ${RED}Failed: $FAIL${NC} | ${YELLOW}Warnings: $WARN${NC}" + separator + + if [[ $FAIL -gt 0 ]]; then + echo -e "\n ${RED}${BOLD}Some tests failed.${NC} Check output above for details." + echo -e " Useful debug commands:" + echo -e " pfctl -a '$PF_ANCHOR' -sr # anchor filter rules" + echo -e " pfctl -a '$PF_ANCHOR' -sn # anchor redirect rules" + echo -e " pfctl -sr | grep controld # main ruleset references" + echo -e " tail -100 $CTRLD_LOG # recent ctrld logs" + else + echo -e "\n ${GREEN}${BOLD}All tests passed!${NC}" + fi +} + +# ============================================================================= +# MAIN +# ============================================================================= + +echo -e "${BOLD}╔═══════════════════════════════════════════════════════╗${NC}" +echo -e "${BOLD}║ ctrld DNS Intercept Mode — macOS Test Suite ║${NC}" +echo -e "${BOLD}║ Tests pf-based DNS interception (route-to + rdr) ║${NC}" +echo -e "${BOLD}╚═══════════════════════════════════════════════════════╝${NC}" + +check_root + +echo "" +echo "Make sure ctrld is running with --dns-intercept before starting." +echo "Log location: $CTRLD_LOG" +wait_for_key + +test_prereqs +test_pf_state +test_dns_interception +test_non_dns_unaffected +test_ctrld_log_health +test_pf_counters + +separator +echo "" +echo "The next tests require manual steps (stop/start ctrld, network changes)." +echo "Press Enter to continue, or Ctrl+C to skip and see results so far." +wait_for_key + +test_cleanup_on_stop +test_restart_resilience +test_network_change + +print_summary diff --git a/test-scripts/darwin/test-pf-group-exemption.sh b/test-scripts/darwin/test-pf-group-exemption.sh new file mode 100755 index 00000000..9f47805b --- /dev/null +++ b/test-scripts/darwin/test-pf-group-exemption.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# Test: pf group-based exemption for DNS intercept +# Run as root: sudo bash test-pf-group-exemption.sh + +set -e + +GROUP_NAME="_ctrld" +ANCHOR="com.controld.test" +TEST_DNS="1.1.1.1" + +echo "=== Step 1: Create test group ===" +if dscl . -read /Groups/$GROUP_NAME PrimaryGroupID &>/dev/null; then + echo "Group $GROUP_NAME already exists" +else + # Find an unused GID in 350-450 range + USED_GIDS=$(dscl . -list /Groups PrimaryGroupID 2>/dev/null | awk '{print $2}' | sort -n) + GROUP_ID="" + for gid in $(seq 350 450); do + if ! echo "$USED_GIDS" | grep -q "^${gid}$"; then + GROUP_ID=$gid + break + fi + done + if [ -z "$GROUP_ID" ]; then + echo "ERROR: Could not find unused GID in 350-450 range" + exit 1 + fi + dscl . -create /Groups/$GROUP_NAME + dscl . -create /Groups/$GROUP_NAME PrimaryGroupID $GROUP_ID + dscl . -create /Groups/$GROUP_NAME RealName "Control D DNS Intercept" + echo "Created group $GROUP_NAME (GID $GROUP_ID)" +fi + +ACTUAL_GID=$(dscl . -read /Groups/$GROUP_NAME PrimaryGroupID | awk '{print $2}') +echo "GID: $ACTUAL_GID" + +echo "" +echo "=== Step 2: Enable pf ===" +pfctl -e 2>&1 || true + +echo "" +echo "=== Step 3: Set up pf anchor with group exemption ===" + +cat > /tmp/pf-group-test-anchor.conf << RULES +# Translation: redirect DNS on loopback to our listener +rdr pass on lo0 inet proto udp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 53 +rdr pass on lo0 inet proto tcp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 53 + +# Exemption: only group _ctrld can talk to $TEST_DNS directly +pass out quick on ! lo0 inet proto { udp, tcp } from any to $TEST_DNS port 53 group $GROUP_NAME + +# Intercept everything else +pass out quick on ! lo0 route-to lo0 inet proto udp from any to ! 127.0.0.1 port 53 +pass out quick on ! lo0 route-to lo0 inet proto tcp from any to ! 127.0.0.1 port 53 +pass in quick on lo0 inet proto { udp, tcp } from any to 127.0.0.1 port 53 +RULES + +pfctl -a $ANCHOR -f /tmp/pf-group-test-anchor.conf 2>/dev/null +echo "Loaded anchor $ANCHOR" + +# Inject anchor refs into running ruleset +NAT_RULES=$(pfctl -sn 2>/dev/null | grep -v "ALTQ" | grep -v "^$") +FILTER_RULES=$(pfctl -sr 2>/dev/null | grep -v "ALTQ" | grep -v "^$") +SCRUB_RULES=$(echo "$FILTER_RULES" | grep "^scrub" || true) +PURE_FILTER=$(echo "$FILTER_RULES" | grep -v "^scrub" | grep -v "com.controld.test" || true) +CLEAN_NAT=$(echo "$NAT_RULES" | grep -v "com.controld.test" || true) + +{ + [ -n "$SCRUB_RULES" ] && echo "$SCRUB_RULES" + [ -n "$CLEAN_NAT" ] && echo "$CLEAN_NAT" + echo "rdr-anchor \"$ANCHOR\"" + echo "anchor \"$ANCHOR\"" + [ -n "$PURE_FILTER" ] && echo "$PURE_FILTER" +} | pfctl -f - 2>/dev/null + +echo "Injected anchor references (no duplicates)" + +echo "" +echo "=== Step 4: Verify rules ===" +echo "NAT rules:" +pfctl -sn 2>/dev/null | grep -v ALTQ +echo "" +echo "Anchor filter rules:" +pfctl -a $ANCHOR -sr 2>/dev/null | grep -v ALTQ +echo "" +echo "Anchor NAT rules:" +pfctl -a $ANCHOR -sn 2>/dev/null | grep -v ALTQ + +echo "" +echo "=== Step 5: Build setgid test binary ===" +# We need a binary that runs with effective group _ctrld. +# sudo -g doesn't work on macOS, so we use a setgid binary. +cat > /tmp/test-dns-group.c << 'EOF' +#include +int main() { + char *args[] = {"dig", "+short", "+timeout=3", "+tries=1", "@1.1.1.1", "popads.net", NULL}; + execvp("dig", args); + return 1; +} +EOF +cc -o /tmp/test-dns-group /tmp/test-dns-group.c +chgrp $GROUP_NAME /tmp/test-dns-group +chmod g+s /tmp/test-dns-group +echo "Built setgid binary /tmp/test-dns-group (group: $GROUP_NAME)" + +echo "" +echo "=== Step 6: Test as regular user (should be INTERCEPTED) ===" +echo "Running: dig @$TEST_DNS popads.net (as root / group wheel — no group exemption)" +echo "If nothing listens on 127.0.0.1:53, this should timeout." +DIG_RESULT=$(dig +short +timeout=3 +tries=1 @$TEST_DNS popads.net 2>&1 || true) +echo "Result: ${DIG_RESULT:-TIMEOUT/INTERCEPTED}" + +echo "" +echo "=== Step 7: Test as group _ctrld (should BYPASS) ===" +echo "Running: setgid binary (effective group: $GROUP_NAME)" +BYPASS_RESULT=$(/tmp/test-dns-group 2>&1 || true) +echo "Result: ${BYPASS_RESULT:-TIMEOUT/BLOCKED}" + +echo "" +echo "=== Results ===" +PASS=true +if [[ -z "$DIG_RESULT" || "$DIG_RESULT" == *"timed out"* || "$DIG_RESULT" == *"connection refused"* ]]; then + echo "✅ Regular query INTERCEPTED (redirected away from $TEST_DNS)" +else + echo "❌ Regular query NOT intercepted — got: $DIG_RESULT" + PASS=false +fi + +if [[ -n "$BYPASS_RESULT" && "$BYPASS_RESULT" != *"timed out"* && "$BYPASS_RESULT" != *"connection refused"* && "$BYPASS_RESULT" != *"TIMEOUT"* ]]; then + echo "✅ Group _ctrld query BYPASSED — got: $BYPASS_RESULT" +else + echo "❌ Group _ctrld query was also intercepted — got: ${BYPASS_RESULT:-TIMEOUT}" + PASS=false +fi + +if $PASS; then + echo "" + echo "🎉 GROUP EXEMPTION WORKS — this approach is viable for dns-intercept mode" +fi + +echo "" +echo "=== Cleanup ===" +pfctl -a $ANCHOR -F all 2>/dev/null +pfctl -f /etc/pf.conf 2>/dev/null +rm -f /tmp/pf-group-test-anchor.conf /tmp/test-dns-group /tmp/test-dns-group.c +echo "Cleaned up. Group $GROUP_NAME left in place." +echo "To remove: sudo dscl . -delete /Groups/$GROUP_NAME" diff --git a/test-scripts/darwin/test-pkg-intercept-mode.sh b/test-scripts/darwin/test-pkg-intercept-mode.sh new file mode 100755 index 00000000..03aad51d --- /dev/null +++ b/test-scripts/darwin/test-pkg-intercept-mode.sh @@ -0,0 +1,227 @@ +#!/bin/sh +set -eu + +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +postinstall_source="$repo_root/scripts/pkg/postinstall" +fixture=$(mktemp -d "${TMPDIR:-/tmp}/ctrld-pkg-intercept.XXXXXX") +trap 'rm -rf "$fixture"' EXIT HUP INT TERM + +bin="$fixture/bin" +mkdir -p "$bin" + +cat >"$bin/defaults" <<'EOF' +#!/bin/sh +key=${3:-} +case "$key" in + ProvisionToken) + [ "${FAKE_TOKEN_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_TOKEN:-test-token}" + ;; + InterceptMode) + [ "${FAKE_MODE_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_MODE:-}" + ;; + ProgramArguments) + [ "${FAKE_ARGS_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_ARGS:-}" + ;; + CustomHostname|UseDevEnvironment) + exit 1 + ;; + *) + exit 1 + ;; +esac +EOF + +cat >"$bin/launchctl" <<'EOF' +#!/bin/sh +printf 'launchctl %s\n' "$*" >>"$CALLS" +exit 0 +EOF + +cat >"$bin/ctrld-client" <<'EOF' +#!/bin/sh +printf 'ctrld-client %s\n' "$*" >>"$CALLS" +if [ "${FAKE_CTRLD_EXIT:-0}" = "0" ]; then + case " $* " in + *" --cd-org="*) : >"$FAKE_PLIST" ;; + esac +fi +exit "${FAKE_CTRLD_EXIT:-0}" +EOF + +chmod +x "$bin/defaults" "$bin/launchctl" "$bin/ctrld-client" + +assert_contains() { + expected=$1 + file=$2 + if ! grep -Fq -- "$expected" "$file"; then + printf 'FAIL: expected %s in %s\n' "$expected" "$file" >&2 + sed -n '1,120p' "$file" >&2 + exit 1 + fi +} + +assert_not_contains() { + unexpected=$1 + file=$2 + if grep -Fq -- "$unexpected" "$file"; then + printf 'FAIL: did not expect %s in %s\n' "$unexpected" "$file" >&2 + sed -n '1,120p' "$file" >&2 + exit 1 + fi +} + +run_case() { + name=$1 + existing=$2 + mode_present=$3 + mode=$4 + ctrld_exit=${5:-0} + expected_status=${6:-0} + case_dir="$fixture/$name" + mkdir -p "$case_dir" + plist="$case_dir/ctrld-client.plist" + prefs="$case_dir/preferences" + calls="$case_dir/calls" + output="$case_dir/output" + postinstall="$case_dir/postinstall" + : >"$calls" + if [ "$existing" = "1" ]; then + : >"$plist" + fi + + sed \ + -e "s|^PLIST=\"/Library/LaunchDaemons/ctrld-client.plist\"$|PLIST=\"$plist\"|" \ + -e "s|^CTRLD=\"/usr/local/bin/ctrld-client\"$|CTRLD=\"$bin/ctrld-client\"|" \ + -e "s|^PREFS=\"/Library/Managed Preferences/com.controld.ctrld\"$|PREFS=\"$prefs\"|" \ + "$postinstall_source" >"$postinstall" + chmod +x "$postinstall" + + # A substitution that stops matching is the dangerous failure here, not a + # noisy one: the generated script would keep the real /usr/local/bin and + # /Library paths and the case would exercise the installed system instead of + # this fixture. Renaming the client once already did exactly that, so check + # that every redirection actually landed. + for expected in "PLIST=\"$plist\"" "CTRLD=\"$bin/ctrld-client\"" "PREFS=\"$prefs\""; do + if ! grep -Fq -- "$expected" "$postinstall"; then + printf 'FAIL: %s: fixture substitution did not apply (%s); postinstall paths changed?\n' \ + "$name" "$expected" >&2 + exit 1 + fi + done + if grep -Eq '^(PLIST|CTRLD|PREFS)="(/Library|/usr/local)' "$postinstall"; then + printf 'FAIL: %s: a real system path survived substitution\n' "$name" >&2 + grep -E '^(PLIST|CTRLD|PREFS)=' "$postinstall" >&2 + exit 1 + fi + + status=0 + PATH="$bin:$PATH" \ + CALLS="$calls" \ + FAKE_TOKEN_PRESENT=1 \ + FAKE_TOKEN=test-token \ + FAKE_MODE_PRESENT="$mode_present" \ + FAKE_MODE="$mode" \ + FAKE_CTRLD_EXIT="$ctrld_exit" \ + FAKE_PLIST="$plist" \ + "$postinstall" >"$output" 2>&1 || status=$? + + if [ "$status" -ne "$expected_status" ]; then + printf 'FAIL: %s exited %s, want %s\n' "$name" "$status" "$expected_status" >&2 + sed -n '1,120p' "$output" >&2 + exit 1 + fi + + printf '%s\n' "$case_dir" +} + +assert_not_contains 'CTRLD_POSTINSTALL_' "$postinstall_source" + +case_dir=$(run_case fresh-legacy 0 0 '') +assert_contains 'ctrld-client start --cd-org=test-token' "$case_dir/calls" +assert_not_contains '--intercept-mode' "$case_dir/calls" + +# Standard must be selected explicitly: with no flag, ctrld defaults to dns +# intercept mode on macOS. +case_dir=$(run_case fresh-standard 0 1 standard) +assert_contains 'ctrld-client start --cd-org=test-token --intercept-mode off' "$case_dir/calls" + +case_dir=$(run_case fresh-intercept 0 1 intercept-dns) +assert_contains 'ctrld-client start --cd-org=test-token --intercept-mode dns' "$case_dir/calls" + +case_dir=$(run_case upgrade-legacy 1 0 '') +assert_contains 'launchctl load' "$case_dir/calls" +assert_not_contains 'ctrld-client start' "$case_dir/calls" + +case_dir=$(run_case upgrade-standard 1 1 standard) +assert_contains 'ctrld-client start --intercept-mode off' "$case_dir/calls" +assert_not_contains 'launchctl load' "$case_dir/calls" + +case_dir=$(run_case upgrade-intercept 1 1 intercept-dns) +assert_contains 'ctrld-client start --intercept-mode dns' "$case_dir/calls" +assert_not_contains 'launchctl load' "$case_dir/calls" + +case_dir=$(run_case fresh-invalid 0 1 invalid 0 1) +assert_contains 'code=INTERCEPT_MODE_INVALID' "$case_dir/output" +assert_not_contains '--intercept-mode' "$case_dir/calls" +assert_not_contains 'ctrld-client start' "$case_dir/calls" + +case_dir=$(run_case upgrade-invalid 1 1 invalid) +assert_contains 'WARNING: unsupported InterceptMode in managed preferences; preserving existing service mode' "$case_dir/output" +assert_contains 'launchctl load' "$case_dir/calls" +assert_not_contains 'ctrld-client start' "$case_dir/calls" + +case_dir=$(run_case upgrade-standard-failure 1 1 standard 1 1) +assert_contains 'ctrld-client start --intercept-mode off' "$case_dir/calls" +assert_contains 'ERROR: upgrade installed but managed InterceptMode could not be applied' "$case_dir/output" +assert_not_contains 'launchctl load' "$case_dir/calls" + +# The legacy upgrade case (#589): a profile without InterceptMode upgrading a +# service installed in Standard mode before ctrld defaulted to dns intercept +# mode. Its arguments select no mode, so it is pinned to Standard +# (--intercept-mode off) instead of switching to the new default. +legacy_args='( + "/usr/local/bin/ctrld-client", + run, + "--cd=abc123", + "--config=/etc/controld/ctrld.toml" +)' +export FAKE_ARGS_PRESENT=1 FAKE_ARGS="$legacy_args" +case_dir=$(run_case upgrade-legacy-standard 1 0 '') +assert_contains 'existing service selects no intercept mode; keeping Standard mode' "$case_dir/output" +assert_contains 'ctrld-client start --intercept-mode off' "$case_dir/calls" +assert_not_contains 'launchctl load' "$case_dir/calls" + +# An invalid profile value warns, and still keeps a legacy service in Standard. +case_dir=$(run_case upgrade-invalid-legacy-standard 1 1 invalid) +assert_contains 'WARNING: unsupported InterceptMode in managed preferences; preserving existing service mode' "$case_dir/output" +assert_contains 'ctrld-client start --intercept-mode off' "$case_dir/calls" + +# A service whose arguments already select a mode keeps it unchanged. +export FAKE_ARGS='( + "/usr/local/bin/ctrld-client", + run, + "--cd=abc123", + "--intercept-mode", + dns +)' +case_dir=$(run_case upgrade-legacy-intercept 1 0 '') +assert_contains 'launchctl load' "$case_dir/calls" +assert_not_contains 'ctrld-client start' "$case_dir/calls" + +# A service a fresh install marked for the default keeps the marker, and the +# new binary applies the default to it. +export FAKE_ARGS='( + "/usr/local/bin/ctrld-client", + run, + "--cd=abc123", + "--intercept-mode-default=true" +)' +case_dir=$(run_case upgrade-default-marker 1 0 '') +assert_contains 'launchctl load' "$case_dir/calls" +assert_not_contains 'ctrld-client start' "$case_dir/calls" +unset FAKE_ARGS_PRESENT FAKE_ARGS + +printf 'PASS: pkg postinstall preserves the existing service mode and applies standard/intercept-dns policy\n' diff --git a/test-scripts/darwin/test-postinstall-provision-failure.sh b/test-scripts/darwin/test-postinstall-provision-failure.sh new file mode 100644 index 00000000..3307e3cc --- /dev/null +++ b/test-scripts/darwin/test-postinstall-provision-failure.sh @@ -0,0 +1,641 @@ +#!/bin/sh +# Test: the MDM pkg postinstall script validates managed prefs before +# running ctrld, classifies every failure (pre-flight or post-invoke) with +# a stable 'stage=package code=' identifier line, best-effort appends +# 'ctrld-client diag' output on failure, and never leaks the provision token. +# +# Self-contained and root-free: each case runs a copy of the real +# postinstall script with its PLIST/CTRLD/PREFS assignments sed-rewritten +# to fixture paths (same seam as test-pkg-intercept-mode.sh), with +# 'defaults', 'launchctl', and 'ctrld-client' stubbed on PATH. No macOS-only API is +# used, so this runs as plain POSIX sh on Linux too. +# +# With POSTINSTALL_TEST_REAL_DEFAULTS=1 on macOS, 'defaults' is not stubbed: +# each case writes its managed-prefs values into a real plist under the +# fixture dir with 'defaults write', and the postinstall copy reads them +# with the real binary. This tests the one boundary a stub cannot: what +# 'defaults read' prints and exits with for an empty string, a boolean, a +# control character, or a missing key. ctrld and launchctl stay stubbed, so +# the run never touches the services of the machine. +# +# The wait loop between profile attempts is shortened to 0 seconds in the +# copy under test. The attempt count and the log lines stay as they are. +# +# Run: sh test-postinstall-provision-failure.sh + +set -eu + +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +postinstall_source="$repo_root/scripts/pkg/postinstall" + +fixture=$(mktemp -d "${TMPDIR:-/tmp}/ctrld-postinstall-test.XXXXXX") +trap 'rm -rf "$fixture"' EXIT HUP INT TERM + +# A well-formed fake token used for cases that must reach ctrld (success, +# ctrld-emitted-identifier passthrough). Obviously fake, but still checked +# for non-leakage like a real one would be. +GOOD_TOKEN="org-v1-FAKE93af0cDONOTLEAK" +# A present-but-malformed value (contains whitespace) for the +# PROVISION_TOKEN_MALFORMED case. +MALFORMED_TOKEN="FAKE TOKEN WITH SPACE DO NOT LEAK" +# A second, differently-shaped org-v1- token: never the value read from +# managed prefs, so it can only be caught by the pattern-based redaction, +# not the exact-match one. +OTHER_ORG_TOKEN="org-v1-DIFFERENTshape001" +export GOOD_TOKEN MALFORMED_TOKEN OTHER_ORG_TOKEN + +# Real mktemp, resolved before the fixture bin/ overrides PATH, so the +# mktemp stub can fall through to it when a test case is not forcing a +# failure. +REAL_MKTEMP=$(command -v mktemp) +export REAL_MKTEMP + +bin="$fixture/bin" +mkdir -p "$bin" + +real_defaults=${POSTINSTALL_TEST_REAL_DEFAULTS:-0} +if [ "$real_defaults" = "1" ] && [ "$(uname -s)" != "Darwin" ]; then + echo "POSTINSTALL_TEST_REAL_DEFAULTS=1 needs macOS: the real 'defaults' binary is the point of this mode" >&2 + exit 2 +fi + +# --- Stubs ----------------------------------------------------------- + +if [ "$real_defaults" != "1" ]; then +cat >"$bin/defaults" <<'EOF' +#!/bin/sh +# Stand-in for macOS 'defaults read '. Every key answers from +# a FAKE_*_PRESENT/FAKE_* pair the test case exports beforehand; a key with +# no *_PRESENT=1 comes back unset, like a profile that never set it. +key=${3:-} +case "$key" in + ProvisionToken) + [ "${FAKE_TOKEN_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_TOKEN_VALUE:-}" + ;; + CustomHostname) + [ "${FAKE_HOST_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_HOST:-}" + ;; + InterceptMode) + [ "${FAKE_MODE_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_MODE:-}" + ;; + UseDevEnvironment) + [ "${FAKE_DEVENV_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_DEVENV:-}" + ;; + *) + exit 1 + ;; +esac +EOF +chmod +x "$bin/defaults" +fi + +# write_real_prefs materializes the FAKE_* managed-prefs values of the +# current case into a real plist at "$1", the path the postinstall copy +# reads as its PREFS domain. A key with no *_PRESENT=1 is not written, like +# a profile that never set it. +write_real_prefs() { + domain=$1 + [ "${FAKE_TOKEN_PRESENT:-0}" = "1" ] && defaults write "$domain" ProvisionToken -string "${FAKE_TOKEN_VALUE:-}" + [ "${FAKE_HOST_PRESENT:-0}" = "1" ] && defaults write "$domain" CustomHostname -string "${FAKE_HOST:-}" + [ "${FAKE_MODE_PRESENT:-0}" = "1" ] && defaults write "$domain" InterceptMode -string "${FAKE_MODE:-}" + [ "${FAKE_DEVENV_PRESENT:-0}" = "1" ] && defaults write "$domain" UseDevEnvironment -string "${FAKE_DEVENV:-}" + return 0 +} + +cat >"$bin/launchctl" <<'EOF' +#!/bin/sh +[ -n "${CALLS:-}" ] && printf 'launchctl %s\n' "$*" >>"$CALLS" +exit "${FAKE_LAUNCHCTL_EXIT:-0}" +EOF +chmod +x "$bin/launchctl" + +cat >"$bin/ctrld-client" <<'EOF' +#!/bin/sh +[ -n "${CALLS:-}" ] && printf 'ctrld-client %s\n' "$*" >>"$CALLS" + +if [ "$1" = "diag" ]; then + if [ "${FAKE_DIAG_TRAPS_TERM:-0}" = "1" ]; then + # Ignores the polite signal, so only SIGKILL can end it. Proves + # postinstall's wait stays bounded either way. + trap '' TERM + sleep 30 + exit 0 + fi + if [ "${FAKE_DIAG_WORKS:-0}" = "1" ]; then + echo "resolver: healthy" + echo "listener: 127.0.0.1:53 bound" + [ -n "${FAKE_DIAG_LEAK_LINE:-}" ] && echo "$FAKE_DIAG_LEAK_LINE" + exit 0 + fi + echo "ctrld-client: unknown command \"diag\"" >&2 + exit 1 +fi + +if [ "${FAKE_CTRLD_EXIT:-0}" = "0" ]; then + : >"$FAKE_PLIST" + exit 0 +fi +# The service manager writes the plist at install time, before the start +# step, so a start failure leaves it behind. Reproduced so the test can +# prove postinstall does not read that leftover as success. +[ "${FAKE_CTRLD_CREATE_PLIST:-0}" = "1" ] && : >"$FAKE_PLIST" +# A real ctrld may echo the raw token on error; reproduced here so the test +# can prove postinstall never lets it reach install.log. +echo "$GOOD_TOKEN" +[ -n "${FAKE_CTRLD_IDENTIFIER:-}" ] && echo "$FAKE_CTRLD_IDENTIFIER" +exit "$FAKE_CTRLD_EXIT" +EOF +chmod +x "$bin/ctrld-client" + +cat >"$bin/mktemp" <<'EOF' +#!/bin/sh +# Stand-in for mktemp: forces the postinstall script's own mktemp failure +# branch on demand, otherwise falls through to the real mktemp. +if [ "${FAKE_MKTEMP_FAIL:-0}" = "1" ]; then + exit 1 +fi +exec "$REAL_MKTEMP" "$@" +EOF +chmod +x "$bin/mktemp" + +# --- Assertion helpers ------------------------------------------------- + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +assert_eq() { + # assert_eq + if [ "$1" != "$2" ]; then + fail "$3 (expected '$2', got '$1')" + fi +} + +assert_contains() { + # assert_contains + case "$1" in + *"$2"*) ;; + *) fail "$3 (expected to find '$2')" ;; + esac +} + +assert_not_contains() { + # assert_not_contains + case "$1" in + *"$2"*) fail "$3 (must not contain '$2')" ;; + *) ;; + esac +} + +assert_not_contains "$(cat "$postinstall_source")" "CTRLD_POSTINSTALL_" "production script must not depend on removed env-var overrides" + +# --- Fixture plumbing --------------------------------------------------- + +# make_postinstall_copy writes a copy of the real postinstall script with +# its PLIST/CTRLD/PREFS assignments rewritten to fixture paths, so the test +# drives the actual production logic instead of a re-implementation. +make_postinstall_copy() { + plist_override=$1 + ctrld_override=$2 + prefs_override=$3 + dest=$4 + sed \ + -e "s|^PLIST=\"/Library/LaunchDaemons/ctrld-client.plist\"\$|PLIST=\"$plist_override\"|" \ + -e "s|^CTRLD=\"/usr/local/bin/ctrld-client\"\$|CTRLD=\"$ctrld_override\"|" \ + -e "s|^PREFS=\"/Library/Managed Preferences/com.controld.ctrld\"\$|PREFS=\"$prefs_override\"|" \ + -e "s|^ sleep 10\$| sleep 0|" \ + "$postinstall_source" >"$dest" + chmod +x "$dest" + # A substitution that stops matching is silent: the copy keeps the real + # system paths and the case runs against the installed machine. Check + # that each redirection landed. + for expected in "PLIST=\"$plist_override\"" "CTRLD=\"$ctrld_override\"" "PREFS=\"$prefs_override\""; do + if ! grep -Fq -- "$expected" "$dest"; then + printf 'FAIL: fixture substitution did not apply (%s); postinstall paths changed?\n' "$expected" >&2 + exit 1 + fi + done + if grep -Eq '^(PLIST|CTRLD|PREFS)="(/Library|/usr/local)' "$dest"; then + printf 'FAIL: a real system path survived substitution\n' >&2 + grep -E '^(PLIST|CTRLD|PREFS)=' "$dest" >&2 + exit 1 + fi +} + +# reset_fakes clears every FAKE_*/CALLS override so one case's env can never +# leak into the next. +reset_fakes() { + unset FAKE_TOKEN_PRESENT FAKE_TOKEN_VALUE FAKE_HOST_PRESENT FAKE_HOST \ + FAKE_MODE_PRESENT FAKE_MODE FAKE_DEVENV_PRESENT FAKE_DEVENV \ + FAKE_CTRLD_EXIT FAKE_CTRLD_IDENTIFIER FAKE_CTRLD_CREATE_PLIST \ + FAKE_DIAG_WORKS FAKE_DIAG_LEAK_LINE FAKE_DIAG_TRAPS_TERM \ + FAKE_LAUNCHCTL_EXIT FAKE_MKTEMP_FAIL CALLS 2>/dev/null || true +} + +# run_case sets up a fresh fixture dir (plist, prefs, private TMPDIR, a +# calls log) for "$name", runs a fixture-rewritten postinstall copy with +# every currently-exported FAKE_* var visible to the stubs, and leaves +# $output/$exit_code/$case_dir set for the caller's assertions. +run_case() { + name=$1 + existing_plist=${2:-0} + case_dir="$fixture/$name" + case_tmp="$case_dir/tmp" + mkdir -p "$case_tmp" + plist="$case_dir/ctrld-client.plist" + prefs="$case_dir/prefs" + calls="$case_dir/calls" + output_file="$case_dir/output" + postinstall="$case_dir/postinstall" + : >"$calls" + [ "$existing_plist" = "1" ] && : >"$plist" + [ "$real_defaults" = "1" ] && write_real_prefs "$prefs" + + make_postinstall_copy "$plist" "$bin/ctrld-client" "$prefs" "$postinstall" + + status=0 + PATH="$bin:$PATH" \ + TMPDIR="$case_tmp" \ + FAKE_PLIST="$plist" \ + CALLS="$calls" \ + "$postinstall" >"$output_file" 2>&1 || status=$? + + exit_code=$status + output=$(cat "$output_file") + calls_out=$(cat "$calls") +} + +# --- Case: missing prefs -> PROFILE_PREFS_MISSING ----------------------- +# Runs postinstall's real wait loop to exhaustion (~110s): no seam shortens +# it, since that timing is untouched by this change. + +reset_fakes +run_case missing-prefs + +assert_eq "$exit_code" "1" "missing-prefs: exit code" +assert_contains "$output" "stage=package code=PROFILE_PREFS_MISSING (exit 1)" "missing-prefs: identifier line" +assert_contains "$output" "Scope the com.controld.ctrld configuration profile" "missing-prefs: remediation hint kept" +assert_not_contains "$calls_out" "ctrld-client start" "missing-prefs: ctrld start never invoked" + +# --- Case: malformed token -> PROVISION_TOKEN_MALFORMED ----------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$MALFORMED_TOKEN" +run_case malformed-token + +assert_eq "$exit_code" "1" "malformed-token: exit code" +assert_contains "$output" "stage=package code=PROVISION_TOKEN_MALFORMED (exit 1)" "malformed-token: identifier line" +assert_not_contains "$output" "$MALFORMED_TOKEN" "malformed-token: token value must not leak" +assert_not_contains "$calls_out" "ctrld-client start" "malformed-token: ctrld start never invoked" + +# --- Case: bad CustomHostname -> CUSTOM_HOSTNAME_INVALID ----------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="bad host!" +run_case bad-hostname + +assert_eq "$exit_code" "1" "bad-hostname: exit code" +assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "bad-hostname: identifier line" +assert_contains "$output" "bad host!" "bad-hostname: message names the hostname" +assert_not_contains "$output" "$GOOD_TOKEN" "bad-hostname: token value must not leak" +assert_not_contains "$calls_out" "ctrld-client start" "bad-hostname: ctrld start never invoked" + +# --- Case: token pasted into CustomHostname -> message is redacted -------- +# An admin can fill any managed-prefs key with the token by mistake. The +# CUSTOM_HOSTNAME_INVALID message echoes the value, so the log line must +# carry the redaction marker and never the token. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="${GOOD_TOKEN}!" +run_case token-in-hostname + +assert_eq "$exit_code" "1" "token-in-hostname: exit code" +assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "token-in-hostname: identifier line" +assert_contains "$output" "[redacted-token]" "token-in-hostname: message carries the redaction marker" +assert_not_contains "$output" "$GOOD_TOKEN" "token-in-hostname: token value must not leak" + +# --- Case: token pasted into UseDevEnvironment -> warning is redacted ------ + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_DEVENV_PRESENT=1 FAKE_DEVENV="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=0 +run_case token-in-dev-env + +assert_eq "$exit_code" "0" "token-in-dev-env: exit code" +assert_contains "$output" "WARNING: UseDevEnvironment value" "token-in-dev-env: still warns on the bad value" +assert_contains "$output" "[redacted-token]" "token-in-dev-env: warning carries the redaction marker" +assert_not_contains "$output" "$GOOD_TOKEN" "token-in-dev-env: token value must not leak" + +# --- Case: CustomHostname structural failures (RFC1123 mirror) ----------- +# Same rule as cmd/cli/hostname.go's validHostname: 3-64 characters, then +# dot-separated labels that start and end on a letter or digit (no +# leading/trailing hyphen or dot, no empty label). None of these contain a +# disallowed character, so only the structural check can catch them. + +bad_hostname_idx=0 +for bad_host in '-' '.abc' 'abc-' 'a'; do + bad_hostname_idx=$((bad_hostname_idx + 1)) + reset_fakes + export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" + export FAKE_HOST_PRESENT=1 FAKE_HOST="$bad_host" + run_case "bad-hostname-struct-$bad_hostname_idx" + assert_eq "$exit_code" "1" "bad-hostname-struct '$bad_host': exit code" + assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "bad-hostname-struct '$bad_host': identifier line" + assert_not_contains "$calls_out" "ctrld-client start" "bad-hostname-struct '$bad_host': ctrld start never invoked" +done + +long_host="" +i=0 +while [ "$i" -lt 90 ]; do + long_host="${long_host}a" + i=$((i + 1)) +done + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="$long_host" +run_case bad-hostname-too-long + +assert_eq "$exit_code" "1" "bad-hostname-too-long: exit code" +assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "bad-hostname-too-long: identifier line" +assert_not_contains "$calls_out" "ctrld-client start" "bad-hostname-too-long: ctrld start never invoked" + +# --- Case: dotted CustomHostname is valid, passes through ---------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="foo.bar" +export FAKE_CTRLD_EXIT=0 +run_case good-hostname-dotted + +assert_eq "$exit_code" "0" "good-hostname-dotted: exit code" +assert_contains "$calls_out" "--custom-hostname=foo.bar" "good-hostname-dotted: passed through to ctrld" + +# --- Case: bad InterceptMode, fresh install -> fails, INTERCEPT_MODE_INVALID + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_MODE_PRESENT=1 FAKE_MODE="bogus" +run_case bad-mode-fresh + +assert_eq "$exit_code" "1" "bad-mode-fresh: exit code" +assert_contains "$output" "stage=package code=INTERCEPT_MODE_INVALID (exit 1)" "bad-mode-fresh: identifier line" +assert_not_contains "$output" "$GOOD_TOKEN" "bad-mode-fresh: token value must not leak" +assert_not_contains "$calls_out" "ctrld-client start" "bad-mode-fresh: ctrld start never invoked" + +# --- Case: bad InterceptMode, upgrade -> warns, keeps mode, exit 0 ------- + +reset_fakes +export FAKE_MODE_PRESENT=1 FAKE_MODE="bogus" +run_case bad-mode-upgrade 1 + +assert_eq "$exit_code" "0" "bad-mode-upgrade: exit code" +assert_contains "$output" "WARNING: unsupported InterceptMode in managed preferences; preserving existing service mode" "bad-mode-upgrade: warns" +assert_contains "$calls_out" "launchctl load" "bad-mode-upgrade: falls back to launchctl load" +assert_not_contains "$calls_out" "ctrld-client " "bad-mode-upgrade: ctrld start never invoked" + +# --- Case: upgrade reload fails -> SERVICE_RELOAD_FAILED ----------------- +# No managed InterceptMode override, so postinstall falls back to +# reloading the existing service; launchctl itself fails. + +reset_fakes +export FAKE_LAUNCHCTL_EXIT=1 +run_case reload-fails 1 + +assert_eq "$exit_code" "1" "reload-fails: exit code" +assert_contains "$output" "stage=package code=SERVICE_RELOAD_FAILED (exit 1)" "reload-fails: identifier line" +assert_contains "$output" "sudo launchctl load" "reload-fails: remediation hint kept" +assert_contains "$calls_out" "launchctl load" "reload-fails: launchctl load was attempted" +assert_contains "$calls_out" "ctrld-client diag" "reload-fails: diag block attempted" + +# --- Case: unrecognized UseDevEnvironment value -> warns, still installs - + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_DEVENV_PRESENT=1 FAKE_DEVENV="banana" +export FAKE_CTRLD_EXIT=0 +run_case bad-dev-env + +assert_eq "$exit_code" "0" "bad-dev-env: exit code" +assert_contains "$output" 'WARNING: UseDevEnvironment value "banana" is not recognized; ignoring' "bad-dev-env: warns on the bad value" +assert_contains "$output" "provisioning complete, service installed" "bad-dev-env: install still proceeds" +assert_not_contains "$calls_out" "--dev" "bad-dev-env: unrecognized value never turns on --dev" +assert_not_contains "$output" "$GOOD_TOKEN" "bad-dev-env: token value must not leak" + +# --- Case: ctrld-emitted identifier passthrough -------------------------- +# The token and hostname both pass postinstall's own shape checks; ctrld +# itself rejects the token and reports its own fixed-format identifier, +# which postinstall must surface unmodified. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +run_case ctrld-identifier-passthrough + +assert_eq "$exit_code" "1" "identifier-passthrough: exit code" +assert_contains "$output" "provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" "identifier-passthrough: surfaces ctrld's own identifier" +assert_contains "$output" "ctrld exit 22" "identifier-passthrough: names ctrld's exit code" +assert_not_contains "$output" "$GOOD_TOKEN" "identifier-passthrough: token value must not leak" + +# --- Case: service start fails after install wrote the plist ------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=52 FAKE_CTRLD_CREATE_PLIST=1 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=service code=SERVICE_START_FAILED (exit 52)" +run_case start-fails-plist-present + +assert_eq "$exit_code" "1" "start-fails-plist-present: exit code" +assert_contains "$output" "stage=service code=SERVICE_START_FAILED (exit 52)" "start-fails-plist-present: identifier line" +assert_not_contains "$output" "provisioning complete" "start-fails-plist-present: leftover plist is not success" +assert_not_contains "$output" "$GOOD_TOKEN" "start-fails-plist-present: token value must not leak" + +# --- Case: empty ProvisionToken -> PROFILE_PREFS_MISSING with its own hint -- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="" +run_case empty-token + +assert_eq "$exit_code" "1" "empty-token: exit code" +assert_contains "$output" "stage=package code=PROFILE_PREFS_MISSING (exit 1)" "empty-token: identifier line" +assert_contains "$output" "ProvisionToken value is empty" "empty-token: names the empty value, not a missing profile" +assert_not_contains "$calls_out" "ctrld-client start" "empty-token: ctrld start never invoked" + +# --- Case: control character in token arrives escaped, passes preflight ---- +# 'defaults read' prints a non-whitespace control character as escaped text +# (a byte 0x01 becomes the four characters \001), so the preflight sees a +# plain 6-64 character value and hands it to ctrld unchanged. The stub emits +# that escaped text; real-defaults mode stores the raw byte and lets the +# real binary escape it. + +reset_fakes +if [ "$real_defaults" = "1" ]; then + export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$(printf 'org-v1-abc\001def')" +else + export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE='org-v1-abc\001def' +fi +export FAKE_CTRLD_EXIT=0 +run_case control-char-token-escaped + +assert_eq "$exit_code" "0" "control-char-token-escaped: preflight passes the escaped text" +assert_contains "$calls_out" 'ctrld-client start --cd-org=org-v1-abc\001def' "control-char-token-escaped: ctrld receives the escaped text unchanged" +assert_not_contains "$output" "PROVISION_TOKEN_MALFORMED" "control-char-token-escaped: preflight does not classify escaped text as malformed" + +# --- Case: UseDevEnvironment false -> no warning, no --dev ----------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_DEVENV_PRESENT=1 FAKE_DEVENV="0" +export FAKE_CTRLD_EXIT=0 +run_case dev-env-false + +assert_eq "$exit_code" "0" "dev-env-false: exit code" +assert_not_contains "$output" "WARNING: UseDevEnvironment" "dev-env-false: a boolean false is a recognized value" +assert_not_contains "$calls_out" "--dev" "dev-env-false: false never turns on --dev" + +# --- Case: success -------------------------------------------------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=0 +export FAKE_DIAG_WORKS=1 +run_case success + +assert_eq "$exit_code" "0" "success: exit code" +assert_contains "$output" "provisioning complete, service installed" "success: reports success" +assert_not_contains "$output" "diag:" "success: no diag block on the success path" +assert_not_contains "$output" "$GOOD_TOKEN" "success: token value must not leak" + +leftover=$(find "$fixture/success/tmp" -type f 2>/dev/null) +assert_eq "$leftover" "" "success: capture temp file removed" + +# --- Case: diag present on failure ---------------------------------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_WORKS=1 +run_case diag-present + +assert_eq "$exit_code" "1" "diag-present: exit code" +assert_contains "$output" "ctrld postinstall: diag: resolver: healthy" "diag-present: diag output appended" +assert_contains "$output" "ctrld postinstall: diag: listener: 127.0.0.1:53 bound" "diag-present: every diag line prefixed" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-present: token value must not leak" + +leftover=$(find "$fixture/diag-present/tmp" -type f 2>/dev/null) +assert_eq "$leftover" "" "diag-present: capture and diag temp files removed" + +# --- Case: diag output redacts the token, fresh install ------------------ +# diag leaks both the real (managed-prefs) token and an unrelated org-v1- +# shaped string; the first is caught by the exact-match swap, the second +# only by the pattern-based one. The rest of the leaked line must survive. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_WORKS=1 +export FAKE_DIAG_LEAK_LINE="leaked: $GOOD_TOKEN and $OTHER_ORG_TOKEN" +run_case diag-redacts-token + +assert_eq "$exit_code" "1" "diag-redacts-token: exit code" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-redacts-token: real token value must not leak" +assert_not_contains "$output" "$OTHER_ORG_TOKEN" "diag-redacts-token: unrelated org-v1- string must not leak" +assert_contains "$output" "leaked: [redacted-token] and org-v1-[redacted]" "diag-redacts-token: both redacted, rest of the line intact" +assert_contains "$output" "ctrld postinstall: diag: resolver: healthy" "diag-redacts-token: unrelated diag lines untouched" + +# --- Case: diag output redacts the token, upgrade path -------------------- +# Same redaction, but on the upgrade branch that applies a managed +# InterceptMode: it never reads ProvisionToken for its own purposes, so +# this proves redact_token is still populated there. + +reset_fakes +export FAKE_MODE_PRESENT=1 FAKE_MODE="standard" +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_WORKS=1 +export FAKE_DIAG_LEAK_LINE="leaked: $GOOD_TOKEN and $OTHER_ORG_TOKEN" +run_case diag-redacts-token-upgrade 1 + +assert_eq "$exit_code" "1" "diag-redacts-token-upgrade: exit code" +assert_contains "$calls_out" "ctrld-client start --intercept-mode off" "diag-redacts-token-upgrade: applied managed InterceptMode" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-redacts-token-upgrade: real token value must not leak" +assert_not_contains "$output" "$OTHER_ORG_TOKEN" "diag-redacts-token-upgrade: unrelated org-v1- string must not leak" +assert_contains "$output" "leaked: [redacted-token] and org-v1-[redacted]" "diag-redacts-token-upgrade: both redacted, rest of the line intact" + +# --- Case: diag absent/failing -> identifier still logged (fallback) ----- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=1 +run_case diag-absent + +assert_eq "$exit_code" "1" "diag-absent: exit code" +assert_contains "$output" "provisioning failed with no identifier; see /var/log/install.log and run ctrld-client diag" "diag-absent: falls back when ctrld reports no identifier" +assert_not_contains "$output" "invalid/expired token, or no network" "diag-absent: old speculative wording is gone" +assert_not_contains "$output" "ctrld postinstall: diag:" "diag-absent: no diag block when diag fails" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-absent: token value must not leak" + +leftover=$(find "$fixture/diag-absent/tmp" -type f 2>/dev/null) +assert_eq "$leftover" "" "diag-absent: capture temp file removed" + +# --- Case: diag ignores SIGTERM -> SIGKILL escalation bounds the wait ---- +# Without the escalation this would hang for the diag stub's full 30s +# sleep (or longer, on a real ignore-forever daemon); the grace loop above +# already spends up to 10s, so 20s is a generous bound that still catches +# a regression back to a plain SIGTERM-and-wait. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_TRAPS_TERM=1 +start_ts=$(date +%s) +run_case diag-ignores-term +end_ts=$(date +%s) +elapsed=$((end_ts - start_ts)) + +assert_eq "$exit_code" "1" "diag-ignores-term: exit code" +if [ "$elapsed" -gt 20 ]; then + fail "diag-ignores-term: took ${elapsed}s, expected the SIGKILL escalation to keep this well under 30s" +fi +assert_not_contains "$output" "$GOOD_TOKEN" "diag-ignores-term: token value must not leak" + +# --- Case: mktemp failure -> TEMP_FILE_UNAVAILABLE, no diag attempted ---- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_MKTEMP_FAIL=1 +run_case tempfile-unavailable + +assert_eq "$exit_code" "1" "tempfile-unavailable: exit code" +assert_contains "$output" "stage=package code=TEMP_FILE_UNAVAILABLE (exit 1)" "tempfile-unavailable: identifier line" +assert_contains "$output" "free space in /tmp or TMPDIR" "tempfile-unavailable: remediation hint" +assert_not_contains "$calls_out" "ctrld-client " "tempfile-unavailable: ctrld never invoked" +assert_not_contains "$output" "ctrld postinstall: diag:" "tempfile-unavailable: no diag block without a capture file" +assert_not_contains "$output" "$GOOD_TOKEN" "tempfile-unavailable: token value must not leak" + +if [ "$FAILURES" -gt 0 ]; then + echo "$FAILURES assertion(s) failed" >&2 + exit 1 +fi + +if [ "$real_defaults" = "1" ]; then + echo "OK (real defaults): postinstall classifies failures, surfaces diagnostics, and never leaks the token" +else + echo "OK (stubbed defaults): postinstall classifies failures, surfaces diagnostics, and never leaks the token" +fi +exit 0 diff --git a/test-scripts/darwin/test-recovery-bypass.sh b/test-scripts/darwin/test-recovery-bypass.sh new file mode 100755 index 00000000..f5aad7e7 --- /dev/null +++ b/test-scripts/darwin/test-recovery-bypass.sh @@ -0,0 +1,301 @@ +#!/bin/bash +# test-recovery-bypass.sh — Test DNS intercept recovery bypass (captive portal simulation) +# +# Simulates a captive portal by: +# 1. Discovering ctrld's upstream IPs from active connections +# 2. Blackholing ALL of them via route table +# 3. Cycling wifi to trigger network change → recovery flow +# 4. Verifying recovery bypass forwards to OS/DHCP resolver +# 5. Unblocking and verifying normal operation resumes +# +# SAFE: Uses route add/delete + networksetup — cleaned up on exit (including Ctrl+C). +# +# Usage: sudo bash test-recovery-bypass.sh [wifi_interface] +# wifi_interface defaults to en0 +# +# Prerequisites: +# - ctrld running with --dns-intercept and -v 1 --log /tmp/dns.log +# - Run as root (sudo) + +set -euo pipefail + +WIFI_IFACE="${1:-en0}" +CTRLD_LOG="/tmp/dns.log" +BLOCKED_IPS=() + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*"; } +pass() { echo -e "${GREEN}[PASS]${NC} $*"; } +fail() { echo -e "${RED}[FAIL]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } + +# ── Safety: always clean up on exit ────────────────────────────────────────── +cleanup() { + echo "" + log "═══ CLEANUP ═══" + + # Ensure wifi is on + log "Ensuring wifi is on..." + networksetup -setairportpower "$WIFI_IFACE" on 2>/dev/null || true + + # Remove all blackhole routes + for ip in "${BLOCKED_IPS[@]}"; do + route delete -host "$ip" 2>/dev/null && log "Removed route for $ip" || true + done + + log "Cleanup complete. Internet should be restored." + log "(If not, run: sudo networksetup -setairportpower $WIFI_IFACE on)" +} +trap cleanup EXIT INT TERM + +# ── Pre-checks ─────────────────────────────────────────────────────────────── +if [[ $EUID -ne 0 ]]; then + echo "Run as root: sudo bash $0 $*" + exit 1 +fi + +if [[ ! -f "$CTRLD_LOG" ]]; then + fail "ctrld log not found at $CTRLD_LOG" + echo "Start ctrld with: ctrld run --dns-intercept --cd -v 1 --log $CTRLD_LOG" + exit 1 +fi + +# Check wifi interface exists +if ! networksetup -getairportpower "$WIFI_IFACE" >/dev/null 2>&1; then + fail "Wifi interface $WIFI_IFACE not found" + echo "Try: networksetup -listallhardwareports" + exit 1 +fi + +log "═══════════════════════════════════════════════════════════" +log " Recovery Bypass Test (Captive Portal Simulation)" +log "═══════════════════════════════════════════════════════════" +log "Wifi interface: $WIFI_IFACE" +log "ctrld log: $CTRLD_LOG" +echo "" + +# ── Phase 1: Discover upstream IPs ────────────────────────────────────────── +log "Phase 1: Discovering ctrld upstream IPs from active connections" + +# Find ctrld's established connections (DoH uses port 443) +CTRLD_CONNS=$(lsof -i -n -P 2>/dev/null | grep -i ctrld | grep ESTABLISHED || true) +if [[ -z "$CTRLD_CONNS" ]]; then + warn "No established ctrld connections found via lsof" + warn "Trying: ss/netstat fallback..." + CTRLD_CONNS=$(netstat -an 2>/dev/null | grep "\.443 " | grep ESTABLISHED || true) +fi + +echo "$CTRLD_CONNS" | head -10 | while read -r line; do + log " $line" +done + +# Extract unique remote IPs from ctrld connections +UPSTREAM_IPS=() +while IFS= read -r ip; do + [[ -n "$ip" ]] && UPSTREAM_IPS+=("$ip") +done < <(echo "$CTRLD_CONNS" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -u | while read -r ip; do + # Filter out local/private IPs — we only want the upstream DoH server IPs + if [[ ! "$ip" =~ ^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.) ]]; then + echo "$ip" + fi +done) + +# Also try to resolve known Control D DoH endpoints +for host in dns.controld.com freedns.controld.com; do + for ip in $(dig +short "$host" 2>/dev/null || true); do + if [[ "$ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + UPSTREAM_IPS+=("$ip") + fi + done +done + +# Deduplicate +UPSTREAM_IPS=($(printf '%s\n' "${UPSTREAM_IPS[@]}" | sort -u)) + +if [[ ${#UPSTREAM_IPS[@]} -eq 0 ]]; then + fail "Could not discover any upstream IPs!" + echo "Check: lsof -i -n -P | grep ctrld" + exit 1 +fi + +log "Found ${#UPSTREAM_IPS[@]} upstream IP(s):" +for ip in "${UPSTREAM_IPS[@]}"; do + log " $ip" +done +echo "" + +# ── Phase 2: Baseline check ───────────────────────────────────────────────── +log "Phase 2: Baseline — verify DNS works normally" +BASELINE=$(dig +short +timeout=5 example.com @127.0.0.1 2>/dev/null || true) +if [[ -z "$BASELINE" ]]; then + fail "DNS not working before test!" + exit 1 +fi +pass "Baseline: example.com → $BASELINE" + +LOG_LINES_BEFORE=$(wc -l < "$CTRLD_LOG" | tr -d ' ') +log "Log position: line $LOG_LINES_BEFORE" +echo "" + +# ── Phase 3: Block all upstream IPs ───────────────────────────────────────── +log "Phase 3: Blackholing all upstream IPs" +for ip in "${UPSTREAM_IPS[@]}"; do + route delete -host "$ip" 2>/dev/null || true # clean slate + route add -host "$ip" 127.0.0.1 2>/dev/null + BLOCKED_IPS+=("$ip") + log " Blocked: $ip → 127.0.0.1" +done +pass "All ${#UPSTREAM_IPS[@]} upstream IPs blackholed" +echo "" + +# ── Phase 4: Cycle wifi to trigger network change ─────────────────────────── +log "Phase 4: Cycling wifi to trigger network change event" +log " Turning wifi OFF..." +networksetup -setairportpower "$WIFI_IFACE" off +sleep 3 + +log " Turning wifi ON..." +networksetup -setairportpower "$WIFI_IFACE" on + +log " Waiting for wifi to reconnect (up to 15s)..." +WIFI_UP=false +for i in $(seq 1 15); do + # Check if we have an IP on the wifi interface + IF_IP=$(ipconfig getifaddr "$WIFI_IFACE" 2>/dev/null || true) + if [[ -n "$IF_IP" ]]; then + WIFI_UP=true + pass "Wifi reconnected: $WIFI_IFACE → $IF_IP" + break + fi + sleep 1 +done + +if [[ "$WIFI_UP" == "false" ]]; then + fail "Wifi did not reconnect in 15s!" + warn "Cleaning up and exiting..." + exit 1 +fi + +log " Waiting 5s for ctrld network monitor to fire..." +sleep 5 +echo "" + +# ── Phase 5: Query and watch for recovery ──────────────────────────────────── +log "Phase 5: Sending queries — upstream is blocked, recovery should activate" +log " (ctrld should detect upstream failure → enable recovery bypass → use DHCP DNS)" +echo "" + +RECOVERY_DETECTED=false +BYPASS_ACTIVE=false +DNS_DURING_BYPASS=false +QUERY_COUNT=0 + +for i in $(seq 1 30); do + QUERY_COUNT=$((QUERY_COUNT + 1)) + RESULT=$(dig +short +timeout=3 "example.com" @127.0.0.1 2>/dev/null || true) + + if [[ -n "$RESULT" ]]; then + log " Query #$QUERY_COUNT: example.com → $RESULT ✓" + else + log " Query #$QUERY_COUNT: example.com → FAIL ✗" + fi + + # Check logs + NEW_LOGS=$(tail -n +$((LOG_LINES_BEFORE + 1)) "$CTRLD_LOG" 2>/dev/null || true) + + if [[ "$RECOVERY_DETECTED" == "false" ]] && echo "$NEW_LOGS" | grep -qiE "enabling DHCP bypass|triggering recovery|No healthy"; then + echo "" + pass "🎯 Recovery flow triggered!" + RECOVERY_DETECTED=true + echo "$NEW_LOGS" | grep -iE "recovery|bypass|DHCP|No healthy|network change" | tail -8 | while read -r line; do + echo " 📋 $line" + done + echo "" + fi + + if [[ "$BYPASS_ACTIVE" == "false" ]] && echo "$NEW_LOGS" | grep -qi "Recovery bypass active"; then + pass "🔄 Recovery bypass is forwarding queries to OS/DHCP resolver" + BYPASS_ACTIVE=true + fi + + if [[ "$RECOVERY_DETECTED" == "true" && -n "$RESULT" ]]; then + pass "✅ DNS resolves during recovery bypass: example.com → $RESULT" + DNS_DURING_BYPASS=true + break + fi + + sleep 2 +done + +# ── Phase 6: Show all recovery-related log entries ────────────────────────── +echo "" +log "Phase 6: All recovery-related ctrld log entries" +log "────────────────────────────────────────────────" +NEW_LOGS=$(tail -n +$((LOG_LINES_BEFORE + 1)) "$CTRLD_LOG" 2>/dev/null || true) +RELEVANT=$(echo "$NEW_LOGS" | grep -iE "recovery|bypass|DHCP|unhealthy|upstream.*fail|No healthy|network change|network monitor|OS resolver" || true) +if [[ -n "$RELEVANT" ]]; then + echo "$RELEVANT" | head -40 | while read -r line; do + echo " $line" + done +else + warn "No recovery-related log entries found!" + log "Last 15 lines of ctrld log:" + tail -15 "$CTRLD_LOG" | while read -r line; do + echo " $line" + done +fi + +# ── Phase 7: Unblock and verify full recovery ─────────────────────────────── +echo "" +log "Phase 7: Unblocking upstream IPs" +for ip in "${BLOCKED_IPS[@]}"; do + route delete -host "$ip" 2>/dev/null && log " Unblocked: $ip" || true +done +BLOCKED_IPS=() # clear so cleanup doesn't double-delete +pass "All upstream IPs unblocked" + +log "Waiting for ctrld to recover (up to 30s)..." +LOG_LINES_UNBLOCK=$(wc -l < "$CTRLD_LOG" | tr -d ' ') +RECOVERY_COMPLETE=false + +for i in $(seq 1 15); do + dig +short +timeout=3 example.com @127.0.0.1 >/dev/null 2>&1 || true + POST_LOGS=$(tail -n +$((LOG_LINES_UNBLOCK + 1)) "$CTRLD_LOG" 2>/dev/null || true) + + if echo "$POST_LOGS" | grep -qiE "recovery complete|disabling DHCP bypass|Upstream.*recovered"; then + RECOVERY_COMPLETE=true + pass "ctrld recovered — normal operation resumed" + echo "$POST_LOGS" | grep -iE "recovery|recovered|bypass|disabling" | head -5 | while read -r line; do + echo " 📋 $line" + done + break + fi + sleep 2 +done + +[[ "$RECOVERY_COMPLETE" == "false" ]] && warn "Recovery completion not detected (may need more time)" + +# Final check +echo "" +log "Phase 8: Final DNS verification" +sleep 2 +FINAL=$(dig +short +timeout=5 example.com @127.0.0.1 2>/dev/null || true) +if [[ -n "$FINAL" ]]; then + pass "DNS working: example.com → $FINAL" +else + fail "DNS not resolving" +fi + +# ── Summary ────────────────────────────────────────────────────────────────── +echo "" +log "═══════════════════════════════════════════════════════════" +log " Test Summary" +log "═══════════════════════════════════════════════════════════" +[[ "$RECOVERY_DETECTED" == "true" ]] && pass "Recovery bypass activated" || fail "Recovery bypass NOT activated" +[[ "$BYPASS_ACTIVE" == "true" ]] && pass "Queries forwarded to OS/DHCP resolver" || warn "OS resolver forwarding not confirmed" +[[ "$DNS_DURING_BYPASS" == "true" ]] && pass "DNS resolved during bypass (proof of OS resolver leak)" || warn "DNS during bypass not confirmed" +[[ "$RECOVERY_COMPLETE" == "true" ]] && pass "Normal operation resumed after unblock" || warn "Recovery completion not confirmed" +[[ -n "${FINAL:-}" ]] && pass "DNS functional at end of test" || fail "DNS broken at end of test" +echo "" +log "Full log since test: tail -n +$LOG_LINES_BEFORE $CTRLD_LOG" +log "Recovery entries: tail -n +$LOG_LINES_BEFORE $CTRLD_LOG | grep -i recovery" diff --git a/test-scripts/darwin/validate-pf-rules.sh b/test-scripts/darwin/validate-pf-rules.sh new file mode 100755 index 00000000..7cd0d0ac --- /dev/null +++ b/test-scripts/darwin/validate-pf-rules.sh @@ -0,0 +1,272 @@ +#!/bin/bash +# validate-pf-rules.sh +# Standalone test of the pf redirect rules for dns-intercept mode. +# Does NOT require ctrld. Loads the pf anchor, validates interception, cleans up. +# Run as root (sudo). + +set -e + +GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +ok() { echo -e "${GREEN}[OK]${NC} $1"; } +fail() { echo -e "${RED}[FAIL]${NC} $1"; FAILURES=$((FAILURES+1)); } +warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +FAILURES=0 + +ANCHOR="com.controld.ctrld.test" +ANCHOR_FILE="/tmp/pf-dns-intercept-test.conf" +# Use a local DNS listener to prove redirect works (python one-liner) +LISTENER_PID="" + +cleanup() { + echo "" + echo -e "${CYAN}--- Cleanup ---${NC}" + # Remove anchor rules + pfctl -a "$ANCHOR" -F all 2>/dev/null && echo " Flushed anchor $ANCHOR" || true + # Remove anchor file + rm -f "$ANCHOR_FILE" "/tmp/pf-combined-test.conf" && echo " Removed temp files" || true + # Reload original pf.conf to remove anchor reference + pfctl -f /etc/pf.conf 2>/dev/null && echo " Reloaded original pf.conf" || true + # Kill test listener + if [ -n "$LISTENER_PID" ]; then + kill "$LISTENER_PID" 2>/dev/null && echo " Stopped test DNS listener" || true + fi + echo " Cleanup complete" +} +trap cleanup EXIT + +resolve() { + dig "@${1}" "$2" A +short +timeout=3 +tries=1 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1 +} + +echo -e "${CYAN}=== pf DNS Redirect Rule Validation ===${NC}" +echo " This loads the exact pf rules from the dns-intercept MR," +echo " starts a tiny DNS listener on 127.0.0.1:53, and verifies" +echo " that queries to external IPs get redirected." +echo "" + +# 0. Check we're root +if [ "$(id -u)" -ne 0 ]; then + fail "Must run as root (sudo)" + exit 1 +fi + +# 1. Start a minimal DNS listener on 127.0.0.1:53 +# Uses socat to echo a fixed response — enough to prove redirect works. +# If port 53 is already in use (mDNSResponder), we'll use that instead. +echo "--- Step 1: DNS Listener on 127.0.0.1:53 ---" +if lsof -i :53 -sTCP:LISTEN 2>/dev/null | grep -q "." || lsof -i UDP:53 2>/dev/null | grep -q "."; then + ok "Something already listening on port 53 (likely mDNSResponder or ctrld)" + HAVE_LISTENER=true +else + # Start a simple Python DNS proxy that forwards to 1.1.1.1 + python3 -c " +import socket, threading, sys +def proxy(data, addr, sock): + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(3) + s.sendto(data, ('1.1.1.1', 53)) + resp, _ = s.recvfrom(4096) + sock.sendto(resp, addr) + s.close() + except: pass + +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.bind(('127.0.0.1', 53)) +print('READY', flush=True) +while True: + data, addr = sock.recvfrom(4096) + threading.Thread(target=proxy, args=(data, addr, sock), daemon=True).start() +" & + LISTENER_PID=$! + sleep 1 + if kill -0 "$LISTENER_PID" 2>/dev/null; then + ok "Started test DNS proxy on 127.0.0.1:53 (PID $LISTENER_PID, forwards to 1.1.1.1)" + HAVE_LISTENER=true + else + fail "Could not start DNS listener on port 53 — port may be in use" + HAVE_LISTENER=false + fi +fi +echo "" + +# 2. Verify baseline: direct query to 8.8.8.8 works (before pf rules) +echo "--- Step 2: Baseline (before pf rules) ---" +IP=$(resolve "8.8.8.8" "example.com") +if [ -n "$IP" ]; then + ok "Direct DNS to 8.8.8.8 works (baseline): $IP" +else + warn "Direct DNS to 8.8.8.8 failed — may be blocked by existing firewall" +fi +echo "" + +# 3. Write and load the pf anchor (exact rules from MR) +echo "--- Step 3: Load pf Anchor Rules ---" +TEST_UPSTREAM="1.1.1.1" +cat > "$ANCHOR_FILE" << PFRULES +# ctrld DNS Intercept Mode (test anchor) +# Two-step: route-to lo0 + rdr on lo0 +# +# In production, ctrld uses DoH (port 443) for upstreams so they're not +# affected by port 53 rules. For this test, we exempt our upstream ($TEST_UPSTREAM) +# explicitly — same mechanism ctrld uses for OS resolver exemptions. + +# --- Translation rules (rdr) --- +rdr pass on lo0 inet proto udp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 53 +rdr pass on lo0 inet proto tcp from any to ! 127.0.0.1 port 53 -> 127.0.0.1 port 53 + +# --- Filtering rules (pass) --- +# Exempt test upstream (in production: ctrld uses DoH, so this isn't needed). +pass out quick on ! lo0 inet proto { udp, tcp } from any to $TEST_UPSTREAM port 53 + +# Force remaining outbound DNS through loopback for interception. +pass out quick on ! lo0 route-to lo0 inet proto udp from any to ! 127.0.0.1 port 53 no state +pass out quick on ! lo0 route-to lo0 inet proto tcp from any to ! 127.0.0.1 port 53 no state + +# Allow redirected traffic through on loopback. +pass in quick on lo0 inet proto { udp, tcp } from any to 127.0.0.1 port 53 no state +PFRULES + +ok "Wrote anchor file: $ANCHOR_FILE" +cat "$ANCHOR_FILE" | sed 's/^/ /' +echo "" + +# Load anchor +OUTPUT=$(pfctl -a "$ANCHOR" -f "$ANCHOR_FILE" 2>&1) || { + fail "Failed to load anchor: $OUTPUT" + exit 1 +} +ok "Loaded anchor: $ANCHOR" + +# Inject anchor references into running pf config. +# pf enforces strict rule ordering: options, normalization, queueing, translation, filtering. +# We must insert rdr-anchor with other rdr-anchors and anchor with other anchors. +TMPCONF="/tmp/pf-combined-test.conf" +python3 -c " +import sys +lines = open('/etc/pf.conf').read().splitlines() +anchor = '$ANCHOR' +rdr_ref = 'rdr-anchor \"' + anchor + '\"' +anchor_ref = 'anchor \"' + anchor + '\"' +out = [] +rdr_done = False +anc_done = False +for line in lines: + s = line.strip() + # Insert our rdr-anchor before the first existing rdr-anchor + if not rdr_done and s.startswith('rdr-anchor'): + out.append(rdr_ref) + rdr_done = True + # Insert our anchor before the first existing anchor (filter-phase) + if not anc_done and s.startswith('anchor') and not s.startswith('anchor \"com.apple'): + out.append(anchor_ref) + anc_done = True + out.append(line) +# Fallback if no existing anchors found +if not rdr_done: + # Insert before first non-comment, non-blank after any 'set' or 'scrub' lines + out.insert(0, rdr_ref) +if not anc_done: + out.append(anchor_ref) +open('$TMPCONF', 'w').write('\n'.join(out) + '\n') +" || { fail "Failed to build combined pf config"; exit 1; } + +INJECT_OUT=$(pfctl -f "$TMPCONF" 2>&1) || { + fail "Failed to inject anchor reference: $INJECT_OUT" + rm -f "$TMPCONF" + exit 1 +} +rm -f "$TMPCONF" +ok "Injected anchor references into running pf ruleset" + +# Enable pf +pfctl -e 2>/dev/null || true + +# Show loaded rules +echo "" +echo " Active NAT rules:" +pfctl -a "$ANCHOR" -sn 2>/dev/null | sed 's/^/ /' +echo " Active filter rules:" +pfctl -a "$ANCHOR" -sr 2>/dev/null | sed 's/^/ /' +echo "" + +# 4. Test: DNS to 8.8.8.8 should now be redirected to 127.0.0.1:53 +echo "--- Step 4: Redirect Test ---" +if [ "$HAVE_LISTENER" = true ]; then + IP=$(resolve "8.8.8.8" "example.com" 5) + if [ -n "$IP" ]; then + ok "DNS to 8.8.8.8 redirected through 127.0.0.1:53: $IP" + else + fail "DNS to 8.8.8.8 failed — redirect may not be working" + fi + + # Also test another random IP + IP2=$(resolve "9.9.9.9" "example.com" 5) + if [ -n "$IP2" ]; then + ok "DNS to 9.9.9.9 also redirected: $IP2" + else + fail "DNS to 9.9.9.9 failed" + fi +else + warn "No listener on port 53 — cannot test redirect" +fi +echo "" + +# 5. Test: DNS to 127.0.0.1 still works (not double-redirected) +echo "--- Step 5: Localhost DNS (no loop) ---" +if [ "$HAVE_LISTENER" = true ]; then + IP=$(resolve "127.0.0.1" "example.com" 5) + if [ -n "$IP" ]; then + ok "DNS to 127.0.0.1 works normally (not caught by redirect): $IP" + else + fail "DNS to 127.0.0.1 failed — possible redirect loop" + fi +fi +echo "" + +# 6. Simulate VPN DNS override +echo "--- Step 6: VPN DNS Override Simulation ---" +IFACE=$(route -n get default 2>/dev/null | awk '/interface:/{print $2}') +SVC="" +for try_svc in "Wi-Fi" "Ethernet" "Thunderbolt Ethernet"; do + if networksetup -getdnsservers "$try_svc" 2>/dev/null >/dev/null; then + SVC="$try_svc" + break + fi +done + +if [ -n "$SVC" ] && [ "$HAVE_LISTENER" = true ]; then + ORIG_DNS=$(networksetup -getdnsservers "$SVC" 2>/dev/null || echo "") + echo " Service: $SVC" + echo " Current DNS: $ORIG_DNS" + + networksetup -setdnsservers "$SVC" 10.50.10.77 + dscacheutil -flushcache 2>/dev/null || true + killall -HUP mDNSResponder 2>/dev/null || true + echo " Set DNS to 10.50.10.77 (simulating F5 VPN)" + sleep 2 + + IP=$(resolve "10.50.10.77" "google.com" 5) + if [ -n "$IP" ]; then + ok "Query to fake VPN DNS (10.50.10.77) redirected to ctrld: $IP" + else + fail "Query to fake VPN DNS failed" + fi + + # Restore + if echo "$ORIG_DNS" | grep -q "There aren't any DNS Servers"; then + networksetup -setdnsservers "$SVC" Empty + else + networksetup -setdnsservers "$SVC" $ORIG_DNS + fi + echo " Restored DNS" +else + warn "Skipping VPN simulation (no service found or no listener)" +fi + +echo "" +if [ "$FAILURES" -eq 0 ]; then + echo -e "${GREEN}=== All tests passed ===${NC}" +else + echo -e "${RED}=== $FAILURES test(s) failed ===${NC}" +fi diff --git a/test-scripts/macos/diag-lo0-capture.sh b/test-scripts/macos/diag-lo0-capture.sh new file mode 100644 index 00000000..902cafd4 --- /dev/null +++ b/test-scripts/macos/diag-lo0-capture.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# diag-lo0-capture.sh — Capture DNS on lo0 to see where the pf chain breaks +# Usage: sudo bash diag-lo0-capture.sh +# Run while Windscribe + ctrld are both active, then dig from another terminal + +set -u +PCAP="/tmp/lo0-dns-$(date +%s).pcap" +echo "=== lo0 DNS Packet Capture ===" +echo "Capturing to: $PCAP" +echo "" + +# Show current rules (verify build) +echo "--- ctrld anchor rdr rules ---" +pfctl -a com.controld.ctrld -sn 2>/dev/null +echo "" +echo "--- ctrld anchor filter rules (lo0 only) ---" +pfctl -a com.controld.ctrld -sr 2>/dev/null | grep lo0 +echo "" + +# Check pf state table for port 53 before +echo "--- port 53 states BEFORE dig ---" +pfctl -ss 2>/dev/null | grep ':53' | head -10 +echo "(total: $(pfctl -ss 2>/dev/null | grep -c ':53'))" +echo "" + +# Start capture on lo0 +echo "Starting tcpdump on lo0 port 53..." +echo ">>> In another terminal, run: dig example.com" +echo ">>> Then press Ctrl-C here" +echo "" +tcpdump -i lo0 -n -v port 53 -w "$PCAP" 2>&1 & +TCPDUMP_PID=$! + +# Also show live output +tcpdump -i lo0 -n port 53 2>&1 & +LIVE_PID=$! + +# Wait for Ctrl-C +trap "kill $TCPDUMP_PID $LIVE_PID 2>/dev/null; echo ''; echo '--- port 53 states AFTER dig ---'; pfctl -ss 2>/dev/null | grep ':53' | head -20; echo '(total: '$(pfctl -ss 2>/dev/null | grep -c ':53')')'; echo ''; echo 'Capture saved to: $PCAP'; echo 'Read with: tcpdump -r $PCAP -n -v'; exit 0" INT +wait diff --git a/test-scripts/macos/diag-pf-poll.sh b/test-scripts/macos/diag-pf-poll.sh new file mode 100644 index 00000000..7a7cb630 --- /dev/null +++ b/test-scripts/macos/diag-pf-poll.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# diag-pf-poll.sh — Polls pf rules, options, states, and DNS every 2s +# Usage: sudo bash diag-pf-poll.sh | tee /tmp/pf-poll.log +# Steps: 1) Run script 2) Connect Windscribe 3) Start ctrld 4) Ctrl-C when done + +set -u +LOG="/tmp/pf-poll-$(date +%s).log" +echo "=== PF Poll Diagnostic — logging to $LOG ===" +echo "Press Ctrl-C to stop" +echo "" + +poll() { + local ts=$(date '+%H:%M:%S.%3N') + echo "======== [$ts] POLL ========" + + # 1. pf options — looking for "set skip on lo0" + echo "--- pf options ---" + pfctl -so 2>/dev/null | grep -i skip || echo "(no skip rules)" + + # 2. Main ruleset anchors — where is ctrld relative to block drop all? + echo "--- main filter rules (summary) ---" + pfctl -sr 2>/dev/null | head -30 + + # 3. Main NAT/rdr rules + echo "--- main nat/rdr rules (summary) ---" + pfctl -sn 2>/dev/null | head -20 + + # 4. ctrld anchor content + echo "--- ctrld anchor (filter) ---" + pfctl -a com.apple.internet-sharing/ctrld -sr 2>/dev/null || echo "(no anchor)" + echo "--- ctrld anchor (nat/rdr) ---" + pfctl -a com.apple.internet-sharing/ctrld -sn 2>/dev/null || echo "(no anchor)" + + # 5. State count for rdr target (10.255.255.3) and loopback + echo "--- states summary ---" + local total=$(pfctl -ss 2>/dev/null | wc -l | tr -d ' ') + local rdr=$(pfctl -ss 2>/dev/null | grep -c '10\.255\.255\.3' || true) + local lo0=$(pfctl -ss 2>/dev/null | grep -c 'lo0' || true) + echo "total=$total rdr_target=$rdr lo0=$lo0" + + # 6. Quick DNS test (1s timeout) + echo "--- DNS tests ---" + local direct=$(dig +short +time=1 +tries=1 example.com @127.0.0.1 2>&1 | head -1) + local system=$(dig +short +time=1 +tries=1 example.com 2>&1 | head -1) + echo "direct @127.0.0.1: $direct" + echo "system DNS: $system" + + # 7. Windscribe tunnel interface + echo "--- tunnel interfaces ---" + ifconfig -l | tr ' ' '\n' | grep -E '^utun' | while read iface; do + echo -n "$iface: " + ifconfig "$iface" 2>/dev/null | grep 'inet ' | awk '{print $2}' || echo "no ip" + done + + echo "" +} + +# Main loop +while true; do + poll 2>&1 | tee -a "$LOG" + sleep 2 +done diff --git a/test-scripts/macos/diag-windscribe-connect.sh b/test-scripts/macos/diag-windscribe-connect.sh new file mode 100644 index 00000000..176f77f5 --- /dev/null +++ b/test-scripts/macos/diag-windscribe-connect.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# diag-windscribe-connect.sh — Diagnostic script for testing ctrld dns-intercept +# during Windscribe VPN connection on macOS. +# +# Usage: sudo ./diag-windscribe-connect.sh +# +# Run this BEFORE connecting Windscribe. It polls every 0.5s and captures: +# 1. pf anchor state (are ctrld anchors present?) +# 2. pf state table entries (rdr interception working?) +# 3. ctrld log events (watchdog, rebootstrap, errors) +# 4. scutil DNS resolver state +# 5. Active tunnel interfaces +# 6. dig test query results +# +# Output goes to /tmp/diag-windscribe-/ +# Press Ctrl-C to stop. A summary is printed at the end. + +set -e + +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: Must run as root (sudo)" + exit 1 +fi + +CTRLD_LOG="${CTRLD_LOG:-/tmp/dns.log}" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +OUTDIR="/tmp/diag-windscribe-${TIMESTAMP}" +mkdir -p "$OUTDIR" + +echo "=== Windscribe + ctrld DNS Intercept Diagnostic ===" +echo "Output: $OUTDIR" +echo "ctrld log: $CTRLD_LOG" +echo "" +echo "1. Start this script" +echo "2. Connect Windscribe" +echo "3. Wait ~30 seconds" +echo "4. Try: dig popads.net / dig @127.0.0.1 popads.net" +echo "5. Ctrl-C to stop and see summary" +echo "" +echo "Polling every 0.5s... Press Ctrl-C to stop." +echo "" + +# Track ctrld log position +if [ -f "$CTRLD_LOG" ]; then + LOG_START_LINE=$(wc -l < "$CTRLD_LOG") +else + LOG_START_LINE=0 +fi + +ITER=0 +DIG_FAIL=0 +DIG_OK=0 +ANCHOR_MISSING=0 +ANCHOR_PRESENT=0 +PF_WIPE_COUNT=0 +FORCE_REBOOT_COUNT=0 +LAST_TUNNEL_IFACES="" + +cleanup() { + echo "" + echo "=== Stopping diagnostic ===" + + # Capture final state + echo "--- Final pf state ---" > "$OUTDIR/final-pfctl.txt" + pfctl -sa 2>/dev/null >> "$OUTDIR/final-pfctl.txt" 2>&1 || true + + echo "--- Final scutil ---" > "$OUTDIR/final-scutil.txt" + scutil --dns >> "$OUTDIR/final-scutil.txt" 2>&1 || true + + # Extract ctrld log events since start + if [ -f "$CTRLD_LOG" ]; then + tail -n +$((LOG_START_LINE + 1)) "$CTRLD_LOG" > "$OUTDIR/ctrld-events.log" 2>/dev/null || true + + # Extract key events + echo "--- Watchdog events ---" > "$OUTDIR/summary-watchdog.txt" + grep -i "watchdog\|anchor.*missing\|anchor.*restored\|force-reset\|re-bootstrapping\|force re-bootstrapping" "$OUTDIR/ctrld-events.log" >> "$OUTDIR/summary-watchdog.txt" 2>/dev/null || true + + echo "--- Errors ---" > "$OUTDIR/summary-errors.txt" + grep '"level":"error"' "$OUTDIR/ctrld-events.log" >> "$OUTDIR/summary-errors.txt" 2>/dev/null || true + + echo "--- Network changes ---" > "$OUTDIR/summary-network.txt" + grep -i "Network change\|tunnel interface\|Ignoring interface" "$OUTDIR/ctrld-events.log" >> "$OUTDIR/summary-network.txt" 2>/dev/null || true + + echo "--- Transport resets ---" > "$OUTDIR/summary-transport.txt" + grep -i "re-bootstrap\|force.*bootstrap\|dialing to\|connected to" "$OUTDIR/ctrld-events.log" >> "$OUTDIR/summary-transport.txt" 2>/dev/null || true + + # Count key events + PF_WIPE_COUNT=$(grep -c "anchor.*missing\|restoring pf" "$OUTDIR/ctrld-events.log" 2>/dev/null || echo 0) + FORCE_REBOOT_COUNT=$(grep -c "force re-bootstrapping\|force-reset" "$OUTDIR/ctrld-events.log" 2>/dev/null || echo 0) + DEADLINE_COUNT=$(grep -c "context deadline exceeded" "$OUTDIR/ctrld-events.log" 2>/dev/null || echo 0) + FALLBACK_COUNT=$(grep -c "OS resolver retry query successful" "$OUTDIR/ctrld-events.log" 2>/dev/null || echo 0) + fi + + echo "" + echo "=========================================" + echo " DIAGNOSTIC SUMMARY" + echo "=========================================" + echo "Duration: $ITER iterations (~$((ITER / 2))s)" + echo "" + echo "pf Anchor Status:" + echo " Present: $ANCHOR_PRESENT times" + echo " Missing: $ANCHOR_MISSING times" + echo "" + echo "dig Tests (popads.net):" + echo " Success: $DIG_OK" + echo " Failed: $DIG_FAIL" + echo "" + echo "ctrld Log Events:" + echo " pf wipes detected: $PF_WIPE_COUNT" + echo " Force rebootstraps: $FORCE_REBOOT_COUNT" + echo " Context deadline errors: ${DEADLINE_COUNT:-0}" + echo " OS resolver fallbacks: ${FALLBACK_COUNT:-0}" + echo "" + echo "Last tunnel interfaces: ${LAST_TUNNEL_IFACES:-none}" + echo "" + echo "Files saved to: $OUTDIR/" + echo " final-pfctl.txt — full pfctl -sa at exit" + echo " final-scutil.txt — scutil --dns at exit" + echo " ctrld-events.log — ctrld log during test" + echo " summary-watchdog.txt — watchdog events" + echo " summary-errors.txt — errors" + echo " summary-transport.txt — transport reset events" + echo " timeline.log — per-iteration state" + echo "=========================================" + exit 0 +} + +trap cleanup INT TERM + +while true; do + ITER=$((ITER + 1)) + NOW=$(date '+%H:%M:%S.%3N' 2>/dev/null || date '+%H:%M:%S') + + # 1. Check pf anchor presence + ANCHOR_STATUS="MISSING" + if pfctl -sr 2>/dev/null | grep -q "com.controld.ctrld"; then + ANCHOR_STATUS="PRESENT" + ANCHOR_PRESENT=$((ANCHOR_PRESENT + 1)) + else + ANCHOR_MISSING=$((ANCHOR_MISSING + 1)) + fi + + # 2. Check tunnel interfaces + TUNNEL_IFACES=$(ifconfig -l 2>/dev/null | tr ' ' '\n' | grep -E '^(utun|ipsec|ppp|tap|tun)' | \ + while read iface; do + # Only list interfaces that are UP and have an IP + if ifconfig "$iface" 2>/dev/null | grep -q "inet "; then + echo -n "$iface " + fi + done) + TUNNEL_IFACES=$(echo "$TUNNEL_IFACES" | xargs) # trim + if [ -n "$TUNNEL_IFACES" ]; then + LAST_TUNNEL_IFACES="$TUNNEL_IFACES" + fi + + # 3. Count rdr states (three-part = intercepted) + RDR_COUNT=$(pfctl -ss 2>/dev/null | grep -c "127.0.0.1:53 <-" || echo 0) + + # 4. Quick dig test (0.5s timeout) + DIG_RESULT="SKIP" + if [ $((ITER % 4)) -eq 0 ]; then # every 2 seconds + if dig +time=1 +tries=1 popads.net A @127.0.0.1 +short >/dev/null 2>&1; then + DIG_RESULT="OK" + DIG_OK=$((DIG_OK + 1)) + else + DIG_RESULT="FAIL" + DIG_FAIL=$((DIG_FAIL + 1)) + fi + fi + + # 5. Check latest ctrld log for recent errors + RECENT_ERR="" + if [ -f "$CTRLD_LOG" ]; then + RECENT_ERR=$(tail -5 "$CTRLD_LOG" 2>/dev/null | grep -o '"message":"[^"]*deadline[^"]*"' | tail -1 || true) + fi + + # Output timeline + LINE="[$NOW] anchor=$ANCHOR_STATUS rdr_states=$RDR_COUNT tunnels=[$TUNNEL_IFACES] dig=$DIG_RESULT $RECENT_ERR" + echo "$LINE" + echo "$LINE" >> "$OUTDIR/timeline.log" + + sleep 0.5 +done diff --git a/test-scripts/windows/diag-intercept.ps1 b/test-scripts/windows/diag-intercept.ps1 new file mode 100644 index 00000000..05e0f1e1 --- /dev/null +++ b/test-scripts/windows/diag-intercept.ps1 @@ -0,0 +1,131 @@ +# diag-intercept.ps1 — Windows DNS Intercept Mode Diagnostic +# Run as Administrator in the same elevated prompt as ctrld +# Usage: .\diag-intercept.ps1 + +Write-Host "=== CTRLD INTERCEPT MODE DIAGNOSTIC ===" -ForegroundColor Cyan +Write-Host "Timestamp: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" +Write-Host "" + +# 1. Check NRPT rules +Write-Host "--- 1. NRPT Rules ---" -ForegroundColor Yellow +try { + $nrptRules = Get-DnsClientNrptRule -ErrorAction Stop + if ($nrptRules) { + $nrptRules | Format-Table Namespace, NameServers, DisplayName -AutoSize + } else { + Write-Host " NO NRPT RULES FOUND — this is the problem!" -ForegroundColor Red + } +} catch { + Write-Host " Get-DnsClientNrptRule failed: $_" -ForegroundColor Red +} +Write-Host "" + +# 2. Check NRPT registry directly +Write-Host "--- 2. NRPT Registry ---" -ForegroundColor Yellow +$regPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig\CtrldCatchAll" +if (Test-Path $regPath) { + Write-Host " Registry key EXISTS" -ForegroundColor Green + Get-ItemProperty $regPath | Format-List Name, GenericDNSServers, ConfigOptions, Version +} else { + Write-Host " Registry key MISSING at $regPath" -ForegroundColor Red + # Check parent + $parentPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig" + if (Test-Path $parentPath) { + Write-Host " Parent key exists. Children:" + Get-ChildItem $parentPath | ForEach-Object { Write-Host " $($_.PSChildName)" } + } else { + Write-Host " Parent DnsPolicyConfig key also missing" -ForegroundColor Red + } +} +Write-Host "" + +# 3. DNS Client service status +Write-Host "--- 3. DNS Client Service ---" -ForegroundColor Yellow +$dnsSvc = Get-Service Dnscache +Write-Host " Status: $($dnsSvc.Status) StartType: $($dnsSvc.StartType)" +Write-Host "" + +# 4. Interface DNS servers +Write-Host "--- 4. Interface DNS Servers ---" -ForegroundColor Yellow +Get-DnsClientServerAddress | Format-Table InterfaceAlias, InterfaceIndex, AddressFamily, ServerAddresses -AutoSize +Write-Host "" + +# 5. WFP filters check +Write-Host "--- 5. WFP Filters (ctrld sublayer) ---" -ForegroundColor Yellow +try { + $wfpOutput = netsh wfp show filters + if (Test-Path "filters.xml") { + $xml = [xml](Get-Content "filters.xml") + $ctrldFilters = $xml.wfpdiag.filters.item | Where-Object { + $_.displayData.name -like "ctrld:*" + } + if ($ctrldFilters) { + Write-Host " Found $($ctrldFilters.Count) ctrld WFP filter(s):" -ForegroundColor Green + $ctrldFilters | ForEach-Object { + Write-Host " $($_.displayData.name) — action: $($_.action.type)" + } + } else { + Write-Host " NO ctrld WFP filters found" -ForegroundColor Red + } + Remove-Item "filters.xml" -ErrorAction SilentlyContinue + } +} catch { + Write-Host " WFP check failed: $_" -ForegroundColor Red +} +Write-Host "" + +# 6. DNS resolution tests +Write-Host "--- 6. DNS Resolution Tests ---" -ForegroundColor Yellow + +# Test A: Resolve-DnsName (uses DNS Client = respects NRPT) +Write-Host " Test A: Resolve-DnsName google.com (DNS Client path)" -ForegroundColor White +try { + $result = Resolve-DnsName google.com -Type A -DnsOnly -ErrorAction Stop + Write-Host " OK: $($result.IPAddress -join ', ')" -ForegroundColor Green +} catch { + Write-Host " FAILED: $_" -ForegroundColor Red +} + +# Test B: Resolve-DnsName to specific server (127.0.0.1) +Write-Host " Test B: Resolve-DnsName google.com -Server 127.0.0.1" -ForegroundColor White +try { + $result = Resolve-DnsName google.com -Type A -Server 127.0.0.1 -DnsOnly -ErrorAction Stop + Write-Host " OK: $($result.IPAddress -join ', ')" -ForegroundColor Green +} catch { + Write-Host " FAILED: $_" -ForegroundColor Red +} + +# Test C: Resolve-DnsName blocked domain (should return 0.0.0.0 or NXDOMAIN via Control D) +Write-Host " Test C: Resolve-DnsName popads.net (should be blocked by Control D)" -ForegroundColor White +try { + $result = Resolve-DnsName popads.net -Type A -DnsOnly -ErrorAction Stop + Write-Host " Result: $($result.IPAddress -join ', ')" -ForegroundColor Yellow +} catch { + Write-Host " FAILED/Blocked: $_" -ForegroundColor Yellow +} + +# Test D: nslookup (bypasses NRPT - expected to fail with intercept) +Write-Host " Test D: nslookup google.com 127.0.0.1 (direct, bypasses NRPT)" -ForegroundColor White +$nslookup = & nslookup google.com 127.0.0.1 2>&1 +Write-Host " $($nslookup -join "`n ")" + +Write-Host "" + +# 7. Try forcing NRPT reload +Write-Host "--- 7. Force NRPT Reload ---" -ForegroundColor Yellow +Write-Host " Running: gpupdate /target:computer /force" -ForegroundColor White +& gpupdate /target:computer /force 2>&1 | ForEach-Object { Write-Host " $_" } +Write-Host "" + +# Re-test after gpupdate +Write-Host " Re-test: Resolve-DnsName google.com" -ForegroundColor White +try { + $result = Resolve-DnsName google.com -Type A -DnsOnly -ErrorAction Stop + Write-Host " OK: $($result.IPAddress -join ', ')" -ForegroundColor Green +} catch { + Write-Host " STILL FAILED: $_" -ForegroundColor Red +} + +Write-Host "" +Write-Host "=== DIAGNOSTIC COMPLETE ===" -ForegroundColor Cyan +Write-Host "Copy all output above and send it back." diff --git a/test-scripts/windows/test-dns-intercept.ps1 b/test-scripts/windows/test-dns-intercept.ps1 new file mode 100644 index 00000000..fc4cc3fd --- /dev/null +++ b/test-scripts/windows/test-dns-intercept.ps1 @@ -0,0 +1,544 @@ +# ============================================================================= +# DNS Intercept Mode Test Script — Windows (WFP) +# ============================================================================= +# Run as Administrator: powershell -ExecutionPolicy Bypass -File test-dns-intercept-win.ps1 +# +# Tests the dns-intercept feature end-to-end with validation at each step. +# Logs are read from C:\tmp\dns.log (ctrld log location on test machine). +# +# Manual steps marked with [MANUAL] require human interaction. +# ============================================================================= + +$ErrorActionPreference = "Continue" + +$CtrldLog = "C:\tmp\dns.log" +$WfpSubLayerName = "ctrld DNS Intercept" +$Pass = 0 +$Fail = 0 +$Warn = 0 +$Results = @() + +# --- Helpers --- + +function Header($text) { Write-Host "`n━━━ $text ━━━" -ForegroundColor Cyan } +function Info($text) { Write-Host " ℹ $text" } +function Manual($text) { Write-Host " [MANUAL] $text" -ForegroundColor Yellow } +function Separator() { Write-Host "─────────────────────────────────────────────────────" -ForegroundColor Cyan } + +function Pass($text) { + Write-Host " ✅ PASS: $text" -ForegroundColor Green + $script:Pass++ + $script:Results += "PASS: $text" +} + +function Fail($text) { + Write-Host " ❌ FAIL: $text" -ForegroundColor Red + $script:Fail++ + $script:Results += "FAIL: $text" +} + +function Warn($text) { + Write-Host " ⚠️ WARN: $text" -ForegroundColor Yellow + $script:Warn++ + $script:Results += "WARN: $text" +} + +function WaitForKey { + Write-Host "`n Press Enter to continue..." -NoNewline + Read-Host +} + +function LogGrep($pattern, $lines = 200) { + if (Test-Path $CtrldLog) { + Get-Content $CtrldLog -Tail $lines -ErrorAction SilentlyContinue | + Select-String -Pattern $pattern -ErrorAction SilentlyContinue + } +} + +function LogGrepCount($pattern, $lines = 200) { + $matches = LogGrep $pattern $lines + if ($matches) { return @($matches).Count } else { return 0 } +} + +# --- Check Admin --- + +function Check-Admin { + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = New-Object Security.Principal.WindowsPrincipal($identity) + if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + Write-Host "This script must be run as Administrator." -ForegroundColor Red + exit 1 + } +} + +# ============================================================================= +# TEST SECTIONS +# ============================================================================= + +function Test-Prereqs { + Header "0. Prerequisites" + + if (Get-Command nslookup -ErrorAction SilentlyContinue) { + Pass "nslookup available" + } else { + Fail "nslookup not found" + } + + if (Get-Command netsh -ErrorAction SilentlyContinue) { + Pass "netsh available" + } else { + Fail "netsh not found" + } + + if (Test-Path $CtrldLog) { + Pass "ctrld log exists at $CtrldLog" + } else { + Warn "ctrld log not found at $CtrldLog — log checks will be skipped" + } + + # Show current DNS config + Info "Current DNS servers:" + Get-DnsClientServerAddress -AddressFamily IPv4 | + Where-Object { $_.ServerAddresses.Count -gt 0 } | + Format-Table InterfaceAlias, ServerAddresses -AutoSize | + Out-String | ForEach-Object { $_.Trim() } | Write-Host +} + +function Test-WfpState { + Header "1. WFP State Validation" + + # Export WFP filters and check for ctrld's sublayer/filters + $wfpExport = "$env:TEMP\wfp_filters.xml" + Info "Exporting WFP filters (this may take a few seconds)..." + + try { + netsh wfp show filters file=$wfpExport 2>$null | Out-Null + + if (Test-Path $wfpExport) { + $wfpContent = Get-Content $wfpExport -Raw -ErrorAction SilentlyContinue + + # Check for ctrld sublayer + if ($wfpContent -match "ctrld") { + Pass "WFP filters contain 'ctrld' references" + + # Count filters + $filterMatches = ([regex]::Matches($wfpContent, "ctrld")).Count + Info "Found $filterMatches 'ctrld' references in WFP export" + } else { + Fail "No 'ctrld' references found in WFP filters" + } + + # Check for DNS port 53 filters + if ($wfpContent -match "port.*53" -or $wfpContent -match "0x0035") { + Pass "Port 53 filter conditions found in WFP" + } else { + Warn "Could not confirm port 53 filters in WFP export" + } + + Remove-Item $wfpExport -ErrorAction SilentlyContinue + } else { + Warn "WFP export file not created" + } + } catch { + Warn "Could not export WFP filters: $_" + } + + Separator + + # Alternative: Check via PowerShell WFP cmdlets if available + Info "Checking WFP via netsh wfp show state..." + $wfpState = netsh wfp show state 2>$null + if ($wfpState) { + Info "WFP state export completed (check $env:TEMP for details)" + } + + # Check Windows Firewall service is running + $fwService = Get-Service -Name "mpssvc" -ErrorAction SilentlyContinue + if ($fwService -and $fwService.Status -eq "Running") { + Pass "Windows Firewall service (BFE/WFP) is running" + } else { + Fail "Windows Firewall service not running — WFP won't work" + } + + # Check BFE (Base Filtering Engine) + $bfeService = Get-Service -Name "BFE" -ErrorAction SilentlyContinue + if ($bfeService -and $bfeService.Status -eq "Running") { + Pass "Base Filtering Engine (BFE) is running" + } else { + Fail "BFE not running — WFP requires this service" + } +} + +function Test-DnsInterception { + Header "2. DNS Interception Tests" + + # Mark log position + $logLinesBefore = 0 + if (Test-Path $CtrldLog) { + $logLinesBefore = @(Get-Content $CtrldLog -ErrorAction SilentlyContinue).Count + } + + # Test 1: Query to external resolver should be intercepted + Info "Test: nslookup example.com 8.8.8.8 (should be intercepted by ctrld)" + $result = $null + try { + $result = nslookup example.com 8.8.8.8 2>&1 | Out-String + } catch { } + + if ($result -and $result -match "\d+\.\d+\.\d+\.\d+") { + Pass "nslookup @8.8.8.8 returned a result" + + # Check which server answered + if ($result -match "Server:\s+(\S+)") { + $server = $Matches[1] + Info "Answered by server: $server" + if ($server -match "127\.0\.0\.1|localhost") { + Pass "Response came from localhost (ctrld intercepted)" + } elseif ($server -match "8\.8\.8\.8") { + Fail "Response came from 8.8.8.8 directly — NOT intercepted" + } + } + } else { + Fail "nslookup @8.8.8.8 failed or returned no address" + } + + # Check ctrld logged it + Start-Sleep -Seconds 1 + if (Test-Path $CtrldLog) { + $newLines = Get-Content $CtrldLog -ErrorAction SilentlyContinue | + Select-Object -Skip $logLinesBefore + $intercepted = $newLines | Select-String "example.com" -ErrorAction SilentlyContinue + if ($intercepted) { + Pass "ctrld logged the intercepted query for example.com" + } else { + Fail "ctrld did NOT log query for example.com" + } + } + + Separator + + # Test 2: Another external resolver + Info "Test: nslookup cloudflare.com 1.1.1.1 (should also be intercepted)" + try { + $result2 = nslookup cloudflare.com 1.1.1.1 2>&1 | Out-String + if ($result2 -match "\d+\.\d+\.\d+\.\d+") { + Pass "nslookup @1.1.1.1 returned result" + } else { + Fail "nslookup @1.1.1.1 failed" + } + } catch { + Fail "nslookup @1.1.1.1 threw exception" + } + + Separator + + # Test 3: Query to localhost should work (no loop) + Info "Test: nslookup example.org 127.0.0.1 (direct to ctrld, no loop)" + try { + $result3 = nslookup example.org 127.0.0.1 2>&1 | Out-String + if ($result3 -match "\d+\.\d+\.\d+\.\d+") { + Pass "nslookup @127.0.0.1 works (no loop)" + } else { + Fail "nslookup @127.0.0.1 failed — possible loop" + } + } catch { + Fail "nslookup @127.0.0.1 exception — possible loop" + } + + Separator + + # Test 4: System DNS via Resolve-DnsName + Info "Test: Resolve-DnsName example.net (system resolver)" + try { + $result4 = Resolve-DnsName example.net -Type A -ErrorAction Stop + if ($result4) { + Pass "System DNS resolution works (Resolve-DnsName)" + } + } catch { + Fail "System DNS resolution failed: $_" + } + + Separator + + # Test 5: TCP DNS + Info "Test: nslookup -vc example.com 9.9.9.9 (TCP DNS)" + try { + $result5 = nslookup -vc example.com 9.9.9.9 2>&1 | Out-String + if ($result5 -match "\d+\.\d+\.\d+\.\d+") { + Pass "TCP DNS query intercepted and resolved" + } else { + Warn "TCP DNS query may not have been intercepted" + } + } catch { + Warn "TCP DNS test inconclusive" + } +} + +function Test-NonDnsUnaffected { + Header "3. Non-DNS Traffic Unaffected" + + # HTTPS + Info "Test: Invoke-WebRequest https://example.com (HTTPS should NOT be affected)" + try { + $web = Invoke-WebRequest -Uri "https://example.com" -UseBasicParsing -TimeoutSec 10 -ErrorAction Stop + if ($web.StatusCode -eq 200) { + Pass "HTTPS works (HTTP 200)" + } else { + Pass "HTTPS returned HTTP $($web.StatusCode)" + } + } catch { + Fail "HTTPS failed: $_" + } + + # Test non-53 port connectivity + Info "Test: Test-NetConnection to github.com:443 (non-DNS port)" + try { + $nc = Test-NetConnection -ComputerName "github.com" -Port 443 -WarningAction SilentlyContinue + if ($nc.TcpTestSucceeded) { + Pass "Port 443 reachable (non-DNS traffic unaffected)" + } else { + Warn "Port 443 unreachable (may be firewall)" + } + } catch { + Warn "Test-NetConnection failed: $_" + } +} + +function Test-CtrldLogHealth { + Header "4. ctrld Log Health Check" + + if (-not (Test-Path $CtrldLog)) { + Warn "Skipping log checks — $CtrldLog not found" + return + } + + # Check for WFP initialization + if (LogGrepCount "initializing Windows Filtering Platform" 500) { + Pass "WFP initialization logged" + } else { + Fail "No WFP initialization in recent logs" + } + + # Check for successful WFP engine open + if (LogGrepCount "WFP engine opened" 500) { + Pass "WFP engine opened successfully" + } else { + Fail "WFP engine open not found in logs" + } + + # Check for sublayer creation + if (LogGrepCount "WFP sublayer created" 500) { + Pass "WFP sublayer created" + } else { + Fail "WFP sublayer creation not logged" + } + + # Check for filter creation + $filterCount = LogGrepCount "added WFP.*filter" 500 + if ($filterCount -gt 0) { + Pass "WFP filters added ($filterCount filter log entries)" + } else { + Fail "No WFP filter creation logged" + } + + # Check for permit-localhost filters + if (LogGrepCount "permit.*localhost\|permit.*127\.0\.0\.1" 500) { + Pass "Localhost permit filters logged" + } else { + Warn "Localhost permit filters not explicitly logged" + } + + Separator + + # Check for errors + Info "Recent errors in ctrld log:" + $errors = LogGrep '"level":"error"' 500 + if ($errors) { + $errors | Select-Object -Last 5 | ForEach-Object { Write-Host " $_" } + Warn "Errors found in recent logs" + } else { + Pass "No errors in recent logs" + } + + # Warnings (excluding expected ones) + $warnings = LogGrep '"level":"warn"' 500 | Where-Object { + $_ -notmatch "skipping self-upgrade" + } + if ($warnings) { + Info "Warnings:" + $warnings | Select-Object -Last 5 | ForEach-Object { Write-Host " $_" } + } + + # VPN DNS detection + $vpnLogs = LogGrep "VPN DNS" 500 + if ($vpnLogs) { + Info "VPN DNS activity:" + $vpnLogs | Select-Object -Last 5 | ForEach-Object { Write-Host " $_" } + } else { + Info "No VPN DNS activity (expected if no VPN connected)" + } +} + +function Test-CleanupOnStop { + Header "5. Cleanup Validation (After ctrld Stop)" + + Manual "Stop ctrld now (ctrld stop or Ctrl+C), then press Enter" + WaitForKey + + Start-Sleep -Seconds 2 + + # Check WFP filters are removed + $wfpExport = "$env:TEMP\wfp_after_stop.xml" + try { + netsh wfp show filters file=$wfpExport 2>$null | Out-Null + if (Test-Path $wfpExport) { + $content = Get-Content $wfpExport -Raw -ErrorAction SilentlyContinue + if ($content -match "ctrld") { + Fail "WFP still contains 'ctrld' filters after stop" + } else { + Pass "WFP filters cleaned up after stop" + } + Remove-Item $wfpExport -ErrorAction SilentlyContinue + } + } catch { + Warn "Could not verify WFP cleanup" + } + + # DNS should work normally + Info "Test: nslookup example.com (should work via system DNS)" + try { + $result = nslookup example.com 2>&1 | Out-String + if ($result -match "\d+\.\d+\.\d+\.\d+") { + Pass "DNS works after ctrld stop" + } else { + Fail "DNS broken after ctrld stop" + } + } catch { + Fail "DNS exception after ctrld stop" + } +} + +function Test-RestartResilience { + Header "6. Restart Resilience" + + Manual "Start ctrld again with --dns-intercept, then press Enter" + WaitForKey + + Start-Sleep -Seconds 3 + + # Quick interception test + Info "Test: nslookup example.com 8.8.8.8 (should be intercepted after restart)" + try { + $result = nslookup example.com 8.8.8.8 2>&1 | Out-String + if ($result -match "\d+\.\d+\.\d+\.\d+") { + Pass "DNS interception works after restart" + } else { + Fail "DNS interception broken after restart" + } + } catch { + Fail "DNS test failed after restart" + } + + # Check WFP filters restored + if (LogGrepCount "WFP engine opened" 100) { + Pass "WFP re-initialized after restart" + } +} + +function Test-NetworkChange { + Header "7. Network Change Recovery" + + Info "This test verifies recovery after network changes." + Manual "Switch Wi-Fi networks, or disable/re-enable network adapter, then press Enter" + WaitForKey + + Start-Sleep -Seconds 5 + + # Test interception still works + Info "Test: nslookup example.com 8.8.8.8 (should still be intercepted)" + try { + $result = nslookup example.com 8.8.8.8 2>&1 | Out-String + if ($result -match "\d+\.\d+\.\d+\.\d+") { + Pass "DNS interception works after network change" + } else { + Fail "DNS interception broken after network change" + } + } catch { + Fail "DNS test failed after network change" + } + + # Check logs for recovery/network events + if (Test-Path $CtrldLog) { + $recoveryLogs = LogGrep "recovery|network change|network monitor" 100 + if ($recoveryLogs) { + Info "Recovery/network log entries:" + $recoveryLogs | Select-Object -Last 5 | ForEach-Object { Write-Host " $_" } + } + } +} + +# ============================================================================= +# SUMMARY +# ============================================================================= + +function Print-Summary { + Header "TEST SUMMARY" + Write-Host "" + foreach ($r in $Results) { + if ($r.StartsWith("PASS")) { + Write-Host " ✅ $($r.Substring(6))" -ForegroundColor Green + } elseif ($r.StartsWith("FAIL")) { + Write-Host " ❌ $($r.Substring(6))" -ForegroundColor Red + } elseif ($r.StartsWith("WARN")) { + Write-Host " ⚠️ $($r.Substring(6))" -ForegroundColor Yellow + } + } + Write-Host "" + Separator + Write-Host " Passed: $Pass | Failed: $Fail | Warnings: $Warn" + Separator + + if ($Fail -gt 0) { + Write-Host "`n Some tests failed. Debug commands:" -ForegroundColor Red + Write-Host " netsh wfp show filters # dump all WFP filters" + Write-Host " Get-Content $CtrldLog -Tail 100 # recent ctrld logs" + Write-Host " Get-DnsClientServerAddress # current DNS config" + Write-Host " netsh wfp show state # WFP state dump" + } else { + Write-Host "`n All tests passed!" -ForegroundColor Green + } +} + +# ============================================================================= +# MAIN +# ============================================================================= + +Write-Host "╔═══════════════════════════════════════════════════════╗" -ForegroundColor White +Write-Host "║ ctrld DNS Intercept Mode — Windows Test Suite ║" -ForegroundColor White +Write-Host "║ Tests WFP-based DNS interception ║" -ForegroundColor White +Write-Host "╚═══════════════════════════════════════════════════════╝" -ForegroundColor White + +Check-Admin + +Write-Host "" +Write-Host "Make sure ctrld is running with --dns-intercept before starting." +Write-Host "Log location: $CtrldLog" +WaitForKey + +Test-Prereqs +Test-WfpState +Test-DnsInterception +Test-NonDnsUnaffected +Test-CtrldLogHealth + +Separator +Write-Host "" +Write-Host "The next tests require manual steps (stop/start ctrld, network changes)." +Write-Host "Press Enter to continue, or Ctrl+C to skip and see results so far." +WaitForKey + +Test-CleanupOnStop +Test-RestartResilience +Test-NetworkChange + +Print-Summary diff --git a/test-scripts/windows/test-recovery-bypass.ps1 b/test-scripts/windows/test-recovery-bypass.ps1 new file mode 100644 index 00000000..005a7feb --- /dev/null +++ b/test-scripts/windows/test-recovery-bypass.ps1 @@ -0,0 +1,289 @@ +# test-recovery-bypass.ps1 — Test DNS intercept recovery bypass (captive portal simulation) +# +# Simulates a captive portal by: +# 1. Discovering ctrld's upstream IPs from active connections +# 2. Blocking them via Windows Firewall rules +# 3. Disabling/re-enabling the wifi adapter to trigger network change +# 4. Verifying recovery bypass forwards to OS/DHCP resolver +# 5. Removing firewall rules and verifying normal operation resumes +# +# SAFE: Uses named firewall rules that are cleaned up on exit. +# +# Usage (run as Administrator): +# .\test-recovery-bypass.ps1 [-WifiAdapter "Wi-Fi"] [-CtrldLog "C:\temp\dns.log"] +# +# Prerequisites: +# - ctrld running with --dns-intercept and -v 1 --log C:\temp\dns.log +# - Run as Administrator + +param( + [string]$WifiAdapter = "Wi-Fi", + [string]$CtrldLog = "C:\temp\dns.log", + [int]$BlockDurationSec = 60 +) + +$ErrorActionPreference = "Stop" +$FwRulePrefix = "ctrld-test-recovery-block" +$BlockedIPs = @() + +function Log($msg) { Write-Host "[$(Get-Date -Format 'HH:mm:ss')] $msg" -ForegroundColor Cyan } +function Pass($msg) { Write-Host "[PASS] $msg" -ForegroundColor Green } +function Fail($msg) { Write-Host "[FAIL] $msg" -ForegroundColor Red } +function Warn($msg) { Write-Host "[WARN] $msg" -ForegroundColor Yellow } + +# ── Safety: cleanup function ───────────────────────────────────────────────── +function Cleanup { + Log "═══ CLEANUP ═══" + + # Ensure wifi is enabled + Log "Ensuring wifi adapter is enabled..." + try { Enable-NetAdapter -Name $WifiAdapter -Confirm:$false -ErrorAction SilentlyContinue } catch {} + + # Remove all test firewall rules + Log "Removing test firewall rules..." + Get-NetFirewallRule -DisplayName "$FwRulePrefix*" -ErrorAction SilentlyContinue | + Remove-NetFirewallRule -ErrorAction SilentlyContinue + Log "Cleanup complete." +} + +# Register cleanup on script exit +$null = Register-EngineEvent -SourceIdentifier PowerShell.Exiting -Action { Cleanup } -ErrorAction SilentlyContinue +trap { Cleanup; break } + +# ── Pre-checks ─────────────────────────────────────────────────────────────── +$isAdmin = ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) +if (-not $isAdmin) { + Fail "Run as Administrator!" + exit 1 +} + +if (-not (Test-Path $CtrldLog)) { + Fail "ctrld log not found at $CtrldLog" + Write-Host "Start ctrld with: ctrld run --dns-intercept --cd -v 1 --log $CtrldLog" + exit 1 +} + +# Check wifi adapter exists +$adapter = Get-NetAdapter -Name $WifiAdapter -ErrorAction SilentlyContinue +if (-not $adapter) { + Fail "Wifi adapter '$WifiAdapter' not found" + Write-Host "Available adapters:" + Get-NetAdapter | Format-Table Name, Status, InterfaceDescription + exit 1 +} + +Log "═══════════════════════════════════════════════════════════" +Log " Recovery Bypass Test (Captive Portal Simulation)" +Log "═══════════════════════════════════════════════════════════" +Log "Wifi adapter: $WifiAdapter" +Log "ctrld log: $CtrldLog" +Write-Host "" + +# ── Phase 1: Discover upstream IPs ────────────────────────────────────────── +Log "Phase 1: Discovering ctrld upstream IPs from active connections" + +$ctrldConns = Get-NetTCPConnection -OwningProcess (Get-Process ctrld* -ErrorAction SilentlyContinue).Id -ErrorAction SilentlyContinue | + Where-Object { $_.State -eq "Established" -and $_.RemotePort -eq 443 } + +$upstreamIPs = @() +if ($ctrldConns) { + $upstreamIPs = $ctrldConns | Select-Object -ExpandProperty RemoteAddress -Unique | + Where-Object { $_ -notmatch "^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)" } + + foreach ($conn in $ctrldConns) { + Log " $($conn.LocalAddress):$($conn.LocalPort) -> $($conn.RemoteAddress):$($conn.RemotePort)" + } +} + +# Also resolve known Control D endpoints +foreach ($host_ in @("dns.controld.com", "freedns.controld.com")) { + try { + $resolved = Resolve-DnsName $host_ -Type A -ErrorAction SilentlyContinue + $resolved | ForEach-Object { if ($_.IPAddress) { $upstreamIPs += $_.IPAddress } } + } catch {} +} + +$upstreamIPs = $upstreamIPs | Sort-Object -Unique + +if ($upstreamIPs.Count -eq 0) { + Fail "Could not discover any upstream IPs!" + exit 1 +} + +Log "Found $($upstreamIPs.Count) upstream IP(s):" +foreach ($ip in $upstreamIPs) { Log " $ip" } +Write-Host "" + +# ── Phase 2: Baseline ─────────────────────────────────────────────────────── +Log "Phase 2: Baseline — verify DNS works normally" +$baseline = Resolve-DnsName example.com -Server 127.0.0.1 -Type A -ErrorAction SilentlyContinue +if ($baseline) { + Pass "Baseline: example.com -> $($baseline[0].IPAddress)" +} else { + Fail "DNS not working!" + exit 1 +} + +$logLinesBefore = (Get-Content $CtrldLog).Count +Log "Log position: line $logLinesBefore" +Write-Host "" + +# ── Phase 3: Block upstream IPs via Windows Firewall ──────────────────────── +Log "Phase 3: Blocking upstream IPs via Windows Firewall" +foreach ($ip in $upstreamIPs) { + $ruleName = "$FwRulePrefix-$ip" + # Remove existing rule if any + Remove-NetFirewallRule -DisplayName $ruleName -ErrorAction SilentlyContinue + # Block outbound to this IP + New-NetFirewallRule -DisplayName $ruleName -Direction Outbound -Action Block ` + -RemoteAddress $ip -Protocol TCP -RemotePort 443 ` + -Description "Temporary test rule for ctrld recovery bypass test" | Out-Null + $BlockedIPs += $ip + Log " Blocked: $ip (outbound TCP 443)" +} +Pass "All $($upstreamIPs.Count) upstream IPs blocked" +Write-Host "" + +# ── Phase 4: Cycle wifi ───────────────────────────────────────────────────── +Log "Phase 4: Cycling wifi to trigger network change event" +Log " Disabling $WifiAdapter..." +Disable-NetAdapter -Name $WifiAdapter -Confirm:$false +Start-Sleep -Seconds 3 + +Log " Enabling $WifiAdapter..." +Enable-NetAdapter -Name $WifiAdapter -Confirm:$false + +Log " Waiting for wifi to reconnect (up to 20s)..." +$wifiUp = $false +for ($i = 0; $i -lt 20; $i++) { + $status = (Get-NetAdapter -Name $WifiAdapter).Status + if ($status -eq "Up") { + # Check for IP + $ipAddr = (Get-NetIPAddress -InterfaceAlias $WifiAdapter -AddressFamily IPv4 -ErrorAction SilentlyContinue).IPAddress + if ($ipAddr) { + $wifiUp = $true + Pass "Wifi reconnected: $WifiAdapter -> $ipAddr" + break + } + } + Start-Sleep -Seconds 1 +} + +if (-not $wifiUp) { + Fail "Wifi did not reconnect in 20s!" + Cleanup + exit 1 +} + +Log " Waiting 5s for ctrld network monitor..." +Start-Sleep -Seconds 5 +Write-Host "" + +# ── Phase 5: Query and watch for recovery ──────────────────────────────────── +Log "Phase 5: Sending queries — upstream blocked, recovery should activate" +Write-Host "" + +$recoveryDetected = $false +$bypassActive = $false +$dnsDuringBypass = $false + +for ($q = 1; $q -le 30; $q++) { + $result = $null + try { + $result = Resolve-DnsName "example.com" -Server 127.0.0.1 -Type A -DnsOnly -ErrorAction SilentlyContinue + } catch {} + + if ($result) { + Log " Query #$q`: example.com -> $($result[0].IPAddress) ✓" + } else { + Log " Query #$q`: example.com -> FAIL ✗" + } + + # Check ctrld log for recovery + $newLogs = Get-Content $CtrldLog | Select-Object -Skip $logLinesBefore + $logText = $newLogs -join "`n" + + if (-not $recoveryDetected -and ($logText -match "enabling DHCP bypass|triggering recovery|No healthy")) { + Write-Host "" + Pass "🎯 Recovery flow triggered!" + $recoveryDetected = $true + } + + if (-not $bypassActive -and ($logText -match "Recovery bypass active")) { + Pass "🔄 Recovery bypass forwarding to OS/DHCP resolver" + $bypassActive = $true + } + + if ($recoveryDetected -and $result) { + Pass "✅ DNS resolves during recovery: example.com -> $($result[0].IPAddress)" + $dnsDuringBypass = $true + break + } + + Start-Sleep -Seconds 2 +} + +# ── Phase 6: Show log entries ──────────────────────────────────────────────── +Write-Host "" +Log "Phase 6: Recovery-related ctrld log entries" +Log "────────────────────────────────────────────" +$newLogs = Get-Content $CtrldLog | Select-Object -Skip $logLinesBefore +$relevant = $newLogs | Where-Object { $_ -match "recovery|bypass|DHCP|unhealthy|upstream.*fail|No healthy|network change|OS resolver" } +if ($relevant) { + $relevant | Select-Object -First 30 | ForEach-Object { Write-Host " $_" } +} else { + Warn "No recovery-related log entries found" + Get-Content $CtrldLog | Select-Object -Last 10 | ForEach-Object { Write-Host " $_" } +} + +# ── Phase 7: Unblock and verify ───────────────────────────────────────────── +Write-Host "" +Log "Phase 7: Removing firewall blocks" +Get-NetFirewallRule -DisplayName "$FwRulePrefix*" -ErrorAction SilentlyContinue | + Remove-NetFirewallRule -ErrorAction SilentlyContinue +$BlockedIPs = @() +Pass "Firewall rules removed" + +Log "Waiting for recovery (up to 30s)..." +$logLinesUnblock = (Get-Content $CtrldLog).Count +$recoveryComplete = $false + +for ($i = 0; $i -lt 15; $i++) { + try { Resolve-DnsName example.com -Server 127.0.0.1 -Type A -DnsOnly -ErrorAction SilentlyContinue } catch {} + $postLogs = (Get-Content $CtrldLog | Select-Object -Skip $logLinesUnblock) -join "`n" + if ($postLogs -match "recovery complete|disabling DHCP bypass|Upstream.*recovered") { + $recoveryComplete = $true + Pass "ctrld recovered — normal operation resumed" + break + } + Start-Sleep -Seconds 2 +} + +if (-not $recoveryComplete) { Warn "Recovery completion not detected (may need more time)" } + +# ── Phase 8: Final check ──────────────────────────────────────────────────── +Write-Host "" +Log "Phase 8: Final DNS verification" +Start-Sleep -Seconds 2 +$final = Resolve-DnsName example.com -Server 127.0.0.1 -Type A -ErrorAction SilentlyContinue +if ($final) { + Pass "DNS working: example.com -> $($final[0].IPAddress)" +} else { + Fail "DNS not resolving" +} + +# ── Summary ────────────────────────────────────────────────────────────────── +Write-Host "" +Log "═══════════════════════════════════════════════════════════" +Log " Test Summary" +Log "═══════════════════════════════════════════════════════════" +if ($recoveryDetected) { Pass "Recovery bypass activated" } else { Fail "Recovery bypass NOT activated" } +if ($bypassActive) { Pass "Queries forwarded to OS/DHCP" } else { Warn "OS resolver forwarding not confirmed" } +if ($dnsDuringBypass) { Pass "DNS resolved during bypass" } else { Warn "DNS during bypass not confirmed" } +if ($recoveryComplete) { Pass "Normal operation resumed" } else { Warn "Recovery completion not confirmed" } +if ($final) { Pass "DNS functional at end of test" } else { Fail "DNS broken at end of test" } +Write-Host "" +Log "Full log: Get-Content $CtrldLog | Select-Object -Skip $logLinesBefore" + +# Cleanup runs via trap +Cleanup diff --git a/transport_replacement_test.go b/transport_replacement_test.go new file mode 100644 index 00000000..9b20ec94 --- /dev/null +++ b/transport_replacement_test.go @@ -0,0 +1,329 @@ +package ctrld + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "errors" + "io" + "net" + "net/http" + "net/url" + "sync" + "syscall" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" +) + +var transportReplacements = []struct { + name string + run func(*UpstreamConfig, context.Context) +}{ + {"SetupTransport", (*UpstreamConfig).SetupTransport}, + {"ForceReBootstrap", (*UpstreamConfig).ForceReBootstrap}, +} + +// Accept multiple connections so a replacement must answer on the same real +// endpoint, while the original query is still held by the server. +func replacementDOTServer(t *testing.T) (*UpstreamConfig, <-chan struct{}, func(), <-chan struct{}) { + t.Helper() + cert := generateTestCertificate(t) + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert.tlsCert}}) + if err != nil { + t.Fatal(err) + } + entered, release, peerClosed, done := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + var mu sync.Mutex + var conns []net.Conn + var wg sync.WaitGroup + go func() { + defer close(done) + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + wg.Add(1) + go func() { + defer wg.Done() + defer c.Close() + dc := &dns.Conn{Conn: c} + for { + q, err := dc.ReadMsg() + if err != nil { + return + } + held := q.Question[0].Name == "held.example." + if held { + close(entered) + defer close(peerClosed) + <-release + } + a := new(dns.Msg) + a.SetReply(q) + if err := dc.WriteMsg(a); err != nil { + return + } + } + }() + } + }() + roots := x509.NewCertPool() + roots.AddCert(cert.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1", Domain: "localhost", certPool: roots} + uc.ensureSetupTransport(context.Background()) + t.Cleanup(func() { + unblock() + uc.CloseTransports() + _ = ln.Close() + retirementWait(t, done, "DoT accept loop cleanup") // No more wg.Add after this. + mu.Lock() + for _, c := range conns { + _ = c.Close() + } + mu.Unlock() + wg.Wait() + }) + return uc, entered, unblock, peerClosed +} + +func replacementAnswer(t *testing.T, resolve func(context.Context, *dns.Msg) (*dns.Msg, error)) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + q := retirementQuery() + a, err := resolve(ctx, q) + if err != nil { + t.Fatalf("replacement query failed: %v", err) + } + if a == nil || !a.Response || a.Id != q.Id || len(a.Question) != 1 || a.Question[0] != q.Question[0] { + t.Fatalf("replacement returned wrong DNS answer: %v", a) + } +} + +func replacementResult(t *testing.T, result <-chan error) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(3 * time.Second): + t.Fatal("old query survived replacement") + return nil + } +} + +func TestReplaceTransportsDoTActive(t *testing.T) { + for _, replace := range transportReplacements { + t.Run(replace.name, func(t *testing.T) { + uc, entered, release, peerClosed := replacementDOTServer(t) + old := uc.dotClientPool + selected := make(chan struct{}) + old.dialer.ControlContext = func(context.Context, string, string, syscall.RawConn) error { + close(selected) + return nil + } + r := &dotResolver{uc: uc} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + q := new(dns.Msg) + q.SetQuestion("held.example.", dns.TypeA) + result := make(chan error, 1) + go func() { _, err := r.Resolve(ctx, q); result <- err }() + retirementWait(t, selected, "DoT transport selection") + retirementWait(t, entered, "active DoT query") + replace.run(uc, context.Background()) + if uc.dotClientPool == old { + t.Fatal("DoT pool was not replaced") + } + if err := replacementResult(t, result); !errors.Is(err, net.ErrClosed) { + t.Fatalf("old query did not fail from socket retirement: %v", err) + } + // This answer must arrive before releasing the old server handler. + replacementAnswer(t, r.Resolve) + release() + retirementWait(t, peerClosed, "old DoT peer closure") + if _, err := old.Resolve(ctx, retirementQuery()); !errors.Is(err, net.ErrClosed) { + t.Fatalf("late query revived replaced DoT pool: %v", err) + } + replacementAnswer(t, r.Resolve) + }) + } +} + +func TestReplaceTransportsDoTLateReturn(t *testing.T) { + for _, replace := range transportReplacements { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + t.Run(replace.name+"/"+slot, func(t *testing.T) { + uc, _, _, _ := replacementDOTServer(t) + old := uc.dotClientPool + r := &dotResolver{uc: uc} + replacementAnswer(t, r.Resolve) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := old.getConn(ctx) + if err != nil { + t.Fatal(err) + } + // Model each old slot, including the IPv6-to-IPv4 alias, without + // requiring an external IPv6 capability probe in this fixture. + switch slot { + case "v4": + uc.dotClientPool, uc.dotClientPool4 = nil, old + case "v6": + uc.dotClientPool, uc.dotClientPool6 = nil, old + case "aliased": + uc.dotClientPool4, uc.dotClientPool6 = old, old + } + replace.run(uc, context.Background()) + // A successful borrower returning after the slots change must not + // put its connection back into the now-orphaned pool. + old.putConn(conn, true) + if len(old.conns) != 0 { + t.Fatal("late return repopulated replaced DoT pool") + } + if _, err := old.getConn(ctx); !errors.Is(err, net.ErrClosed) { + t.Fatalf("replaced DoT pool accepted late work: %v", err) + } + replacementAnswer(t, r.Resolve) + }) + } + } +} + +func TestReplaceTransportsDoH3IdleAndLateWork(t *testing.T) { + for _, replace := range transportReplacements { + t.Run(replace.name, func(t *testing.T) { + srv := newTestHTTP3Server(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + roots := x509.NewCertPool() + roots.AddCert(srv.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOH3, Endpoint: "https://" + srv.addr, BootstrapIP: "127.0.0.1", certPool: roots} + defer uc.CloseTransports() + // The real warmup path initializes the transport and leaves it idle. + if err := uc.ErrorPing(context.Background()); err != nil { + t.Fatal(err) + } + old := uc.http3RoundTripper + defer old.(io.Closer).Close() + replace.run(uc, context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(ctx, "HEAD", uc.Endpoint, nil) + resp, err := old.RoundTrip(req) + if resp != nil { + resp.Body.Close() + } + if err == nil || ctx.Err() != nil { + t.Fatalf("late work revived idle replaced HTTP/3 transport: %v (context: %v)", err, ctx.Err()) + } + if err := uc.ErrorPing(context.Background()); err != nil { + t.Fatalf("replacement HTTP/3 transport is not usable: %v", err) + } + }) + } +} + +func TestReplaceTransportsDoH3ActiveAndLateWork(t *testing.T) { + for _, replace := range transportReplacements { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + t.Run(replace.name+"/"+slot, func(t *testing.T) { + entered, release, peerCanceled := make(chan struct{}), make(chan struct{}), make(chan struct{}) + srv := newTestHTTP3Server(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/late" { + w.WriteHeader(http.StatusOK) + return + } + data, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns")) + q := new(dns.Msg) + if err != nil || q.Unpack(data) != nil || len(q.Question) != 1 { + http.Error(w, "bad DNS query", http.StatusBadRequest) + return + } + if q.Question[0].Name == "held.example." { + close(entered) + select { + case <-r.Context().Done(): + close(peerCanceled) + case <-release: + } + return + } + a := new(dns.Msg) + a.SetReply(q) + data, _ = a.Pack() + w.Header().Set("Content-Type", headerApplicationDNS) + _, _ = w.Write(data) + })) + defer close(release) + roots := x509.NewCertPool() + roots.AddCert(srv.cert) + u, err := url.Parse("https://" + srv.addr + "/dns-query") + if err != nil { + t.Fatal(err) + } + uc := &UpstreamConfig{Type: ResolverTypeDOH3, Endpoint: u.String(), u: u, BootstrapIP: "127.0.0.1", certPool: roots} + uc.ensureSetupTransport(context.Background()) + defer uc.CloseTransports() + old := uc.http3RoundTripper + // Clean the old transport even if a regression leaves it orphaned. + defer old.(io.Closer).Close() + selected := make(chan struct{}) + rt := old.(*http3.Transport) + dial := rt.Dial + var selectedOnce sync.Once + rt.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { + selectedOnce.Do(func() { close(selected) }) + return dial(ctx, addr, tlsCfg, cfg) + } + r := newDohResolver(uc) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + q := new(dns.Msg) + q.SetQuestion("held.example.", dns.TypeA) + result := make(chan error, 1) + go func() { _, err := r.Resolve(ctx, q); result <- err }() + // A network exchange alone isn't a Go happens-before edge. Observe + // selection before writing slots; still execute the production dial. + retirementWait(t, selected, "HTTP/3 transport selection") + retirementWait(t, entered, "active HTTP/3 DNS query") + switch slot { + case "v4": + uc.http3RoundTripper, uc.http3RoundTripper4 = nil, old + case "v6": + uc.http3RoundTripper, uc.http3RoundTripper6 = nil, old + case "aliased": + uc.http3RoundTripper4, uc.http3RoundTripper6 = old, old + } + replace.run(uc, context.Background()) + if uc.http3RoundTripper == old { + t.Fatal("HTTP/3 transport was not replaced") + } + retirementWait(t, peerCanceled, "old HTTP/3 request cancellation") + if err := replacementResult(t, result); err == nil || ctx.Err() != nil { + t.Fatalf("old HTTP/3 query not aborted by replacement: %v (context: %v)", err, ctx.Err()) + } + replacementAnswer(t, r.Resolve) + req, _ := http.NewRequestWithContext(ctx, "GET", "https://"+srv.addr+"/late", nil) + resp, err := old.RoundTrip(req) + if resp != nil { + resp.Body.Close() + } + if err == nil || ctx.Err() != nil { + t.Fatalf("late request revived replaced HTTP/3 transport: %v (context: %v)", err, ctx.Err()) + } + replacementAnswer(t, r.Resolve) + }) + } + } +} diff --git a/transport_retirement_test.go b/transport_retirement_test.go new file mode 100644 index 00000000..e95e537c --- /dev/null +++ b/transport_retirement_test.go @@ -0,0 +1,471 @@ +package ctrld + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "sync" + "syscall" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/quic-go/quic-go/http3" +) + +func retirementWait(t *testing.T, ch <-chan struct{}, what string) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatalf("timed out waiting for %s", what) + } +} + +// One real TLS connection, with the response optionally held until release. +// Cleanup joins the server goroutine and never touches host DNS or global state. +func retirementDOTServer(t *testing.T, hold bool) (*UpstreamConfig, <-chan struct{}, func(), <-chan struct{}) { + t.Helper() + cert := generateTestCertificate(t) + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert.tlsCert}}) + if err != nil { + t.Fatal(err) + } + entered, release, peerClosed, done := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + if !hold { + unblock() + } + var mu sync.Mutex + var accepted net.Conn + go func() { + defer close(done) + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + accepted = c + mu.Unlock() + defer c.Close() + dc := &dns.Conn{Conn: c} + q, err := dc.ReadMsg() + if err != nil { + return + } + close(entered) + <-release + a := new(dns.Msg) + a.SetReply(q) + _ = dc.WriteMsg(a) + // No more queries are sent on this fixture; EOF proves peer retirement. + _, _ = dc.ReadMsg() + close(peerClosed) + }() + roots := x509.NewCertPool() + roots.AddCert(cert.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1", Domain: "localhost", certPool: roots} + uc.SetupTransport(context.Background()) + uc.transportOnce.Do(func() {}) // SetupTransport above is the resolver's initial setup. + t.Cleanup(func() { + unblock() + uc.CloseTransports() + _ = ln.Close() + mu.Lock() + if accepted != nil { + _ = accepted.Close() + } + mu.Unlock() + retirementWait(t, done, "DoT server cleanup") + }) + return uc, entered, unblock, peerClosed +} + +func retirementQuery() *dns.Msg { + q := new(dns.Msg) + q.SetQuestion("retirement.example.", dns.TypeA) + return q +} + +func TestCloseTransportsDoHIdleSlots(t *testing.T) { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + t.Run(slot, func(t *testing.T) { + closed := make(chan struct{}) + var once sync.Once + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("ok")) + })) + srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateClosed { + once.Do(func() { close(closed) }) + } + } + srv.Start() + defer srv.Close() + transport := http.DefaultTransport.(*http.Transport).Clone() + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 3 * time.Second} + resp, err := client.Get(srv.URL) + if err != nil { + t.Fatal(err) + } + _, err = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if err != nil { + t.Fatal(err) + } + select { + case <-closed: + t.Fatal("fixture did not retain an idle connection") + default: + } + uc := &UpstreamConfig{} + switch slot { + case "default": + uc.transport = transport + case "v4": + uc.transport4 = transport + case "v6": + uc.transport6 = transport + case "aliased": + uc.transport, uc.transport4, uc.transport6 = transport, transport, transport + } + uc.CloseTransports() + retirementWait(t, closed, "idle HTTP connection retirement") + uc.CloseTransports() + }) + } +} + +func TestCloseTransportsDoTActive(t *testing.T) { + uc, entered, release, peerClosed := retirementDOTServer(t, true) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := (&dotResolver{uc: uc}).Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "active DoT query") + uc.CloseTransports() + select { + case err := <-result: + if !errors.Is(err, net.ErrClosed) { + // miekg/dns has its own two-second read timeout, independent of + // the longer context deadline. A timeout is not retirement proof. + t.Fatalf("active DoT query was not closed by retirement: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("active DoT query survived retirement until its own deadline") + } + release() + retirementWait(t, peerClosed, "DoT peer socket closure") + if _, err := (&dotResolver{uc: uc}).Resolve(ctx, retirementQuery()); !errors.Is(err, net.ErrClosed) { + t.Fatalf("retired resolver accepted another query: %v", err) + } + if n := len(uc.dotClientPool.conns); n != 0 { + t.Fatalf("late return repopulated pool: %d", n) + } +} + +func TestCloseTransportsDoTIdleAndLateReturn(t *testing.T) { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + for _, checkedOut := range []bool{false, true} { + t.Run(slot+map[bool]string{false: "/idle", true: "/checked-out"}[checkedOut], func(t *testing.T) { + uc, _, _, peerClosed := retirementDOTServer(t, false) + p := uc.dotClientPool + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := p.Resolve(ctx, retirementQuery()); err != nil { + t.Fatal(err) + } + var conn net.Conn + if checkedOut { + var err error + conn, err = p.getConn(ctx) + if err != nil { + t.Fatal(err) + } + } + switch slot { + case "v4": + uc.dotClientPool = nil + uc.dotClientPool4 = p + case "v6": + uc.dotClientPool = nil + uc.dotClientPool6 = p + case "aliased": + uc.dotClientPool4 = p + uc.dotClientPool6 = p + } + uc.CloseTransports() + retirementWait(t, peerClosed, "idle/checked-out DoT socket closure") + // Simulate a response that completed just before retirement and returns late. + if conn != nil { + p.putConn(conn, true) + } + uc.CloseTransports() // aliased slots and repeated retirement are safe. + if n := len(p.conns); n != 0 { + t.Fatalf("retired pool has %d connections", n) + } + if _, err := p.getConn(ctx); !errors.Is(err, net.ErrClosed) { + t.Fatalf("get after close: %v", err) + } + }) + } + } +} + +func TestCloseTransportsDoTHandshake(t *testing.T) { + for _, mode := range []string{"bootstrap", "parallel", "fallback"} { + t.Run(mode, func(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + entered, peerClosed := make(chan struct{}), make(chan struct{}) + go func() { + c, err := ln.Accept() + if err != nil { + return + } + defer c.Close() + var b [1]byte + if _, err := c.Read(b[:]); err != nil { + return + } + close(entered) // ClientHello was received, but no TLS response is sent. + _, _ = io.Copy(io.Discard, c) + close(peerClosed) + }() + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1", Domain: "localhost"} + var addrs []string + if mode != "bootstrap" { + uc.BootstrapIP = "" + } + if mode == "parallel" { + addrs = []string{"127.0.0.1"} + } + p := newDOTClientPool(context.Background(), uc, addrs) + // The raw fallback fixture has no bootstrap hostname to infer SNI from. + p.tlsConfig.ServerName = "localhost" + uc.dotClientPool = p + defer uc.CloseTransports() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := p.Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "stalled TLS handshake") + uc.CloseTransports() + retirementWait(t, peerClosed, "handshaking socket closure") + select { + case err := <-result: + if err == nil { + t.Fatal("handshake succeeded after retirement") + } + case <-time.After(3 * time.Second): + t.Fatal("handshake was not canceled") + } + }) + } +} + +func TestCloseTransportsDoTPendingDial(t *testing.T) { + // Per-pool dialer seam holds a real TCP dial before connect; no global hooks. + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: "127.0.0.1:853", BootstrapIP: "127.0.0.1"} + p := newDOTClientPool(context.Background(), uc, nil) + uc.dotClientPool = p + defer uc.CloseTransports() + entered, canceled := make(chan struct{}), make(chan struct{}) + p.dialer.ControlContext = func(ctx context.Context, _, _ string, _ syscall.RawConn) error { + close(entered) + <-ctx.Done() + close(canceled) + return ctx.Err() + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := p.Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "pending TCP dial") + uc.CloseTransports() + retirementWait(t, canceled, "TCP dial cancellation") + select { + case err := <-result: + if err == nil { + t.Fatal("dial succeeded after close") + } + case <-time.After(3 * time.Second): + t.Fatal("dial did not return") + } +} + +func TestDoTParallelDialClosesLosers(t *testing.T) { + const attempts = 4 + cert := generateTestCertificate(t) + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert.tlsCert}}) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + allAccepted, release := make(chan struct{}), make(chan struct{}) + closed, received := make(chan struct{}, attempts), make(chan struct{}, attempts) + var unblock sync.Once + releaseAll := func() { unblock.Do(func() { close(release) }) } + defer releaseAll() + var wg sync.WaitGroup + go func() { + for i := 0; i < attempts; i++ { + c, err := ln.Accept() + if err != nil { + return + } + wg.Add(1) + go func() { + defer wg.Done() + defer c.Close() + defer func() { closed <- struct{}{} }() + _ = c.SetDeadline(time.Now().Add(5 * time.Second)) + <-release + var b [1]byte + if _, err := c.Read(b[:]); err == nil { + received <- struct{}{} + _, _ = io.Copy(io.Discard, c) + } + }() + } + close(allAccepted) + }() + roots := x509.NewCertPool() + roots.AddCert(cert.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), Domain: "localhost", certPool: roots} + addrs := make([]string, attempts) + for i := range addrs { + addrs[i] = "127.0.0.1" + } + p := newDOTClientPool(context.Background(), uc, addrs) + uc.dotClientPool = p + defer uc.CloseTransports() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + type result struct { + conn net.Conn + err error + } + resultCh := make(chan result, 1) + go func() { conn, err := p.getConn(ctx); resultCh <- result{conn, err} }() + retirementWait(t, allAccepted, "all parallel TCP connections") + releaseAll() + var conn net.Conn + select { + case res := <-resultCh: + if res.err != nil { + t.Fatal(res.err) + } + conn = res.conn + case <-time.After(3 * time.Second): + t.Fatal("parallel dial failed to return a winner") + } + // A dial-context cancellation must close losers, not the winning connection. + if _, err := conn.Write([]byte{42}); err != nil { + t.Fatal(err) + } + retirementWait(t, received, "winner data") + for i := 0; i < attempts-1; i++ { + retirementWait(t, closed, "losing parallel socket closure") + } + uc.CloseTransports() + retirementWait(t, closed, "winning socket retirement") + p.putConn(conn, true) + wg.Wait() +} + +func TestDoTIdleCleanupPreservesActive(t *testing.T) { + uc, entered, release, _ := retirementDOTServer(t, true) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := uc.dotClientPool.Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "active query") + uc.closeTransports() // Idle-only cleanup, not transport replacement. + release() + select { + case err := <-result: + if err != nil { + t.Fatalf("idle cleanup aborted active query: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("query did not finish") + } + conn, err := uc.dotClientPool.getConn(ctx) + if err != nil { + t.Fatalf("idle cleanup retired pool: %v", err) + } + uc.dotClientPool.putConn(conn, true) +} + +func TestCloseTransportsDoH3Active(t *testing.T) { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + t.Run(slot, func(t *testing.T) { + entered, release, peerCanceled := make(chan struct{}), make(chan struct{}), make(chan struct{}) + srv := newTestHTTP3Server(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(entered) + select { + case <-r.Context().Done(): + close(peerCanceled) + case <-release: + } + })) + defer close(release) + roots := x509.NewCertPool() + roots.AddCert(srv.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOH3, BootstrapIP: "127.0.0.1", certPool: roots} + rt := uc.newDOH3Transport(context.Background(), nil).(*http3.Transport) + defer rt.Close() + switch slot { + case "default": + uc.http3RoundTripper = rt + case "v4": + uc.http3RoundTripper4 = rt + case "v6": + uc.http3RoundTripper6 = rt + case "aliased": + uc.http3RoundTripper = rt + uc.http3RoundTripper4 = rt + uc.http3RoundTripper6 = rt + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + request := func() error { + req, _ := http.NewRequestWithContext(ctx, "GET", "https://"+srv.addr, nil) + resp, err := rt.RoundTrip(req) + if err == nil { + defer resp.Body.Close() + _, err = io.ReadAll(resp.Body) + } + return err + } + result := make(chan error, 1) + go func() { result <- request() }() + retirementWait(t, entered, "active HTTP/3 request") + uc.CloseTransports() + retirementWait(t, peerCanceled, "HTTP/3 server request cancellation") + select { + case err := <-result: + if err == nil { + t.Fatal("active HTTP/3 request survived close") + } + case <-time.After(3 * time.Second): + t.Fatal("HTTP/3 request was not aborted") + } + if err := request(); err == nil { + t.Fatal("retired HTTP/3 transport accepted another request") + } + uc.CloseTransports() + }) + } +} diff --git a/vpn_dns_config.go b/vpn_dns_config.go new file mode 100644 index 00000000..f1bf91d3 --- /dev/null +++ b/vpn_dns_config.go @@ -0,0 +1,11 @@ +package ctrld + +// VPNDNSConfig represents DNS configuration discovered from a VPN interface. +// Used by the dns-intercept mode to detect VPN split DNS settings and +// route matching queries to VPN DNS servers automatically. +type VPNDNSConfig struct { + InterfaceName string // VPN adapter name (e.g., "F5 Networks VPN") + Servers []string // DNS server IPs (e.g., ["10.50.10.77"]) + Domains []string // Search/match domains (e.g., ["provisur.local"]) + IsExitMode bool // True if this VPN is also the system default resolver (exit node mode) +} diff --git a/vpn_dns_darwin.go b/vpn_dns_darwin.go new file mode 100644 index 00000000..86b72938 --- /dev/null +++ b/vpn_dns_darwin.go @@ -0,0 +1,244 @@ +//go:build darwin + +package ctrld + +import ( + "bufio" + "context" + "net" + "os/exec" + "regexp" + "strconv" + "strings" +) + +// DiscoverVPNDNS discovers DNS servers and search domains from VPN interfaces on macOS. +// Parses `scutil --dns` output to find VPN resolver configurations. +func DiscoverVPNDNS(ctx context.Context) []VPNDNSConfig { + logger := LoggerFromCtx(ctx) + + Log(ctx, logger.Debug(), "Discovering VPN DNS configurations on macOS") + + cmd := exec.CommandContext(ctx, "scutil", "--dns") + output, err := cmd.Output() + if err != nil { + Log(ctx, logger.Error().Err(err), "Failed to execute scutil --dns") + return nil + } + + return parseScutilOutput(ctx, string(output)) +} + +// parseScutilOutput parses the output of `scutil --dns` to extract VPN DNS configurations. +func parseScutilOutput(ctx context.Context, output string) []VPNDNSConfig { + logger := LoggerFromCtx(ctx) + + Log(ctx, logger.Debug(), "Parsing scutil --dns output") + + resolverBlockRe := regexp.MustCompile(`resolver #(\d+)`) + searchDomainRe := regexp.MustCompile(`search domain\[\d+\] : (.+)`) + // Matches singular "domain : value" entries (e.g., Tailscale per-domain resolvers). + singleDomainRe := regexp.MustCompile(`^domain\s+:\s+(.+)`) + nameserverRe := regexp.MustCompile(`nameserver\[\d+\] : (.+)`) + ifIndexRe := regexp.MustCompile(`if_index : (\d+) \((.+)\)`) + + var vpnConfigs []VPNDNSConfig + var currentResolver *resolverInfo + var allResolvers []resolverInfo + + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + if match := resolverBlockRe.FindStringSubmatch(line); match != nil { + if currentResolver != nil { + allResolvers = append(allResolvers, *currentResolver) + } + resolverNum, _ := strconv.Atoi(match[1]) + currentResolver = &resolverInfo{ + Number: resolverNum, + } + continue + } + + if currentResolver == nil { + continue + } + + if match := searchDomainRe.FindStringSubmatch(line); match != nil { + domain := strings.TrimSpace(match[1]) + if domain != "" { + currentResolver.Domains = append(currentResolver.Domains, domain) + } + continue + } + + // Parse singular "domain : value" (used by Tailscale per-domain resolvers). + if match := singleDomainRe.FindStringSubmatch(line); match != nil { + domain := strings.TrimSpace(match[1]) + if domain != "" { + currentResolver.Domains = append(currentResolver.Domains, domain) + } + continue + } + + if match := nameserverRe.FindStringSubmatch(line); match != nil { + server := strings.TrimSpace(match[1]) + if ip := net.ParseIP(server); ip != nil && !ip.IsLoopback() { + currentResolver.Servers = append(currentResolver.Servers, server) + } + continue + } + + if match := ifIndexRe.FindStringSubmatch(line); match != nil { + currentResolver.InterfaceName = strings.TrimSpace(match[2]) + continue + } + + if strings.HasPrefix(line, "flags") { + if idx := strings.Index(line, ":"); idx >= 0 { + currentResolver.Flags = strings.TrimSpace(line[idx+1:]) + } + continue + } + } + + if currentResolver != nil { + allResolvers = append(allResolvers, *currentResolver) + } + + for _, resolver := range allResolvers { + if isSplitDNSResolver(ctx, &resolver) { + ifaceName := resolver.InterfaceName + + // When scutil doesn't provide if_index (common with Tailscale MagicDNS + // per-domain resolvers), look up the outbound interface from the routing + // table. This is needed for interface-scoped pf exemptions — without the + // interface name, we can't generate rules that let the VPN's Network + // Extension handle DNS queries from all processes. + if ifaceName == "" && len(resolver.Servers) > 0 { + if routeIface := resolveInterfaceForIP(ctx, resolver.Servers[0]); routeIface != "" { + ifaceName = routeIface + Log(ctx, logger.Debug(), "Resolver #%d: resolved interface %q from routing table for %s", + resolver.Number, routeIface, resolver.Servers[0]) + } + } + + config := VPNDNSConfig{ + InterfaceName: ifaceName, + Servers: resolver.Servers, + Domains: resolver.Domains, + } + + vpnConfigs = append(vpnConfigs, config) + + Log(ctx, logger.Debug(), "Found VPN DNS config - Interface: %s, Servers: %v, Domains: %v", + config.InterfaceName, config.Servers, config.Domains) + } + } + + // Detect exit mode: if a VPN DNS server IP also appears as the system's default + // resolver (no search domains, no Supplemental flag), the VPN is routing ALL traffic + // (not just specific domains). In exit mode, ctrld must continue intercepting DNS + // on the VPN interface to enforce its profile on all queries. + defaultResolverIPs := make(map[string]bool) + for _, resolver := range allResolvers { + if len(resolver.Servers) > 0 && len(resolver.Domains) == 0 && + !strings.Contains(resolver.Flags, "Supplemental") && + !strings.Contains(resolver.Flags, "Scoped") { + for _, server := range resolver.Servers { + defaultResolverIPs[server] = true + } + } + } + for i := range vpnConfigs { + for _, server := range vpnConfigs[i].Servers { + if defaultResolverIPs[server] { + vpnConfigs[i].IsExitMode = true + Log(ctx, logger.Info(), "VPN DNS config on %s detected as EXIT MODE — server %s is also the system default resolver", + vpnConfigs[i].InterfaceName, server) + break + } + } + } + + Log(ctx, logger.Debug(), "VPN DNS discovery completed: found %d VPN interfaces", len(vpnConfigs)) + return vpnConfigs +} + +// resolveInterfaceForIP uses the macOS routing table to determine which network +// interface would be used to reach the given IP address. This is a fallback for +// when scutil --dns doesn't include if_index in the resolver entry (common with +// Tailscale MagicDNS per-domain resolvers). +// +// Runs: route -n get and parses the "interface:" line from the output. +// Returns empty string on any error (callers should treat as "unknown interface"). +func resolveInterfaceForIP(ctx context.Context, ip string) string { + logger := LoggerFromCtx(ctx) + + cmd := exec.CommandContext(ctx, "route", "-n", "get", ip) + output, err := cmd.Output() + if err != nil { + Log(ctx, logger.Debug(), "route -n get %s failed: %v", ip, err) + return "" + } + + // Parse "interface: utun11" from route output. + for _, line := range strings.Split(string(output), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "interface:") { + iface := strings.TrimSpace(strings.TrimPrefix(line, "interface:")) + if iface != "" && iface != "lo0" { + return iface + } + } + } + return "" +} + +// resolverInfo holds information about a resolver block from scutil --dns output. +type resolverInfo struct { + Number int + InterfaceName string + Servers []string + Domains []string + Flags string // Raw flags line (e.g., "Supplemental, Request A records") +} + +// isSplitDNSResolver reports whether a scutil --dns resolver entry represents a +// split DNS configuration that ctrld should forward to. Any resolver with both +// non-loopback DNS servers and search domains qualifies — this covers VPN adapters +// (F5, Tailscale, Cisco AnyConnect, etc.) and any other virtual interface that +// registers search domains (e.g., corporate proxies, containers). +// +// We intentionally avoid heuristics about interface names or domain suffixes: +// if an interface declares "these domains resolve via these servers," we honor it. +// The only exclusions are mDNS entries (bare ".local" without an interface binding). +// +// Note: loopback servers are already filtered out during parsing in parseScutilOutput. +func isSplitDNSResolver(ctx context.Context, resolver *resolverInfo) bool { + logger := LoggerFromCtx(ctx) + + // Must have both DNS servers and search domains to be a useful split DNS route. + if len(resolver.Servers) == 0 || len(resolver.Domains) == 0 { + Log(ctx, logger.Debug(), "Resolver #%d: skipping — no servers (%d) or no domains (%d)", + resolver.Number, len(resolver.Servers), len(resolver.Domains)) + return false + } + + // Skip multicast DNS entries. scutil --dns shows a resolver for ".local" that + // handles mDNS — it has no interface binding and the sole domain is "local". + // Real VPN entries with ".local" suffix (e.g., "provisur.local") will have an + // interface name or additional domains. + if len(resolver.Domains) == 1 { + domain := strings.ToLower(strings.TrimSpace(resolver.Domains[0])) + if domain == "local" || domain == ".local" { + Log(ctx, logger.Debug(), "Resolver #%d: skipping — mDNS resolver", resolver.Number) + return false + } + } + + Log(ctx, logger.Debug(), "Resolver #%d: split DNS resolver — interface: %q, servers: %v, domains: %v", + resolver.Number, resolver.InterfaceName, resolver.Servers, resolver.Domains) + return true +} diff --git a/vpn_dns_linux.go b/vpn_dns_linux.go new file mode 100644 index 00000000..dbff6cc7 --- /dev/null +++ b/vpn_dns_linux.go @@ -0,0 +1,240 @@ +//go:build linux + +package ctrld + +import ( + "bufio" + "context" + "net" + "os/exec" + "regexp" + "strings" + +) + +// DiscoverVPNDNS discovers DNS servers and search domains from VPN interfaces on Linux. +// Uses resolvectl status to find per-link DNS configurations. +func DiscoverVPNDNS(ctx context.Context) []VPNDNSConfig { + logger := LoggerFromCtx(ctx) + + Log(ctx, logger.Debug(), "Discovering VPN DNS configurations on Linux") + + // Try resolvectl first (systemd-resolved) + if configs := parseResolvectlStatus(ctx); len(configs) > 0 { + return configs + } + + // Fallback: check for VPN interfaces with DNS in /etc/resolv.conf + Log(ctx, logger.Debug(), "resolvectl not available or no results, trying fallback method") + return parseVPNInterfacesDNS(ctx) +} + +// parseResolvectlStatus parses the output of `resolvectl status` to extract VPN DNS configurations. +func parseResolvectlStatus(ctx context.Context) []VPNDNSConfig { + logger := LoggerFromCtx(ctx) + + cmd := exec.CommandContext(ctx, "resolvectl", "status") + output, err := cmd.Output() + if err != nil { + Log(ctx, logger.Debug(), "Failed to execute resolvectl status: %v", err) + return nil + } + + Log(ctx, logger.Debug(), "Parsing resolvectl status output") + + // Regular expressions to match link sections and their properties + linkRe := regexp.MustCompile(`^Link (\d+) \((.+)\):`) + dnsServersRe := regexp.MustCompile(`^\s+DNS Servers?: (.+)`) + dnsDomainsRe := regexp.MustCompile(`^\s+DNS Domain: (.+)`) + + var vpnConfigs []VPNDNSConfig + var currentLink *linkInfo + + scanner := bufio.NewScanner(strings.NewReader(string(output))) + for scanner.Scan() { + line := scanner.Text() + + // Check for new link section + if match := linkRe.FindStringSubmatch(line); match != nil { + // Process previous link if it's a VPN + if currentLink != nil && isVPNLink(ctx, currentLink) { + config := VPNDNSConfig{ + InterfaceName: currentLink.InterfaceName, + Servers: currentLink.Servers, + Domains: currentLink.Domains, + } + vpnConfigs = append(vpnConfigs, config) + + Log(ctx, logger.Debug(), "Found VPN DNS config - Interface: %s, Servers: %v, Domains: %v", + config.InterfaceName, config.Servers, config.Domains) + } + + // Start new link + currentLink = &linkInfo{ + InterfaceName: strings.TrimSpace(match[2]), + } + continue + } + + if currentLink == nil { + continue + } + + // Parse DNS servers + if match := dnsServersRe.FindStringSubmatch(line); match != nil { + serverList := strings.TrimSpace(match[1]) + for _, server := range strings.Fields(serverList) { + if ip := net.ParseIP(server); ip != nil && !ip.IsLoopback() { + currentLink.Servers = append(currentLink.Servers, server) + } + } + continue + } + + // Parse DNS domains + if match := dnsDomainsRe.FindStringSubmatch(line); match != nil { + domainList := strings.TrimSpace(match[1]) + for _, domain := range strings.Fields(domainList) { + domain = strings.TrimSpace(domain) + if domain != "" { + currentLink.Domains = append(currentLink.Domains, domain) + } + } + continue + } + } + + // Don't forget the last link + if currentLink != nil && isVPNLink(ctx, currentLink) { + config := VPNDNSConfig{ + InterfaceName: currentLink.InterfaceName, + Servers: currentLink.Servers, + Domains: currentLink.Domains, + } + vpnConfigs = append(vpnConfigs, config) + + Log(ctx, logger.Debug(), "Found VPN DNS config - Interface: %s, Servers: %v, Domains: %v", + config.InterfaceName, config.Servers, config.Domains) + } + + Log(ctx, logger.Debug(), "resolvectl parsing completed: found %d VPN interfaces", len(vpnConfigs)) + return vpnConfigs +} + +// parseVPNInterfacesDNS is a fallback method that looks for VPN interfaces and tries to +// find their DNS configuration from various sources. +func parseVPNInterfacesDNS(ctx context.Context) []VPNDNSConfig { + logger := LoggerFromCtx(ctx) + + Log(ctx, logger.Debug(), "Using fallback method to detect VPN DNS") + + // Get list of network interfaces + interfaces, err := net.Interfaces() + if err != nil { + Log(ctx, logger.Error().Err(err), "Failed to get network interfaces") + return nil + } + + var vpnConfigs []VPNDNSConfig + + for _, iface := range interfaces { + if !isVPNInterfaceName(iface.Name) { + continue + } + + // Check if interface is up + if iface.Flags&net.FlagUp == 0 { + continue + } + + Log(ctx, logger.Debug(), "Found potential VPN interface: %s", iface.Name) + + // For VPN interfaces, we can't easily determine their specific DNS settings + // without more complex parsing of network manager configurations. + // This is a basic implementation that could be extended. + + // For now, we'll skip this fallback as it's complex and platform-specific + Log(ctx, logger.Debug(), "Fallback DNS detection not implemented for interface: %s", iface.Name) + } + + Log(ctx, logger.Debug(), "Fallback method completed: found %d VPN interfaces", len(vpnConfigs)) + return vpnConfigs +} + +// linkInfo holds information about a network link from resolvectl status. +type linkInfo struct { + InterfaceName string + Servers []string + Domains []string +} + +// isVPNLink determines if a network link configuration looks like it belongs to a VPN. +func isVPNLink(ctx context.Context, link *linkInfo) bool { + logger := LoggerFromCtx(ctx) + + // Must have both DNS servers and domains + if len(link.Servers) == 0 || len(link.Domains) == 0 { + Log(ctx, logger.Debug(), "Link %s: insufficient config (servers: %d, domains: %d)", + link.InterfaceName, len(link.Servers), len(link.Domains)) + return false + } + + // Check interface name patterns + if isVPNInterfaceName(link.InterfaceName) { + Log(ctx, logger.Debug(), "Link %s: identified as VPN based on interface name", link.InterfaceName) + return true + } + + // Look for routing domains (prefixed with ~) + hasRoutingDomain := false + for _, domain := range link.Domains { + if strings.HasPrefix(domain, "~") { + hasRoutingDomain = true + break + } + } + + if hasRoutingDomain { + Log(ctx, logger.Debug(), "Link %s: identified as VPN based on routing domain", link.InterfaceName) + return true + } + + // Additional heuristics similar to macOS + hasPrivateDNS := false + for _, server := range link.Servers { + if ip := net.ParseIP(server); ip != nil && ip.IsPrivate() { + hasPrivateDNS = true + break + } + } + + hasVPNDomains := false + for _, domain := range link.Domains { + domain = strings.ToLower(strings.TrimPrefix(domain, "~")) + if strings.HasSuffix(domain, ".local") || + strings.HasSuffix(domain, ".corp") || + strings.HasSuffix(domain, ".internal") || + strings.Contains(domain, "vpn") { + hasVPNDomains = true + break + } + } + + if hasPrivateDNS && hasVPNDomains { + Log(ctx, logger.Debug(), "Link %s: identified as VPN based on private DNS + VPN domains", link.InterfaceName) + return true + } + + Log(ctx, logger.Debug(), "Link %s: not identified as VPN link", link.InterfaceName) + return false +} + +// isVPNInterfaceName checks if an interface name looks like a VPN interface. +func isVPNInterfaceName(name string) bool { + name = strings.ToLower(name) + return strings.HasPrefix(name, "tun") || + strings.HasPrefix(name, "tap") || + strings.HasPrefix(name, "ppp") || + strings.HasPrefix(name, "vpn") || + strings.Contains(name, "vpn") +} \ No newline at end of file diff --git a/vpn_dns_others.go b/vpn_dns_others.go new file mode 100644 index 00000000..8bf8b9e6 --- /dev/null +++ b/vpn_dns_others.go @@ -0,0 +1,15 @@ +//go:build !windows && !darwin && !linux + +package ctrld + +import ( + "context" +) + +// DiscoverVPNDNS is a stub implementation for unsupported platforms. +// Returns nil to indicate no VPN DNS configurations found. +func DiscoverVPNDNS(ctx context.Context) []VPNDNSConfig { + logger := LoggerFromCtx(ctx) + Log(ctx, logger.Debug(), "VPN DNS discovery not implemented for this platform") + return nil +} \ No newline at end of file diff --git a/vpn_dns_windows.go b/vpn_dns_windows.go new file mode 100644 index 00000000..e02cf15b --- /dev/null +++ b/vpn_dns_windows.go @@ -0,0 +1,130 @@ +//go:build windows + +package ctrld + +import ( + "context" + "strings" + "syscall" + + "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" +) + +// DiscoverVPNDNS discovers DNS servers and search domains from non-physical (VPN) interfaces. +// Only called when dnsIntercept is active. +func DiscoverVPNDNS(ctx context.Context) []VPNDNSConfig { + logger := LoggerFromCtx(ctx) + + Log(ctx, logger.Debug(), "Discovering VPN DNS configurations on Windows") + + flags := winipcfg.GAAFlagIncludeGateways | winipcfg.GAAFlagIncludePrefix + aas, err := winipcfg.GetAdaptersAddresses(syscall.AF_UNSPEC, flags) + if err != nil { + Log(ctx, logger.Error().Err(err), "Failed to get adapters addresses") + return nil + } + + Log(ctx, logger.Debug(), "Found %d network adapters", len(aas)) + + // Get valid (physical/hardware) interfaces to filter them out + validInterfacesMap := ValidInterfaces(ctx) + + var vpnConfigs []VPNDNSConfig + + for _, aa := range aas { + if aa.OperStatus != winipcfg.IfOperStatusUp { + Log(ctx, logger.Debug(), "Skipping adapter %s - not up, status: %d", + aa.FriendlyName(), aa.OperStatus) + continue + } + + if aa.IfType == winipcfg.IfTypeSoftwareLoopback { + Log(ctx, logger.Debug(), "Skipping %s (software loopback)", aa.FriendlyName()) + continue + } + + // INVERT the ValidInterfaces filter: we want non-physical/non-hardware adapters. + _, isValidPhysical := validInterfacesMap[aa.FriendlyName()] + if isValidPhysical { + Log(ctx, logger.Debug(), "Skipping %s (physical/hardware adapter)", aa.FriendlyName()) + continue + } + + // Skip adapters that have no routable unicast addresses. An adapter + // with only link-local (fe80::) or APIPA (169.254.x.x) addresses is + // not actually connected — its DNS servers are stale. This prevents + // picking up e.g. Tailscale's adapter when the app is installed but + // disconnected (OperStatus reports Up but only APIPA addresses exist). + hasRoutableAddr := false + for a := aa.FirstUnicastAddress; a != nil; a = a.Next { + ip := a.Address.IP() + if ip == nil { + continue + } + if !ip.IsLinkLocalUnicast() { + hasRoutableAddr = true + break + } + } + if !hasRoutableAddr { + Log(ctx, logger.Debug(), "Skipping %s - no routable addresses (likely disconnected)", aa.FriendlyName()) + continue + } + + var servers []string + for dns := aa.FirstDNSServerAddress; dns != nil; dns = dns.Next { + ip := dns.Address.IP() + if ip == nil { + continue + } + + ipStr := ip.String() + if ip.IsLoopback() { + continue + } + + servers = append(servers, ipStr) + } + + // Check adapter-specific (connection-specific) DNS suffix first, + // since we want to map per-adapter DNS servers to per-adapter suffixes. + // This is what most traditional VPNs set (F5, Cisco AnyConnect, GlobalProtect). + var domains []string + if connSuffix := strings.TrimSpace(aa.DNSSuffix()); connSuffix != "" { + domains = append(domains, connSuffix) + Log(ctx, logger.Debug(), "Using connection-specific DNS suffix for %s: %s", + aa.FriendlyName(), connSuffix) + } + + // Then check supplemental DNS suffix list (used by Tailscale and + // VPN clients that register search domains via the DNS Client API). + for suffix := aa.FirstDNSSuffix; suffix != nil; suffix = suffix.Next { + domain := strings.TrimSpace(suffix.String()) + if domain != "" { + domains = append(domains, domain) + } + } + + // Accept VPN adapters with DNS servers even without domains. + // Domain-less configs still provide useful DNS server IPs that + // can serve existing split-rules and OS resolver queries. + if len(servers) > 0 { + config := VPNDNSConfig{ + InterfaceName: aa.FriendlyName(), + Servers: servers, + Domains: domains, + } + + vpnConfigs = append(vpnConfigs, config) + + Log(ctx, logger.Debug(), "Found VPN DNS config - Interface: %s, Servers: %v, Domains: %v", + config.InterfaceName, config.Servers, config.Domains) + } else { + Log(ctx, logger.Debug(), "Skipping %s - no DNS servers found", + aa.FriendlyName()) + } + } + + Log(ctx, logger.Debug(), "VPN DNS discovery completed: found %d VPN interfaces", len(vpnConfigs)) + return vpnConfigs +}